fix: make mapping of generics work correctly

fixes #27

.gitlab-ci.yml

@@ -65,8 +65,8 @@ mapping:
       alias: elasticsearch
   script:
     - npm install @openstapps/core
-    - node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength"
+    - node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength,tjs-format"
-    - curl http://elasticsearch:9200/stapps --upload-file mapping.json -o response.json
+    - curl http://elasticsearch:9200/_template --upload-file mapping.json -o response.json
     - cat response.json
     - grep -q "\"acknowledged\":true" response.json
     # - curl --show-error --fail http://elasticsearch:9200/stapps --upload-file mapping.json

src/mapping.ts

@@ -31,13 +31,13 @@ import {
 import {AggregationSchema, ESNestedAggregation} from './mappings/aggregation-definitions';
 import {fieldmap, filterableMap, filterableTagName} from './mappings/definitions/fieldmap';
 import {premaps} from './mappings/definitions/premap';
+import {settings} from './mappings/definitions/settings';
 import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap';
 import {
   ElasticsearchDynamicTemplate,
   ElasticsearchMappings,
-  ElasticsearchObject,
+  ElasticsearchObject, ElasticsearchType,
   ElasticsearchValue,
-  ReflectionGeneric,
 } from './mappings/mapping-definitions';
 
 let dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
@@ -116,29 +116,29 @@ function composeErrorMessage(path: string, topTypeName: string, typeName: string
  * @param type the ReferenceType of a DeclarationReflection
  * @param out the previous reflection, it then overrides all parameters or keeps old ones
  * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
  * @param tags any tags attached to the type
  */
 function getReflectionGeneric(type: ReferenceType,
-                              out: ReflectionGeneric[],
+                              out: Map<string, ElasticsearchValue>,
-                              path: string, topTypeName: string, tags: CommentTag[]): ReflectionGeneric[] {
+                              topTypeName: string,
+                              path: string, tags: CommentTag[]): Map<string, ElasticsearchValue> {
   if (typeof type.typeArguments !== 'undefined'
     && type.reflection instanceof DeclarationReflection
-    && typeof type.reflection.typeParameters !== 'undefined'
+    && typeof type.reflection.typeParameters !== 'undefined') {
-    && type.typeArguments.length === type.reflection.typeParameters.length) {
+    for (let i = 0; i < type.reflection.typeParameters.length; i++) {
-    for (let i = 0; i < type.typeArguments.length; i++) {
+      if (i < type.typeArguments.length) {
-      let replaced = false;
+        out
-      for (const old of out) {
+          .set(type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags));
-        if (old.name === type.reflection.typeParameters[i].name) {
+      } else {
-          old.value = handleType(type.typeArguments[i], out, path, topTypeName, tags);
+        // this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061
-          replaced = true;
+        // we have no way to know the type here, so we have to use this.
-        }
+        out.set(type.reflection.typeParameters[i].name, {
-      }
+          dynamic: true,
-      if (!replaced) {
+          properties: {},
-        out.push({
-          name: type.reflection.typeParameters[i].name,
-          value: handleType(type.typeArguments[i], out, path, topTypeName, tags),
         });
+
+        Logger.warn(`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
+          ` in TypeDoc. It has been replaced by a dynamic type.`);
       }
     }
   }
@@ -149,13 +149,15 @@ function getReflectionGeneric(type: ReferenceType,
 /**
  * Handles a ReferenceType that has no value
  *
+ * Most of the times that is an external type.
+ *
  * @param ref the ReferenceType
  * @param generics the generics from levels above, so we can use them without having access to the parent
  * @param path the current path to the object we are in
  * @param topTypeName the name of the SCThingType
  * @param tags any tags attached to the type
  */
-function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[],
+function handleExternalType(ref: ReferenceType, generics: Map<string, ElasticsearchValue>,
                             path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
   for (const premap in premaps) {
     if (premap === ref.name) {
@@ -170,7 +172,8 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
       return {type: ElasticsearchDataType.parse_error};
     }
 
-    return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path, topTypeName, tags),
+    return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, new Map(generics), path,
+      topTypeName, tags),
       path, topTypeName, tags),
       path, topTypeName, tags);
   }
@@ -195,17 +198,17 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
  * @param topTypeName the name of the SCThingType
  */
 function handleDeclarationReflection(decl: DeclarationReflection,
-                                     generics: ReflectionGeneric[],
+                                     generics: Map<string, ElasticsearchValue>,
                                      path: string,
                                      topTypeName: string,
                                      inheritedTags?: CommentTag[]):
   ElasticsearchValue {
   // check if we have an object referencing a generic
-  for (const gRefl of generics) {
+  if (generics.has(decl.name)) { // if the object name is the same as the generic name
-    if (gRefl.name === decl.name) { // if the object name is the same as the generic name
+    return readFieldTags(generics.get(decl.name) as ElasticsearchObject | ElasticsearchType, path, topTypeName,
-      return readFieldTags(gRefl.value, path, topTypeName, getCommentTags(decl));
+      typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
     // use the value defined by the generic
-    }
+
   }
 
   // start the actual handling process
@@ -221,7 +224,7 @@ function handleDeclarationReflection(decl: DeclarationReflection,
       empty = false;
       const template: ElasticsearchDynamicTemplate = {};
       template[decl.name] = {
-        mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path, topTypeName),
+        mapping: handleDeclarationReflection(param as DeclarationReflection, new Map(generics), path, topTypeName),
         match: '*',
         match_mapping_type: '*',
         path_match: `${path}*`,
@@ -238,11 +241,12 @@ function handleDeclarationReflection(decl: DeclarationReflection,
   if (typeof decl.children !== 'undefined' && decl.children.length > 0) {
     for (const child of decl.children) {
       empty = false;
-      out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`, topTypeName);
+      out.properties[child.name] =
+        handleDeclarationReflection(child, new Map(generics), `${path}${child.name}.`, topTypeName);
     }
   } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY
     // get inherited tags
-    return handleType(decl.type, generics, path, topTypeName, getCommentTags(decl));
+    return handleType(decl.type, new Map(generics), path, topTypeName, getCommentTags(decl));
   } else if (decl.kindString === 'Enumeration member') {
     return readTypeTags(typeof decl.defaultValue, path, topTypeName, getCommentTags(decl, inheritedTags));
   }
@@ -258,6 +262,7 @@ function handleDeclarationReflection(decl: DeclarationReflection,
  * Reads all comment tags, including inherited ones
  *
  * @param decl the DeclarationReflection to read the tags from
+ * @param inheritedTags any tags that might have been inherited by a parent
  */
 function getCommentTags(decl: DeclarationReflection, inheritedTags: CommentTag[] = []): CommentTag[] {
   let out: CommentTag[] = decl.comment instanceof Comment ?
@@ -305,7 +310,7 @@ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): Co
  * @param tags any tags attached to the type
  */
 function handleUnionType(type: UnionType,
-                         generics: ReflectionGeneric[],
+                         generics: Map<string, ElasticsearchValue>,
                          path: string,
                          topTypeName: string,
                          tags: CommentTag[]): ElasticsearchValue {
@@ -315,7 +320,7 @@ function handleUnionType(type: UnionType,
     if (subType instanceof IntrinsicType && subType.name === 'undefined') {
       continue;
     }
-    list.push(handleType(subType, generics, path, topTypeName, tags));
+    list.push(handleType(subType, new Map(generics), path, topTypeName, tags));
   }
 
   if (list.length > 0) {
@@ -343,11 +348,12 @@ function handleUnionType(type: UnionType,
  * @param topTypeName the name of the SCThingType
  * @param tags any tags attached to the type
  */
-function handleType(type: Type, generics: ReflectionGeneric[], path: string, topTypeName: string, tags: CommentTag[]):
+function handleType(type: Type, generics: Map<string, ElasticsearchValue>, path: string, topTypeName: string,
+                    tags: CommentTag[]):
   ElasticsearchValue {
   // logger.log((type as any).name);
   if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type
-    return handleType(type.elementType, generics, path, topTypeName, tags);
+    return handleType(type.elementType, new Map(generics), path, topTypeName, tags);
   }
   if (type.type === 'stringLiteral') { // a string literal, usually for type
     return readTypeTags(type.type, path, topTypeName, tags);
@@ -356,23 +362,21 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top
     return readTypeTags(type.name, path, topTypeName, tags);
   }
   if (type instanceof UnionType) { // the union type...
-    return handleUnionType(type, generics, path, topTypeName, tags);
+    return handleUnionType(type, new Map(generics), path, topTypeName, tags);
   }
   if (type instanceof ReferenceType) {
     if (typeof type.reflection !== 'undefined') {
       // there is really no way to make this typesafe, every element in DeclarationReflection is optional.
       return handleDeclarationReflection(type.reflection as DeclarationReflection,
-        getReflectionGeneric(type, generics, path, topTypeName, tags), path, topTypeName, tags);
+        getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags);
     }
 
-    return handleRefWithoutReflection(type, generics, path, topTypeName, tags);
+    return handleExternalType(type, new Map(generics), path, topTypeName, tags);
   }
   if (type instanceof TypeParameterType) {
     // check if we have an object referencing a generic
-    for (const gRefl of generics) {
+    if (generics.has(type.name)) {
-      if (gRefl.name === type.name) { // if the object name is the same as the generic name
+      return generics.get(type.name) as ElasticsearchObject | ElasticsearchType;
-        return gRefl.value; // use the value defined by the generic
-      }
     }
     composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
 
@@ -380,7 +384,7 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top
 
   }
   if (type instanceof ReflectionType) {
-    return readFieldTags(handleDeclarationReflection(type.declaration, generics, path, topTypeName),
+    return readFieldTags(handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
       path, topTypeName, tags);
   }
 
@@ -502,7 +506,7 @@ function readTypeTags(type: string, path: string, topTypeName: string, tags: Com
 
     return out;
   }
-  if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type
+  if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types
     return {
       dynamic: true,
       properties: {},
@@ -587,21 +591,33 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
       },
     };
 
-    out[typeName] =
+    let typeNameWithoutSpaces = typeName.toLowerCase();
-      handleDeclarationReflection(_interface, [], '', typeName) as ElasticsearchObject;
+    while (typeNameWithoutSpaces.includes(' ')) {
-    out[typeName].properties.creation_date = {
+      typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_');
+    }
+    const templateName = `template_${typeNameWithoutSpaces}`;
+
+    out[templateName] = {
+      mappings: {
+        [typeName]: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject,
+      },
+      settings: settings,
+      template: `stapps_${typeNameWithoutSpaces}*`,
+    }
+      ;
+    out[templateName].mappings[typeName].properties.creation_date = {
       type: ElasticsearchDataType.date,
     };
 
-    out[typeName].dynamic_templates = dynamicTemplates;
+    out[templateName].mappings[typeName].dynamic_templates = dynamicTemplates;
 
     // Set some properties
-    out[typeName]._source = {
+    out[templateName].mappings[typeName]._source = {
       excludes: [
         'creation_date',
       ],
     };
-    out[typeName].date_detection = false;
+    out[templateName].mappings[typeName].date_detection = false;
 
     dynamicTemplates = [];
 

src/mappings/definitions/premap.ts

@@ -12,15 +12,17 @@
  * You should have received a copy of the GNU General Public License along with
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
-import {Logger} from '@openstapps/logger';
+import {ElasticsearchPremap} from '../mapping-definitions';
-import {ElasticsearchPremap, ElasticsearchValue} from '../mapping-definitions';
 import {ElasticsearchDataType} from './typemap';
 
 export const premaps: ElasticsearchPremap = {
   CoordinateReferenceSystem: {
-    precision: '1m',
+    dynamic: true,
-    tree: 'quadtree',
+    properties: {
-    type: ElasticsearchDataType.geo_shape,
+      type: {
+        type: ElasticsearchDataType.keyword,
+      },
+    },
   },
   LineString: {
     precision: '1m',
@@ -50,27 +52,9 @@ export const premaps: ElasticsearchPremap = {
         type: ElasticsearchDataType.keyword,
       },
       value: {
-        dynamic: true,
+        // this is actually an 'any' type, however ES does not really support that.
-        properties: {},
+        type: ElasticsearchDataType.keyword,
       },
     },
   },
 };
-
-/**
- * Gets an ElasticsearchValue for a name
- *
- * @param name the name of the premap
- */
-export function getPremap(name: string): ElasticsearchValue {
-  for (const premap in premaps) {
-    if (premap === name) {
-      return premaps[premap];
-    }
-  }
-
-  // tslint:disable-next-line:no-floating-promises
-  Logger.error(`Missing pre-map for external type ${name}`);
-
-  return {type: ElasticsearchDataType.missing_premap};
-}

src/mappings/mapping-definitions.ts

@@ -24,23 +24,6 @@ import {ElasticsearchDataType} from './definitions/typemap';
  */
 export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject | ElasticsearchGeoShape;
 
-/**
- * Used internally for saving a generic value contained in a reflection
- */
-export interface ReflectionGeneric {
-  /**
-   * The name of the generic
-   *
-   * For example in `<number A>` the name would be 'A'
-   */
-  name: string;
-
-  /**
-   * The value of the generic
-   */
-  value: ElasticsearchValue;
-}
-
 /**
  * The Typemap is used to get the corresponding ElasicsearchDataType for a name provided by the ProjectReflection
  */
@@ -273,7 +256,24 @@ export interface ElasticsearchObject {
   };
 }
 
-export type ElasticsearchMapping = ElasticsearchObject;
+export interface ElasticsearchMapping {
+  /**
+   * The mappings of the index
+   */
+  mappings: {
+    [name: string]: ElasticsearchObject;
+  };
+
+  /**
+   * The settings for the index
+   */
+  settings: unknown;
+
+  /**
+   * The name of the index
+   */
+  template: string;
+}
 
 // TODO: docs
 export interface ElasticsearchMappings {