fix: make mapping tags work for overwritten values

src/cli.ts

@@ -102,7 +102,7 @@ commander
 
     // write documentation to file
     // tslint:disable-next-line:no-magic-numbers
-    writeFileSync(mappingPath, JSON.stringify(result.template, null, 2));
+    writeFileSync(mappingPath, JSON.stringify(result, null, 2));
 
     Logger.ok(`Elasticsearch mapping written to ${mappingPath}.`);
   });

src/mapping.ts

@@ -18,33 +18,37 @@ import {stringify} from 'flatted';
 import {DeclarationReflection, ProjectReflection} from 'typedoc';
 import {
   ArrayType,
+  Comment,
   CommentTag,
   IntrinsicType,
-  ReferenceType, ReflectionType,
+  ReferenceType,
+  ReflectionType,
   StringLiteralType,
-  Type, TypeParameterType,
+  Type,
+  TypeParameterType,
   UnionType,
 } from 'typedoc/dist/lib/models';
+import {AggregationSchema, ESNestedAggregation} from './mappings/aggregation-definitions';
 import {fieldmap, filterableMap, filterableTagName} from './mappings/definitions/fieldmap';
 import {premaps} from './mappings/definitions/premap';
-import {settings} from './mappings/definitions/settings';
 import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap';
 import {
   ElasticsearchDynamicTemplate,
+  ElasticsearchMappings,
   ElasticsearchObject,
-  ElasticsearchTemplate,
   ElasticsearchValue,
   ReflectionGeneric,
 } from './mappings/mapping-definitions';
 
-const dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
+let dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
 let errors: string[] = [];
 let showErrors = true;
 
-let aggregatablePaths: string[] = [];
+let aggregations: AggregationSchema = {};
 
 const indexableTag = 'indexable';
 const aggregatableTag = 'aggregatable';
+const aggregatableTagParameterGlobal = 'global';
 
 let ignoredTagsList = ['indexable', 'validatable'];
 
@@ -193,13 +197,13 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
 function handleDeclarationReflection(decl: DeclarationReflection,
                                      generics: ReflectionGeneric[],
                                      path: string,
-                                     topTypeName: string):
+                                     topTypeName: string,
+                                     inheritedTags?: CommentTag[]):
   ElasticsearchValue {
   // check if we have an object referencing a generic
   for (const gRefl of generics) {
     if (gRefl.name === decl.name) { // if the object name is the same as the generic name
-      return readFieldTags(gRefl.value, path, topTypeName,
-        typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
+      return readFieldTags(gRefl.value, path, topTypeName, getCommentTags(decl));
       // use the value defined by the generic
     }
   }
@@ -226,6 +230,10 @@ function handleDeclarationReflection(decl: DeclarationReflection,
     }
   }
 
+  if (decl.kindString === 'Enumeration') {
+    return readTypeTags('string', path, topTypeName, getCommentTags(decl, inheritedTags));
+  }
+
   // check all the children, so in this case we are dealing with an OBJECT
   if (typeof decl.children !== 'undefined' && decl.children.length > 0) {
     for (const child of decl.children) {
@@ -233,19 +241,55 @@ function handleDeclarationReflection(decl: DeclarationReflection,
       out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`, topTypeName);
     }
   } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY
-    return handleType(decl.type, generics, path, topTypeName,
-      typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
+    // get inherited tags
+    return handleType(decl.type, generics, path, topTypeName, getCommentTags(decl));
   } else if (decl.kindString === 'Enumeration member') {
-    return readTypeTags(typeof decl.defaultValue, path, topTypeName,
-      typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
+    return readTypeTags(typeof decl.defaultValue, path, topTypeName, getCommentTags(decl, inheritedTags));
   }
 
   if (empty) {
     composeErrorMessage(path, topTypeName, 'object', decl.name, 'Empty object');
   }
 
-  return readFieldTags(out, path, topTypeName,
-    typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
+  return readFieldTags(out, path, topTypeName, getCommentTags(decl));
+}
+
+/**
+ * Reads all comment tags, including inherited ones
+ *
+ * @param decl the DeclarationReflection to read the tags from
+ */
+function getCommentTags(decl: DeclarationReflection, inheritedTags: CommentTag[] = []): CommentTag[] {
+  let out: CommentTag[] = decl.comment instanceof Comment ?
+    typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : inheritedTags : inheritedTags;
+  if (decl.overwrites instanceof ReferenceType && decl.overwrites.reflection instanceof DeclarationReflection) {
+    out = arrayPriorityJoin(out, getCommentTags(decl.overwrites.reflection));
+  }
+
+  return out;
+}
+
+/**
+ * Joins two arrays of CommentTags, but overrides all original CommentTags with the same tagName
+ *
+ * @param originals the original array
+ * @param overrider the array that should be appended and provide the override values
+ */
+function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] {
+  const out: CommentTag[] = overrider;
+
+  originals.forEach((original) => {
+    const result = overrider.find((element) => {
+      return original.tagName === element.tagName;
+    });
+
+    // no support for multiple tags with the same name
+    if (!(result instanceof CommentTag)) {
+      out.push(original);
+    }
+  });
+
+  return out;
 }
 
 /**
@@ -318,7 +362,7 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top
     if (typeof type.reflection !== 'undefined') {
       // there is really no way to make this typesafe, every element in DeclarationReflection is optional.
       return handleDeclarationReflection(type.reflection as DeclarationReflection,
-        getReflectionGeneric(type, generics, path, topTypeName, tags), path, topTypeName);
+        getReflectionGeneric(type, generics, path, topTypeName, tags), path, topTypeName, tags);
     }
 
     return handleRefWithoutReflection(type, generics, path, topTypeName, tags);
@@ -345,6 +389,27 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top
   return {type: ElasticsearchDataType.parse_error};
 }
 
+/**
+ * Adds an aggregatable to the aggregations list
+ *
+ * @param path the current path
+ * @param topTypeName the name of the top type
+ * @param global whether the topTypeName will be used
+ */
+function addAggregatable(path: string, topTypeName: string, global: boolean) {
+  // push type.path and remove the '.' at the end of the path
+  const property = path.slice(0, -1)
+    .split('.')
+    .pop() as string; // cannot be undefined
+
+  (aggregations[global ? '@all' : topTypeName] as ESNestedAggregation).aggs[property] = {
+    terms: {
+      field: `${property}.keyword`,
+      size: 1000,
+    },
+  };
+}
+
 /**
  * Reads all tags related to Elasticsearch fields from the fieldMap
  *
@@ -361,8 +426,7 @@ function readFieldTags(prev: ElasticsearchValue,
                        dataType?: string): ElasticsearchValue {
   for (const tag of tags) {
     if (tag.tagName === aggregatableTag) {
-      // push type.path and remove the '.' at the end of the path
-      aggregatablePaths.push(`${topTypeName}.${path.slice(0, -1)}`);
+      addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal);
     }
 
     if (!ignoredTagsList.includes(tag.tagName)) {
@@ -463,9 +527,16 @@ function readTypeTags(type: string, path: string, topTypeName: string, tags: Com
  */
 export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true):
 // tslint:disable-next-line:completed-docs
-  { aggregations: string[]; errors: string[]; template: ElasticsearchTemplate; } {
+  { aggregations: AggregationSchema; errors: string[]; mappings: ElasticsearchMappings; } {
   errors = [];
-  aggregatablePaths = [];
+  aggregations = {
+    '@all': {
+      aggs: {},
+      filter: {
+        match_all: {},
+      },
+    },
+  };
   showErrors = showErrorOutput;
 
   ignoredTagsList = ['indexable', 'validatable'];
@@ -473,22 +544,7 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
 
   const indexableInterfaces = getAllIndexableInterfaces(projectReflection);
 
-  const out: ElasticsearchTemplate = {
-    mappings: {
-      _default_: {
-        _source: {
-          excludes: [
-            'creation_date',
-          ],
-        },
-        date_detection: false,
-        dynamic_templates: [],
-        properties: {},
-      },
-    },
-    settings: settings,
-    template: 'stapps_*',
-  };
+  const out: ElasticsearchMappings = {};
 
   for (const _interface of indexableInterfaces) {
     if (!Array.isArray(_interface.children) || _interface.children.length === 0) {
@@ -521,11 +577,38 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
       Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`);
     }
 
-    out.mappings._default_.properties[typeName] =
+    // init aggregation schema for type
+    aggregations[typeName] = {
+      aggs: {},
+      filter: {
+        type: {
+          value: typeName,
+        },
+      },
+    };
+
+    out[typeName] =
       handleDeclarationReflection(_interface, [], '', typeName) as ElasticsearchObject;
+    out[typeName].properties.creation_date = {
+      type: ElasticsearchDataType.date,
+    };
+
+    out[typeName].dynamic_templates = dynamicTemplates;
+
+    // Set some properties
+    out[typeName]._source = {
+      excludes: [
+        'creation_date',
+      ],
+    };
+    out[typeName].date_detection = false;
+
+    dynamicTemplates = [];
+
+    if (Object.keys((aggregations[typeName] as ESNestedAggregation).aggs).length === 0) {
+      delete aggregations[typeName];
+    }
   }
 
-  out.mappings._default_.dynamic_templates = dynamicTemplates;
-
-  return {aggregations: aggregatablePaths, template: out, errors};
+  return {aggregations, mappings: out, errors};
 }

src/mappings/aggregation-definitions.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * An elasticsearch bucket aggregation
+ * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
+ */
+export interface AggregationSchema {
+  [aggregationName: string]: ESTermsFilter | ESNestedAggregation;
+}
+
+/**
+ * An elasticsearch terms filter
+ */
+export interface ESTermsFilter {
+  /**
+   * Terms filter definition
+   */
+  terms: {
+    /**
+     * Field to apply filter to
+     */
+    field: string;
+
+    /**
+     * Number of results
+     */
+    size?: number;
+  };
+}
+
+/**
+ * Filter that filters by name of the the field type
+ */
+export interface ESAggTypeFilter {
+  /**
+   * The type of the object to find
+   */
+  type: {
+    /**
+     * The name of the type
+     */
+    value: string;
+  };
+}
+
+/**
+ * Filter that matches everything
+ */
+export interface ESAggMatchAllFilter {
+  /**
+   * Filter that matches everything
+   */
+  match_all: {};
+}
+
+/**
+ * For nested aggregations
+ */
+export interface ESNestedAggregation {
+  /**
+   * Possible nested Aggregations
+   */
+  aggs: AggregationSchema;
+  /**
+   * Possible filter for types
+   */
+  filter: ESAggTypeFilter | ESAggMatchAllFilter;
+}

src/mappings/definitions/fieldmap.ts

@@ -28,7 +28,7 @@ export const fieldmap: ElasticsearchFieldmap = {
         type: ElasticsearchDataType.keyword,
       },
     },
-    ignore: [],
+    ignore: ['global'],
   },
   sortable: {
     default: {

src/mappings/definitions/premap.ts

@@ -18,69 +18,24 @@ import {ElasticsearchDataType} from './typemap';
 
 export const premaps: ElasticsearchPremap = {
   CoordinateReferenceSystem: {
-    dynamic: 'strict',
-    properties: {
-      properties: {
-        dynamic: true,
-        properties: {},
-      },
-      type: {
-        type: ElasticsearchDataType.keyword,
-      },
-    },
+    precision: '1m',
+    tree: 'quadtree',
+    type: ElasticsearchDataType.geo_shape,
   },
   LineString: {
-    dynamic: 'strict',
-    properties: {
-      coordinates: {
-        type: ElasticsearchDataType.float,
-      },
-      type: {
-        type: ElasticsearchDataType.keyword,
-      },
-    },
+    precision: '1m',
+    tree: 'quadtree',
+    type: ElasticsearchDataType.geo_shape,
   },
   Point: {
-    dynamic: 'strict',
-    properties: {
-      bbox: {type: ElasticsearchDataType.float},
-      coordinates: {
-        fields: {raw: {type: ElasticsearchDataType.keyword}},
-        type: ElasticsearchDataType.geo_point,
-      },
-      crs: {
-        dynamic: 'strict',
-        properties: {
-          properties: {
-            dynamic: true,
-            properties: {},
-          },
-          type: {type: ElasticsearchDataType.keyword},
-        },
-      },
-      type: {type: ElasticsearchDataType.keyword},
-    },
+    precision: '1m',
+    tree: 'quadtree',
+    type: ElasticsearchDataType.geo_shape,
   },
-  Polygon: { // a Polygon is mapped the same way as a Point is, you can just copy & paste
-    dynamic: 'strict',
-    properties: {
-      bbox: {type: ElasticsearchDataType.float},
-      coordinates: {
-        fields: {raw: {type: ElasticsearchDataType.keyword}},
-        type: ElasticsearchDataType.geo_point,
-      },
-      crs: {
-        dynamic: 'strict',
-        properties: {
-          properties: {
-            dynamic: true,
-            properties: {},
-          },
-          type: {type: ElasticsearchDataType.keyword},
-        },
-      },
-      type: {type: ElasticsearchDataType.keyword},
-    },
+  Polygon: {
+    precision: '1m',
+    tree: 'quadtree',
+    type: ElasticsearchDataType.geo_shape,
   },
   'jsonpatch.OpPatch': {
     dynamic: 'strict',

src/mappings/mapping-definitions.ts

@@ -22,7 +22,7 @@ import {ElasticsearchDataType} from './definitions/typemap';
  * Both are composed similarly, and can be the value of a propery
  * of an Elasticsearch Object.
  */
-export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject;
+export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject | ElasticsearchGeoShape;
 
 /**
  * Used internally for saving a generic value contained in a reflection
@@ -178,12 +178,61 @@ export interface ElasticsearchType {
   type: ElasticsearchDataType;
 }
 
+/**
+ * A GeoShape data type
+ *
+ * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/geo-shape.html
+ */
+export interface ElasticsearchGeoShape {
+  /**
+   * Does not exist; here for TypeScript compiler
+   */
+  fields?: undefined;
+
+  /**
+   * This parameter may be used instead of tree_levels to set an appropriate value for the tree_levels parameter.
+   *
+   * The value specifies the desired precision and Elasticsearch will calculate the best tree_levels value to honor
+   * this precision. The value should be a number followed by an optional distance unit. Valid distance units include:
+   * in, inch, yd, yard, mi, miles, km, kilometers, m,meters, cm,centimeters, mm, millimeters.
+   */
+  precision: string;
+
+  /**
+   * Name of the PrefixTree implementation to be used: geohash for GeohashPrefixTree and quadtree for QuadPrefixTree.
+   */
+  tree: 'quadtree' | 'geohash';
+
+  /**
+   * The type of the object, obviously geo_shape
+   */
+  type: ElasticsearchDataType.geo_shape;
+}
+
 /**
  * An object data type
  *
  * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/object.html
  */
 export interface ElasticsearchObject {
+
+  /**
+   * Only for the top type
+   */
+  _source?: {
+    /**
+     * Fields that should be excluded in the _source field
+     */
+    excludes: [
+      'creation_date',
+    ];
+  };
+
+  /**
+   * Whether the creation date should be set automatically
+   */
+  date_detection?: boolean;
+
   /**
    * If the object is a dynamic
    *
@@ -224,6 +273,13 @@ export interface ElasticsearchObject {
   };
 }
 
+export type ElasticsearchMapping = ElasticsearchObject;
+
+// TODO: docs
+export interface ElasticsearchMappings {
+  [indexName: string]: ElasticsearchMapping;
+}
+
 /**
  * An Elasticsearch template
  *