ci: make mapping test work for new mappings

.gitlab-ci.yml

@@ -65,12 +65,7 @@ mapping:
       alias: elasticsearch
   script:
     - npm install @openstapps/core
-    - node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength,tjs-format"
+    - node lib/cli.js put-es-templates ./node_modules/@openstapps/core/src http://elasticsearch:9200/ "pattern,see,minlength,tjs-format"
-    - curl http://elasticsearch:9200/_template --upload-file mapping.json -o response.json
-    - cat response.json
-    - grep -q "\"acknowledged\":true" response.json
-    # - curl --show-error --fail http://elasticsearch:9200/stapps --upload-file mapping.json
-
 
 package:
   dependencies:

src/cli.ts

@@ -15,7 +15,9 @@
 import {Logger} from '@openstapps/logger';
 import * as commander from 'commander';
 import {existsSync, readFileSync, writeFileSync} from 'fs';
+import * as got from 'got';
 import {join, resolve} from 'path';
+import {exit} from 'process';
 import {
   getProjectReflection,
   mkdirPromisified,
@@ -102,11 +104,53 @@ commander
 
     // write documentation to file
     // tslint:disable-next-line:no-magic-numbers
-    writeFileSync(mappingPath, JSON.stringify(result, null, 2));
+    writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
 
     Logger.ok(`Elasticsearch mapping written to ${mappingPath}.`);
   });
 
+commander
+  .command('put-es-templates <srcPath> <esAddress> [ignoredTags]')
+  .action(async (relativeSrcPath, esAddress, ignoredTags) => {
+    // get absolute paths
+    const srcPath = resolve(relativeSrcPath);
+
+    let ignoredTagsList: string[] = [];
+    if (typeof ignoredTags === 'string') {
+      ignoredTagsList = ignoredTags.split(',');
+    }
+
+    // get project reflection
+    const projectReflection = getProjectReflection(srcPath);
+
+    const result = generateTemplate(projectReflection, ignoredTagsList, true);
+    if (result.errors.length !== 0) {
+      await Logger.error(`Mapping generated with errors:\n${JSON.stringify(result.errors)}`);
+      exit(-1);
+    } else {
+      Logger.ok('Mapping generated without errors!');
+    }
+
+    for (const template in result.mappings) {
+      if (!result.mappings.hasOwnProperty(template)) {
+        continue;
+      }
+
+      const response = await got.put(`${esAddress}_template/${template}`, {
+        body: result.mappings[template],
+        json: true,
+      });
+
+      const HTTP_STATUS_OK = 200;
+      if (response.statusCode !== HTTP_STATUS_OK) {
+        await Logger.error(`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`);
+        exit(-1);
+      }
+    }
+
+    Logger.ok(`Templates accepted by Elasticsearch.`);
+  });
+
 commander
   .command('schema <srcPath> <schemaPath>')
   .action(async (relativeSrcPath, relativeSchemaPath) => {

src/mapping.ts

@@ -35,8 +35,8 @@ import {settings} from './mappings/definitions/settings';
 import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap';
 import {
   ElasticsearchDynamicTemplate,
-  ElasticsearchMappings,
+  ElasticsearchObject, ElasticsearchTemplateCollection,
-  ElasticsearchObject, ElasticsearchType,
+  ElasticsearchType,
   ElasticsearchValue,
 } from './mappings/mapping-definitions';
 
@@ -172,9 +172,11 @@ function handleExternalType(ref: ReferenceType, generics: Map<string, Elasticsea
       return {type: ElasticsearchDataType.parse_error};
     }
 
-    return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, new Map(generics), path,
+    return readFieldTags(
-      topTypeName, tags),
+      handleType(
-      path, topTypeName, tags),
+        ref.typeArguments[0], getReflectionGeneric(
+          ref, new Map(generics), path, topTypeName, tags),
+        path, topTypeName, tags),
       path, topTypeName, tags);
   }
   if (ref.name === '__type') { // empty object
@@ -531,7 +533,7 @@ function readTypeTags(type: string, path: string, topTypeName: string, tags: Com
  */
 export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true):
 // tslint:disable-next-line:completed-docs
-  { aggregations: AggregationSchema; errors: string[]; mappings: ElasticsearchMappings; } {
+  { aggregations: AggregationSchema; errors: string[]; mappings: ElasticsearchTemplateCollection; } {
   errors = [];
   aggregations = {
     '@all': {
@@ -548,7 +550,7 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
 
   const indexableInterfaces = getAllIndexableInterfaces(projectReflection);
 
-  const out: ElasticsearchMappings = {};
+  const out: ElasticsearchTemplateCollection = {};
 
   for (const _interface of indexableInterfaces) {
     if (!Array.isArray(_interface.children) || _interface.children.length === 0) {
@@ -604,7 +606,7 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
       settings: settings,
       template: `stapps_${typeNameWithoutSpaces}*`,
     }
-      ;
+    ;
     out[templateName].mappings[typeName].properties.creation_date = {
       type: ElasticsearchDataType.date,
     };

src/mappings/mapping-definitions.ts

@@ -256,28 +256,11 @@ export interface ElasticsearchObject {
   };
 }
 
-export interface ElasticsearchMapping {
+/**
-  /**
+ * A collection of Elasticsearch Templates
-   * The mappings of the index
+ */
-   */
+export interface ElasticsearchTemplateCollection {
-  mappings: {
+  [indexName: string]: ElasticsearchTemplate;
-    [name: string]: ElasticsearchObject;
-  };
-
-  /**
-   * The settings for the index
-   */
-  settings: unknown;
-
-  /**
-   * The name of the index
-   */
-  template: string;
-}
-
-// TODO: docs
-export interface ElasticsearchMappings {
-  [indexName: string]: ElasticsearchMapping;
 }
 
 /**
@@ -291,39 +274,7 @@ export interface ElasticsearchTemplate {
    * This is a pre-defined structure you should use for your mapping
    */
   mappings: {
-    /**
+    [typeName: string]: ElasticsearchObject;
-     * This mapping will be used by default for everything
-     */
-    _default_: {
-      /**
-       * Contains the original JSON document body
-       *
-       * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-source-field.html
-       */
-      _source: {
-        /**
-         * Any fields that are excluded from the source
-         */
-        excludes: string[];
-      };
-
-      /**
-       * Whether Elasticsearch should automatically add date fields to objects
-       */
-      date_detection: false;
-
-      /**
-       * This is where all the dynamic templates should go
-       */
-      dynamic_templates: ElasticsearchDynamicTemplate[];
-
-      /**
-       * This is where all the mappings should go
-       */
-      properties: {
-        [name: string]: ElasticsearchObject;
-      };
-    };
   };
 
   /**