feat: improve monorepo dev experience

packages/es-mapping-generator/src/app.ts

@@ -1,99 +0,0 @@
-/*
- * Copyright (C) 2018-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {Command} from 'commander';
-import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'fs';
-import path from 'path';
-import {generateTemplate} from './mapping';
-import {getProjectReflection} from './project-reflection';
-
-// handle unhandled promise rejections
-process.on('unhandledRejection', async (reason: unknown) => {
-  if (reason instanceof Error) {
-    await console.error(reason.message);
-    console.info(reason.stack);
-  }
-  process.exit(1);
-});
-
-const commander = new Command('openstapps-core-tools');
-
-// eslint-disable-next-line unicorn/prefer-module
-commander.version(JSON.parse(readFileSync(path.resolve(__dirname, '..', 'package.json')).toString()).version);
-
-commander
-  .command('mapping <relativeSrcPath>')
-  .option('-m, --mappingPath <relativeMappingPath>', 'Mapping Path')
-  .option('-i, --ignoredTags <ignoredTags>', 'Ignored Tags (comma-separated)')
-  .option('-a, --aggPath <relativeAggregationPath>', 'Aggregations Path')
-  .option('-e, --errorPath <relativeErrorPath>', 'Error Path')
-  .action(async (relativeSourcePath, options) => {
-    // get absolute paths
-    const sourcePath = path.resolve(relativeSourcePath);
-
-    let ignoredTagsList: string[] = [];
-    if (typeof options.ignoredTags === 'string') {
-      ignoredTagsList = options.ignoredTags.split(',');
-    }
-
-    // get project reflection
-    const projectReflection = getProjectReflection(sourcePath);
-
-    const result = generateTemplate(projectReflection, ignoredTagsList, true);
-    if (result.errors.length > 0) {
-      await console.error('Mapping generated with errors!');
-    } else {
-      console.log('Mapping generated without errors!');
-    }
-
-    // write documentation to file
-    if (options.aggPath !== undefined) {
-      const aggPath = path.resolve(options.aggPath);
-      mkdirSync(path.dirname(aggPath), {recursive: true});
-      // tslint:disable-next-line:no-magic-numbers
-      writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
-      copyFileSync(
-        // eslint-disable-next-line unicorn/prefer-module
-        require.resolve('../schema/aggregations.d.ts'),
-        path.join(path.dirname(aggPath), 'aggregations.json.d.ts'),
-      );
-      console.log(`Elasticsearch aggregations written to ${aggPath}.`);
-    }
-    if (options.mappingPath !== undefined) {
-      const mappingPath = path.resolve(options.mappingPath);
-      mkdirSync(path.dirname(mappingPath), {recursive: true});
-      writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
-      copyFileSync(
-        // eslint-disable-next-line unicorn/prefer-module
-        require.resolve('../schema/mappings.d.ts'),
-        path.join(path.dirname(mappingPath), 'mappings.json.d.ts'),
-      );
-      console.log(`Elasticsearch mappings written to ${mappingPath}.`);
-    }
-    if (options.errorPath !== undefined) {
-      const errorPath = path.resolve(options.errorPath);
-      mkdirSync(path.dirname(errorPath), {recursive: true});
-      // tslint:disable-next-line:no-magic-numbers
-      writeFileSync(errorPath, JSON.stringify(result.errors, null, 2));
-      console.log(`Mapping errors written to ${errorPath}.`);
-    } else if (result.errors.length > 0) {
-      for (const error of result.errors) {
-        await console.error(error);
-      }
-
-      throw new Error('Mapping generation failed');
-    }
-  });
-
-commander.parse(process.argv);

packages/es-mapping-generator/src/config/fieldmap.ts

@@ -1,55 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
-import type {ElasticsearchFieldmap, SimpleType} from '../../schema/mappings.js';
-
-const ducetSort = {
-  type: 'icu_collation_keyword',
-  language: 'de',
-  country: 'DE',
-  variant: '@collation=phonebook',
-};
-
-const keyword: MappingProperty['type'] = 'keyword';
-
-export const fieldmap: ElasticsearchFieldmap = {
-  aggregatable: {
-    default: {
-      raw: {
-        ignore_above: 10_000,
-        type: keyword,
-      },
-    },
-    ignore: ['global'],
-  },
-  sortable: {
-    default: {
-      sort: ducetSort,
-    },
-    ducet: {
-      sort: ducetSort,
-    },
-    ignore: ['price'],
-  },
-};
-
-export const filterableTagName = 'filterable';
-
-export const filterableMap: Record<string, SimpleType> = {
-  date: 'keyword',
-  keyword: 'keyword',
-  text: 'keyword',
-  integer: 'integer',
-};

packages/es-mapping-generator/src/config/premap.ts

@@ -1,64 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
-
-export const premaps: Record<string, MappingProperty> = {
-  'CoordinateReferenceSystem': {
-    dynamic: true,
-    properties: {
-      type: {
-        type: 'keyword',
-      },
-    },
-  },
-  'LineString': {
-    type: 'geo_shape',
-  },
-  'Point': {
-    properties: {
-      type: {
-        type: 'keyword',
-      },
-      coordinates: {
-        type: 'geo_point',
-      },
-    },
-    dynamic: 'strict',
-  },
-  'Polygon': {
-    type: 'geo_shape',
-  },
-  'SCISO8601DateRange': {
-    type: 'date_range',
-  },
-  'jsonpatch.OpPatch': {
-    dynamic: 'strict',
-    properties: {
-      from: {
-        type: 'keyword',
-      },
-      op: {
-        type: 'keyword',
-      },
-      path: {
-        type: 'keyword',
-      },
-      value: {
-        // this is actually an 'any' type; however, ES does not really support that.
-        type: 'keyword',
-      },
-    },
-  },
-};

packages/es-mapping-generator/src/config/settings.ts

@@ -1,22 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {IndicesPutTemplateRequest} from '@elastic/elasticsearch/lib/api/types';
-
-export const settings: IndicesPutTemplateRequest['settings'] = {
-  'mapping.total_fields.limit': 10_000,
-  'max_result_window': 30_000,
-  'number_of_replicas': 0,
-  'number_of_shards': 1,
-};

packages/es-mapping-generator/src/config/typemap.ts

@@ -1,62 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {MappingFloatNumberProperty} from '@elastic/elasticsearch/lib/api/types';
-import type {ElasticsearchTypemap} from '../../schema/mappings';
-
-export const PARSE_ERROR = 'PARSE_ERROR' as MappingFloatNumberProperty['type'];
-export const MISSING_PREMAP = 'MISSING_PREMAP' as MappingFloatNumberProperty['type'];
-export const TYPE_CONFLICT = 'TYPE_CONFLICT' as MappingFloatNumberProperty['type'];
-
-export const typemap: ElasticsearchTypemap = {
-  boolean: {
-    default: 'boolean',
-  },
-  false: {
-    default: 'boolean',
-  },
-  number: {
-    default: 'integer',
-    float: 'float',
-    integer: 'integer',
-    date: 'date',
-  },
-  string: {
-    default: 'text',
-    keyword: 'keyword',
-    text: 'text',
-    date: 'date',
-  },
-  stringLiteral: {
-    default: 'keyword',
-  },
-  true: {
-    default: 'boolean',
-  },
-};
-
-/**
- * If the string is a tag type
- */
-export function isTagType(string_: string): boolean {
-  for (const key in typemap) {
-    if (typemap.hasOwnProperty(key) && typemap[key][string_] !== undefined) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-export const dynamicTypes = ['any', 'unknown'];

packages/es-mapping-generator/src/dsl/schema.ts

@@ -0,0 +1,60 @@
+import {MappingProperty, SearchRequest} from '@elastic/elasticsearch/lib/api/types.js';
+import Ajv from 'ajv';
+import {readFile} from 'fs/promises';
+import {fileURLToPath} from 'url';
+import path from 'path';
+import {Context} from '../generator/context.js';
+
+/**
+ * @validatable
+ */
+export interface ElasticsearchOptionsDSL {
+  /**
+   * Mark an interface as indexable
+   */
+  indexable?: true;
+  /**
+   * Inherit customization options from another item
+   */
+  extends?: string[];
+  /**
+   * Completely override the property
+   */
+  override?: MappingProperty;
+  /**
+   * Merge property values
+   */
+  merge?: MappingProperty;
+  /**
+   * Modify the search request
+   *
+   * Supports `{name}`, `{type}` and `{prop}` templates substitutions anywhere
+   */
+  search?: Partial<SearchRequest>;
+}
+
+const schema = JSON.parse(
+  await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), 'index.schema.json'), 'utf8'),
+);
+const ajv = new Ajv.default({schemas: [schema], allowUnionTypes: true});
+
+/**
+ * Validate that the options are valid
+ */
+export function validateElasticsearchOptionsDsl(
+  context: Context,
+  value: unknown,
+): value is ElasticsearchOptionsDSL {
+  return ajv.validate('#/definitions/ElasticsearchOptionsDSL', value)
+    ? true
+    : context.bail(JSON.stringify(ajv.errors));
+}
+
+/**
+ * Validate that the mapping result is correct
+ */
+export function validateMappingResult(context: Context, value: unknown): value is MappingProperty {
+  return ajv.validate('#/definitions/MappingProperty', value)
+    ? true
+    : context.bail(JSON.stringify(ajv.errors));
+}

packages/es-mapping-generator/src/generator/base.ts

@@ -0,0 +1,47 @@
+import {JSONSchema7} from 'json-schema';
+import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
+import {transformObject} from './transformers/object.js';
+import {transformString} from './transformers/string.js';
+import {Context} from './context.js';
+import {transformDefinition} from './definition.js';
+
+/**
+ * Transform JSONSchema without applying custom tag logic
+ */
+export function transformBase(context: Context, definition: JSONSchema7): MappingProperty {
+  if (definition.anyOf) {
+    return context.resolveUnion(definition.anyOf as JSONSchema7[]);
+  }
+
+  switch (definition.type) {
+    case 'array': {
+      if (Array.isArray(definition.items)) {
+        return context.resolveUnion(definition.items as JSONSchema7[]);
+      } else if (typeof definition.items === 'object') {
+        return transformDefinition(context, definition.items);
+      } else {
+        return context.bail(`Not implemented array type ${typeof definition.items}`);
+      }
+    }
+    case 'object': {
+      return transformObject(context, definition);
+    }
+    case 'string': {
+      return transformString(definition);
+    }
+    case 'number': {
+      return {type: 'float'};
+    }
+    case 'integer': {
+      return {type: 'integer'};
+    }
+    case 'boolean': {
+      return {type: 'boolean'};
+    }
+    default: {
+      return {
+        dynamic: false,
+      };
+    }
+  }
+}

packages/es-mapping-generator/src/generator/context.ts

@@ -0,0 +1,115 @@
+import {JSONSchema7} from 'json-schema';
+import {transformDefinition} from './definition.js';
+import crypto from 'crypto';
+import {
+  MappingProperty,
+  MappingDynamicProperty,
+  SearchRequest,
+} from '@elastic/elasticsearch/lib/api/types.js';
+import deepmerge from 'deepmerge';
+import {MappingGenerator, sanitizeTypeName} from './mapping-generator.js';
+import {renderTemplate} from '../template.js';
+
+/**
+ * Get the name from a $ref such as `#/definitions/SCThing`
+ */
+function getNameFromRef(ref: string): string {
+  return decodeURI(ref).replace(/^#\/definitions\//, '');
+}
+
+export class Context {
+  constructor(
+    readonly generator: MappingGenerator,
+    readonly thingType: string,
+    readonly path: string[],
+    readonly propertyPath: string[],
+    readonly dependencies: Map<string, Set<string>>,
+  ) {}
+
+  resolveReference(reference: string): MappingProperty {
+    return this.deriveContext(reference)[1];
+  }
+
+  resolveUnion(types: JSONSchema7[]): MappingDynamicProperty {
+    for (const type of types) {
+      const [name] = this.deriveContext(type.$ref ?? type);
+      this.addDependency(name, this.propertyPath.join('.'));
+    }
+
+    return {
+      type: '{dynamic_property}',
+    };
+  }
+
+  private deriveContext(reference: string | JSONSchema7): [dependencyName: string, mapping: MappingProperty] {
+    let referenceName = typeof reference === 'string' ? getNameFromRef(reference) : undefined;
+    let definition = typeof reference === 'string' ? undefined : reference;
+
+    if (!definition && !this.generator.cache.has(referenceName!)) {
+      definition = this.generator.project.definitions![referenceName!] as JSONSchema7;
+      if (typeof definition !== 'object') this.bail(`Invalid path ${referenceName!}`);
+    }
+
+    if (!referenceName || !this.generator.cache.has(referenceName)) {
+      const derivedContext = new Context(
+        this.generator,
+        this.thingType,
+        referenceName ? [referenceName] : [],
+        this.propertyPath,
+        new Map(),
+      );
+      const result = transformDefinition(derivedContext, definition);
+      referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
+
+      this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});
+    }
+
+    const {mapping, dependencies} = this.generator.cache.get(referenceName)!;
+    for (const [name, paths] of dependencies) {
+      for (const path of paths) {
+        this.addDependency(name, [...this.propertyPath.slice(0, -1), path].join('.'));
+      }
+    }
+    return [referenceName, mapping];
+  }
+
+  registerSearchMod(modification: Partial<SearchRequest>) {
+    this.generator.searchMods.mods = deepmerge<Partial<SearchRequest>>(
+      this.generator.searchMods.mods,
+      renderTemplate(modification, [
+        ['{name}', this.path[0]],
+        ['{prop}', this.propertyPath.join('.')],
+        ['{type}', this.thingType],
+        ['{_type}', sanitizeTypeName(this.thingType)],
+      ]),
+    );
+  }
+
+  private addDependency(name: string, path: string) {
+    if (!this.dependencies.has(name)) {
+      this.dependencies.set(name, new Set([path]));
+    } else {
+      this.dependencies.get(name)!.add(path);
+    }
+  }
+
+  /**
+   * Step down into a property
+   */
+  step(property: string): Context {
+    return new Context(
+      this.generator,
+      this.thingType,
+      [...this.path, property],
+      [...this.propertyPath, property],
+      this.dependencies,
+    );
+  }
+
+  /**
+   * Bail and throw
+   */
+  bail(reason: string): never {
+    throw new Error(`${this.path.join('.')} ${reason}`);
+  }
+}

packages/es-mapping-generator/src/generator/definition.ts

@@ -0,0 +1,32 @@
+import {JSONSchema7} from 'json-schema';
+import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
+import {transformBase} from './base.js';
+import {Context} from './context.js';
+import deepmerge from 'deepmerge';
+import {resolveDsl} from './dsl.js';
+import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
+
+/**
+ * Transform JSONSchema
+ */
+export function transformDefinition(context: Context, definition: JSONSchema7): MappingProperty {
+  if (definition.$ref) return context.resolveReference(definition.$ref);
+  const tags = getTags(definition);
+  tags.delete(INDEXABLE_TAG_NAME);
+  let base = transformBase(context, definition);
+
+  if (tags.size > 0) {
+    const options = resolveDsl(context, {extends: [...tags]});
+    if (options.override) {
+      base = options.override;
+    }
+    if (options.merge) {
+      base = deepmerge<MappingProperty>(base, options.merge);
+    }
+    if (options.search) {
+      context.registerSearchMod(options.search);
+    }
+  }
+
+  return base;
+}

packages/es-mapping-generator/src/generator/dsl.ts

@@ -0,0 +1,48 @@
+import {Context} from './context.js';
+import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
+import deepmerge from 'deepmerge';
+
+type ResolvedOptions = Omit<ElasticsearchOptionsDSL, 'extends'>;
+
+/**
+ * Resolve DSL inheritance
+ */
+export function resolveDsl(
+  context: Context,
+  {extends: parents, ...result}: ElasticsearchOptionsDSL,
+): ResolvedOptions {
+  for (const reference of parents ?? []) {
+    result = deepmerge<ResolvedOptions>(
+      result,
+      reference.startsWith('@')
+        ? resolveReferencePath(context, reference)
+        : resolvePresetReference(context, reference),
+    );
+  }
+
+  return result;
+}
+
+/**
+ * Resolve preset references
+ */
+function resolvePresetReference(context: Context, reference: string): ResolvedOptions {
+  if (!context.generator.presets.has(reference)) return context.bail(`Missing preset ${reference}`);
+  return resolveDsl(context, context.generator.presets.get(reference)!);
+}
+
+/**
+ * Resolve @references
+ */
+function resolveReferencePath(context: Context, reference: string): ResolvedOptions {
+  const [type, ...path] = reference.replace(/^@/, '').split('.');
+  let declaration = context.resolveReference(type);
+  while (path.length > 0) {
+    const property = path.shift()!;
+    if (!('properties' in declaration && declaration.properties && property in declaration.properties))
+      context.bail(`Invalid reference ${reference}`);
+
+    declaration = declaration.properties[property];
+  }
+  return {override: declaration};
+}

packages/es-mapping-generator/src/generator/index.ts

@@ -0,0 +1,171 @@
+import type {JSONSchema7} from 'json-schema';
+import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
+import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
+import {MappingGenerator} from './mapping-generator.js';
+import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
+
+export interface GeneratorOptions {
+  /**
+   * Presets you can extend
+   */
+  presets: Record<string, ElasticsearchOptionsDSL>;
+  /**
+   * Override specific types
+   */
+  overrides: Record<string, MappingProperty>;
+  /**
+   * Template for the generated index request
+   *
+   * Supports `{type}` and `{sanitized_type}` (same as `{type}`, but no spaces) template substitutions
+   */
+  template: Partial<IndicesPutTemplateRequest>;
+}
+
+/**
+ * Fully transform a project
+ */
+export function transformProject(project: JSONSchema7) {
+  const context = new MappingGenerator(project, OPTIONS);
+
+  const results = [];
+  for (const name in project.definitions) {
+    const definition = project.definitions[name];
+    if (typeof definition !== 'object' || !getTags(definition).has(INDEXABLE_TAG_NAME)) continue;
+
+    results.push(context.buildTemplate(name));
+  }
+  return {
+    mappings: results,
+    search: context.searchMods.mods,
+  };
+}
+
+const OPTIONS: GeneratorOptions = {
+  template: {
+    name: 'template_{_type}',
+    index_patterns: 'stapps_{_type}*',
+    settings: {
+      'mapping.total_fields.limit': 10_000,
+      'max_result_window': 30_000,
+      'number_of_replicas': 0,
+      'number_of_shards': 1,
+    },
+    mappings: {
+      date_detection: false,
+      _source: {
+        excludes: ['creation_date'],
+      },
+      properties: {
+        creation_date: {
+          type: 'date',
+        },
+      },
+    },
+  },
+  overrides: {
+    'CoordinateReferenceSystem': {
+      dynamic: true,
+      properties: {
+        type: {
+          type: 'keyword',
+        },
+      },
+    },
+    'LineString': {
+      type: 'geo_shape',
+    },
+    'Point': {
+      properties: {
+        type: {
+          type: 'keyword',
+        },
+        coordinates: {
+          type: 'geo_point',
+        },
+      },
+      dynamic: 'strict',
+    },
+    'Polygon': {
+      type: 'geo_shape',
+    },
+    'SCISO8601DateRange': {
+      type: 'date_range',
+    },
+    'jsonpatch.OpPatch': {
+      dynamic: 'strict',
+      properties: {
+        from: {
+          type: 'keyword',
+        },
+        op: {
+          type: 'keyword',
+        },
+        path: {
+          type: 'keyword',
+        },
+        value: {
+          // this is actually an 'any' type; however, ES does not really support that.
+          type: 'keyword',
+        },
+      },
+    },
+  },
+  presets: {
+    'type': {extends: ['sortable:ducet', 'filterable', 'aggregatable:global']},
+    'text': {
+      merge: {type: 'text'},
+    },
+    'filterable': {
+      merge: {
+        fields: {
+          raw: {type: 'keyword'},
+        },
+      },
+    },
+    'sortable:ducet': {
+      merge: {
+        fields: {
+          sort: {
+            type: 'icu_collation_keyword',
+            language: 'de',
+            country: 'DE',
+            variant: '@collation=phonebook',
+          } as never,
+        },
+      },
+    },
+    'sortable': {
+      extends: ['sortable:ducet'],
+    },
+    'sortable:price': {},
+    'aggregatable': {
+      search: {
+        aggs: {
+          '{type}': {
+            aggs: {
+              '{prop}': {
+                terms: {
+                  field: '{prop}.raw',
+                  size: 1000,
+                },
+              },
+            },
+            filter: {term: {type: '{type}'}},
+          },
+        },
+      },
+    },
+    'aggregatable:global': {
+      search: {
+        aggs: {
+          '@all.{prop}': {
+            terms: {
+              field: '{prop}.raw',
+              size: 1000,
+            },
+          },
+        },
+      },
+    },
+  },
+};

packages/es-mapping-generator/src/generator/mapping-generator.ts

@@ -0,0 +1,76 @@
+import {GeneratorOptions} from './index.js';
+import {JSONSchema7} from 'json-schema';
+import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
+import {
+  IndicesPutTemplateRequest,
+  MappingProperty,
+  MappingTypeMapping,
+  SearchRequest,
+} from '@elastic/elasticsearch/lib/api/types.js';
+import {Context} from './context.js';
+import deepmerge from 'deepmerge';
+import {renderTemplate} from '../template.js';
+
+/**
+ * Sanitize a type name
+ */
+export function sanitizeTypeName(typeName: string): string {
+  return typeName.replaceAll(' ', '_');
+}
+
+export class MappingGenerator {
+  readonly presets: Map<string, ElasticsearchOptionsDSL>;
+
+  readonly cache: Map<string, {mapping: MappingProperty; dependencies: Map<string, Set<string>>}>;
+
+  readonly searchMods: {mods: Partial<SearchRequest>} = {mods: {}};
+
+  readonly template: Partial<IndicesPutTemplateRequest>;
+
+  constructor(readonly project: JSONSchema7, options: GeneratorOptions) {
+    this.template = options.template ?? {};
+    this.presets = new Map(Object.entries(options.presets));
+    this.cache = new Map(
+      Object.entries(options.overrides).map(([name, mapping]) => [
+        name,
+        {
+          mapping,
+          dependencies: new Map(),
+        },
+      ]),
+    );
+  }
+
+  buildTemplate(name: string): IndicesPutTemplateRequest {
+    const thingType = ((this.project.definitions![name] as JSONSchema7).properties!.type as JSONSchema7)
+      .const;
+    if (typeof thingType !== 'string') {
+      throw new TypeError(`${name} needs a valid thing type`);
+    }
+    const mappingContext = new Context(this, thingType, [name], [], new Map());
+    const mappings = mappingContext.resolveReference(name);
+
+    const request: IndicesPutTemplateRequest = deepmerge(
+      {mappings: mappings as MappingTypeMapping},
+      renderTemplate(this.template, [
+        ['{name}', name],
+        ['{type}', thingType],
+        ['{_type}', sanitizeTypeName(thingType)],
+      ]),
+    );
+
+    if (mappingContext.dependencies.size > 0) {
+      request.mappings!.dynamic_templates = [];
+      for (const [name, paths] of mappingContext.dependencies) {
+        request.mappings!.dynamic_templates.push({
+          [name]: {
+            path_match: paths.size > 1 ? [...paths] : paths.values().next().value,
+            mapping: mappingContext.resolveReference(name),
+          },
+        });
+      }
+    }
+
+    return request;
+  }
+}

packages/es-mapping-generator/src/generator/tags.ts

@@ -0,0 +1,13 @@
+import {JSONSchema7} from 'json-schema';
+
+export const INDEXABLE_TAG_NAME = 'indexable';
+
+/**
+ * Get elasticsearch tags
+ */
+export function getTags(definition: JSONSchema7): Set<string> {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  return new Set(
+    (definition as {elasticsearch: string}).elasticsearch?.split(/\s/).map(it => it.trim()) ?? [],
+  );
+}

packages/es-mapping-generator/src/generator/transformers/object.ts

@@ -0,0 +1,20 @@
+import {JSONSchema7} from 'json-schema';
+import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
+import {transformDefinition} from '../definition.js';
+import {Context} from '../context.js';
+
+/**
+ * Transform a JSON Schema with `object` type
+ */
+export function transformObject(context: Context, definition: JSONSchema7): MappingProperty {
+  const value = {
+    dynamic: definition.additionalProperties === true ? undefined : 'strict',
+    properties: {} as Record<string, MappingProperty>,
+  } satisfies MappingProperty;
+
+  for (const key in definition.properties!) {
+    value.properties[key] = transformDefinition(context.step(key), definition.properties[key] as JSONSchema7);
+  }
+
+  return value;
+}

packages/es-mapping-generator/src/generator/transformers/string.ts

@@ -0,0 +1,19 @@
+import {JSONSchema7} from 'json-schema';
+import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
+
+const stringFormats = new Map<string, MappingProperty>([
+  ['date', {type: 'date', format: 'date'}],
+  ['time', {type: 'date', format: 'time'}],
+  ['date-time', {type: 'date', format: 'date_optional_time'}],
+  ['ipv4', {type: 'ip'}],
+  ['ipv6', {type: 'ip'}],
+]);
+
+/**
+ * Transform a JSON Schema with `string` type
+ */
+export function transformString(definition: JSONSchema7): MappingProperty {
+  return definition.format && stringFormats.has(definition.format)
+    ? stringFormats.get(definition.format)!
+    : {type: 'keyword'};
+}

packages/es-mapping-generator/src/index.ts

@@ -1,7 +1,16 @@
-export * from './mapping';
-export * from './project-reflection';
+import {transformProject} from './generator/index.js';
+import {SchemaConsumer} from '@openstapps/json-schema-generator';
 
-export * from './config/premap';
-export * from './config/fieldmap';
-export * from './config/settings';
-export * from './config/typemap';
+/**
+ * JSON Schema Generator plugin for Elasticsearch Mappings
+ */
+export function elasticsearchMappingGenerator(fileName: string): [string, SchemaConsumer] {
+  return [
+    'Elasticsearch-Mappings',
+    function (schema) {
+      return {
+        [fileName]: JSON.stringify(transformProject(schema)),
+      };
+    },
+  ];
+}

packages/es-mapping-generator/src/mapping.ts

@@ -1,894 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {
-  MappingDynamicTemplate,
-  MappingObjectProperty,
-  MappingProperty,
-} from '@elastic/elasticsearch/lib/api/types';
-import merge from 'deepmerge';
-import {stringify} from 'flatted';
-import {DeclarationReflection, ProjectReflection, SignatureReflection} from 'typedoc';
-import {
-  ArrayType,
-  Comment,
-  CommentTag,
-  IntrinsicType,
-  ReferenceType,
-  ReflectionType,
-  StringLiteralType,
-  Type,
-  TypeParameterType,
-  UnionType,
-} from 'typedoc/dist/lib/models';
-import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap';
-import {premaps} from './config/premap';
-import {settings} from './config/settings';
-import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap';
-import type {AggregationSchema, ESNestedAggregation} from '../schema/aggregations';
-import type {ElasticsearchTemplateCollection, MappingGenTemplate} from '../schema/mappings';
-import * as console from 'console';
-
-let dynamicTemplates: Record<string, MappingDynamicTemplate>[] = [];
-let errors: string[] = [];
-let showErrors = true;
-
-let aggregations: AggregationSchema = {};
-
-const indexableTag = 'indexable';
-const aggregatableTag = 'aggregatable';
-const aggregatableTagParameterGlobal = 'global';
-const inheritTagsName = 'inherittags';
-
-// clamp printed object to 1000 chars to keep error messages readable
-const maxErrorObjectChars = 1000;
-
-let ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
-let inheritTagsMap: {[path: string]: CommentTag[]} = {};
-
-/**
- * Gets all interfaces that have an @indexable tag
- * @param projectReflection the project reflection from which to extract the indexable interfaces
- */
-export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] {
-  let indexableInterfaces: DeclarationReflection[] = [];
-
-  if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) {
-    throw new Error('No DeclarationReflections found. Please check your input path');
-  }
-
-  // push all declaration reflections into one array
-  for (const declarationReflection of projectReflection.children) {
-    if (Array.isArray(declarationReflection.children)) {
-      indexableInterfaces = [...indexableInterfaces, ...declarationReflection.children];
-    }
-  }
-
-  // filter all declaration reflections with an @indexable tag
-  indexableInterfaces = indexableInterfaces.filter(declarationReflection => {
-    if (declarationReflection.comment === undefined || declarationReflection.comment.tags === undefined) {
-      return false;
-    }
-
-    return declarationReflection.comment.tags.some(commentTag => {
-      return commentTag.tagName === indexableTag;
-    });
-  });
-
-  return indexableInterfaces;
-}
-
-/**
- * Composes error messages, that are readable and contain a certain minimum of information
- * @param path the path where the error took place
- * @param topTypeName the name of the SCThingType
- * @param typeName the name of the object, with which something went wrong
- * @param object the object or name
- * @param message the error message
- */
-function composeErrorMessage(
-  path: string,
-  topTypeName: string,
-  typeName: string,
-  object: string,
-  message: string,
-) {
-  const error = `At "${topTypeName}::${path.slice(
-    0,
-    Math.max(0, path.length - 1),
-  )}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
-  errors.push(error);
-  if (showErrors) {
-    // tslint:disable-next-line:no-floating-promises
-    void console.error(error);
-  }
-}
-
-/**
- * Trims a string to a readable size and appends "..."
- * @param value the string to trim
- * @param maxLength the maximum allowed length before it is clamped
- */
-function trimString(value: string, maxLength: number): string {
-  return value.length > maxLength ? `${value.slice(0, Math.max(0, maxLength))}...` : value;
-}
-
-/**
- * Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection
- *
- * Warning to future maintainers: The code for generics doesn't account for depth. when there is a new generic, it will
- * override the previous one, if there isn't, it will just continue passing it down.
- * @param type the ReferenceType of a DeclarationReflection
- * @param out the previous reflection, it then overrides all parameters or keeps old ones
- * @param topTypeName the name of the object, with which something went wrong
- * @param path the current path to the object we are in
- * @param tags any tags attached to the type
- */
-function getReflectionGeneric(
-  type: ReferenceType,
-  out: Map<string, MappingProperty>,
-  topTypeName: string,
-  path: string,
-  tags: CommentTag[],
-): Map<string, MappingProperty> {
-  if (
-    type.typeArguments !== undefined &&
-    type.reflection instanceof DeclarationReflection &&
-    type.reflection.typeParameters !== undefined
-  ) {
-    for (let i = 0; i < type.reflection.typeParameters.length; i++) {
-      if (i < type.typeArguments.length) {
-        out.set(
-          type.reflection.typeParameters[i].name,
-          handleType(type.typeArguments[i], out, topTypeName, path, tags),
-        );
-      } else {
-        // this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061
-        // we have no way to know the type here, so we have to use this.
-        out.set(type.reflection.typeParameters[i].name, {
-          dynamic: true,
-          properties: {},
-        });
-
-        console.warn(
-          `Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
-            ` in TypeDoc. It has been replaced by a dynamic type.`,
-        );
-      }
-    }
-  }
-
-  return out;
-}
-
-/**
- * Handles a ReferenceType that has no value
- *
- * Most of the times that is an external type.
- * @param ref the ReferenceType
- * @param generics the generics from levels above, so we can use them without having access to the parent
- * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- * @param tags any tags attached to the type
- */
-function handleExternalType(
-  ref: ReferenceType,
-  generics: Map<string, MappingProperty>,
-  path: string,
-  topTypeName: string,
-  tags: CommentTag[],
-): MappingProperty {
-  for (const premap of Object.keys(premaps)) {
-    if (premap === ref.name) {
-      return readFieldTags(premaps[premap], path, topTypeName, tags);
-    }
-  }
-
-  if (ref.name === 'Array') {
-    // basically an external type, but Array is quite common, especially with generics
-    if (ref.typeArguments === undefined || ref.typeArguments[0] === undefined) {
-      composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse');
-
-      return {type: PARSE_ERROR};
-    }
-
-    return readFieldTags(
-      handleType(
-        ref.typeArguments[0],
-        getReflectionGeneric(ref, new Map(generics), path, topTypeName, tags),
-        path,
-        topTypeName,
-        tags,
-      ),
-      path,
-      topTypeName,
-      tags,
-    );
-  }
-  if (ref.name === '__type') {
-    // empty object
-    return {
-      dynamic: 'strict',
-      properties: {},
-    };
-  }
-
-  composeErrorMessage(path, topTypeName, 'external type', ref.name, 'Missing pre-map');
-
-  return readFieldTags({type: MISSING_PREMAP}, path, topTypeName, tags);
-}
-
-/**
- * Handles an object
- * @param decl the DeclarationReflection of the object
- * @param generics the generics from levels above, so we can use them without having access to the parent
- * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- * @param inheritedTags the inherited tags
- */
-function handleDeclarationReflection(
-  decl: DeclarationReflection,
-  generics: Map<string, MappingProperty>,
-  path: string,
-  topTypeName: string,
-  inheritedTags?: CommentTag[],
-): MappingProperty {
-  // check if we have an object referencing a generic
-  if (generics.has(decl.name)) {
-    // if the object name is the same as the generic name
-    return readFieldTags(generics.get(decl.name)!, path, topTypeName, decl.comment?.tags ?? []);
-    // use the value defined by the generic
-  }
-
-  // start the actual handling process
-  const out: MappingProperty = {
-    dynamic: 'strict',
-    properties: {},
-  };
-
-  let empty = true;
-  // first check if there are any index signatures, so for example `[name: string]: Foo`
-  if (decl.indexSignature !== undefined) {
-    out.dynamic = true;
-
-    if (decl.indexSignature.type !== undefined) {
-      empty = false;
-      const template: Record<string, MappingDynamicTemplate> = {};
-      template[decl.name] = {
-        mapping: handleType(
-          decl.indexSignature.type,
-          new Map(generics),
-          path,
-          topTypeName,
-          getCommentTags(decl.indexSignature, path, topTypeName),
-        ),
-        match: '*',
-        match_mapping_type: '*',
-        path_match: `${path}*`,
-      };
-      dynamicTemplates.push(template);
-    }
-  }
-
-  if (decl.kindString === 'Enumeration') {
-    return readTypeTags('string', path, topTypeName, getCommentTags(decl, path, topTypeName, inheritedTags));
-  }
-
-  // check all the children, so in this case we are dealing with an OBJECT
-  if (decl.children !== undefined && decl.children.length > 0) {
-    for (const child of decl.children) {
-      empty = false;
-      out.properties![child.name] = handleDeclarationReflection(
-        child,
-        new Map(generics),
-        `${path}${child.name}.`,
-        topTypeName,
-      );
-    }
-  } else if (decl.type instanceof Type) {
-    // if the object is a type, so we are dealing with a PROPERTY
-    // get inherited tags
-    const tags =
-      (inheritedTags ?? []).length > 0
-        ? inheritedTags!.some(it => isTagType(it.tagName))
-          ? inheritedTags!
-          : [...(inheritedTags ?? []), ...getCommentTags(decl, path, topTypeName)]
-        : getCommentTags(decl, path, topTypeName);
-
-    return handleType(decl.type, new Map(generics), path, topTypeName, tags);
-  } else if (decl.kindString === 'Enumeration member') {
-    return readTypeTags(
-      typeof decl.defaultValue,
-      path,
-      topTypeName,
-      getCommentTags(decl, path, topTypeName, inheritedTags),
-    );
-  }
-
-  if (empty) {
-    composeErrorMessage(path, topTypeName, 'object', decl.name, 'Empty object');
-  }
-
-  return readFieldTags(out, path, topTypeName, getCommentTags(decl, path, topTypeName));
-}
-
-/**
- * Reads all comment tags, including inherited ones
- * @param decl the DeclarationReflection to read the tags from
- * @param path the path on which the comments lie
- * @param topTypeName the name of the SCThingType
- * @param inheritedTags any tags that might have been inherited by a parent
- * @param breakId the id of the previous reflection to prevent infinite recursion in some cases
- */
-function getCommentTags(
-  decl: DeclarationReflection | SignatureReflection,
-  path: string,
-  topTypeName: string,
-  inheritedTags: CommentTag[] = [],
-  // tslint:disable-next-line:no-unnecessary-initializer
-  breakId: number | undefined = undefined,
-): CommentTag[] {
-  if (decl.id === breakId) {
-    return [];
-  }
-
-  let out: CommentTag[] =
-    decl.comment instanceof Comment
-      ? decl.comment.tags !== undefined
-        ? decl.comment.tags
-        : inheritedTags
-      : inheritedTags;
-  if (
-    decl.overwrites instanceof ReferenceType &&
-    decl.overwrites.reflection instanceof DeclarationReflection
-  ) {
-    out = arrayPriorityJoin(
-      getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id),
-      out,
-    );
-  }
-  if (
-    decl.inheritedFrom instanceof ReferenceType &&
-    decl.inheritedFrom.reflection instanceof DeclarationReflection
-  ) {
-    out = arrayPriorityJoin(
-      getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id),
-      out,
-    );
-  }
-
-  saveCommentTags(out, path, topTypeName);
-  const inheritTag = out.find(value => value.tagName === inheritTagsName);
-  if (inheritTag !== undefined) {
-    out = arrayPriorityJoin(out, retrieveCommentTags(inheritTag.text.trim(), path, topTypeName));
-  }
-
-  return out;
-}
-
-/**
- * Saves all comment tags to the map
- * @param tags all tags to be saved (@see and @[inheritTags] will be stripped)
- * @param path the path of field
- * @param topTypeName the name of the SCThingType
- */
-function saveCommentTags(tags: CommentTag[], path: string, topTypeName: string) {
-  inheritTagsMap[`${topTypeName}::${path.slice(0, Math.max(0, path.length - 1))}`] = tags.filter(
-    value => value.tagName !== 'see' && value.tagName !== inheritTagsName,
-  );
-}
-
-/**
- * Retrieves any saved tags
- * @param path the path to the original field
- * @param currentPath the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- */
-function retrieveCommentTags(path: string, currentPath: string, topTypeName: string): CommentTag[] {
-  if (!(path in inheritTagsMap)) {
-    composeErrorMessage(currentPath, topTypeName, path, 'Comment', 'Referenced path to tags does not exist!');
-
-    return [];
-  }
-
-  return inheritTagsMap[path];
-}
-
-/**
- * Joins two arrays of CommentTags, but overrides all original CommentTags with the same tagName
- * @param originals the original array
- * @param overrider the array that should be appended and provide the override values
- */
-function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] {
-  const out: CommentTag[] = overrider;
-
-  for (const original of originals) {
-    const result = overrider.find(element => original.tagName === element.tagName);
-
-    // no support for multiple tags with the same name
-    if (!(result instanceof CommentTag)) {
-      out.push(original);
-    }
-  }
-
-  return out;
-}
-
-/**
- * Handles UnionTypes
- *
- * Put into a separate function as it is a little bit more complex
- * Works fairly reliable, although there are issues with primitive union types, which don't work at all (And never will)
- * @param type the type object
- * @param generics the generics from levels above, so we can use them without having access to the parent
- * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- * @param tags any tags attached to the type
- */
-function handleUnionType(
-  type: UnionType,
-  generics: Map<string, MappingProperty>,
-  path: string,
-  topTypeName: string,
-  tags: CommentTag[],
-): MappingProperty {
-  const list: MappingProperty[] = [];
-
-  for (const subType of type.types) {
-    if (subType instanceof IntrinsicType && subType.name === 'undefined') {
-      continue;
-    }
-    list.push(handleType(subType, new Map(generics), path, topTypeName, tags));
-  }
-
-  if (list.length > 0) {
-    let out = list[0];
-
-    for (const item of list) {
-      out = merge<MappingProperty>(out, item);
-    }
-
-    return out;
-  }
-
-  composeErrorMessage(
-    path,
-    topTypeName,
-    'Union Type',
-    stringify(list),
-    'Empty union type. This is likely not a user error.',
-  );
-
-  return {type: PARSE_ERROR};
-}
-
-/**
- * Serves as a kind of distributor for the different types, should not contain any specific code
- * @param type the type object
- * @param generics the generics from levels above, so we can use them without having access to the parent
- * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- * @param tags any tags attached to the type
- */
-function handleType(
-  type: Type,
-  generics: Map<string, MappingProperty>,
-  path: string,
-  topTypeName: string,
-  tags: CommentTag[],
-): MappingProperty {
-  // logger.log((type as any).name);
-  if (type instanceof ArrayType) {
-    // array is irrelevant in Elasticsearch, so just go with the element type
-    const esType = handleType(type.elementType, new Map(generics), path, topTypeName, tags);
-    // also merge tags of the array to the element type
-    // filter out the type tags lazily, this can lead to double messages for "Not implemented tag"
-    let newTags = tags;
-    if ('type' in esType) {
-      newTags = tags.filter(tag => {
-        return !(tag.tagName === esType.type);
-      });
-    }
-
-    return readFieldTags(esType, path, topTypeName, newTags);
-  }
-  if (type.type === 'stringLiteral') {
-    // a string literal, usually for type
-    return readTypeTags(type.type, path, topTypeName, tags);
-  }
-  if (type instanceof IntrinsicType) {
-    // the absolute default type, like strings
-    return readTypeTags(type.name, path, topTypeName, tags);
-  }
-  if (type instanceof UnionType) {
-    // the union type...
-    return handleUnionType(type, new Map(generics), path, topTypeName, tags);
-  }
-  if (type instanceof ReferenceType) {
-    if (premaps[type.name] === undefined && type.reflection !== undefined) {
-      // there is really no way to make this typesafe, every element in DeclarationReflection is optional.
-      return handleDeclarationReflection(
-        type.reflection as DeclarationReflection,
-        getReflectionGeneric(type, new Map(generics), path, topTypeName, tags),
-        path,
-        topTypeName,
-        tags,
-      );
-    }
-
-    return handleExternalType(type, new Map(generics), path, topTypeName, tags);
-  }
-  if (type instanceof TypeParameterType) {
-    // check if we have an object referencing a generic
-    if (generics.has(type.name)) {
-      return generics.get(type.name)!;
-    }
-    composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
-
-    return {type: PARSE_ERROR};
-  }
-  if (type instanceof ReflectionType) {
-    return readFieldTags(
-      handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
-      path,
-      topTypeName,
-      tags,
-    );
-  }
-
-  composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type');
-
-  return {type: PARSE_ERROR};
-}
-
-/**
- * Adds an aggregatable to the aggregations list
- * @param path the current path
- * @param topTypeName the name of the top type
- * @param global whether the topTypeName will be used
- */
-function addAggregatable(path: string, topTypeName: string, global: boolean) {
-  // push type.path and remove the '.' at the end of the path
-
-  if (global) {
-    const property_ = path.slice(0, -1).split('.').pop() as string; // cannot be undefined
-
-    return ((aggregations['@all'] as ESNestedAggregation).aggs[property_.split('.').pop() as string] = {
-      terms: {
-        field: `${property_}.raw`,
-        size: 1000,
-      },
-    });
-  }
-
-  const property = path.slice(0, -1);
-
-  return ((aggregations[topTypeName] as ESNestedAggregation).aggs[property] = {
-    terms: {
-      field: `${property}.raw`,
-      size: 1000,
-    },
-  });
-}
-
-/**
- * Reads all tags related to Elasticsearch fields from the fieldMap
- * @param previous the previous ElasticsearchValue, for example and object
- * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- * @param tags tags attached to the value
- * @param dataType the ElasticsearchDataType, for checking if a tag is a type tag
- */
-function readFieldTags(
-  previous: MappingProperty,
-  path: string,
-  topTypeName: string,
-  tags: CommentTag[],
-  dataType?: string,
-): MappingProperty {
-  for (const tag of tags) {
-    if (tag.tagName === aggregatableTag) {
-      addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal);
-    }
-
-    if (!ignoredTagsList.includes(tag.tagName)) {
-      if (fieldmap[tag.tagName] !== undefined) {
-        if (previous.fields === undefined) {
-          // create in case it doesn't exist
-          previous.fields = {};
-        }
-        if (tag.text.trim() === '') {
-          // merge fields
-          previous.fields = {...previous.fields, ...fieldmap[tag.tagName].default};
-        } else if (fieldmap[tag.tagName][tag.text.trim()] !== undefined) {
-          // merge fields
-          previous.fields = {...previous.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
-        } else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) {
-          // when there is an unidentified tag
-          composeErrorMessage(
-            path,
-            topTypeName,
-            'tag',
-            tag.tagName,
-            `Not implemented tag param "${tag.text.trim()}"`,
-          );
-        }
-      } else if (tag.tagName === filterableTagName) {
-        if (previous.fields === undefined) {
-          previous.fields = {};
-        }
-        if ('type' in previous) {
-          const type = filterableMap[previous.type!];
-          if (type !== undefined) {
-            // merge fields
-            previous.fields = {...previous.fields, raw: {type: type}};
-          } else {
-            composeErrorMessage(
-              path,
-              topTypeName,
-              'tag',
-              tag.tagName,
-              `Not implemented for ${previous.type}`,
-            );
-          }
-        } else {
-          composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types');
-        }
-      } else if (dataType === undefined || typemap[dataType][tag.tagName] === undefined) {
-        composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag`);
-      }
-    }
-  }
-
-  return previous;
-}
-
-/**
- * Reads all types related to Elasticsearch fields from the fieldMap
- * @param type the type of the value
- * @param path the current path to the object we are in
- * @param topTypeName the name of the SCThingType
- * @param tags tags attached to the value
- */
-function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): MappingProperty {
-  let out: MappingProperty = {type: PARSE_ERROR};
-
-  if (typemap[type] !== undefined) {
-    // first look if the value has a definition in the typemap
-    for (let i = tags.length - 1; i >= 0; i--) {
-      if (!ignoredTagsList.includes(tags[i].tagName) && typemap[type][tags[i].tagName] !== undefined) {
-        // if we have a tag that indicates a type
-        if (out.type !== PARSE_ERROR) {
-          composeErrorMessage(
-            path,
-            topTypeName,
-            'type',
-            type,
-            `Type conflict; "${typemap[type][tags[i].tagName]}" would override "${
-              (out as MappingProperty).type
-            }"`,
-          );
-          (out as MappingProperty).type = TYPE_CONFLICT;
-          continue;
-        }
-        (out as MappingProperty).type = typemap[type][tags[i].tagName];
-      }
-    }
-
-    if (out.type === PARSE_ERROR) {
-      (out as MappingProperty).type = typemap[type].default;
-    }
-
-    out = readFieldTags(out, path, topTypeName, tags, type);
-
-    return out;
-  }
-  if (dynamicTypes.includes(type)) {
-    // Elasticsearch dynamic type TODO: doesn't work for direct types
-    return {
-      dynamic: true,
-      properties: {},
-    };
-  }
-
-  composeErrorMessage(path, topTypeName, 'type', type, 'Not implemented type');
-
-  return out;
-}
-
-/**
- * Reset the state
- *
- * This is kind of a suboptimal solution and should be changed in the future.
- * https://gitlab.com/openstapps/core-tools/-/issues/49
- * @param resetInheritTags whether inherited tags should be reset as well
- */
-function reset(resetInheritTags = true) {
-  errors = [];
-  dynamicTemplates = [];
-  aggregations = {
-    '@all': {
-      aggs: {},
-      filter: {
-        match_all: {},
-      },
-    },
-  };
-
-  if (resetInheritTags) {
-    inheritTagsMap = {};
-  }
-}
-
-/**
- * Takes a project reflection and generates an ElasticsearchTemplate from it
- *
- * Serves as the entry point for getting the mapping, so if you just want to get the mapping files for Elasticsearch,
- * you can do so by calling this function, `RETURNED_VALUE.template` contains the mapping in a fashion that is directly
- * readable by Elasticsearch.
- * @param projectReflection a reflection of the project you want to get the ES Mappings from
- * @param ignoredTags the tag names for which the error output should be suppressed
- * @param showErrorOutput whether to print all errors in the command line or not
- * @param interfaceFilter only parse specific interfaces, this is for testing purposes
- */
-export function generateTemplate(
-  projectReflection: ProjectReflection,
-  ignoredTags: string[],
-  showErrorOutput = true,
-  interfaceFilter: string[] = [],
-): MappingGenTemplate {
-  reset();
-
-  showErrors = showErrorOutput;
-
-  ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
-  // eslint-disable-next-line prefer-spread
-  ignoredTagsList.push.apply(ignoredTagsList, ignoredTags);
-
-  const indexableInterfaces = getAllIndexableInterfaces(projectReflection);
-
-  const out: ElasticsearchTemplateCollection = {};
-
-  for (const _interface of indexableInterfaces) {
-    // TODO: lots of duplicate code, this all needs to be changed https://gitlab.com/openstapps/core-tools/-/issues/49
-    if (!Array.isArray(_interface.children) || _interface.children.length === 0) {
-      throw new Error('Interface needs at least some properties to be indexable');
-    }
-
-    const typeObject = _interface.children.find(declarationReflection => {
-      return declarationReflection.name === 'type';
-    });
-
-    if (typeObject === undefined || typeObject.type === undefined) {
-      throw new TypeError('Interface needs a type to be indexable');
-    }
-
-    let typeName = 'INVALID_TYPE';
-    if (typeObject.type instanceof ReferenceType) {
-      if (
-        typeObject.type.reflection instanceof DeclarationReflection &&
-        typeof typeObject.type.reflection.defaultValue === 'string'
-      ) {
-        typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
-      } else {
-        // tslint:disable-next-line:no-floating-promises
-        void console.error(
-          'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
-        );
-      }
-    } else if (typeObject.type instanceof StringLiteralType) {
-      console.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
-      typeName = typeObject.type.value;
-    } else {
-      // tslint:disable-next-line:no-floating-promises
-      void console.error(
-        `The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
-      );
-    }
-    // init aggregation schema for type
-    aggregations[typeName] = {
-      aggs: {},
-      filter: {
-        term: {
-          type: typeName,
-        },
-      },
-    };
-    handleDeclarationReflection(_interface, new Map(), '', typeName);
-  }
-
-  // second traversal
-  reset(false);
-
-  for (const _interface of indexableInterfaces) {
-    if (!Array.isArray(_interface.children) || _interface.children.length === 0) {
-      throw new Error('Interface needs at least some properties to be indexable');
-    }
-
-    const typeObject = _interface.children.find(declarationReflection => {
-      return declarationReflection.name === 'type';
-    });
-
-    if (typeObject === undefined || typeObject.type === undefined) {
-      throw new TypeError('Interface needs a type to be indexable');
-    }
-
-    let typeName = 'INVALID_TYPE';
-    if (typeObject.type instanceof ReferenceType) {
-      if (
-        typeObject.type.reflection instanceof DeclarationReflection &&
-        typeof typeObject.type.reflection.defaultValue === 'string'
-      ) {
-        typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
-      } else {
-        // tslint:disable-next-line:no-floating-promises
-        void console.error(
-          'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
-        );
-      }
-    } else if (typeObject.type instanceof StringLiteralType) {
-      console.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
-      typeName = typeObject.type.value;
-    } else {
-      // tslint:disable-next-line:no-floating-promises
-      void console.error(
-        `The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
-      );
-    }
-
-    // filter out
-    if (interfaceFilter.length > 0 && !interfaceFilter.includes(typeName)) {
-      continue;
-    }
-
-    // init aggregation schema for type
-    aggregations[typeName] = {
-      aggs: {},
-      filter: {
-        term: {
-          type: typeName,
-        },
-      },
-    };
-
-    // eslint-disable-next-line unicorn/prefer-string-replace-all
-    const typeNameWithoutSpaces = typeName.toLowerCase().replace(/\s/g, '_');
-    const templateName = `template_${typeNameWithoutSpaces}`;
-
-    out[templateName] = {
-      mappings: handleDeclarationReflection(_interface, new Map(), '', typeName) as MappingObjectProperty,
-      settings: settings,
-      index_patterns: [`stapps_${typeNameWithoutSpaces}*`],
-    };
-    out[templateName].mappings!.properties!.creation_date = {
-      type: 'date',
-    };
-
-    out[templateName].mappings!.dynamic_templates = dynamicTemplates;
-
-    // Set some properties
-    out[templateName].mappings!._source = {
-      excludes: ['creation_date'],
-    };
-    out[templateName].mappings!.date_detection = false;
-
-    dynamicTemplates = [];
-
-    if (Object.keys((aggregations[typeName] as ESNestedAggregation).aggs).length === 0) {
-      delete aggregations[typeName];
-    }
-  }
-
-  return {aggregations, mappings: out, errors};
-}

packages/es-mapping-generator/src/project-reflection.ts

@@ -1,85 +0,0 @@
-/*
- * Copyright (C) 2018-2021 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {existsSync, PathLike} from 'fs';
-import {platform} from 'os';
-import path from 'path';
-import {Application, ProjectReflection} from 'typedoc';
-
-/**
- * Get a project reflection from a path
- * @param sourcePath Path to get reflection from
- * @param excludeExternals Exclude external dependencies
- */
-export function getProjectReflection(sourcePath: PathLike, excludeExternals = true): ProjectReflection {
-  console.info(`Generating project reflection for ${sourcePath.toString()}.`);
-
-  const tsconfigPath = getTsconfigPath(sourcePath.toString());
-
-  // initialize new Typedoc application
-  const app = new Application();
-
-  app.bootstrap({
-    excludeExternals: excludeExternals,
-    ignoreCompilerErrors: true,
-    includeDeclarations: true,
-    tsconfig: path.join(tsconfigPath, 'tsconfig.json'),
-  });
-
-  let inputFilePath = sourcePath;
-  if (inputFilePath === tsconfigPath) {
-    inputFilePath = path.join(tsconfigPath, 'src');
-  }
-
-  // get input files
-  const inputFiles = app.expandInputFiles([inputFilePath.toString()]);
-
-  // get project reflection from input files
-  const result = app.convert(inputFiles);
-
-  if (result === undefined) {
-    throw new TypeError('Project reflection could not be generated.');
-  }
-
-  return result;
-}
-
-/**
- * Get path that contains a tsconfig.json
- * @param startPath Path from where to start searching "upwards"
- */
-export function getTsconfigPath(startPath: string): string {
-  let tsconfigPath = startPath;
-
-  // see https://stackoverflow.com/questions/9652043/identifying-the-file-system-root-with-node-js
-  const root = platform() === 'win32' ? process.cwd().split(path.sep)[0] : '/';
-
-  // repeat until a tsconfig.json is found
-  while (!existsSync(path.join(tsconfigPath, 'tsconfig.json'))) {
-    if (tsconfigPath === root) {
-      throw new Error(
-        `Reached file system root ${root} while searching for 'tsconfig.json' in ${startPath}!`,
-      );
-    }
-
-    // pop last directory
-    const tsconfigPathParts = tsconfigPath.split(path.sep);
-    tsconfigPathParts.pop();
-    tsconfigPath = tsconfigPathParts.join(path.sep);
-  }
-
-  console.info(`Using 'tsconfig.json' from ${tsconfigPath}.`);
-
-  return tsconfigPath;
-}

packages/es-mapping-generator/src/template.ts

@@ -0,0 +1,13 @@
+/**
+ * Render a template
+ * @param template The template to render (must be stringify-able)
+ * @param substitutions the substitutions
+ */
+export function renderTemplate<T>(template: T, substitutions: [string, string][]): T {
+  return JSON.parse(
+    substitutions.reduce(
+      (template, [search, replace]) => template.replaceAll(search, replace),
+      JSON.stringify(template),
+    ),
+  );
+}