refactor: migrate es mapping types from es-mapping-generator to .d.ts next to generated mappings

.changeset/quick-houses-count.md

@@ -0,0 +1,7 @@
+---
+'@openstapps/es-mapping-generator': major
+---
+
+Remove put-es-templates
+
+The `put-es-templates` functionality has been removed.

.changeset/sour-coins-visit.md

@@ -0,0 +1,7 @@
+---
+'@openstapps/es-mapping-generator': major
+'@openstapps/backend': major
+'@openstapps/core': major
+---
+
+Migrate es mapping types from es-mapping-generator to .d.ts next to generated mappings

.gitlab-ci.yml

@@ -47,6 +47,7 @@ default:
 stages:
   - build
   - test
+  - audit
   - publish
 
 build:
@@ -116,3 +117,12 @@ unit:
         path: coverage.xml
   rules:
     - when: on_success
+
+audit:
+  stage: audit
+  needs: []
+  script:
+    - pnpm audit --prod
+  rules:
+    - when: on_success
+      allow_failure: true

.syncpackrc.cjs

@@ -25,6 +25,7 @@ const config = {
     'dependencies',
     'devDependencies',
     'peerDependencies',
+    'pnpm',
     'tsup',
     'prettier',
     'eslintConfig',

backend/backend/package.json

@@ -45,7 +45,6 @@
     "@elastic/elasticsearch": "8.4.0",
     "@openstapps/core": "workspace:*",
     "@openstapps/core-tools": "workspace:*",
-    "@openstapps/es-mapping-generator": "workspace:*",
     "@openstapps/logger": "workspace:*",
     "@types/body-parser": "1.19.2",
     "@types/cors": "2.8.13",
@@ -89,7 +88,6 @@
     "c8": "7.14.0",
     "chai": "4.3.7",
     "chai-as-promised": "7.1.1",
-    "conventional-changelog-cli": "2.2.2",
     "cross-env": "7.0.3",
     "get-port": "5.1.1",
     "mocha": "10.2.0",

backend/backend/src/storage/elasticsearch/templating.ts

@@ -15,8 +15,8 @@
  */
 import {Client} from '@elastic/elasticsearch';
 import {SCThingType} from '@openstapps/core';
-import {AggregationSchema} from '@openstapps/es-mapping-generator/src/types/aggregation.js';
+import type {AggregationSchema} from '@openstapps/core/lib/mappings/aggregations.json.js';
-import {ElasticsearchTemplateCollection} from '@openstapps/es-mapping-generator/src/types/mapping.js';
+import type {ElasticsearchTemplateCollection} from '@openstapps/core/lib/mappings/mappings.json.js';
 import {readFileSync} from 'fs';
 import path from 'path';
 

configuration/eslint-config/package.json

@@ -17,15 +17,15 @@
   "devDependencies": {
     "@openstapps/tsconfig": "workspace:*",
     "@types/node": "18.15.3",
-    "eslint": "8.41.0",
+    "eslint": "8.43.0",
     "typescript": "4.9.5"
   },
   "peerDependencies": {
-    "@typescript-eslint/eslint-plugin": "5.59.8",
+    "@typescript-eslint/eslint-plugin": "5.60.1",
-    "@typescript-eslint/parser": "5.59.8",
+    "@typescript-eslint/parser": "5.60.1",
-    "eslint": "8.41.0",
+    "eslint": "8.43.0",
     "eslint-config-prettier": "8.8.0",
-    "eslint-plugin-jsdoc": "46.1.0",
+    "eslint-plugin-jsdoc": "46.4.2",
     "eslint-plugin-prettier": "4.2.1",
     "eslint-plugin-unicorn": "47.0.0"
   }

packages/core/package.json

@@ -38,7 +38,6 @@
     "lint": "eslint --ext .ts src/",
     "lint:fix": "eslint --fix --ext .ts src/",
     "mappings": "openstapps-es-mapping-generator mapping ../core/src -i minlength,pattern,see,tjs-format -m lib/mappings/mappings.json -a lib/mappings/aggregations.json",
-    "mappings-integration": "openstapps-es-mapping-generator put-es-templates lib/mappings/mappings.json http://elasticsearch:9200/",
     "openapi": "openstapps-core-tools openapi lib lib && node -e \"assert(JSON.parse(require('fs').readFileSync('lib/openapi.json', 'utf8')).paths['/search'] !== undefined)\"",
     "schema": "node --max-old-space-size=8192 --stack-size=10240 ./node_modules/@openstapps/core-tools/lib/app.js schema src lib/schema",
     "test": "c8 mocha"
@@ -120,11 +119,5 @@
   "eslintIgnore": [
     "resources",
     "openapi"
-  ],
+  ]
-  "openstapps-configuration": {
-    "overrides": [
-      "lint",
-      "build"
-    ]
-  }
 }

packages/core/turbo.json

@@ -0,0 +1,9 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    "test": {
+      "dependsOn": ["build"],
+      "outputs": ["coverage/cobertura-coverage.xml", "coverage/report-junit.xml"]
+    }
+  }
+}

packages/es-mapping-generator/package.json

@@ -13,6 +13,7 @@
   "files": [
     "app.js",
     "lib",
+    "schema",
     "README.md"
   ],
   "scripts": {
@@ -28,7 +29,6 @@
     "commander": "10.0.0",
     "deepmerge": "4.3.1",
     "flatted": "3.2.7",
-    "got": "11.8.6",
     "typedoc": "0.18.0",
     "typescript": "3.8.3"
   },

packages/es-mapping-generator/schema/aggregations.d.ts

@@ -12,6 +12,10 @@
  * You should have received a copy of the GNU General Public License along with
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
+declare module 'aggregations.json' {
+  const value: AggregationSchema;
+  export default value;
+}
 
 /**
  * An elasticsearch bucket aggregation

packages/es-mapping-generator/schema/mappings.d.ts

@@ -14,7 +14,12 @@ import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch
  * You should have received a copy of the GNU General Public License along with
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
-import {AggregationSchema} from './aggregation';
+import {AggregationSchema} from './aggregations';
+
+declare module 'mappings.json' {
+  const value: ElasticsearchTemplateCollection;
+  export default value;
+}
 
 /**
  * Template output of the mapping generation

packages/es-mapping-generator/src/app.ts

@@ -13,13 +13,10 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {Command} from 'commander';
-import {mkdirSync, readFileSync, writeFileSync} from 'fs';
+import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'fs';
-import got from 'got';
 import path from 'path';
-import {exit} from 'process';
 import {generateTemplate} from './mapping';
 import {getProjectReflection} from './project-reflection';
-import {ElasticsearchTemplateCollection} from './types/mapping';
 
 // handle unhandled promise rejections
 process.on('unhandledRejection', async (reason: unknown) => {
@@ -66,13 +63,22 @@ commander
       mkdirSync(path.dirname(aggPath), {recursive: true});
       // tslint:disable-next-line:no-magic-numbers
       writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
+      copyFileSync(
+        // eslint-disable-next-line unicorn/prefer-module
+        require.resolve('../schema/aggregations.d.ts'),
+        path.join(path.dirname(aggPath), 'aggregations.json.d.ts'),
+      );
       console.log(`Elasticsearch aggregations written to ${aggPath}.`);
     }
     if (options.mappingPath !== undefined) {
       const mappingPath = path.resolve(options.mappingPath);
       mkdirSync(path.dirname(mappingPath), {recursive: true});
-      // tslint:disable-next-line:no-magic-numbers
       writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
+      copyFileSync(
+        // eslint-disable-next-line unicorn/prefer-module
+        require.resolve('../schema/mappings.d.ts'),
+        path.join(path.dirname(mappingPath), 'mappings.json.d.ts'),
+      );
       console.log(`Elasticsearch mappings written to ${mappingPath}.`);
     }
     if (options.errorPath !== undefined) {
@@ -90,35 +96,4 @@ commander
     }
   });
 
-commander
-  .command('put-es-templates <srcPath> <esAddress> [ignoredTags]')
-  .action(async (relativeSourcePath, esAddress) => {
-    // get absolute paths
-    const sourcePath = path.resolve(relativeSourcePath);
-
-    // get project reflection
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module
-    const templates = require(sourcePath) as ElasticsearchTemplateCollection;
-
-    for (const template in templates) {
-      if (!templates.hasOwnProperty(template)) {
-        continue;
-      }
-
-      const response = await got.put(`${esAddress}_template/${template}`, {
-        json: templates[template],
-      });
-
-      const HTTP_STATUS_OK = 200;
-      if (response.statusCode !== HTTP_STATUS_OK) {
-        await console.error(
-          `Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`,
-        );
-        exit(-1);
-      }
-    }
-
-    console.log(`Templates accepted by Elasticsearch.`);
-  });
-
 commander.parse(process.argv);

packages/es-mapping-generator/src/config/fieldmap.ts

@@ -13,7 +13,7 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
-import {ElasticsearchFieldmap, SimpleType} from '../types/mapping.js';
+import type {ElasticsearchFieldmap, SimpleType} from '../../schema/mappings.js';
 
 const ducetSort = {
   type: 'icu_collation_keyword',

packages/es-mapping-generator/src/config/typemap.ts

@@ -13,7 +13,7 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {MappingFloatNumberProperty} from '@elastic/elasticsearch/lib/api/types';
-import {ElasticsearchTypemap} from '../types/mapping';
+import type {ElasticsearchTypemap} from '../../schema/mappings';
 
 export const PARSE_ERROR = 'PARSE_ERROR' as MappingFloatNumberProperty['type'];
 export const MISSING_PREMAP = 'MISSING_PREMAP' as MappingFloatNumberProperty['type'];

packages/es-mapping-generator/src/index.ts

@@ -5,6 +5,3 @@ export * from './config/premap';
 export * from './config/fieldmap';
 export * from './config/settings';
 export * from './config/typemap';
-
-export * from './types/mapping';
-export * from './types/aggregation';

packages/es-mapping-generator/src/mapping.ts

@@ -36,8 +36,8 @@ import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap';
 import {premaps} from './config/premap';
 import {settings} from './config/settings';
 import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap';
-import {AggregationSchema, ESNestedAggregation} from './types/aggregation';
+import type {AggregationSchema, ESNestedAggregation} from '../schema/aggregations';
-import {ElasticsearchTemplateCollection, MappingGenTemplate} from './types/mapping';
+import type {ElasticsearchTemplateCollection, MappingGenTemplate} from '../schema/mappings';
 import * as console from 'console';
 
 let dynamicTemplates: Record<string, MappingDynamicTemplate>[] = [];
@@ -862,10 +862,8 @@ export function generateTemplate(
       },
     };
 
-    let typeNameWithoutSpaces = typeName.toLowerCase();
+    // eslint-disable-next-line unicorn/prefer-string-replace-all
-    while (typeNameWithoutSpaces.includes(' ')) {
+    const typeNameWithoutSpaces = typeName.toLowerCase().replace(/\s/g, '_');
-      typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_');
-    }
     const templateName = `template_${typeNameWithoutSpaces}`;
 
     out[templateName] = {

packages/es-mapping-generator/test/mapping-model/map-agg-test.ts

@@ -12,18 +12,13 @@
  * You should have received a copy of the GNU General Public License along with
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
-import {
+import {generateTemplate, getProjectReflection, settings} from '../../src';
-  generateTemplate,
-  getProjectReflection,
-  AggregationSchema,
-  ESNestedAggregation,
-  ElasticsearchTemplateCollection,
-  settings,
-} from '../../src';
 import path from 'path';
 import {expect} from 'chai';
 import {ProjectReflection} from 'typedoc';
 import {MapAggTestOptions, MinimalMappingDescription} from './map-agg-test-options';
+import type {AggregationSchema, ESNestedAggregation} from '../../schema/aggregations';
+import type {ElasticsearchTemplateCollection} from '../../schema/mappings';
 
 export class MapAggTest {
   mapping_model_path!: string;