feat: pipeline improvements

.syncpackrc.cjs

@@ -40,12 +40,6 @@ const config = {
       packages: ['**'],
       isIgnored: true,
     },
-    {
-      label: 'ES Mapping Generator Special Dependencies',
-      dependencies: ['typescript', 'typedoc', 'ts-node', '@types/node', 'got'],
-      packages: ['@openstapps/es-mapping-generator'],
-      isIgnored: true,
-    },
     {
       label: 'Packages should use workspace version',
       dependencies: ['@openstapps/**'],

backend/e2e-connector/package.json

@@ -70,8 +70,7 @@
   },
   "tsup": {
     "entry": [
-      "src/app.ts",
+      "src/cli.ts"
-      "src/index.ts"
     ],
     "sourcemap": true,
     "clean": true,

packages/collection-utils/package.json

@@ -3,15 +3,15 @@
   "version": "3.0.0",
   "type": "module",
   "license": "GPL-3.0-only",
-  "main": "src/index.js",
+  "main": "./src/index.js",
-  "types": "src/types.d.ts",
+  "types": "./src/types.d.ts",
   "files": [
-    "lib",
+    "src",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.ts",
+    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/types.d.ts",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
     "lint": "tsc --noEmit && eslint --ext .js src/",
@@ -38,6 +38,5 @@
     "extends": [
       "@openstapps"
     ]
-  },
+  }
-  "exports": "./lib/index.js"
 }

packages/core-validator/package.json

@@ -16,18 +16,18 @@
   "main": "./src/index.js",
   "types": "./src/types.d.ts",
   "files": [
-    "lib",
+    "src",
     "schema",
     "Dockerfile",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.ts",
+    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/types.d.ts",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
-    "lint": "eslint --ext .ts src/",
+    "lint": "tsc --noEmit && eslint --ext .js src/",
-    "lint:fix": "eslint --fix --ext .ts src/",
+    "lint:fix": "eslint --fix --ext .js src/",
     "test": "c8 mocha"
   },
   "dependencies": {
@@ -51,6 +51,7 @@
     "mocha": "10.2.0",
     "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
+    "ts-node": "10.9.1",
     "typedoc": "0.24.8",
     "typescript": "5.1.6"
   },

packages/core-validator/test/validate.spec.js

@@ -0,0 +1,58 @@
+/* eslint-disable unicorn/prefer-module */
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {expect} from 'chai';
+import path from 'path';
+import {fileURLToPath} from 'url';
+import {stat, readFile} from 'fs/promises';
+import {Validator} from '../src/index.js';
+
+describe('Validator', function () {
+  this.timeout(40_000);
+  this.slow(5000);
+
+  /** @type {import('json-schema').JSONSchema7} */
+  let schema;
+  /** @type {import('@openstapps/core').SCThings[]} */
+  let instances = [];
+
+  beforeEach(async function () {
+    const resourceDirectory = path.join(
+      path.dirname(fileURLToPath(import.meta.url)),
+      '..',
+      '..',
+      'core',
+      'test',
+      'resources',
+    );
+    /** @type {import('@openstapps/core').SCThings[]} */
+    instances = [];
+    for (const file in await stat(resourceDirectory)) {
+      if (file.endsWith('.json')) {
+        const {instance} = JSON.parse(await readFile(path.join(resourceDirectory, file), 'utf8'));
+        instances.push(instance);
+      }
+    }
+  });
+
+  it('should validate schemas by their SCThingType', async function () {
+    const validator = new Validator();
+    for (const instance of instances) {
+      const validationResult = validator.validate(instance, instance.type);
+      expect(validator.errors).to.be.empty;
+      expect(validationResult).to.be.true;
+    }
+  });
+});

packages/core-validator/test/validate.spec.ts

@@ -1,85 +0,0 @@
-/* eslint-disable unicorn/prefer-module */
-/*
- * Copyright (C) 2019 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {Logger} from 'packages/logger/lib/index.js';
-import {expect} from 'chai';
-import {existsSync} from 'fs';
-import {JSONSchema7 as Schema} from 'json-schema';
-import {Foo} from '@openstapps/core-tools/src/resources/foo.js';
-import {Validator} from '../src/validate.js';
-import path from 'path';
-import {fileURLToPath} from 'url';
-import {rm, mkdir, writeFile} from 'fs/promises';
-import {Converter} from '@openstapps/core-tools/src/index.js';
-
-process.on('unhandledRejection', (error: unknown) => {
-  if (error instanceof Error) {
-    void Logger.error('UNHANDLED REJECTION', error.stack);
-  }
-  process.exit(1);
-});
-
-const tmpdir = path.join(path.dirname(fileURLToPath(import.meta.url)), 'tmp');
-const fooInstance: Foo = {
-  lorem: 'ipsum',
-  type: 'Foo',
-};
-
-describe('Validator', function () {
-  this.timeout(40_000);
-  this.slow(5000);
-
-  let schema: Schema;
-  let converter: Converter;
-
-  beforeEach(async function () {
-    converter = new Converter(
-      path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'src', 'resources'),
-    );
-    schema = converter.getSchema('Foo', '0.0.1');
-    if (!existsSync(tmpdir)) {
-      await mkdir(tmpdir);
-    }
-    await writeFile(path.join(tmpdir, 'SCFoo.json'), JSON.stringify(schema, undefined, 2));
-  });
-
-  afterEach(async function () {
-    try {
-      await rm(tmpdir, {recursive: true});
-    } catch (error) {
-      expect(error, `Unable to remove temporary directory for tests at: ${tmpdir}`).to.be.null;
-    }
-  });
-
-  it('should validate by schema identifying string', async function () {
-    const validator = new Validator();
-    await validator.addSchemas(tmpdir);
-    const validationResult = validator.validate(fooInstance, 'SCFoo');
-    expect(validationResult.errors, JSON.stringify(validationResult.errors, undefined, 2)).to.be.empty;
-  });
-
-  it('should validate by schema instance', async function () {
-    const validator = new Validator();
-    const validationResult = validator.validate(fooInstance, schema);
-    expect(validationResult.errors, JSON.stringify(validationResult.errors, undefined, 2)).to.be.empty;
-  });
-
-  it('should validate intrinsic', async function () {
-    const validator = new Validator();
-    await validator.addSchemas(tmpdir);
-    const validationResult = validator.validate(fooInstance);
-    expect(validationResult.errors, JSON.stringify(validationResult.errors, undefined, 2)).to.be.empty;
-  });
-});

packages/core-validator/tsconfig.json

@@ -1,7 +1,3 @@
 {
-  "extends": "@openstapps/tsconfig",
+  "extends": "@openstapps/tsconfig"
-  "compilerOptions": {
-    "noUnusedLocals": false,
-    "stripInternal": true
-  }
 }

packages/es-mapping-generator/package.json

@@ -8,13 +8,13 @@
   "main": "./src/index.js",
   "types": "./src/types.d.ts",
   "files": [
-    "app.js",
+    "src",
-    "lib",
     "schema",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
+    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/types.d.ts",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
     "lint": "tsc --noEmit && eslint --ext .js src/",
@@ -25,9 +25,7 @@
     "@elastic/elasticsearch": "8.10.0",
     "@openstapps/json-schema-generator": "workspace:*",
     "@openstapps/tsup-plugin": "workspace:*",
-    "@types/json-schema": "7.0.14",
+    "@types/json-schema": "7.0.14"
-    "ajv": "8.12.0",
-    "ajv-formats": "2.1.1"
   },
   "devDependencies": {
     "@openstapps/eslint-config": "workspace:*",

packages/es-mapping-generator/src/generator/context.js

@@ -71,11 +71,10 @@ export class Context {
    */
   deriveContext(reference) {
     let referenceName = typeof reference === 'string' ? getNameFromRef(reference) : undefined;
-    if (referenceName === undefined) this.bail(`Can't find reference name for ${reference}`);
     let definition = /** @type {JSONSchema} */ (typeof reference === 'string' ? undefined : reference);
 
-    if (!definition && !this.generator.cache.has(referenceName)) {
+    if (!definition && !this.generator.cache.has(/** @type {string} */ (referenceName))) {
-      const reference = this.generator.project.definitions[referenceName];
+      const reference = this.generator.project.definitions[/** @type {string} */ (referenceName)];
       if (typeof reference === 'boolean') this.bail('Invalid schema');
       definition = reference;
       if (typeof definition !== 'object') this.bail(`Invalid path ${referenceName}`);

packages/es-mapping-generator/src/generator/mapping-generator.js

@@ -26,12 +26,12 @@ export class MappingGenerator {
   searchMods = {mods: {}};
 
   /**
-   * @param project {JSONSchema & Pick<Required<JSONSchema>, 'definitions'>}
+   * @param project {JSONSchema}
    * @param options {import('../types.js').GeneratorOptions}
    */
   constructor(project, options) {
     if (!project.definitions) throw new Error('Invalid schema');
-    this.project = project;
+    this.project = /** @type {JSONSchema & Pick<Required<JSONSchema>, 'definitions'>} */ (project);
     this.template = options.template ?? {};
     this.presets = new Map(Object.entries(options.presets));
     this.cache = new Map(

packages/es-mapping-generator/test/mapping-model/map-agg-test-options.ts

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU General Public License along with
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
-import {MappingDynamicTemplate, MappingProperty} from '@elastic/elasticsearch/lib/api/types';
+import {MappingDynamicTemplate, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
 
 export interface MapAggTestOptions {
   testName: string;

packages/es-mapping-generator/test/mapping-model/map-agg-test.ts

@@ -16,9 +16,9 @@ import {generateTemplate, getProjectReflection, settings} from '../../src';
 import path from 'path';
 import {expect} from 'chai';
 import {ProjectReflection} from 'typedoc';
-import {MapAggTestOptions, MinimalMappingDescription} from './map-agg-test-options';
+import {MapAggTestOptions, MinimalMappingDescription} from './map-agg-test-options.js';
-import type {AggregationSchema, ESNestedAggregation} from '../../schema/aggregations';
+import type {AggregationSchema, ESNestedAggregation} from '../../schema/aggregations.js';
-import type {ElasticsearchTemplateCollection} from '../../schema/mappings';
+import type {ElasticsearchTemplateCollection} from '../../schema/mappings.js';
 
 export class MapAggTest {
   mapping_model_path!: string;

packages/es-mapping-generator/test/mapping-model/mappings/mapping-test.schema.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema",
+  "$ref": "#/definitions/MappingSchema",
+  "definitions": {
+    "MappingSchema": {
+      "type": "object",
+      "properties": {
+        "schema": {
+          "$ref": "https://json-schema.org/draft-07/schema"
+        },
+        "config": {
+          "type": "object"
+        }
+      },
+      "required": ["schema", "config"]
+    }
+  }
+}

packages/es-mapping-generator/test/mapping-model/mappings/src/any-unknown.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "../mapping-test.schema.json",
+  "schema": {
+    "definitions": ""
+  },
+  "config": {}
+}

packages/es-mapping-generator/tsconfig.spec.json

@@ -1,4 +0,0 @@
-{
-  "extends": "./tsconfig.json",
-  "exclude": ["./lib/", "./src/"]
-}

packages/es-mapping-generator/tsup.config.ts

@@ -1,9 +0,0 @@
-import {defineConfig} from 'tsup';
-
-export default defineConfig({
-  entry: ['src/index.ts'],
-  sourcemap: true,
-  clean: true,
-  format: 'esm',
-  outDir: 'lib',
-});

packages/json-schema-generator/package.json

@@ -16,16 +16,16 @@
   "main": "src/index.js",
   "types": "src/types.d.ts",
   "files": [
-    "lib",
+    "src",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.ts",
+    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/types.d.ts",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
-    "lint": "eslint --ext .ts src/",
+    "lint": "tsc --noEmit && eslint --ext .js src/",
-    "lint:fix": "eslint --fix --ext .ts src/",
+    "lint:fix": "eslint --fix --ext .js src/",
     "test": "c8 mocha"
   },
   "dependencies": {
@@ -50,8 +50,8 @@
     "mocha": "10.2.0",
     "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
-    "typedoc": "0.24.8",
     "ts-node": "10.9.1",
+    "typedoc": "0.24.8",
     "typescript": "5.1.6"
   },
   "prettier": "@openstapps/prettier-config",

packages/openapi-generator/package.json

@@ -16,12 +16,12 @@
   "main": "src/index.js",
   "types": "src/types.d.ts",
   "files": [
-    "lib",
+    "src",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/generator/index.ts",
+    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/types.d.ts",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
     "lint": "tsc --noEmit && eslint --ext .js src/",

packages/tsup-plugin/package.json

@@ -16,12 +16,12 @@
   "main": "src/index.js",
   "types": "src/types.d.ts",
   "files": [
-    "lib",
+    "src",
     "README.md",
     "CHANGELOG.md"
   ],
   "scripts": {
-    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.js",
+    "docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/types.d.ts",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
     "lint": "tsc --noEmit && eslint --ext .js src/",

pnpm-lock.yaml

@@ -1831,6 +1831,9 @@ importers:
       prettier:
         specifier: 2.8.6
         version: 2.8.6
+      ts-node:
+        specifier: 10.9.1
+        version: 10.9.1(@types/node@18.15.3)(typescript@5.1.6)
       typedoc:
         specifier: 0.24.8
         version: 0.24.8(typescript@5.1.6)
@@ -1931,12 +1934,6 @@ importers:
       '@types/json-schema':
         specifier: 7.0.14
         version: 7.0.14
-      ajv:
-        specifier: 8.12.0
-        version: 8.12.0
-      ajv-formats:
-        specifier: 2.1.1
-        version: 2.1.1(ajv@8.12.0)
     devDependencies:
       '@openstapps/eslint-config':
         specifier: workspace:*