feat: generator updates

This commit is contained in:
2023-11-07 17:14:58 +01:00
parent 9ef77ab3ed
commit 8a421cb2fb
51 changed files with 1385 additions and 1241 deletions

View File

@@ -50,18 +50,18 @@
"@openstapps/tsconfig": "workspace:*",
"@types/chai": "4.3.5",
"@types/chai-as-promised": "7.1.5",
"@types/chai-spies": "1.0.3",
"@types/chai-spies": "1.0.6",
"@types/mocha": "10.0.1",
"c8": "7.14.0",
"chai": "4.3.7",
"chai-as-promised": "7.1.1",
"chai-spies": "1.0.0",
"chai-spies": "1.1.0",
"conventional-changelog-cli": "2.2.2",
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"nock": "13.3.1",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},

View File

@@ -42,20 +42,20 @@
"@openstapps/tsconfig": "workspace:*",
"@types/chai": "4.3.5",
"@types/chai-as-promised": "7.1.5",
"@types/chai-spies": "1.0.3",
"@types/chai-spies": "1.0.6",
"@types/mocha": "10.0.1",
"@types/traverse": "0.6.32",
"c8": "7.14.0",
"chai": "4.3.7",
"chai-as-promised": "7.1.1",
"chai-spies": "1.0.0",
"chai-spies": "1.1.0",
"conventional-changelog-cli": "2.2.2",
"date-fns": "2.30.0",
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"traverse": "0.6.7",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6",
"undici": "5.22.1"

View File

@@ -30,7 +30,7 @@
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},

View File

@@ -13,8 +13,8 @@
"core",
"validator"
],
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"main": "./src/index.js",
"types": "./src/types.d.ts",
"files": [
"lib",
"schema",
@@ -23,7 +23,6 @@
"CHANGELOG.md"
],
"scripts": {
"build": "tsup-node --dts",
"docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.ts",
"format": "prettier . -c --ignore-path ../../.gitignore",
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
@@ -52,21 +51,9 @@
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"nock": "13.3.1",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},
"tsup": {
"entry": [
"src/app.ts",
"src/index.ts"
],
"sourcemap": true,
"clean": true,
"format": "esm",
"outDir": "lib"
},
"prettier": "@openstapps/prettier-config",
"eslintConfig": {
"extends": [

View File

@@ -0,0 +1,62 @@
import Ajv from 'ajv';
import addFormats from 'ajv-formats';
import schema from '@openstapps/core/schema.json' assert {type: 'json'};
/**
* StAppsCore validator
* @typedef {import('ajv').AnySchema} AnySchema
* @typedef {import('@openstapps/core/schema.json').SchemaMap} SchemaMap
*/
export class Validator {
/**
* @private
* @readonly
* @type {Ajv.default}
*/
ajv;
get errors() {
return this.ajv.errors;
}
/**
* @param additionalSchemas {AnySchema[]}
*/
constructor(additionalSchemas = []) {
this.ajv = new Ajv.default({
schemas: [schema, ...additionalSchemas],
verbose: true,
keywords: ['elasticsearch'],
allowUnionTypes: true,
});
addFormats.default(this.ajv, {
formats: ['date-time', 'time', 'uuid', 'duration'],
mode: 'fast',
});
}
/**
* Add additional schemas to the validator
* @param schema {AnySchema[]}
*/
addSchema(...schema) {
this.ajv.addSchema(schema);
}
/**
* Validates anything against a given schema name or infers schema name from object
* @template {SchemaMap[keyof SchemaMap]} T
*
* @param instance {unknown} Instance to validate
* @param schema {import('./types.js').NameOf<T>} Name of schema to validate instance against or the schema itself
* @returns {instance is T}
*
*/ /**
* @param instance {unknown}
* @param schema {Ajv.Schema | string}
* @returns {boolean}
*/
validate(instance, schema) {
return this.ajv.validate(typeof schema === 'string' ? `#/definitions/${schema}` : schema, instance);
}
}

View File

@@ -1,63 +0,0 @@
import Ajv, {AnySchema} from 'ajv';
import addFormats from 'ajv-formats';
import schema from '@openstapps/core/schema.json' assert {type: 'json'};
import type {SchemaMap} from '@openstapps/core/schema.json';
export type RemoveNeverProperties<T> = {
[K in Exclude<
keyof T,
{
// eslint-disable-next-line @typescript-eslint/ban-types
[P in keyof T]: T[P] extends Function ? P : never;
}[keyof T]
>]: T[K];
};
export type IncludeProperty<T extends object, E> = RemoveNeverProperties<{
[K in keyof T]: T[K] extends E ? T[K] : never;
}>;
type NameOf<I extends SchemaMap[keyof SchemaMap]> = keyof IncludeProperty<SchemaMap, I>;
/**
* StAppsCore validator
*/
export class Validator {
private readonly ajv: Ajv.default;
get errors() {
return this.ajv.errors;
}
constructor(additionalSchemas: AnySchema[] = []) {
this.ajv = new Ajv.default({
schemas: [schema, ...additionalSchemas],
verbose: true,
keywords: ['elasticsearch'],
allowUnionTypes: true,
});
addFormats.default(this.ajv, {
formats: ['date-time', 'time', 'uuid', 'duration'],
mode: 'fast',
});
}
/**
* Add additional schemas to the validator
*/
public addSchema(...schema: AnySchema[]) {
this.ajv.addSchema(schema);
}
/**
* Validates anything against a given schema name or infers schema name from object
* @param instance Instance to validate
* @param schema Name of schema to validate instance against or the schema itself
*/
public validate<T extends SchemaMap[keyof SchemaMap]>(instance: unknown, schema: NameOf<T>): instance is T;
public validate(instance: unknown, schema: Ajv.Schema): boolean;
public validate(instance: unknown, schema: string): boolean;
public validate(instance: unknown, schema: Ajv.Schema | string): boolean {
return this.ajv.validate(typeof schema === 'string' ? `#/definitions/${schema}` : schema, instance);
}
}

19
packages/core-validator/src/types.d.ts vendored Normal file
View File

@@ -0,0 +1,19 @@
import {SchemaMap} from '@openstapps/core/schema.json';
export type RemoveNeverProperties<T> = {
[K in Exclude<
keyof T,
{
// eslint-disable-next-line @typescript-eslint/ban-types
[P in keyof T]: T[P] extends Function ? P : never;
}[keyof T]
>]: T[K];
};
export type IncludeProperty<T extends object, E> = RemoveNeverProperties<{
[K in keyof T]: T[K] extends E ? T[K] : never;
}>;
export type NameOf<I extends SchemaMap[keyof SchemaMap]> = keyof IncludeProperty<SchemaMap, I>;
export {Validator} from './index.js';

View File

@@ -25,20 +25,6 @@
],
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"exports": {
".": {
"import": "./lib/index.js",
"types": "./lib/index.d.ts"
},
"./schema.json": {
"import": "./lib/schema.json",
"types": "./lib/schema.json.d.ts"
},
"./elasticsearch.json": {
"import": "./lib/elasticsearch.json",
"types": "./lib/elasticsearch.json.d.ts"
}
},
"files": [
"lib",
"test/resources",
@@ -84,7 +70,7 @@
"source-map-support": "0.5.21",
"surge": "0.23.1",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},
@@ -122,5 +108,19 @@
"eslintIgnore": [
"resources",
"openapi"
]
],
"exports": {
".": {
"import": "./lib/index.js",
"types": "./lib/index.d.ts"
},
"./schema.json": {
"import": "./lib/schema.json",
"types": "./lib/schema.json.d.ts"
},
"./elasticsearch.json": {
"import": "./lib/elasticsearch.json",
"types": "./lib/elasticsearch.json.d.ts"
}
}
}

View File

@@ -40,7 +40,7 @@
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8"
},
"tsup": {

View File

@@ -5,8 +5,8 @@
"type": "module",
"license": "GPL-3.0-only",
"author": "Thea Schöbl <dev@theaninova.de>",
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"main": "./src/index.js",
"types": "./src/types.d.ts",
"files": [
"app.js",
"lib",
@@ -15,20 +15,19 @@
"CHANGELOG.md"
],
"scripts": {
"build": "tsup-node --dts",
"format": "prettier . -c --ignore-path ../../.gitignore",
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
"lint": "eslint --ext .ts src/",
"lint:fix": "eslint --ext .ts src/",
"lint": "tsc --noEmit && eslint --ext .js src/",
"lint:fix": "eslint --ext .js src/",
"test": "c8 mocha"
},
"dependencies": {
"ajv": "8.12.0",
"ajv-formats": "2.1.1",
"@elastic/elasticsearch": "8.10.0",
"@openstapps/json-schema-generator": "workspace:*",
"@openstapps/tsup-plugin": "workspace:*",
"@types/json-schema": "7.0.14"
"@types/json-schema": "7.0.14",
"ajv": "8.12.0",
"ajv-formats": "2.1.1"
},
"devDependencies": {
"@openstapps/eslint-config": "workspace:*",
@@ -47,7 +46,7 @@
"mocha-junit-reporter": "2.2.0",
"nock": "13.3.1",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},

View File

@@ -1,60 +0,0 @@
import {MappingProperty, SearchRequest} from '@elastic/elasticsearch/lib/api/types.js';
import Ajv from 'ajv';
import {readFile} from 'fs/promises';
import {fileURLToPath} from 'url';
import path from 'path';
import {Context} from '../generator/context.js';
/**
* @validatable
*/
export interface ElasticsearchOptionsDSL {
/**
* Mark an interface as indexable
*/
indexable?: true;
/**
* Inherit customization options from another item
*/
extends?: string[];
/**
* Completely override the property
*/
override?: MappingProperty;
/**
* Merge property values
*/
merge?: MappingProperty;
/**
* Modify the search request
*
* Supports `{name}`, `{type}` and `{prop}` templates substitutions anywhere
*/
search?: Partial<SearchRequest>;
}
const schema = JSON.parse(
await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), 'index.schema.json'), 'utf8'),
);
const ajv = new Ajv.default({schemas: [schema], allowUnionTypes: true});
/**
* Validate that the options are valid
*/
export function validateElasticsearchOptionsDsl(
context: Context,
value: unknown,
): value is ElasticsearchOptionsDSL {
return ajv.validate('#/definitions/ElasticsearchOptionsDSL', value)
? true
: context.bail(JSON.stringify(ajv.errors));
}
/**
* Validate that the mapping result is correct
*/
export function validateMappingResult(context: Context, value: unknown): value is MappingProperty {
return ajv.validate('#/definitions/MappingProperty', value)
? true
: context.bail(JSON.stringify(ajv.errors));
}

View File

@@ -1,22 +1,24 @@
import {JSONSchema7} from 'json-schema';
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
import {transformObject} from './transformers/object.js';
import {transformString} from './transformers/string.js';
import {Context} from './context.js';
import {transformDefinition} from './definition.js';
/**
* Transform JSONSchema without applying custom tag logic
* @typedef {import('json-schema').JSONSchema7} JSONSchema
*
* @param context {import('./context.js').Context}
* @param definition {JSONSchema}
* @returns {import('../types.js').MappingProperty}
*/
export function transformBase(context: Context, definition: JSONSchema7): MappingProperty {
export function transformBase(context, definition) {
if (definition.anyOf) {
return context.resolveUnion(definition.anyOf as JSONSchema7[]);
return context.resolveUnion(/** @type {JSONSchema[]} */ (definition.anyOf));
}
switch (definition.type) {
case 'array': {
if (Array.isArray(definition.items)) {
return context.resolveUnion(definition.items as JSONSchema7[]);
return context.resolveUnion(/** @type {JSONSchema[]} */ (definition.items));
} else if (typeof definition.items === 'object') {
return transformDefinition(context, definition.items);
} else {

View File

@@ -0,0 +1,159 @@
import {transformDefinition} from './definition.js';
import crypto from 'crypto';
import deepmerge from 'deepmerge';
import {sanitizeTypeName} from './mapping-generator.js';
import {renderTemplate} from '../template.js';
/**
* Get the name from a $ref such as `#/definitions/SCThing`
* @param ref {string}
* @returns {string}
*/
function getNameFromRef(ref) {
return decodeURI(ref).replace(/^#\/definitions\//, '');
}
/**
* @typedef {import('@elastic/elasticsearch/lib/api/types.d.ts').MappingProperty} MappingProperty
* @typedef {import('@elastic/elasticsearch/lib/api/types.d.ts').MappingDynamicProperty} MappingDynamicProperty
* @typedef {import('@elastic/elasticsearch/lib/api/types.d.ts').SearchRequest} SearchRequest
* @typedef {import('json-schema').JSONSchema7} JSONSchema
*
*
* @property {string} thingType
* @property {string[]} path
* @property {string[]} propertyPath
* @property {Map<string, Set<string>>} dependencies
*/
export class Context {
/**
* @param generator {import('./mapping-generator.js').MappingGenerator}
* @param thingType {string}
* @param path {string[]}
* @param propertyPath {string[]}
* @param dependencies {Map<string, Set<string>>}
*/
constructor(generator, thingType, path, propertyPath, dependencies) {
this.generator = generator;
this.thingType = thingType;
this.path = path;
this.propertyPath = propertyPath;
this.dependencies = dependencies;
}
/**
* @param reference {string}
* @returns {MappingProperty}
*/
resolveReference(reference) {
return this.deriveContext(reference)[1];
}
/**
* @param types {JSONSchema[]}
* @returns {MappingDynamicProperty}
*/
resolveUnion(types) {
for (const type of types) {
const [name] = this.deriveContext(type.$ref ?? type);
this.addDependency(name, this.propertyPath.join('.'));
}
return {
type: '{dynamic_property}',
};
}
/**
* @private
* @param reference {string | JSONSchema}
* @returns {[dependency: string, mapping: MappingProperty]}
*/
deriveContext(reference) {
let referenceName = typeof reference === 'string' ? getNameFromRef(reference) : undefined;
if (referenceName === undefined) this.bail(`Can't find reference name for ${reference}`);
let definition = /** @type {JSONSchema} */ (typeof reference === 'string' ? undefined : reference);
if (!definition && !this.generator.cache.has(referenceName)) {
const reference = this.generator.project.definitions[referenceName];
if (typeof reference === 'boolean') this.bail('Invalid schema');
definition = reference;
if (typeof definition !== 'object') this.bail(`Invalid path ${referenceName}`);
}
if (!referenceName || !this.generator.cache.has(referenceName)) {
const derivedContext = new Context(
this.generator,
this.thingType,
referenceName ? [referenceName] : [],
this.propertyPath,
new Map(),
);
const result = transformDefinition(derivedContext, definition);
referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});
}
const cached = this.generator.cache.get(referenceName);
if (!cached) this.bail('Internal error');
for (const [name, paths] of cached.dependencies) {
for (const path of paths) {
this.addDependency(name, [...this.propertyPath.slice(0, -1), path].join('.'));
}
}
return [referenceName, cached.mapping];
}
/**
* @param modification {Partial<SearchRequest>}
*/
registerSearchMod(modification) {
this.generator.searchMods.mods = deepmerge(
this.generator.searchMods.mods,
renderTemplate(modification, [
['{name}', this.path[0]],
['{prop}', this.propertyPath.join('.')],
['{type}', this.thingType],
['{_type}', sanitizeTypeName(this.thingType)],
]),
);
}
/**
* @private
* @param name {string}
* @param path {string}
*/
addDependency(name, path) {
if (!this.dependencies.has(name)) {
this.dependencies.set(name, new Set([path]));
} else {
this.dependencies.get(name)?.add(path);
}
}
/**
* Step down into a property
* @param property {string}
* @returns {Context}
*/
step(property) {
return new Context(
this.generator,
this.thingType,
[...this.path, property],
[...this.propertyPath, property],
this.dependencies,
);
}
/**
* Bail and throw
* @param reason {string}
* @returns {never}
*/
bail(reason) {
throw new Error(`${this.path.join('.')} ${reason}`);
}
}

View File

@@ -1,115 +0,0 @@
import {JSONSchema7} from 'json-schema';
import {transformDefinition} from './definition.js';
import crypto from 'crypto';
import {
MappingProperty,
MappingDynamicProperty,
SearchRequest,
} from '@elastic/elasticsearch/lib/api/types.js';
import deepmerge from 'deepmerge';
import {MappingGenerator, sanitizeTypeName} from './mapping-generator.js';
import {renderTemplate} from '../template.js';
/**
* Get the name from a $ref such as `#/definitions/SCThing`
*/
function getNameFromRef(ref: string): string {
return decodeURI(ref).replace(/^#\/definitions\//, '');
}
export class Context {
constructor(
readonly generator: MappingGenerator,
readonly thingType: string,
readonly path: string[],
readonly propertyPath: string[],
readonly dependencies: Map<string, Set<string>>,
) {}
resolveReference(reference: string): MappingProperty {
return this.deriveContext(reference)[1];
}
resolveUnion(types: JSONSchema7[]): MappingDynamicProperty {
for (const type of types) {
const [name] = this.deriveContext(type.$ref ?? type);
this.addDependency(name, this.propertyPath.join('.'));
}
return {
type: '{dynamic_property}',
};
}
private deriveContext(reference: string | JSONSchema7): [dependencyName: string, mapping: MappingProperty] {
let referenceName = typeof reference === 'string' ? getNameFromRef(reference) : undefined;
let definition = typeof reference === 'string' ? undefined : reference;
if (!definition && !this.generator.cache.has(referenceName!)) {
definition = this.generator.project.definitions![referenceName!] as JSONSchema7;
if (typeof definition !== 'object') this.bail(`Invalid path ${referenceName!}`);
}
if (!referenceName || !this.generator.cache.has(referenceName)) {
const derivedContext = new Context(
this.generator,
this.thingType,
referenceName ? [referenceName] : [],
this.propertyPath,
new Map(),
);
const result = transformDefinition(derivedContext, definition!);
referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});
}
const {mapping, dependencies} = this.generator.cache.get(referenceName)!;
for (const [name, paths] of dependencies) {
for (const path of paths) {
this.addDependency(name, [...this.propertyPath.slice(0, -1), path].join('.'));
}
}
return [referenceName, mapping];
}
registerSearchMod(modification: Partial<SearchRequest>) {
this.generator.searchMods.mods = deepmerge<Partial<SearchRequest>>(
this.generator.searchMods.mods,
renderTemplate(modification, [
['{name}', this.path[0]],
['{prop}', this.propertyPath.join('.')],
['{type}', this.thingType],
['{_type}', sanitizeTypeName(this.thingType)],
]),
);
}
private addDependency(name: string, path: string) {
if (!this.dependencies.has(name)) {
this.dependencies.set(name, new Set([path]));
} else {
this.dependencies.get(name)!.add(path);
}
}
/**
* Step down into a property
*/
step(property: string): Context {
return new Context(
this.generator,
this.thingType,
[...this.path, property],
[...this.propertyPath, property],
this.dependencies,
);
}
/**
* Bail and throw
*/
bail(reason: string): never {
throw new Error(`${this.path.join('.')} ${reason}`);
}
}

View File

@@ -1,27 +1,29 @@
import {JSONSchema7} from 'json-schema';
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
import {transformBase} from './base.js';
import {Context} from './context.js';
import deepmerge from 'deepmerge';
import {resolveDsl} from './dsl.js';
import {resolveOptions} from './options.js';
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
/**
* Transform JSONSchema
* @typedef {import('../types.js').MappingProperty} MappingProperty
* @param context {Context}
* @param definition {import('json-schema').JSONSchema7}
* @returns {MappingProperty}
*/
export function transformDefinition(context: Context, definition: JSONSchema7): MappingProperty {
export function transformDefinition(context, definition) {
if (definition.$ref) return context.resolveReference(definition.$ref);
const tags = getTags(definition);
tags.delete(INDEXABLE_TAG_NAME);
let base = transformBase(context, definition);
if (tags.size > 0) {
const options = resolveDsl(context, {extends: [...tags]});
const options = resolveOptions(context, {extends: [...tags]});
if (options.override) {
base = options.override;
}
if (options.merge) {
base = deepmerge<MappingProperty>(base, options.merge);
base = /** @type {typeof deepmerge<MappingProperty>} */ (deepmerge)(base, options.merge);
}
if (options.search) {
context.registerSearchMod(options.search);

View File

@@ -1,34 +1,16 @@
import type {JSONSchema7} from 'json-schema';
import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
import {MappingGenerator} from './mapping-generator.js';
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
import {ElasticsearchConfig} from '../../schema/mappings.js';
export interface GeneratorOptions {
/**
* Presets you can extend
*/
presets: Record<string, ElasticsearchOptionsDSL>;
/**
* Override specific types
*/
overrides: Record<string, MappingProperty>;
/**
* Template for the generated index request
*
* Supports `{type}` and `{sanitized_type}` (same as `{type}`, but no spaces) template substitutions
*/
template: Partial<IndicesPutTemplateRequest>;
}
/**
* Fully transform a project
* @param project {import('json-schema').JSONSchema7}
* @returns {import('../../schema/mappings.js').default}
*/
export function transformProject(project: JSONSchema7): ElasticsearchConfig {
export function transformProject(project) {
const context = new MappingGenerator(project, OPTIONS);
const results: Record<string, IndicesPutTemplateRequest> = {};
/** @type {Record<string, import('../types.js').IndicesPutTemplateRequest>} */
const results = {};
for (const name in project.definitions) {
const definition = project.definitions[name];
if (typeof definition !== 'object' || !getTags(definition).has(INDEXABLE_TAG_NAME)) continue;
@@ -42,7 +24,8 @@ export function transformProject(project: JSONSchema7): ElasticsearchConfig {
};
}
const OPTIONS: GeneratorOptions = {
/** @type {import('../types.js').GeneratorOptions} */
const OPTIONS = {
template: {
name: 'template_{_type}',
index_patterns: 'stapps_{_type}*',
@@ -132,7 +115,7 @@ const OPTIONS: GeneratorOptions = {
language: 'de',
country: 'DE',
variant: '@collation=phonebook',
} as never,
},
},
},
},

View File

@@ -0,0 +1,88 @@
import {Context} from './context.js';
import deepmerge from 'deepmerge';
import {renderTemplate} from '../template.js';
/**
* Sanitize a type name
* @param typeName {string}
* @returns {string}
*/
export function sanitizeTypeName(typeName) {
return typeName.replaceAll(' ', '_');
}
/**
* @typedef {import('json-schema').JSONSchema7} JSONSchema
* @typedef {import('../types.js').MappingProperty} MappingProperty
* @typedef {import('../types.js').MappingTypeMapping} MappingTypeMapping
* @typedef {import('../types.js').IndicesPutTemplateRequest} IndicesPutTemplateRequest
* @typedef {import('../types.js').SearchRequest} SearchRequest
*/
export class MappingGenerator {
/** @type {Map<string, {mapping: MappingProperty; dependencies: Map<string, Set<string>>}>} */
cache;
/** @type {{mods: Partial<SearchRequest>}} */
searchMods = {mods: {}};
/**
* @param project {JSONSchema & Pick<Required<JSONSchema>, 'definitions'>}
* @param options {import('../types.js').GeneratorOptions}
*/
constructor(project, options) {
if (!project.definitions) throw new Error('Invalid schema');
this.project = project;
this.template = options.template ?? {};
this.presets = new Map(Object.entries(options.presets));
this.cache = new Map(
Object.entries(options.overrides).map(([name, mapping]) => [
name,
{
mapping,
dependencies: new Map(),
},
]),
);
}
/**
* @param name {string}
* @returns {[string, IndicesPutTemplateRequest]}
*/
buildTemplate(name) {
const definition = this.project.definitions[name];
if (typeof definition === 'boolean' || !definition.properties) throw new Error('Invalid schema');
const thingTypeDefinition = definition.properties.type;
if (typeof thingTypeDefinition === 'boolean') throw new Error('Invalid schema');
const thingType = thingTypeDefinition.const;
if (typeof thingType !== 'string') {
throw new TypeError(`${name} needs a valid thing type`);
}
const mappingContext = new Context(this, thingType, [name], [], new Map());
const mappings = mappingContext.resolveReference(name);
const request = deepmerge(
{mappings: mappings},
renderTemplate(this.template, [
['{name}', name],
['{type}', thingType],
['{_type}', sanitizeTypeName(thingType)],
]),
);
if (mappingContext.dependencies.size > 0) {
request.mappings.dynamic_templates = [];
for (const [name, paths] of mappingContext.dependencies) {
request.mappings.dynamic_templates.push({
[name]: {
path_match: paths.size > 1 ? [...paths] : paths.values().next().value,
mapping: mappingContext.resolveReference(name),
},
});
}
}
return [thingType, request];
}
}

View File

@@ -1,76 +0,0 @@
import {GeneratorOptions} from './index.js';
import {JSONSchema7} from 'json-schema';
import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
import {
IndicesPutTemplateRequest,
MappingProperty,
MappingTypeMapping,
SearchRequest,
} from '@elastic/elasticsearch/lib/api/types.js';
import {Context} from './context.js';
import deepmerge from 'deepmerge';
import {renderTemplate} from '../template.js';
/**
* Sanitize a type name
*/
export function sanitizeTypeName(typeName: string): string {
return typeName.replaceAll(' ', '_');
}
export class MappingGenerator {
readonly presets: Map<string, ElasticsearchOptionsDSL>;
readonly cache: Map<string, {mapping: MappingProperty; dependencies: Map<string, Set<string>>}>;
readonly searchMods: {mods: Partial<SearchRequest>} = {mods: {}};
readonly template: Partial<IndicesPutTemplateRequest>;
constructor(readonly project: JSONSchema7, options: GeneratorOptions) {
this.template = options.template ?? {};
this.presets = new Map(Object.entries(options.presets));
this.cache = new Map(
Object.entries(options.overrides).map(([name, mapping]) => [
name,
{
mapping,
dependencies: new Map(),
},
]),
);
}
buildTemplate(name: string): [string, IndicesPutTemplateRequest] {
const thingType = ((this.project.definitions![name] as JSONSchema7).properties!.type as JSONSchema7)
.const;
if (typeof thingType !== 'string') {
throw new TypeError(`${name} needs a valid thing type`);
}
const mappingContext = new Context(this, thingType, [name], [], new Map());
const mappings = mappingContext.resolveReference(name);
const request: IndicesPutTemplateRequest = deepmerge(
{mappings: mappings as MappingTypeMapping},
renderTemplate(this.template, [
['{name}', name],
['{type}', thingType],
['{_type}', sanitizeTypeName(thingType)],
]),
);
if (mappingContext.dependencies.size > 0) {
request.mappings!.dynamic_templates = [];
for (const [name, paths] of mappingContext.dependencies) {
request.mappings!.dynamic_templates.push({
[name]: {
path_match: paths.size > 1 ? [...paths] : paths.values().next().value,
mapping: mappingContext.resolveReference(name),
},
});
}
}
return [thingType, request];
}
}

View File

@@ -1,18 +1,15 @@
import {Context} from './context.js';
import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
import deepmerge from 'deepmerge';
type ResolvedOptions = Omit<ElasticsearchOptionsDSL, 'extends'>;
/**
* Resolve DSL inheritance
* Resolve option inheritance
* @param context {Context}
* @param options {import('../types.js').ElasticsearchOptions}
* @returns {import('../types.js').ResolvedOptions}
*/
export function resolveDsl(
context: Context,
{extends: parents, ...result}: ElasticsearchOptionsDSL,
): ResolvedOptions {
export function resolveOptions(context, {extends: parents, ...result}) {
for (const reference of parents ?? []) {
result = deepmerge<ResolvedOptions>(
result = deepmerge(
result,
reference.startsWith('@')
? resolveReferencePath(context, reference)
@@ -25,20 +22,27 @@ export function resolveDsl(
/**
* Resolve preset references
* @param context {Context}
* @param reference {string}
* @returns {import('../types.js').ResolvedOptions}
*/
function resolvePresetReference(context: Context, reference: string): ResolvedOptions {
function resolvePresetReference(context, reference) {
if (!context.generator.presets.has(reference)) return context.bail(`Missing preset ${reference}`);
return resolveDsl(context, context.generator.presets.get(reference)!);
return resolveOptions(context, context.generator.presets.get(reference));
}
/**
* Resolve @references
* @param context {Context}
* @param reference {string}
* @returns {import('../types.js').ResolvedOptions}
*/
function resolveReferencePath(context: Context, reference: string): ResolvedOptions {
function resolveReferencePath(context, reference) {
const [type, ...path] = reference.replace(/^@/, '').split('.');
let declaration = context.resolveReference(type);
while (path.length > 0) {
const property = path.shift()!;
const property = path.shift();
if (!property) context.bail('internal logic error');
if (!('properties' in declaration && declaration.properties && property in declaration.properties))
context.bail(`Invalid reference ${reference}`);

View File

@@ -0,0 +1,14 @@
export const INDEXABLE_TAG_NAME = 'indexable';
/**
* Get elasticsearch tags
* @param definition {import('json-schema').JSONSchema7}
* @returns {Set<string>}
*/
export function getTags(definition) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return new Set(
/** @type {{elasticsearch?: string}} */ (definition).elasticsearch?.split(/\s/).map(it => it.trim()) ??
[],
);
}

View File

@@ -1,13 +0,0 @@
import {JSONSchema7} from 'json-schema';
export const INDEXABLE_TAG_NAME = 'indexable';
/**
* Get elasticsearch tags
*/
export function getTags(definition: JSONSchema7): Set<string> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return new Set(
(definition as {elasticsearch: string}).elasticsearch?.split(/\s/).map(it => it.trim()) ?? [],
);
}

View File

@@ -0,0 +1,27 @@
import {transformDefinition} from '../definition.js';
/**
* Transform a JSON Schema with `object` type
* @typedef {import('../../types.js').MappingProperty} MappingProperty
* @typedef {import('json-schema').JSONSchema7} JSONSchema
*
* @param context {import('../context.js').Context}
* @param definition {JSONSchema}
* @returns {MappingProperty}
*/
export function transformObject(context, definition) {
/** @satisfies {MappingProperty} */
const value = {
dynamic: definition.additionalProperties === true ? undefined : 'strict',
properties: /** @type {Record<string, MappingProperty>} */ ({}),
};
for (const key in definition.properties) {
value.properties[key] = transformDefinition(
context.step(key),
/** @type {JSONSchema} */ (definition.properties[key]),
);
}
return value;
}

View File

@@ -1,20 +0,0 @@
import {JSONSchema7} from 'json-schema';
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
import {transformDefinition} from '../definition.js';
import {Context} from '../context.js';
/**
* Transform a JSON Schema with `object` type
*/
export function transformObject(context: Context, definition: JSONSchema7): MappingProperty {
const value = {
dynamic: definition.additionalProperties === true ? undefined : 'strict',
properties: {} as Record<string, MappingProperty>,
} satisfies MappingProperty;
for (const key in definition.properties!) {
value.properties[key] = transformDefinition(context.step(key), definition.properties[key] as JSONSchema7);
}
return value;
}

View File

@@ -1,7 +1,7 @@
import {JSONSchema7} from 'json-schema';
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
const stringFormats = new Map<string, MappingProperty>([
/**
* @type {Map<string, import('../../types.js').MappingProperty>}
*/
const stringFormats = new Map([
['date', {type: 'date', format: 'date'}],
['time', {type: 'date', format: 'time'}],
['date-time', {type: 'date', format: 'date_optional_time'}],
@@ -11,9 +11,11 @@ const stringFormats = new Map<string, MappingProperty>([
/**
* Transform a JSON Schema with `string` type
* @param definition {import('json-schema').JSONSchema7}
* @returns {import('../../types.js').MappingProperty}
*/
export function transformString(definition: JSONSchema7): MappingProperty {
export function transformString(definition) {
return definition.format && stringFormats.has(definition.format)
? stringFormats.get(definition.format)!
? stringFormats.get(definition.format) ?? {type: 'keyword'}
: {type: 'keyword'};
}

View File

@@ -1,18 +1,19 @@
import {transformProject} from './generator/index.js';
import {SchemaConsumer} from '@openstapps/json-schema-generator';
import {readFile} from 'fs/promises';
import path from 'path';
import {join, dirname} from 'path';
import {fileURLToPath} from 'url';
const mappingTypes = await readFile(
path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'mappings.d.ts'),
join(dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'mappings.d.ts'),
'utf8',
);
/**
* JSON Schema Generator plugin for Elasticsearch Mappings
* @param fileName {string}
* @returns {[string, import('@openstapps/json-schema-generator').SchemaConsumer]}
*/
export function elasticsearchMappingGenerator(fileName: string): [string, SchemaConsumer] {
export function elasticsearchMappingGenerator(fileName) {
return [
'Elasticsearch-Mappings',
function (schema) {

View File

@@ -0,0 +1,15 @@
/**
* Render a template
* @template T
* @param template {T} The template to render (must be stringify-able)
* @param substitutions {[string, string][]} the substitutions
* @returns {T}
*/
export function renderTemplate(template, substitutions) {
return JSON.parse(
substitutions.reduce(
(template, [search, replace]) => template.replaceAll(search, replace),
JSON.stringify(template),
),
);
}

View File

@@ -1,13 +0,0 @@
/**
* Render a template
* @param template The template to render (must be stringify-able)
* @param substitutions the substitutions
*/
export function renderTemplate<T>(template: T, substitutions: [string, string][]): T {
return JSON.parse(
substitutions.reduce(
(template, [search, replace]) => template.replaceAll(search, replace),
JSON.stringify(template),
),
);
}

View File

@@ -0,0 +1,56 @@
import {MappingProperty, SearchRequest} from '@elastic/elasticsearch/lib/api/types.d.ts';
import {IndicesPutTemplateRequest} from '@elastic/elasticsearch/lib/api/types.js';
export {
MappingProperty,
SearchRequest,
MappingDynamicProperty,
MappingTypeMapping,
IndicesPutTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types.d.ts';
export interface ElasticsearchOptions {
/**
* Mark an interface as indexable
*/
indexable?: true;
/**
* Inherit customization options from another item
*/
extends?: string[];
/**
* Completely override the property
*/
override?: MappingProperty;
/**
* Merge property values
*/
merge?: MappingProperty;
/**
* Modify the search request
*
* Supports `{name}`, `{type}` and `{prop}` templates substitutions anywhere
*/
search?: Partial<SearchRequest>;
}
export interface GeneratorOptions {
/**
* Presets you can extend
*/
presets: Record<string, ElasticsearchOptionsDSL>;
/**
* Override specific types
*/
overrides: Record<string, MappingProperty>;
/**
* Template for the generated index request
*
* Supports `{type}` and `{sanitized_type}` (same as `{type}`, but no spaces) template substitutions
*/
template: Partial<IndicesPutTemplateRequest>;
}
export type ResolvedOptions = Omit<ElasticsearchOptions, 'extends'>;
export {elasticsearchMappingGenerator} from './index.js';

View File

@@ -39,7 +39,7 @@
"@openstapps/tsconfig": "workspace:*",
"@types/node": "18.15.3",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},

View File

@@ -13,15 +13,14 @@
"core",
"validator"
],
"main": "lib/index.js",
"types": "lib/index.d.ts",
"main": "src/index.js",
"types": "src/types.d.ts",
"files": [
"lib",
"README.md",
"CHANGELOG.md"
],
"scripts": {
"build": "tsup-node --dts",
"docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.ts",
"format": "prettier . -c --ignore-path ../../.gitignore",
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
@@ -51,20 +50,10 @@
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"nock": "13.3.1",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"typedoc": "0.24.8",
"ts-node": "10.9.1",
"typescript": "5.1.6"
},
"tsup": {
"entry": [
"src/index.ts"
],
"sourcemap": true,
"clean": true,
"format": "esm",
"outDir": "lib"
},
"prettier": "@openstapps/prettier-config",
"eslintConfig": {
"extends": [

View File

@@ -1,11 +1,13 @@
import {createGenerator} from 'ts-json-schema-generator';
import {getValidatableTypes} from './get-validatable-types.js';
import {JSONSchema7} from 'json-schema';
/**
* Compile the JSON schema for a path
* @param path {string}
* @param tsconfig {string}
* @returns {[schema: import('json-schema').JSONSchema7, type: string]}
*/
export function compileSchema(path: string, tsconfig: string): [schma: JSONSchema7, type: string] {
export function compileSchema(path, tsconfig) {
const generator = createGenerator({
path,
tsconfig,

View File

@@ -2,10 +2,13 @@ import {ts} from 'ts-json-schema-generator';
/**
* Get all types with `@validatable` annotations
* @param program {ts.Program}
* @returns {Set<string>}
*/
export function getValidatableTypes(program: ts.Program) {
export function getValidatableTypes(program) {
const checker = program.getTypeChecker();
const declarationNames = new Set<string>();
/** @type {Set<string>} */
const declarationNames = new Set();
for (const sourceFile of program.getSourceFiles()) {
const sourceFileSymbol = checker.getSymbolAtLocation(sourceFile);
@@ -27,8 +30,10 @@ export function getValidatableTypes(program: ts.Program) {
/**
* Type predicate for if a JSDoc tag is `@validatable`
* @param tag {ts.JSDocTag}
* @returns {tag is ts.JSDocTag}
*/
// eslint-disable-next-line unicorn/prevent-abbreviations
function isValidatableJSDocTag(tag: ts.JSDocTag): tag is ts.JSDocTag {
function isValidatableJSDocTag(tag) {
return tag.tagName.escapedText === 'validatable';
}

View File

@@ -1,11 +1,7 @@
import {compileSchema} from './generator/compile-schema.js';
import {generateFiles, Plugin, PluginContext} from '@openstapps/tsup-plugin';
import {JSONSchema7} from 'json-schema';
import {Plugin as EsbuildPlugin} from 'esbuild';
import {generateFiles} from '@openstapps/tsup-plugin';
import {createGenerator} from 'ts-json-schema-generator';
export type SchemaConsumer = (this: PluginContext, schema: JSONSchema7) => Record<string, string | Buffer>;
/**
* ESBuild Plugin for directly importing schemas
*
@@ -19,13 +15,16 @@ export type SchemaConsumer = (this: PluginContext, schema: JSONSchema7) => Recor
* interface Bar {}
* // ./schema-consumer.ts
* import {default as barSchema} from 'schema:./my-type.js#Bar'
*
* @type {import('esbuild').Plugin}
*/
export const esbuildJsonSchemaPlugin: EsbuildPlugin = {
export const esbuildJsonSchemaPlugin = {
name: 'json-schema',
setup(build) {
const fileRegex = /^schema:/;
const namespace = 'json-schema-ns';
const schemas = new Map<string, string>();
/** @type {Map<string, string>} */
const schemas = new Map();
build.onResolve({filter: fileRegex}, ({path, importer}) => {
const [from, name] = path.replace(fileRegex, '').split('#', 1);
@@ -56,22 +55,22 @@ export const esbuildJsonSchemaPlugin: EsbuildPlugin = {
/**
* TSUp plugin for generating JSONSchema files
* @param schemaName the name of the generated schema
* @param schemaConsumers any consumers that can directly use the schema
* @param schemaName {string} the name of the generated schema
* @param schemaConsumers {Array<[string, import('./types.js').SchemaConsumer]>} any consumers
* that can directly use the schema
* @returns {import('@openstapps/tsup-plugin').Plugin}
*/
export function jsonSchemaPlugin(
schemaName: string,
...schemaConsumers: Array<[string, SchemaConsumer]>
): Plugin {
export function jsonSchemaPlugin(schemaName, ...schemaConsumers) {
return {
name: 'json-schema-generator',
async buildEnd() {
let schema: JSONSchema7;
/** @type {import('json-schema').JSONSchema7} */
let schema;
await generateFiles('JSON-Schema', async function () {
const [jsonSchema, types] = compileSchema(
(this.options.entry as string[])[0],
this.options.tsconfig!,
);
if (!this.options.tsconfig) throw new Error('Must supply a tsconfig');
if (!Array.isArray(this.options.entry)) throw new Error('Must supply entry as an array');
const [jsonSchema, types] = compileSchema(this.options.entry[0], this.options.tsconfig);
schema = jsonSchema;
return {

View File

@@ -0,0 +1,6 @@
import {JSONSchema7} from 'json-schema';
import {PluginContext} from '@openstapps/tsup-plugin/src/types.js';
export type SchemaConsumer = (this: PluginContext, schema: JSONSchema7) => Record<string, string | Buffer>;
export {compileSchema, jsonSchemaPlugin, esbuildJsonSchemaPlugin} from './index.js';

View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2023 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* This is a simple interface declaration for
* testing the schema generation and validation.
*
* @validatable
*/
export interface Foo {
/**
* lorem parameter
*/
lorem: 'lorem' | 'ipsum';
/**
* String literal type property
*/
type: FooType;
}
/**
* This is a simple type declaration for
* usage in the Foo interface.
*/
export type FooType = 'Foo';

View File

@@ -13,46 +13,36 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from 'packages/logger/lib/index.js';
import {expect} from 'chai';
import {Converter} from '../../core-tools/src/schema.js';
import {compileSchema} from '../src/index.js';
import path from 'path';
import {fileURLToPath} from 'url';
process.on('unhandledRejection', (error: unknown) => {
if (error instanceof Error) {
void Logger.error('UNHANDLED REJECTION', error.stack);
}
process.exit(1);
});
describe('Schema', function () {
this.timeout(40_000);
this.slow(10_000);
it('should create schema', function () {
const converter = new Converter(
const [schema, _types] = compileSchema(
path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'src', 'resources'),
path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'tsconfig.json'),
);
const schema = converter.getSchema('Foo', '0.0.1');
expect(schema).to.be.deep.equal({
$id: 'https://core.stapps.tu-berlin.de/v0.0.1/lib/schema/Foo.json',
$schema: 'http://json-schema.org/draft-07/schema#',
additionalProperties: false,
definitions: {
FooType: {
description: 'This is a simple type declaration for usage in the Foo interface.',
const: 'Foo',
type: 'string',
},
SCFoo: {
Foo: {
additionalProperties: false,
description:
'This is a simple interface declaration for testing the schema generation and validation.',
properties: {
lorem: {
description: 'Dummy parameter',
description: 'lorem parameter',
enum: ['lorem', 'ipsum'],
type: 'string',
},
@@ -65,20 +55,6 @@ describe('Schema', function () {
type: 'object',
},
},
description: 'This is a simple interface declaration for testing the schema generation and validation.',
properties: {
lorem: {
description: 'Dummy parameter',
enum: ['lorem', 'ipsum'],
type: 'string',
},
type: {
$ref: '#/definitions/FooType',
description: 'String literal type property',
},
},
required: ['lorem', 'type'],
type: 'object',
});
});
});

View File

@@ -39,17 +39,17 @@
"@openstapps/tsconfig": "workspace:*",
"@types/chai": "4.3.5",
"@types/chai-as-promised": "7.1.5",
"@types/chai-spies": "1.0.3",
"@types/chai-spies": "1.0.6",
"@types/mocha": "10.0.1",
"@types/node": "18.15.3",
"c8": "7.14.0",
"chai": "4.3.7",
"chai-as-promised": "7.1.1",
"chai-spies": "1.0.0",
"chai-spies": "1.1.0",
"mocha": "10.2.0",
"mocha-junit-reporter": "2.2.0",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"tsup": "7.2.0",
"typedoc": "0.24.8",
"typescript": "5.1.6"
},

View File

@@ -13,8 +13,8 @@
"core",
"validator"
],
"types": "src/types.d.ts",
"main": "src/index.js",
"types": "src/types.d.ts",
"files": [
"lib",
"README.md",