mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-21 17:12:43 +00:00
feat: improve monorepo dev experience
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
require('./lib/app.js');
|
||||
@@ -2,14 +2,11 @@
|
||||
"name": "@openstapps/es-mapping-generator",
|
||||
"description": "Tool to convert TypeScript Interfaces to Elasticsearch Mappings",
|
||||
"version": "3.0.0",
|
||||
"type": "commonjs",
|
||||
"type": "module",
|
||||
"license": "GPL-3.0-only",
|
||||
"author": "Thea Schöbl <dev@theaninova.de>",
|
||||
"main": "./lib/index.js",
|
||||
"types": "./lib/index.d.ts",
|
||||
"bin": {
|
||||
"openstapps-es-mapping-generator": "app.js"
|
||||
},
|
||||
"files": [
|
||||
"app.js",
|
||||
"lib",
|
||||
@@ -18,7 +15,8 @@
|
||||
"CHANGELOG.md"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "rimraf lib && tsc",
|
||||
"build": "tsup-node --dts",
|
||||
"dev": "tsup --watch --onSuccess \"node lib/index.js\"",
|
||||
"format": "prettier . -c --ignore-path ../../.gitignore",
|
||||
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
|
||||
"lint": "eslint --ext .ts src/ test/",
|
||||
@@ -26,28 +24,33 @@
|
||||
"test": "c8 mocha"
|
||||
},
|
||||
"dependencies": {
|
||||
"@elastic/elasticsearch": "8.4.0",
|
||||
"commander": "10.0.0",
|
||||
"deepmerge": "4.3.1",
|
||||
"flatted": "3.2.7",
|
||||
"typedoc": "0.18.0",
|
||||
"typescript": "3.8.3"
|
||||
"@elastic/elasticsearch": "8.10.0",
|
||||
"@openstapps/json-schema-generator": "workspace:*",
|
||||
"@openstapps/tsup-plugin": "workspace:*",
|
||||
"@types/json-schema": "7.0.14",
|
||||
"ajv": "8.12.0",
|
||||
"better-ajv-errors": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openstapps/eslint-config": "workspace:*",
|
||||
"@openstapps/prettier-config": "workspace:*",
|
||||
"@testdeck/mocha": "0.3.3",
|
||||
"@openstapps/tsconfig": "workspace:*",
|
||||
"@types/chai": "4.3.5",
|
||||
"@types/fs-extra": "9.0.13",
|
||||
"@types/glob": "8.0.1",
|
||||
"@types/mocha": "10.0.1",
|
||||
"@types/node": "14.18.38",
|
||||
"@types/rimraf": "3.0.2",
|
||||
"@types/mustache": "4.2.2",
|
||||
"@types/node": "18.15.3",
|
||||
"c8": "7.14.0",
|
||||
"chai": "4.3.7",
|
||||
"esbuild": "0.17.19",
|
||||
"mocha": "10.2.0",
|
||||
"mocha-junit-reporter": "2.2.0",
|
||||
"nock": "13.3.1",
|
||||
"rimraf": "5.0.0",
|
||||
"ts-node": "10.9.1"
|
||||
"ts-node": "10.9.1",
|
||||
"tsup": "6.7.0",
|
||||
"typedoc": "0.24.8",
|
||||
"typescript": "5.1.6"
|
||||
},
|
||||
"prettier": "@openstapps/prettier-config",
|
||||
"eslintConfig": {
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {Command} from 'commander';
|
||||
import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'fs';
|
||||
import path from 'path';
|
||||
import {generateTemplate} from './mapping';
|
||||
import {getProjectReflection} from './project-reflection';
|
||||
|
||||
// handle unhandled promise rejections
|
||||
process.on('unhandledRejection', async (reason: unknown) => {
|
||||
if (reason instanceof Error) {
|
||||
await console.error(reason.message);
|
||||
console.info(reason.stack);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const commander = new Command('openstapps-core-tools');
|
||||
|
||||
// eslint-disable-next-line unicorn/prefer-module
|
||||
commander.version(JSON.parse(readFileSync(path.resolve(__dirname, '..', 'package.json')).toString()).version);
|
||||
|
||||
commander
|
||||
.command('mapping <relativeSrcPath>')
|
||||
.option('-m, --mappingPath <relativeMappingPath>', 'Mapping Path')
|
||||
.option('-i, --ignoredTags <ignoredTags>', 'Ignored Tags (comma-separated)')
|
||||
.option('-a, --aggPath <relativeAggregationPath>', 'Aggregations Path')
|
||||
.option('-e, --errorPath <relativeErrorPath>', 'Error Path')
|
||||
.action(async (relativeSourcePath, options) => {
|
||||
// get absolute paths
|
||||
const sourcePath = path.resolve(relativeSourcePath);
|
||||
|
||||
let ignoredTagsList: string[] = [];
|
||||
if (typeof options.ignoredTags === 'string') {
|
||||
ignoredTagsList = options.ignoredTags.split(',');
|
||||
}
|
||||
|
||||
// get project reflection
|
||||
const projectReflection = getProjectReflection(sourcePath);
|
||||
|
||||
const result = generateTemplate(projectReflection, ignoredTagsList, true);
|
||||
if (result.errors.length > 0) {
|
||||
await console.error('Mapping generated with errors!');
|
||||
} else {
|
||||
console.log('Mapping generated without errors!');
|
||||
}
|
||||
|
||||
// write documentation to file
|
||||
if (options.aggPath !== undefined) {
|
||||
const aggPath = path.resolve(options.aggPath);
|
||||
mkdirSync(path.dirname(aggPath), {recursive: true});
|
||||
// tslint:disable-next-line:no-magic-numbers
|
||||
writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
|
||||
copyFileSync(
|
||||
// eslint-disable-next-line unicorn/prefer-module
|
||||
require.resolve('../schema/aggregations.d.ts'),
|
||||
path.join(path.dirname(aggPath), 'aggregations.json.d.ts'),
|
||||
);
|
||||
console.log(`Elasticsearch aggregations written to ${aggPath}.`);
|
||||
}
|
||||
if (options.mappingPath !== undefined) {
|
||||
const mappingPath = path.resolve(options.mappingPath);
|
||||
mkdirSync(path.dirname(mappingPath), {recursive: true});
|
||||
writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
|
||||
copyFileSync(
|
||||
// eslint-disable-next-line unicorn/prefer-module
|
||||
require.resolve('../schema/mappings.d.ts'),
|
||||
path.join(path.dirname(mappingPath), 'mappings.json.d.ts'),
|
||||
);
|
||||
console.log(`Elasticsearch mappings written to ${mappingPath}.`);
|
||||
}
|
||||
if (options.errorPath !== undefined) {
|
||||
const errorPath = path.resolve(options.errorPath);
|
||||
mkdirSync(path.dirname(errorPath), {recursive: true});
|
||||
// tslint:disable-next-line:no-magic-numbers
|
||||
writeFileSync(errorPath, JSON.stringify(result.errors, null, 2));
|
||||
console.log(`Mapping errors written to ${errorPath}.`);
|
||||
} else if (result.errors.length > 0) {
|
||||
for (const error of result.errors) {
|
||||
await console.error(error);
|
||||
}
|
||||
|
||||
throw new Error('Mapping generation failed');
|
||||
}
|
||||
});
|
||||
|
||||
commander.parse(process.argv);
|
||||
@@ -1,55 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type {ElasticsearchFieldmap, SimpleType} from '../../schema/mappings.js';
|
||||
|
||||
const ducetSort = {
|
||||
type: 'icu_collation_keyword',
|
||||
language: 'de',
|
||||
country: 'DE',
|
||||
variant: '@collation=phonebook',
|
||||
};
|
||||
|
||||
const keyword: MappingProperty['type'] = 'keyword';
|
||||
|
||||
export const fieldmap: ElasticsearchFieldmap = {
|
||||
aggregatable: {
|
||||
default: {
|
||||
raw: {
|
||||
ignore_above: 10_000,
|
||||
type: keyword,
|
||||
},
|
||||
},
|
||||
ignore: ['global'],
|
||||
},
|
||||
sortable: {
|
||||
default: {
|
||||
sort: ducetSort,
|
||||
},
|
||||
ducet: {
|
||||
sort: ducetSort,
|
||||
},
|
||||
ignore: ['price'],
|
||||
},
|
||||
};
|
||||
|
||||
export const filterableTagName = 'filterable';
|
||||
|
||||
export const filterableMap: Record<string, SimpleType> = {
|
||||
date: 'keyword',
|
||||
keyword: 'keyword',
|
||||
text: 'keyword',
|
||||
integer: 'integer',
|
||||
};
|
||||
@@ -1,64 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export const premaps: Record<string, MappingProperty> = {
|
||||
'CoordinateReferenceSystem': {
|
||||
dynamic: true,
|
||||
properties: {
|
||||
type: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
'LineString': {
|
||||
type: 'geo_shape',
|
||||
},
|
||||
'Point': {
|
||||
properties: {
|
||||
type: {
|
||||
type: 'keyword',
|
||||
},
|
||||
coordinates: {
|
||||
type: 'geo_point',
|
||||
},
|
||||
},
|
||||
dynamic: 'strict',
|
||||
},
|
||||
'Polygon': {
|
||||
type: 'geo_shape',
|
||||
},
|
||||
'SCISO8601DateRange': {
|
||||
type: 'date_range',
|
||||
},
|
||||
'jsonpatch.OpPatch': {
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
from: {
|
||||
type: 'keyword',
|
||||
},
|
||||
op: {
|
||||
type: 'keyword',
|
||||
},
|
||||
path: {
|
||||
type: 'keyword',
|
||||
},
|
||||
value: {
|
||||
// this is actually an 'any' type; however, ES does not really support that.
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,22 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {IndicesPutTemplateRequest} from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export const settings: IndicesPutTemplateRequest['settings'] = {
|
||||
'mapping.total_fields.limit': 10_000,
|
||||
'max_result_window': 30_000,
|
||||
'number_of_replicas': 0,
|
||||
'number_of_shards': 1,
|
||||
};
|
||||
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {MappingFloatNumberProperty} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type {ElasticsearchTypemap} from '../../schema/mappings';
|
||||
|
||||
export const PARSE_ERROR = 'PARSE_ERROR' as MappingFloatNumberProperty['type'];
|
||||
export const MISSING_PREMAP = 'MISSING_PREMAP' as MappingFloatNumberProperty['type'];
|
||||
export const TYPE_CONFLICT = 'TYPE_CONFLICT' as MappingFloatNumberProperty['type'];
|
||||
|
||||
export const typemap: ElasticsearchTypemap = {
|
||||
boolean: {
|
||||
default: 'boolean',
|
||||
},
|
||||
false: {
|
||||
default: 'boolean',
|
||||
},
|
||||
number: {
|
||||
default: 'integer',
|
||||
float: 'float',
|
||||
integer: 'integer',
|
||||
date: 'date',
|
||||
},
|
||||
string: {
|
||||
default: 'text',
|
||||
keyword: 'keyword',
|
||||
text: 'text',
|
||||
date: 'date',
|
||||
},
|
||||
stringLiteral: {
|
||||
default: 'keyword',
|
||||
},
|
||||
true: {
|
||||
default: 'boolean',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* If the string is a tag type
|
||||
*/
|
||||
export function isTagType(string_: string): boolean {
|
||||
for (const key in typemap) {
|
||||
if (typemap.hasOwnProperty(key) && typemap[key][string_] !== undefined) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export const dynamicTypes = ['any', 'unknown'];
|
||||
60
packages/es-mapping-generator/src/dsl/schema.ts
Normal file
60
packages/es-mapping-generator/src/dsl/schema.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import {MappingProperty, SearchRequest} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import Ajv from 'ajv';
|
||||
import {readFile} from 'fs/promises';
|
||||
import {fileURLToPath} from 'url';
|
||||
import path from 'path';
|
||||
import {Context} from '../generator/context.js';
|
||||
|
||||
/**
|
||||
* @validatable
|
||||
*/
|
||||
export interface ElasticsearchOptionsDSL {
|
||||
/**
|
||||
* Mark an interface as indexable
|
||||
*/
|
||||
indexable?: true;
|
||||
/**
|
||||
* Inherit customization options from another item
|
||||
*/
|
||||
extends?: string[];
|
||||
/**
|
||||
* Completely override the property
|
||||
*/
|
||||
override?: MappingProperty;
|
||||
/**
|
||||
* Merge property values
|
||||
*/
|
||||
merge?: MappingProperty;
|
||||
/**
|
||||
* Modify the search request
|
||||
*
|
||||
* Supports `{name}`, `{type}` and `{prop}` templates substitutions anywhere
|
||||
*/
|
||||
search?: Partial<SearchRequest>;
|
||||
}
|
||||
|
||||
const schema = JSON.parse(
|
||||
await readFile(path.join(path.dirname(fileURLToPath(import.meta.url)), 'index.schema.json'), 'utf8'),
|
||||
);
|
||||
const ajv = new Ajv.default({schemas: [schema], allowUnionTypes: true});
|
||||
|
||||
/**
|
||||
* Validate that the options are valid
|
||||
*/
|
||||
export function validateElasticsearchOptionsDsl(
|
||||
context: Context,
|
||||
value: unknown,
|
||||
): value is ElasticsearchOptionsDSL {
|
||||
return ajv.validate('#/definitions/ElasticsearchOptionsDSL', value)
|
||||
? true
|
||||
: context.bail(JSON.stringify(ajv.errors));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that the mapping result is correct
|
||||
*/
|
||||
export function validateMappingResult(context: Context, value: unknown): value is MappingProperty {
|
||||
return ajv.validate('#/definitions/MappingProperty', value)
|
||||
? true
|
||||
: context.bail(JSON.stringify(ajv.errors));
|
||||
}
|
||||
47
packages/es-mapping-generator/src/generator/base.ts
Normal file
47
packages/es-mapping-generator/src/generator/base.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import {transformObject} from './transformers/object.js';
|
||||
import {transformString} from './transformers/string.js';
|
||||
import {Context} from './context.js';
|
||||
import {transformDefinition} from './definition.js';
|
||||
|
||||
/**
|
||||
* Transform JSONSchema without applying custom tag logic
|
||||
*/
|
||||
export function transformBase(context: Context, definition: JSONSchema7): MappingProperty {
|
||||
if (definition.anyOf) {
|
||||
return context.resolveUnion(definition.anyOf as JSONSchema7[]);
|
||||
}
|
||||
|
||||
switch (definition.type) {
|
||||
case 'array': {
|
||||
if (Array.isArray(definition.items)) {
|
||||
return context.resolveUnion(definition.items as JSONSchema7[]);
|
||||
} else if (typeof definition.items === 'object') {
|
||||
return transformDefinition(context, definition.items);
|
||||
} else {
|
||||
return context.bail(`Not implemented array type ${typeof definition.items}`);
|
||||
}
|
||||
}
|
||||
case 'object': {
|
||||
return transformObject(context, definition);
|
||||
}
|
||||
case 'string': {
|
||||
return transformString(definition);
|
||||
}
|
||||
case 'number': {
|
||||
return {type: 'float'};
|
||||
}
|
||||
case 'integer': {
|
||||
return {type: 'integer'};
|
||||
}
|
||||
case 'boolean': {
|
||||
return {type: 'boolean'};
|
||||
}
|
||||
default: {
|
||||
return {
|
||||
dynamic: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
115
packages/es-mapping-generator/src/generator/context.ts
Normal file
115
packages/es-mapping-generator/src/generator/context.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
import {transformDefinition} from './definition.js';
|
||||
import crypto from 'crypto';
|
||||
import {
|
||||
MappingProperty,
|
||||
MappingDynamicProperty,
|
||||
SearchRequest,
|
||||
} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import deepmerge from 'deepmerge';
|
||||
import {MappingGenerator, sanitizeTypeName} from './mapping-generator.js';
|
||||
import {renderTemplate} from '../template.js';
|
||||
|
||||
/**
|
||||
* Get the name from a $ref such as `#/definitions/SCThing`
|
||||
*/
|
||||
function getNameFromRef(ref: string): string {
|
||||
return decodeURI(ref).replace(/^#\/definitions\//, '');
|
||||
}
|
||||
|
||||
export class Context {
|
||||
constructor(
|
||||
readonly generator: MappingGenerator,
|
||||
readonly thingType: string,
|
||||
readonly path: string[],
|
||||
readonly propertyPath: string[],
|
||||
readonly dependencies: Map<string, Set<string>>,
|
||||
) {}
|
||||
|
||||
resolveReference(reference: string): MappingProperty {
|
||||
return this.deriveContext(reference)[1];
|
||||
}
|
||||
|
||||
resolveUnion(types: JSONSchema7[]): MappingDynamicProperty {
|
||||
for (const type of types) {
|
||||
const [name] = this.deriveContext(type.$ref ?? type);
|
||||
this.addDependency(name, this.propertyPath.join('.'));
|
||||
}
|
||||
|
||||
return {
|
||||
type: '{dynamic_property}',
|
||||
};
|
||||
}
|
||||
|
||||
private deriveContext(reference: string | JSONSchema7): [dependencyName: string, mapping: MappingProperty] {
|
||||
let referenceName = typeof reference === 'string' ? getNameFromRef(reference) : undefined;
|
||||
let definition = typeof reference === 'string' ? undefined : reference;
|
||||
|
||||
if (!definition && !this.generator.cache.has(referenceName!)) {
|
||||
definition = this.generator.project.definitions![referenceName!] as JSONSchema7;
|
||||
if (typeof definition !== 'object') this.bail(`Invalid path ${referenceName!}`);
|
||||
}
|
||||
|
||||
if (!referenceName || !this.generator.cache.has(referenceName)) {
|
||||
const derivedContext = new Context(
|
||||
this.generator,
|
||||
this.thingType,
|
||||
referenceName ? [referenceName] : [],
|
||||
this.propertyPath,
|
||||
new Map(),
|
||||
);
|
||||
const result = transformDefinition(derivedContext, definition);
|
||||
referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
|
||||
|
||||
this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});
|
||||
}
|
||||
|
||||
const {mapping, dependencies} = this.generator.cache.get(referenceName)!;
|
||||
for (const [name, paths] of dependencies) {
|
||||
for (const path of paths) {
|
||||
this.addDependency(name, [...this.propertyPath.slice(0, -1), path].join('.'));
|
||||
}
|
||||
}
|
||||
return [referenceName, mapping];
|
||||
}
|
||||
|
||||
registerSearchMod(modification: Partial<SearchRequest>) {
|
||||
this.generator.searchMods.mods = deepmerge<Partial<SearchRequest>>(
|
||||
this.generator.searchMods.mods,
|
||||
renderTemplate(modification, [
|
||||
['{name}', this.path[0]],
|
||||
['{prop}', this.propertyPath.join('.')],
|
||||
['{type}', this.thingType],
|
||||
['{_type}', sanitizeTypeName(this.thingType)],
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
private addDependency(name: string, path: string) {
|
||||
if (!this.dependencies.has(name)) {
|
||||
this.dependencies.set(name, new Set([path]));
|
||||
} else {
|
||||
this.dependencies.get(name)!.add(path);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Step down into a property
|
||||
*/
|
||||
step(property: string): Context {
|
||||
return new Context(
|
||||
this.generator,
|
||||
this.thingType,
|
||||
[...this.path, property],
|
||||
[...this.propertyPath, property],
|
||||
this.dependencies,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bail and throw
|
||||
*/
|
||||
bail(reason: string): never {
|
||||
throw new Error(`${this.path.join('.')} ${reason}`);
|
||||
}
|
||||
}
|
||||
32
packages/es-mapping-generator/src/generator/definition.ts
Normal file
32
packages/es-mapping-generator/src/generator/definition.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import {transformBase} from './base.js';
|
||||
import {Context} from './context.js';
|
||||
import deepmerge from 'deepmerge';
|
||||
import {resolveDsl} from './dsl.js';
|
||||
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
|
||||
|
||||
/**
|
||||
* Transform JSONSchema
|
||||
*/
|
||||
export function transformDefinition(context: Context, definition: JSONSchema7): MappingProperty {
|
||||
if (definition.$ref) return context.resolveReference(definition.$ref);
|
||||
const tags = getTags(definition);
|
||||
tags.delete(INDEXABLE_TAG_NAME);
|
||||
let base = transformBase(context, definition);
|
||||
|
||||
if (tags.size > 0) {
|
||||
const options = resolveDsl(context, {extends: [...tags]});
|
||||
if (options.override) {
|
||||
base = options.override;
|
||||
}
|
||||
if (options.merge) {
|
||||
base = deepmerge<MappingProperty>(base, options.merge);
|
||||
}
|
||||
if (options.search) {
|
||||
context.registerSearchMod(options.search);
|
||||
}
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
48
packages/es-mapping-generator/src/generator/dsl.ts
Normal file
48
packages/es-mapping-generator/src/generator/dsl.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
import {Context} from './context.js';
|
||||
import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
|
||||
import deepmerge from 'deepmerge';
|
||||
|
||||
type ResolvedOptions = Omit<ElasticsearchOptionsDSL, 'extends'>;
|
||||
|
||||
/**
|
||||
* Resolve DSL inheritance
|
||||
*/
|
||||
export function resolveDsl(
|
||||
context: Context,
|
||||
{extends: parents, ...result}: ElasticsearchOptionsDSL,
|
||||
): ResolvedOptions {
|
||||
for (const reference of parents ?? []) {
|
||||
result = deepmerge<ResolvedOptions>(
|
||||
result,
|
||||
reference.startsWith('@')
|
||||
? resolveReferencePath(context, reference)
|
||||
: resolvePresetReference(context, reference),
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve preset references
|
||||
*/
|
||||
function resolvePresetReference(context: Context, reference: string): ResolvedOptions {
|
||||
if (!context.generator.presets.has(reference)) return context.bail(`Missing preset ${reference}`);
|
||||
return resolveDsl(context, context.generator.presets.get(reference)!);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve @references
|
||||
*/
|
||||
function resolveReferencePath(context: Context, reference: string): ResolvedOptions {
|
||||
const [type, ...path] = reference.replace(/^@/, '').split('.');
|
||||
let declaration = context.resolveReference(type);
|
||||
while (path.length > 0) {
|
||||
const property = path.shift()!;
|
||||
if (!('properties' in declaration && declaration.properties && property in declaration.properties))
|
||||
context.bail(`Invalid reference ${reference}`);
|
||||
|
||||
declaration = declaration.properties[property];
|
||||
}
|
||||
return {override: declaration};
|
||||
}
|
||||
171
packages/es-mapping-generator/src/generator/index.ts
Normal file
171
packages/es-mapping-generator/src/generator/index.ts
Normal file
@@ -0,0 +1,171 @@
|
||||
import type {JSONSchema7} from 'json-schema';
|
||||
import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
|
||||
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import {MappingGenerator} from './mapping-generator.js';
|
||||
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
|
||||
|
||||
export interface GeneratorOptions {
|
||||
/**
|
||||
* Presets you can extend
|
||||
*/
|
||||
presets: Record<string, ElasticsearchOptionsDSL>;
|
||||
/**
|
||||
* Override specific types
|
||||
*/
|
||||
overrides: Record<string, MappingProperty>;
|
||||
/**
|
||||
* Template for the generated index request
|
||||
*
|
||||
* Supports `{type}` and `{sanitized_type}` (same as `{type}`, but no spaces) template substitutions
|
||||
*/
|
||||
template: Partial<IndicesPutTemplateRequest>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fully transform a project
|
||||
*/
|
||||
export function transformProject(project: JSONSchema7) {
|
||||
const context = new MappingGenerator(project, OPTIONS);
|
||||
|
||||
const results = [];
|
||||
for (const name in project.definitions) {
|
||||
const definition = project.definitions[name];
|
||||
if (typeof definition !== 'object' || !getTags(definition).has(INDEXABLE_TAG_NAME)) continue;
|
||||
|
||||
results.push(context.buildTemplate(name));
|
||||
}
|
||||
return {
|
||||
mappings: results,
|
||||
search: context.searchMods.mods,
|
||||
};
|
||||
}
|
||||
|
||||
const OPTIONS: GeneratorOptions = {
|
||||
template: {
|
||||
name: 'template_{_type}',
|
||||
index_patterns: 'stapps_{_type}*',
|
||||
settings: {
|
||||
'mapping.total_fields.limit': 10_000,
|
||||
'max_result_window': 30_000,
|
||||
'number_of_replicas': 0,
|
||||
'number_of_shards': 1,
|
||||
},
|
||||
mappings: {
|
||||
date_detection: false,
|
||||
_source: {
|
||||
excludes: ['creation_date'],
|
||||
},
|
||||
properties: {
|
||||
creation_date: {
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
overrides: {
|
||||
'CoordinateReferenceSystem': {
|
||||
dynamic: true,
|
||||
properties: {
|
||||
type: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
'LineString': {
|
||||
type: 'geo_shape',
|
||||
},
|
||||
'Point': {
|
||||
properties: {
|
||||
type: {
|
||||
type: 'keyword',
|
||||
},
|
||||
coordinates: {
|
||||
type: 'geo_point',
|
||||
},
|
||||
},
|
||||
dynamic: 'strict',
|
||||
},
|
||||
'Polygon': {
|
||||
type: 'geo_shape',
|
||||
},
|
||||
'SCISO8601DateRange': {
|
||||
type: 'date_range',
|
||||
},
|
||||
'jsonpatch.OpPatch': {
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
from: {
|
||||
type: 'keyword',
|
||||
},
|
||||
op: {
|
||||
type: 'keyword',
|
||||
},
|
||||
path: {
|
||||
type: 'keyword',
|
||||
},
|
||||
value: {
|
||||
// this is actually an 'any' type; however, ES does not really support that.
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
presets: {
|
||||
'type': {extends: ['sortable:ducet', 'filterable', 'aggregatable:global']},
|
||||
'text': {
|
||||
merge: {type: 'text'},
|
||||
},
|
||||
'filterable': {
|
||||
merge: {
|
||||
fields: {
|
||||
raw: {type: 'keyword'},
|
||||
},
|
||||
},
|
||||
},
|
||||
'sortable:ducet': {
|
||||
merge: {
|
||||
fields: {
|
||||
sort: {
|
||||
type: 'icu_collation_keyword',
|
||||
language: 'de',
|
||||
country: 'DE',
|
||||
variant: '@collation=phonebook',
|
||||
} as never,
|
||||
},
|
||||
},
|
||||
},
|
||||
'sortable': {
|
||||
extends: ['sortable:ducet'],
|
||||
},
|
||||
'sortable:price': {},
|
||||
'aggregatable': {
|
||||
search: {
|
||||
aggs: {
|
||||
'{type}': {
|
||||
aggs: {
|
||||
'{prop}': {
|
||||
terms: {
|
||||
field: '{prop}.raw',
|
||||
size: 1000,
|
||||
},
|
||||
},
|
||||
},
|
||||
filter: {term: {type: '{type}'}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
'aggregatable:global': {
|
||||
search: {
|
||||
aggs: {
|
||||
'@all.{prop}': {
|
||||
terms: {
|
||||
field: '{prop}.raw',
|
||||
size: 1000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,76 @@
|
||||
import {GeneratorOptions} from './index.js';
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
|
||||
import {
|
||||
IndicesPutTemplateRequest,
|
||||
MappingProperty,
|
||||
MappingTypeMapping,
|
||||
SearchRequest,
|
||||
} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import {Context} from './context.js';
|
||||
import deepmerge from 'deepmerge';
|
||||
import {renderTemplate} from '../template.js';
|
||||
|
||||
/**
|
||||
* Sanitize a type name
|
||||
*/
|
||||
export function sanitizeTypeName(typeName: string): string {
|
||||
return typeName.replaceAll(' ', '_');
|
||||
}
|
||||
|
||||
export class MappingGenerator {
|
||||
readonly presets: Map<string, ElasticsearchOptionsDSL>;
|
||||
|
||||
readonly cache: Map<string, {mapping: MappingProperty; dependencies: Map<string, Set<string>>}>;
|
||||
|
||||
readonly searchMods: {mods: Partial<SearchRequest>} = {mods: {}};
|
||||
|
||||
readonly template: Partial<IndicesPutTemplateRequest>;
|
||||
|
||||
constructor(readonly project: JSONSchema7, options: GeneratorOptions) {
|
||||
this.template = options.template ?? {};
|
||||
this.presets = new Map(Object.entries(options.presets));
|
||||
this.cache = new Map(
|
||||
Object.entries(options.overrides).map(([name, mapping]) => [
|
||||
name,
|
||||
{
|
||||
mapping,
|
||||
dependencies: new Map(),
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
buildTemplate(name: string): IndicesPutTemplateRequest {
|
||||
const thingType = ((this.project.definitions![name] as JSONSchema7).properties!.type as JSONSchema7)
|
||||
.const;
|
||||
if (typeof thingType !== 'string') {
|
||||
throw new TypeError(`${name} needs a valid thing type`);
|
||||
}
|
||||
const mappingContext = new Context(this, thingType, [name], [], new Map());
|
||||
const mappings = mappingContext.resolveReference(name);
|
||||
|
||||
const request: IndicesPutTemplateRequest = deepmerge(
|
||||
{mappings: mappings as MappingTypeMapping},
|
||||
renderTemplate(this.template, [
|
||||
['{name}', name],
|
||||
['{type}', thingType],
|
||||
['{_type}', sanitizeTypeName(thingType)],
|
||||
]),
|
||||
);
|
||||
|
||||
if (mappingContext.dependencies.size > 0) {
|
||||
request.mappings!.dynamic_templates = [];
|
||||
for (const [name, paths] of mappingContext.dependencies) {
|
||||
request.mappings!.dynamic_templates.push({
|
||||
[name]: {
|
||||
path_match: paths.size > 1 ? [...paths] : paths.values().next().value,
|
||||
mapping: mappingContext.resolveReference(name),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
}
|
||||
13
packages/es-mapping-generator/src/generator/tags.ts
Normal file
13
packages/es-mapping-generator/src/generator/tags.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
|
||||
export const INDEXABLE_TAG_NAME = 'indexable';
|
||||
|
||||
/**
|
||||
* Get elasticsearch tags
|
||||
*/
|
||||
export function getTags(definition: JSONSchema7): Set<string> {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return new Set(
|
||||
(definition as {elasticsearch: string}).elasticsearch?.split(/\s/).map(it => it.trim()) ?? [],
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
import {transformDefinition} from '../definition.js';
|
||||
import {Context} from '../context.js';
|
||||
|
||||
/**
|
||||
* Transform a JSON Schema with `object` type
|
||||
*/
|
||||
export function transformObject(context: Context, definition: JSONSchema7): MappingProperty {
|
||||
const value = {
|
||||
dynamic: definition.additionalProperties === true ? undefined : 'strict',
|
||||
properties: {} as Record<string, MappingProperty>,
|
||||
} satisfies MappingProperty;
|
||||
|
||||
for (const key in definition.properties!) {
|
||||
value.properties[key] = transformDefinition(context.step(key), definition.properties[key] as JSONSchema7);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import {JSONSchema7} from 'json-schema';
|
||||
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
||||
|
||||
const stringFormats = new Map<string, MappingProperty>([
|
||||
['date', {type: 'date', format: 'date'}],
|
||||
['time', {type: 'date', format: 'time'}],
|
||||
['date-time', {type: 'date', format: 'date_optional_time'}],
|
||||
['ipv4', {type: 'ip'}],
|
||||
['ipv6', {type: 'ip'}],
|
||||
]);
|
||||
|
||||
/**
|
||||
* Transform a JSON Schema with `string` type
|
||||
*/
|
||||
export function transformString(definition: JSONSchema7): MappingProperty {
|
||||
return definition.format && stringFormats.has(definition.format)
|
||||
? stringFormats.get(definition.format)!
|
||||
: {type: 'keyword'};
|
||||
}
|
||||
@@ -1,7 +1,16 @@
|
||||
export * from './mapping';
|
||||
export * from './project-reflection';
|
||||
import {transformProject} from './generator/index.js';
|
||||
import {SchemaConsumer} from '@openstapps/json-schema-generator';
|
||||
|
||||
export * from './config/premap';
|
||||
export * from './config/fieldmap';
|
||||
export * from './config/settings';
|
||||
export * from './config/typemap';
|
||||
/**
|
||||
* JSON Schema Generator plugin for Elasticsearch Mappings
|
||||
*/
|
||||
export function elasticsearchMappingGenerator(fileName: string): [string, SchemaConsumer] {
|
||||
return [
|
||||
'Elasticsearch-Mappings',
|
||||
function (schema) {
|
||||
return {
|
||||
[fileName]: JSON.stringify(transformProject(schema)),
|
||||
};
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@@ -1,894 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
MappingDynamicTemplate,
|
||||
MappingObjectProperty,
|
||||
MappingProperty,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import merge from 'deepmerge';
|
||||
import {stringify} from 'flatted';
|
||||
import {DeclarationReflection, ProjectReflection, SignatureReflection} from 'typedoc';
|
||||
import {
|
||||
ArrayType,
|
||||
Comment,
|
||||
CommentTag,
|
||||
IntrinsicType,
|
||||
ReferenceType,
|
||||
ReflectionType,
|
||||
StringLiteralType,
|
||||
Type,
|
||||
TypeParameterType,
|
||||
UnionType,
|
||||
} from 'typedoc/dist/lib/models';
|
||||
import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap';
|
||||
import {premaps} from './config/premap';
|
||||
import {settings} from './config/settings';
|
||||
import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap';
|
||||
import type {AggregationSchema, ESNestedAggregation} from '../schema/aggregations';
|
||||
import type {ElasticsearchTemplateCollection, MappingGenTemplate} from '../schema/mappings';
|
||||
import * as console from 'console';
|
||||
|
||||
let dynamicTemplates: Record<string, MappingDynamicTemplate>[] = [];
|
||||
let errors: string[] = [];
|
||||
let showErrors = true;
|
||||
|
||||
let aggregations: AggregationSchema = {};
|
||||
|
||||
const indexableTag = 'indexable';
|
||||
const aggregatableTag = 'aggregatable';
|
||||
const aggregatableTagParameterGlobal = 'global';
|
||||
const inheritTagsName = 'inherittags';
|
||||
|
||||
// clamp printed object to 1000 chars to keep error messages readable
|
||||
const maxErrorObjectChars = 1000;
|
||||
|
||||
let ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
|
||||
let inheritTagsMap: {[path: string]: CommentTag[]} = {};
|
||||
|
||||
/**
|
||||
* Gets all interfaces that have an @indexable tag
|
||||
* @param projectReflection the project reflection from which to extract the indexable interfaces
|
||||
*/
|
||||
export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] {
|
||||
let indexableInterfaces: DeclarationReflection[] = [];
|
||||
|
||||
if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) {
|
||||
throw new Error('No DeclarationReflections found. Please check your input path');
|
||||
}
|
||||
|
||||
// push all declaration reflections into one array
|
||||
for (const declarationReflection of projectReflection.children) {
|
||||
if (Array.isArray(declarationReflection.children)) {
|
||||
indexableInterfaces = [...indexableInterfaces, ...declarationReflection.children];
|
||||
}
|
||||
}
|
||||
|
||||
// filter all declaration reflections with an @indexable tag
|
||||
indexableInterfaces = indexableInterfaces.filter(declarationReflection => {
|
||||
if (declarationReflection.comment === undefined || declarationReflection.comment.tags === undefined) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return declarationReflection.comment.tags.some(commentTag => {
|
||||
return commentTag.tagName === indexableTag;
|
||||
});
|
||||
});
|
||||
|
||||
return indexableInterfaces;
|
||||
}
|
||||
|
||||
/**
|
||||
* Composes error messages, that are readable and contain a certain minimum of information
|
||||
* @param path the path where the error took place
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param typeName the name of the object, with which something went wrong
|
||||
* @param object the object or name
|
||||
* @param message the error message
|
||||
*/
|
||||
function composeErrorMessage(
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
typeName: string,
|
||||
object: string,
|
||||
message: string,
|
||||
) {
|
||||
const error = `At "${topTypeName}::${path.slice(
|
||||
0,
|
||||
Math.max(0, path.length - 1),
|
||||
)}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
|
||||
errors.push(error);
|
||||
if (showErrors) {
|
||||
// tslint:disable-next-line:no-floating-promises
|
||||
void console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trims a string to a readable size and appends "..."
|
||||
* @param value the string to trim
|
||||
* @param maxLength the maximum allowed length before it is clamped
|
||||
*/
|
||||
function trimString(value: string, maxLength: number): string {
|
||||
return value.length > maxLength ? `${value.slice(0, Math.max(0, maxLength))}...` : value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection
|
||||
*
|
||||
* Warning to future maintainers: The code for generics doesn't account for depth. when there is a new generic, it will
|
||||
* override the previous one, if there isn't, it will just continue passing it down.
|
||||
* @param type the ReferenceType of a DeclarationReflection
|
||||
* @param out the previous reflection, it then overrides all parameters or keeps old ones
|
||||
* @param topTypeName the name of the object, with which something went wrong
|
||||
* @param path the current path to the object we are in
|
||||
* @param tags any tags attached to the type
|
||||
*/
|
||||
function getReflectionGeneric(
|
||||
type: ReferenceType,
|
||||
out: Map<string, MappingProperty>,
|
||||
topTypeName: string,
|
||||
path: string,
|
||||
tags: CommentTag[],
|
||||
): Map<string, MappingProperty> {
|
||||
if (
|
||||
type.typeArguments !== undefined &&
|
||||
type.reflection instanceof DeclarationReflection &&
|
||||
type.reflection.typeParameters !== undefined
|
||||
) {
|
||||
for (let i = 0; i < type.reflection.typeParameters.length; i++) {
|
||||
if (i < type.typeArguments.length) {
|
||||
out.set(
|
||||
type.reflection.typeParameters[i].name,
|
||||
handleType(type.typeArguments[i], out, topTypeName, path, tags),
|
||||
);
|
||||
} else {
|
||||
// this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061
|
||||
// we have no way to know the type here, so we have to use this.
|
||||
out.set(type.reflection.typeParameters[i].name, {
|
||||
dynamic: true,
|
||||
properties: {},
|
||||
});
|
||||
|
||||
console.warn(
|
||||
`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
|
||||
` in TypeDoc. It has been replaced by a dynamic type.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles a ReferenceType that has no value
|
||||
*
|
||||
* Most of the times that is an external type.
|
||||
* @param ref the ReferenceType
|
||||
* @param generics the generics from levels above, so we can use them without having access to the parent
|
||||
* @param path the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param tags any tags attached to the type
|
||||
*/
|
||||
function handleExternalType(
|
||||
ref: ReferenceType,
|
||||
generics: Map<string, MappingProperty>,
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
tags: CommentTag[],
|
||||
): MappingProperty {
|
||||
for (const premap of Object.keys(premaps)) {
|
||||
if (premap === ref.name) {
|
||||
return readFieldTags(premaps[premap], path, topTypeName, tags);
|
||||
}
|
||||
}
|
||||
|
||||
if (ref.name === 'Array') {
|
||||
// basically an external type, but Array is quite common, especially with generics
|
||||
if (ref.typeArguments === undefined || ref.typeArguments[0] === undefined) {
|
||||
composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse');
|
||||
|
||||
return {type: PARSE_ERROR};
|
||||
}
|
||||
|
||||
return readFieldTags(
|
||||
handleType(
|
||||
ref.typeArguments[0],
|
||||
getReflectionGeneric(ref, new Map(generics), path, topTypeName, tags),
|
||||
path,
|
||||
topTypeName,
|
||||
tags,
|
||||
),
|
||||
path,
|
||||
topTypeName,
|
||||
tags,
|
||||
);
|
||||
}
|
||||
if (ref.name === '__type') {
|
||||
// empty object
|
||||
return {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
}
|
||||
|
||||
composeErrorMessage(path, topTypeName, 'external type', ref.name, 'Missing pre-map');
|
||||
|
||||
return readFieldTags({type: MISSING_PREMAP}, path, topTypeName, tags);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles an object
|
||||
* @param decl the DeclarationReflection of the object
|
||||
* @param generics the generics from levels above, so we can use them without having access to the parent
|
||||
* @param path the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param inheritedTags the inherited tags
|
||||
*/
|
||||
function handleDeclarationReflection(
|
||||
decl: DeclarationReflection,
|
||||
generics: Map<string, MappingProperty>,
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
inheritedTags?: CommentTag[],
|
||||
): MappingProperty {
|
||||
// check if we have an object referencing a generic
|
||||
if (generics.has(decl.name)) {
|
||||
// if the object name is the same as the generic name
|
||||
return readFieldTags(generics.get(decl.name)!, path, topTypeName, decl.comment?.tags ?? []);
|
||||
// use the value defined by the generic
|
||||
}
|
||||
|
||||
// start the actual handling process
|
||||
const out: MappingProperty = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
let empty = true;
|
||||
// first check if there are any index signatures, so for example `[name: string]: Foo`
|
||||
if (decl.indexSignature !== undefined) {
|
||||
out.dynamic = true;
|
||||
|
||||
if (decl.indexSignature.type !== undefined) {
|
||||
empty = false;
|
||||
const template: Record<string, MappingDynamicTemplate> = {};
|
||||
template[decl.name] = {
|
||||
mapping: handleType(
|
||||
decl.indexSignature.type,
|
||||
new Map(generics),
|
||||
path,
|
||||
topTypeName,
|
||||
getCommentTags(decl.indexSignature, path, topTypeName),
|
||||
),
|
||||
match: '*',
|
||||
match_mapping_type: '*',
|
||||
path_match: `${path}*`,
|
||||
};
|
||||
dynamicTemplates.push(template);
|
||||
}
|
||||
}
|
||||
|
||||
if (decl.kindString === 'Enumeration') {
|
||||
return readTypeTags('string', path, topTypeName, getCommentTags(decl, path, topTypeName, inheritedTags));
|
||||
}
|
||||
|
||||
// check all the children, so in this case we are dealing with an OBJECT
|
||||
if (decl.children !== undefined && decl.children.length > 0) {
|
||||
for (const child of decl.children) {
|
||||
empty = false;
|
||||
out.properties![child.name] = handleDeclarationReflection(
|
||||
child,
|
||||
new Map(generics),
|
||||
`${path}${child.name}.`,
|
||||
topTypeName,
|
||||
);
|
||||
}
|
||||
} else if (decl.type instanceof Type) {
|
||||
// if the object is a type, so we are dealing with a PROPERTY
|
||||
// get inherited tags
|
||||
const tags =
|
||||
(inheritedTags ?? []).length > 0
|
||||
? inheritedTags!.some(it => isTagType(it.tagName))
|
||||
? inheritedTags!
|
||||
: [...(inheritedTags ?? []), ...getCommentTags(decl, path, topTypeName)]
|
||||
: getCommentTags(decl, path, topTypeName);
|
||||
|
||||
return handleType(decl.type, new Map(generics), path, topTypeName, tags);
|
||||
} else if (decl.kindString === 'Enumeration member') {
|
||||
return readTypeTags(
|
||||
typeof decl.defaultValue,
|
||||
path,
|
||||
topTypeName,
|
||||
getCommentTags(decl, path, topTypeName, inheritedTags),
|
||||
);
|
||||
}
|
||||
|
||||
if (empty) {
|
||||
composeErrorMessage(path, topTypeName, 'object', decl.name, 'Empty object');
|
||||
}
|
||||
|
||||
return readFieldTags(out, path, topTypeName, getCommentTags(decl, path, topTypeName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads all comment tags, including inherited ones
|
||||
* @param decl the DeclarationReflection to read the tags from
|
||||
* @param path the path on which the comments lie
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param inheritedTags any tags that might have been inherited by a parent
|
||||
* @param breakId the id of the previous reflection to prevent infinite recursion in some cases
|
||||
*/
|
||||
function getCommentTags(
|
||||
decl: DeclarationReflection | SignatureReflection,
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
inheritedTags: CommentTag[] = [],
|
||||
// tslint:disable-next-line:no-unnecessary-initializer
|
||||
breakId: number | undefined = undefined,
|
||||
): CommentTag[] {
|
||||
if (decl.id === breakId) {
|
||||
return [];
|
||||
}
|
||||
|
||||
let out: CommentTag[] =
|
||||
decl.comment instanceof Comment
|
||||
? decl.comment.tags !== undefined
|
||||
? decl.comment.tags
|
||||
: inheritedTags
|
||||
: inheritedTags;
|
||||
if (
|
||||
decl.overwrites instanceof ReferenceType &&
|
||||
decl.overwrites.reflection instanceof DeclarationReflection
|
||||
) {
|
||||
out = arrayPriorityJoin(
|
||||
getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id),
|
||||
out,
|
||||
);
|
||||
}
|
||||
if (
|
||||
decl.inheritedFrom instanceof ReferenceType &&
|
||||
decl.inheritedFrom.reflection instanceof DeclarationReflection
|
||||
) {
|
||||
out = arrayPriorityJoin(
|
||||
getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id),
|
||||
out,
|
||||
);
|
||||
}
|
||||
|
||||
saveCommentTags(out, path, topTypeName);
|
||||
const inheritTag = out.find(value => value.tagName === inheritTagsName);
|
||||
if (inheritTag !== undefined) {
|
||||
out = arrayPriorityJoin(out, retrieveCommentTags(inheritTag.text.trim(), path, topTypeName));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves all comment tags to the map
|
||||
* @param tags all tags to be saved (@see and @[inheritTags] will be stripped)
|
||||
* @param path the path of field
|
||||
* @param topTypeName the name of the SCThingType
|
||||
*/
|
||||
function saveCommentTags(tags: CommentTag[], path: string, topTypeName: string) {
|
||||
inheritTagsMap[`${topTypeName}::${path.slice(0, Math.max(0, path.length - 1))}`] = tags.filter(
|
||||
value => value.tagName !== 'see' && value.tagName !== inheritTagsName,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves any saved tags
|
||||
* @param path the path to the original field
|
||||
* @param currentPath the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
*/
|
||||
function retrieveCommentTags(path: string, currentPath: string, topTypeName: string): CommentTag[] {
|
||||
if (!(path in inheritTagsMap)) {
|
||||
composeErrorMessage(currentPath, topTypeName, path, 'Comment', 'Referenced path to tags does not exist!');
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
return inheritTagsMap[path];
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins two arrays of CommentTags, but overrides all original CommentTags with the same tagName
|
||||
* @param originals the original array
|
||||
* @param overrider the array that should be appended and provide the override values
|
||||
*/
|
||||
function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] {
|
||||
const out: CommentTag[] = overrider;
|
||||
|
||||
for (const original of originals) {
|
||||
const result = overrider.find(element => original.tagName === element.tagName);
|
||||
|
||||
// no support for multiple tags with the same name
|
||||
if (!(result instanceof CommentTag)) {
|
||||
out.push(original);
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles UnionTypes
|
||||
*
|
||||
* Put into a separate function as it is a little bit more complex
|
||||
* Works fairly reliable, although there are issues with primitive union types, which don't work at all (And never will)
|
||||
* @param type the type object
|
||||
* @param generics the generics from levels above, so we can use them without having access to the parent
|
||||
* @param path the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param tags any tags attached to the type
|
||||
*/
|
||||
function handleUnionType(
|
||||
type: UnionType,
|
||||
generics: Map<string, MappingProperty>,
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
tags: CommentTag[],
|
||||
): MappingProperty {
|
||||
const list: MappingProperty[] = [];
|
||||
|
||||
for (const subType of type.types) {
|
||||
if (subType instanceof IntrinsicType && subType.name === 'undefined') {
|
||||
continue;
|
||||
}
|
||||
list.push(handleType(subType, new Map(generics), path, topTypeName, tags));
|
||||
}
|
||||
|
||||
if (list.length > 0) {
|
||||
let out = list[0];
|
||||
|
||||
for (const item of list) {
|
||||
out = merge<MappingProperty>(out, item);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
composeErrorMessage(
|
||||
path,
|
||||
topTypeName,
|
||||
'Union Type',
|
||||
stringify(list),
|
||||
'Empty union type. This is likely not a user error.',
|
||||
);
|
||||
|
||||
return {type: PARSE_ERROR};
|
||||
}
|
||||
|
||||
/**
|
||||
* Serves as a kind of distributor for the different types, should not contain any specific code
|
||||
* @param type the type object
|
||||
* @param generics the generics from levels above, so we can use them without having access to the parent
|
||||
* @param path the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param tags any tags attached to the type
|
||||
*/
|
||||
function handleType(
|
||||
type: Type,
|
||||
generics: Map<string, MappingProperty>,
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
tags: CommentTag[],
|
||||
): MappingProperty {
|
||||
// logger.log((type as any).name);
|
||||
if (type instanceof ArrayType) {
|
||||
// array is irrelevant in Elasticsearch, so just go with the element type
|
||||
const esType = handleType(type.elementType, new Map(generics), path, topTypeName, tags);
|
||||
// also merge tags of the array to the element type
|
||||
// filter out the type tags lazily, this can lead to double messages for "Not implemented tag"
|
||||
let newTags = tags;
|
||||
if ('type' in esType) {
|
||||
newTags = tags.filter(tag => {
|
||||
return !(tag.tagName === esType.type);
|
||||
});
|
||||
}
|
||||
|
||||
return readFieldTags(esType, path, topTypeName, newTags);
|
||||
}
|
||||
if (type.type === 'stringLiteral') {
|
||||
// a string literal, usually for type
|
||||
return readTypeTags(type.type, path, topTypeName, tags);
|
||||
}
|
||||
if (type instanceof IntrinsicType) {
|
||||
// the absolute default type, like strings
|
||||
return readTypeTags(type.name, path, topTypeName, tags);
|
||||
}
|
||||
if (type instanceof UnionType) {
|
||||
// the union type...
|
||||
return handleUnionType(type, new Map(generics), path, topTypeName, tags);
|
||||
}
|
||||
if (type instanceof ReferenceType) {
|
||||
if (premaps[type.name] === undefined && type.reflection !== undefined) {
|
||||
// there is really no way to make this typesafe, every element in DeclarationReflection is optional.
|
||||
return handleDeclarationReflection(
|
||||
type.reflection as DeclarationReflection,
|
||||
getReflectionGeneric(type, new Map(generics), path, topTypeName, tags),
|
||||
path,
|
||||
topTypeName,
|
||||
tags,
|
||||
);
|
||||
}
|
||||
|
||||
return handleExternalType(type, new Map(generics), path, topTypeName, tags);
|
||||
}
|
||||
if (type instanceof TypeParameterType) {
|
||||
// check if we have an object referencing a generic
|
||||
if (generics.has(type.name)) {
|
||||
return generics.get(type.name)!;
|
||||
}
|
||||
composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
|
||||
|
||||
return {type: PARSE_ERROR};
|
||||
}
|
||||
if (type instanceof ReflectionType) {
|
||||
return readFieldTags(
|
||||
handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
|
||||
path,
|
||||
topTypeName,
|
||||
tags,
|
||||
);
|
||||
}
|
||||
|
||||
composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type');
|
||||
|
||||
return {type: PARSE_ERROR};
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an aggregatable to the aggregations list
|
||||
* @param path the current path
|
||||
* @param topTypeName the name of the top type
|
||||
* @param global whether the topTypeName will be used
|
||||
*/
|
||||
function addAggregatable(path: string, topTypeName: string, global: boolean) {
|
||||
// push type.path and remove the '.' at the end of the path
|
||||
|
||||
if (global) {
|
||||
const property_ = path.slice(0, -1).split('.').pop() as string; // cannot be undefined
|
||||
|
||||
return ((aggregations['@all'] as ESNestedAggregation).aggs[property_.split('.').pop() as string] = {
|
||||
terms: {
|
||||
field: `${property_}.raw`,
|
||||
size: 1000,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const property = path.slice(0, -1);
|
||||
|
||||
return ((aggregations[topTypeName] as ESNestedAggregation).aggs[property] = {
|
||||
terms: {
|
||||
field: `${property}.raw`,
|
||||
size: 1000,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads all tags related to Elasticsearch fields from the fieldMap
|
||||
* @param previous the previous ElasticsearchValue, for example and object
|
||||
* @param path the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param tags tags attached to the value
|
||||
* @param dataType the ElasticsearchDataType, for checking if a tag is a type tag
|
||||
*/
|
||||
function readFieldTags(
|
||||
previous: MappingProperty,
|
||||
path: string,
|
||||
topTypeName: string,
|
||||
tags: CommentTag[],
|
||||
dataType?: string,
|
||||
): MappingProperty {
|
||||
for (const tag of tags) {
|
||||
if (tag.tagName === aggregatableTag) {
|
||||
addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal);
|
||||
}
|
||||
|
||||
if (!ignoredTagsList.includes(tag.tagName)) {
|
||||
if (fieldmap[tag.tagName] !== undefined) {
|
||||
if (previous.fields === undefined) {
|
||||
// create in case it doesn't exist
|
||||
previous.fields = {};
|
||||
}
|
||||
if (tag.text.trim() === '') {
|
||||
// merge fields
|
||||
previous.fields = {...previous.fields, ...fieldmap[tag.tagName].default};
|
||||
} else if (fieldmap[tag.tagName][tag.text.trim()] !== undefined) {
|
||||
// merge fields
|
||||
previous.fields = {...previous.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
|
||||
} else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) {
|
||||
// when there is an unidentified tag
|
||||
composeErrorMessage(
|
||||
path,
|
||||
topTypeName,
|
||||
'tag',
|
||||
tag.tagName,
|
||||
`Not implemented tag param "${tag.text.trim()}"`,
|
||||
);
|
||||
}
|
||||
} else if (tag.tagName === filterableTagName) {
|
||||
if (previous.fields === undefined) {
|
||||
previous.fields = {};
|
||||
}
|
||||
if ('type' in previous) {
|
||||
const type = filterableMap[previous.type!];
|
||||
if (type !== undefined) {
|
||||
// merge fields
|
||||
previous.fields = {...previous.fields, raw: {type: type}};
|
||||
} else {
|
||||
composeErrorMessage(
|
||||
path,
|
||||
topTypeName,
|
||||
'tag',
|
||||
tag.tagName,
|
||||
`Not implemented for ${previous.type}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types');
|
||||
}
|
||||
} else if (dataType === undefined || typemap[dataType][tag.tagName] === undefined) {
|
||||
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return previous;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads all types related to Elasticsearch fields from the fieldMap
|
||||
* @param type the type of the value
|
||||
* @param path the current path to the object we are in
|
||||
* @param topTypeName the name of the SCThingType
|
||||
* @param tags tags attached to the value
|
||||
*/
|
||||
function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): MappingProperty {
|
||||
let out: MappingProperty = {type: PARSE_ERROR};
|
||||
|
||||
if (typemap[type] !== undefined) {
|
||||
// first look if the value has a definition in the typemap
|
||||
for (let i = tags.length - 1; i >= 0; i--) {
|
||||
if (!ignoredTagsList.includes(tags[i].tagName) && typemap[type][tags[i].tagName] !== undefined) {
|
||||
// if we have a tag that indicates a type
|
||||
if (out.type !== PARSE_ERROR) {
|
||||
composeErrorMessage(
|
||||
path,
|
||||
topTypeName,
|
||||
'type',
|
||||
type,
|
||||
`Type conflict; "${typemap[type][tags[i].tagName]}" would override "${
|
||||
(out as MappingProperty).type
|
||||
}"`,
|
||||
);
|
||||
(out as MappingProperty).type = TYPE_CONFLICT;
|
||||
continue;
|
||||
}
|
||||
(out as MappingProperty).type = typemap[type][tags[i].tagName];
|
||||
}
|
||||
}
|
||||
|
||||
if (out.type === PARSE_ERROR) {
|
||||
(out as MappingProperty).type = typemap[type].default;
|
||||
}
|
||||
|
||||
out = readFieldTags(out, path, topTypeName, tags, type);
|
||||
|
||||
return out;
|
||||
}
|
||||
if (dynamicTypes.includes(type)) {
|
||||
// Elasticsearch dynamic type TODO: doesn't work for direct types
|
||||
return {
|
||||
dynamic: true,
|
||||
properties: {},
|
||||
};
|
||||
}
|
||||
|
||||
composeErrorMessage(path, topTypeName, 'type', type, 'Not implemented type');
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the state
|
||||
*
|
||||
* This is kind of a suboptimal solution and should be changed in the future.
|
||||
* https://gitlab.com/openstapps/core-tools/-/issues/49
|
||||
* @param resetInheritTags whether inherited tags should be reset as well
|
||||
*/
|
||||
function reset(resetInheritTags = true) {
|
||||
errors = [];
|
||||
dynamicTemplates = [];
|
||||
aggregations = {
|
||||
'@all': {
|
||||
aggs: {},
|
||||
filter: {
|
||||
match_all: {},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (resetInheritTags) {
|
||||
inheritTagsMap = {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a project reflection and generates an ElasticsearchTemplate from it
|
||||
*
|
||||
* Serves as the entry point for getting the mapping, so if you just want to get the mapping files for Elasticsearch,
|
||||
* you can do so by calling this function, `RETURNED_VALUE.template` contains the mapping in a fashion that is directly
|
||||
* readable by Elasticsearch.
|
||||
* @param projectReflection a reflection of the project you want to get the ES Mappings from
|
||||
* @param ignoredTags the tag names for which the error output should be suppressed
|
||||
* @param showErrorOutput whether to print all errors in the command line or not
|
||||
* @param interfaceFilter only parse specific interfaces, this is for testing purposes
|
||||
*/
|
||||
export function generateTemplate(
|
||||
projectReflection: ProjectReflection,
|
||||
ignoredTags: string[],
|
||||
showErrorOutput = true,
|
||||
interfaceFilter: string[] = [],
|
||||
): MappingGenTemplate {
|
||||
reset();
|
||||
|
||||
showErrors = showErrorOutput;
|
||||
|
||||
ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
|
||||
// eslint-disable-next-line prefer-spread
|
||||
ignoredTagsList.push.apply(ignoredTagsList, ignoredTags);
|
||||
|
||||
const indexableInterfaces = getAllIndexableInterfaces(projectReflection);
|
||||
|
||||
const out: ElasticsearchTemplateCollection = {};
|
||||
|
||||
for (const _interface of indexableInterfaces) {
|
||||
// TODO: lots of duplicate code, this all needs to be changed https://gitlab.com/openstapps/core-tools/-/issues/49
|
||||
if (!Array.isArray(_interface.children) || _interface.children.length === 0) {
|
||||
throw new Error('Interface needs at least some properties to be indexable');
|
||||
}
|
||||
|
||||
const typeObject = _interface.children.find(declarationReflection => {
|
||||
return declarationReflection.name === 'type';
|
||||
});
|
||||
|
||||
if (typeObject === undefined || typeObject.type === undefined) {
|
||||
throw new TypeError('Interface needs a type to be indexable');
|
||||
}
|
||||
|
||||
let typeName = 'INVALID_TYPE';
|
||||
if (typeObject.type instanceof ReferenceType) {
|
||||
if (
|
||||
typeObject.type.reflection instanceof DeclarationReflection &&
|
||||
typeof typeObject.type.reflection.defaultValue === 'string'
|
||||
) {
|
||||
typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
|
||||
} else {
|
||||
// tslint:disable-next-line:no-floating-promises
|
||||
void console.error(
|
||||
'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
|
||||
);
|
||||
}
|
||||
} else if (typeObject.type instanceof StringLiteralType) {
|
||||
console.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
|
||||
typeName = typeObject.type.value;
|
||||
} else {
|
||||
// tslint:disable-next-line:no-floating-promises
|
||||
void console.error(
|
||||
`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
|
||||
);
|
||||
}
|
||||
// init aggregation schema for type
|
||||
aggregations[typeName] = {
|
||||
aggs: {},
|
||||
filter: {
|
||||
term: {
|
||||
type: typeName,
|
||||
},
|
||||
},
|
||||
};
|
||||
handleDeclarationReflection(_interface, new Map(), '', typeName);
|
||||
}
|
||||
|
||||
// second traversal
|
||||
reset(false);
|
||||
|
||||
for (const _interface of indexableInterfaces) {
|
||||
if (!Array.isArray(_interface.children) || _interface.children.length === 0) {
|
||||
throw new Error('Interface needs at least some properties to be indexable');
|
||||
}
|
||||
|
||||
const typeObject = _interface.children.find(declarationReflection => {
|
||||
return declarationReflection.name === 'type';
|
||||
});
|
||||
|
||||
if (typeObject === undefined || typeObject.type === undefined) {
|
||||
throw new TypeError('Interface needs a type to be indexable');
|
||||
}
|
||||
|
||||
let typeName = 'INVALID_TYPE';
|
||||
if (typeObject.type instanceof ReferenceType) {
|
||||
if (
|
||||
typeObject.type.reflection instanceof DeclarationReflection &&
|
||||
typeof typeObject.type.reflection.defaultValue === 'string'
|
||||
) {
|
||||
typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
|
||||
} else {
|
||||
// tslint:disable-next-line:no-floating-promises
|
||||
void console.error(
|
||||
'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
|
||||
);
|
||||
}
|
||||
} else if (typeObject.type instanceof StringLiteralType) {
|
||||
console.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
|
||||
typeName = typeObject.type.value;
|
||||
} else {
|
||||
// tslint:disable-next-line:no-floating-promises
|
||||
void console.error(
|
||||
`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
|
||||
);
|
||||
}
|
||||
|
||||
// filter out
|
||||
if (interfaceFilter.length > 0 && !interfaceFilter.includes(typeName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// init aggregation schema for type
|
||||
aggregations[typeName] = {
|
||||
aggs: {},
|
||||
filter: {
|
||||
term: {
|
||||
type: typeName,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// eslint-disable-next-line unicorn/prefer-string-replace-all
|
||||
const typeNameWithoutSpaces = typeName.toLowerCase().replace(/\s/g, '_');
|
||||
const templateName = `template_${typeNameWithoutSpaces}`;
|
||||
|
||||
out[templateName] = {
|
||||
mappings: handleDeclarationReflection(_interface, new Map(), '', typeName) as MappingObjectProperty,
|
||||
settings: settings,
|
||||
index_patterns: [`stapps_${typeNameWithoutSpaces}*`],
|
||||
};
|
||||
out[templateName].mappings!.properties!.creation_date = {
|
||||
type: 'date',
|
||||
};
|
||||
|
||||
out[templateName].mappings!.dynamic_templates = dynamicTemplates;
|
||||
|
||||
// Set some properties
|
||||
out[templateName].mappings!._source = {
|
||||
excludes: ['creation_date'],
|
||||
};
|
||||
out[templateName].mappings!.date_detection = false;
|
||||
|
||||
dynamicTemplates = [];
|
||||
|
||||
if (Object.keys((aggregations[typeName] as ESNestedAggregation).aggs).length === 0) {
|
||||
delete aggregations[typeName];
|
||||
}
|
||||
}
|
||||
|
||||
return {aggregations, mappings: out, errors};
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {existsSync, PathLike} from 'fs';
|
||||
import {platform} from 'os';
|
||||
import path from 'path';
|
||||
import {Application, ProjectReflection} from 'typedoc';
|
||||
|
||||
/**
|
||||
* Get a project reflection from a path
|
||||
* @param sourcePath Path to get reflection from
|
||||
* @param excludeExternals Exclude external dependencies
|
||||
*/
|
||||
export function getProjectReflection(sourcePath: PathLike, excludeExternals = true): ProjectReflection {
|
||||
console.info(`Generating project reflection for ${sourcePath.toString()}.`);
|
||||
|
||||
const tsconfigPath = getTsconfigPath(sourcePath.toString());
|
||||
|
||||
// initialize new Typedoc application
|
||||
const app = new Application();
|
||||
|
||||
app.bootstrap({
|
||||
excludeExternals: excludeExternals,
|
||||
ignoreCompilerErrors: true,
|
||||
includeDeclarations: true,
|
||||
tsconfig: path.join(tsconfigPath, 'tsconfig.json'),
|
||||
});
|
||||
|
||||
let inputFilePath = sourcePath;
|
||||
if (inputFilePath === tsconfigPath) {
|
||||
inputFilePath = path.join(tsconfigPath, 'src');
|
||||
}
|
||||
|
||||
// get input files
|
||||
const inputFiles = app.expandInputFiles([inputFilePath.toString()]);
|
||||
|
||||
// get project reflection from input files
|
||||
const result = app.convert(inputFiles);
|
||||
|
||||
if (result === undefined) {
|
||||
throw new TypeError('Project reflection could not be generated.');
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get path that contains a tsconfig.json
|
||||
* @param startPath Path from where to start searching "upwards"
|
||||
*/
|
||||
export function getTsconfigPath(startPath: string): string {
|
||||
let tsconfigPath = startPath;
|
||||
|
||||
// see https://stackoverflow.com/questions/9652043/identifying-the-file-system-root-with-node-js
|
||||
const root = platform() === 'win32' ? process.cwd().split(path.sep)[0] : '/';
|
||||
|
||||
// repeat until a tsconfig.json is found
|
||||
while (!existsSync(path.join(tsconfigPath, 'tsconfig.json'))) {
|
||||
if (tsconfigPath === root) {
|
||||
throw new Error(
|
||||
`Reached file system root ${root} while searching for 'tsconfig.json' in ${startPath}!`,
|
||||
);
|
||||
}
|
||||
|
||||
// pop last directory
|
||||
const tsconfigPathParts = tsconfigPath.split(path.sep);
|
||||
tsconfigPathParts.pop();
|
||||
tsconfigPath = tsconfigPathParts.join(path.sep);
|
||||
}
|
||||
|
||||
console.info(`Using 'tsconfig.json' from ${tsconfigPath}.`);
|
||||
|
||||
return tsconfigPath;
|
||||
}
|
||||
13
packages/es-mapping-generator/src/template.ts
Normal file
13
packages/es-mapping-generator/src/template.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Render a template
|
||||
* @param template The template to render (must be stringify-able)
|
||||
* @param substitutions the substitutions
|
||||
*/
|
||||
export function renderTemplate<T>(template: T, substitutions: [string, string][]): T {
|
||||
return JSON.parse(
|
||||
substitutions.reduce(
|
||||
(template, [search, replace]) => template.replaceAll(search, replace),
|
||||
JSON.stringify(template),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -15,25 +15,28 @@
|
||||
import {ThingType} from './types';
|
||||
import {MapAggTestOptions} from '../../map-agg-test-options';
|
||||
|
||||
/** @integer */
|
||||
export type Integer = number;
|
||||
|
||||
/**
|
||||
* @indexable
|
||||
*/
|
||||
export interface InheritedProperty extends Bar {
|
||||
foo: number;
|
||||
/** @filterable */
|
||||
foo: Integer;
|
||||
|
||||
type: ThingType.InheritedProperty;
|
||||
// type: ThingType.InheritedProperty;
|
||||
}
|
||||
|
||||
interface Bar {
|
||||
/**
|
||||
* @keyword
|
||||
*/
|
||||
bar: string;
|
||||
|
||||
//bar: string;
|
||||
/**
|
||||
* @float
|
||||
*/
|
||||
baz: number;
|
||||
// baz: number;
|
||||
}
|
||||
|
||||
export const testConfig: MapAggTestOptions = {
|
||||
|
||||
@@ -1,25 +1,3 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"alwaysStrict": true,
|
||||
"charset": "utf8",
|
||||
"declaration": true,
|
||||
"esModuleInterop": true,
|
||||
"experimentalDecorators": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"inlineSourceMap": true,
|
||||
"skipLibCheck": true,
|
||||
"module": "CommonJS",
|
||||
"moduleResolution": "Node",
|
||||
"noErrorTruncation": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"outDir": "./lib/",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"target": "ES2020"
|
||||
},
|
||||
"exclude": ["./lib/", "./test/"]
|
||||
"extends": "@openstapps/tsconfig"
|
||||
}
|
||||
|
||||
12
packages/es-mapping-generator/tsup.config.ts
Normal file
12
packages/es-mapping-generator/tsup.config.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import {defineConfig} from 'tsup';
|
||||
import {jsonSchemaPlugin} from '@openstapps/json-schema-generator';
|
||||
|
||||
export default defineConfig({
|
||||
entry: ['src/index.ts'],
|
||||
sourcemap: true,
|
||||
clean: true,
|
||||
format: 'esm',
|
||||
outDir: 'lib',
|
||||
external: ['./index.schema.json'],
|
||||
plugins: [jsonSchemaPlugin('index.schema.json')],
|
||||
});
|
||||
Reference in New Issue
Block a user