Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-19 08:02:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: mapping generation for `Elasticsearch 8.4

Browse Source
This commit is contained in:
Thea Schöbl
2023-04-27 09:33:09 +00:00
committed by Rainer Killinger
parent 08ec4e4381
commit 61fee2bbf3
52 changed files with 1859 additions and 972 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/cli.ts
View File

@@ -1,3 +1,4 @@
/* eslint-disable unicorn/no-null */
/*
* Copyright (C) 2018-2021 StApps
* This program is free software: you can redistribute it and/or modify it
@@ -16,7 +17,7 @@ import {Logger} from '@openstapps/logger';
import {Command} from 'commander';
import {readFileSync, writeFileSync} from 'fs';
import got from 'got';
import {resolve} from 'path';
import path from 'path';
import {exit} from 'process';
import {generateTemplate} from './mapping';
import {getProjectReflection} from './project-reflection';
@@ -33,11 +34,8 @@ process.on('unhandledRejection', async (reason: unknown) => {
const commander = new Command('openstapps-core-tools');
commander
.version(JSON.parse(
readFileSync(resolve(__dirname, '..', 'package.json'))
.toString(),
).version);
// eslint-disable-next-line unicorn/prefer-module
commander.version(JSON.parse(readFileSync(path.resolve(__dirname, '..', 'package.json')).toString()).version);
commander
.command('mapping <relativeSrcPath>')
@@ -45,9 +43,9 @@ commander
.option('-i, --ignoredTags <ignoredTags>', 'Ignored Tags (comma-separated)')
.option('-a, --aggPath <relativeAggregationPath>', 'Aggregations Path')
.option('-e, --errorPath <relativeErrorPath>', 'Error Path')
.action(async (relativeSrcPath, options) => {
.action(async (relativeSourcePath, options) => {
// get absolute paths
const srcPath = resolve(relativeSrcPath);
const sourcePath = path.resolve(relativeSourcePath);
let ignoredTagsList: string[] = [];
if (typeof options.ignoredTags === 'string') {
@@ -55,10 +53,10 @@ commander
}
// get project reflection
const projectReflection = getProjectReflection(srcPath);
const projectReflection = getProjectReflection(sourcePath);
const result = generateTemplate(projectReflection, ignoredTagsList, true);
if (result.errors.length !== 0) {
if (result.errors.length > 0) {
await Logger.error('Mapping generated with errors!');
} else {
Logger.ok('Mapping generated without errors!');
@@ -66,22 +64,22 @@ commander
// write documentation to file
if (typeof options.aggPath !== 'undefined') {
const aggPath = resolve(options.aggPath);
const aggPath = path.resolve(options.aggPath);
// tslint:disable-next-line:no-magic-numbers
writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
Logger.ok(`Elasticsearch aggregations written to ${aggPath}.`);
}
if (typeof options.mappingPath !== 'undefined') {
const mappingPath = resolve(options.mappingPath);
const mappingPath = path.resolve(options.mappingPath);
// tslint:disable-next-line:no-magic-numbers
writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
Logger.ok(`Elasticsearch mappings written to ${mappingPath}.`);
}
if (typeof options.errorPath !== 'undefined') {
const errPath = resolve(options.errorPath);
const errorPath = path.resolve(options.errorPath);
// tslint:disable-next-line:no-magic-numbers
writeFileSync(errPath, JSON.stringify(result.errors, null, 2));
Logger.ok(`Mapping errors written to ${errPath}.`);
writeFileSync(errorPath, JSON.stringify(result.errors, null, 2));
Logger.ok(`Mapping errors written to ${errorPath}.`);
} else if (result.errors.length > 0) {
for (const error of result.errors) {
await Logger.error(error);
@@ -93,12 +91,13 @@ commander
commander
.command('put-es-templates <srcPath> <esAddress> [ignoredTags]')
.action(async (relativeSrcPath, esAddress) => {
.action(async (relativeSourcePath, esAddress) => {
// get absolute paths
const srcPath = resolve(relativeSrcPath);
const sourcePath = path.resolve(relativeSourcePath);
// get project reflection
const templates = require(srcPath) as ElasticsearchTemplateCollection;
// eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module
const templates = require(sourcePath) as ElasticsearchTemplateCollection;
for (const template in templates) {
if (!templates.hasOwnProperty(template)) {
@@ -111,7 +110,9 @@ commander
const HTTP_STATUS_OK = 200;
if (response.statusCode !== HTTP_STATUS_OK) {
await Logger.error(`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`);
await Logger.error(
`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`,
);
exit(-1);
}
}

24
src/config/fieldmap.ts
View File

@@ -15,16 +15,18 @@
import {ElasticsearchFieldmap, ElasticsearchFilterableMap} from '../types/mapping';
import {ElasticsearchDataType} from './typemap';
export enum analyzers {
ducet_sort = 'ducet_sort',
search_german = 'search_german',
}
const ducetSort = {
type: 'icu_collation_keyword',
language: 'de',
country: 'DE',
variant: '@collation=phonebook',
};
export const fieldmap: ElasticsearchFieldmap = {
aggregatable: {
default: {
raw: {
ignore_above: 10000,
ignore_above: 10_000,
type: ElasticsearchDataType.keyword,
},
},
@@ -32,18 +34,10 @@ export const fieldmap: ElasticsearchFieldmap = {
},
sortable: {
default: {
sort: {
analyzer: analyzers.ducet_sort,
fielddata: true,
type: ElasticsearchDataType.text,
},
sort: ducetSort,
},
ducet: {
sort: {
analyzer: analyzers.ducet_sort,
fielddata: true,
type: ElasticsearchDataType.text,
},
sort: ducetSort,
},
ignore: ['price'],
},

14
src/config/premap.ts
View File

@@ -16,7 +16,7 @@ import {ElasticsearchPremap} from '../types/mapping';
import {ElasticsearchDataType} from './typemap';
export const premaps: ElasticsearchPremap = {
CoordinateReferenceSystem: {
'CoordinateReferenceSystem': {
dynamic: true,
properties: {
type: {
@@ -24,12 +24,10 @@ export const premaps: ElasticsearchPremap = {
},
},
},
LineString: {
precision: '1m',
tree: 'quadtree',
'LineString': {
type: ElasticsearchDataType.geo_shape,
},
Point: {
'Point': {
properties: {
type: {
type: ElasticsearchDataType.keyword,
@@ -40,12 +38,10 @@ export const premaps: ElasticsearchPremap = {
},
dynamic: 'strict',
},
Polygon: {
precision: '1m',
tree: 'quadtree',
'Polygon': {
type: ElasticsearchDataType.geo_shape,
},
SCISO8601DateRange: {
'SCISO8601DateRange': {
type: ElasticsearchDataType.date_range,
},
'jsonpatch.OpPatch': {

51
src/config/settings.ts
View File

@@ -15,51 +15,8 @@
import {ElasticsearchSettings} from '../types/mapping';
export const settings: ElasticsearchSettings = {
analysis: {
analyzer: {
ducet_sort: {
filter: [
'german_phonebook',
],
tokenizer: 'keyword',
type: 'custom',
},
search_german: {
filter: [
'lowercase',
'german_stop',
'german_stemmer',
],
tokenizer: 'stapps_ngram',
type: 'custom',
},
},
filter: {
german_phonebook: {
country: 'DE',
language: 'de',
type: 'icu_collation',
variant: '@collation=phonebook',
},
german_stemmer: {
language: 'german',
type: 'stemmer',
},
german_stop: {
stopwords: '_german_',
type: 'stop',
},
},
tokenizer: {
stapps_ngram: {
max_gram: 7,
min_gram: 4,
type: 'ngram',
},
},
},
'mapping.total_fields.limit': 10000,
max_result_window: 30000,
number_of_replicas: 0,
number_of_shards: 1,
'mapping.total_fields.limit': 10_000,
'max_result_window': 30_000,
'number_of_replicas': 0,
'number_of_shards': 1,
};

4
src/config/typemap.ts
View File

@@ -70,9 +70,9 @@ export const typemap: ElasticsearchTypemap = {
/**
* If the string is a tag type
*/
export function isTagType(str: string): boolean {
export function isTagType(string_: string): boolean {
for (const key in typemap) {
if (typemap.hasOwnProperty(key) && typeof typemap[key][str] !== 'undefined') {
if (typemap.hasOwnProperty(key) && typeof typemap[key][string_] !== 'undefined') {
return true;
}
}

441
src/mapping.ts
View File

@@ -38,7 +38,8 @@ import {
ElasticsearchObject,
ElasticsearchTemplateCollection,
ElasticsearchType,
ElasticsearchValue, MappingGenTemplate,
ElasticsearchValue,
MappingGenTemplate,
} from './types/mapping';
let dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
@@ -56,7 +57,7 @@ const inheritTagsName = 'inherittags';
const maxErrorObjectChars = 1000;
let ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
let inheritTagsMap: { [path: string]: CommentTag[]; } = {};
let inheritTagsMap: {[path: string]: CommentTag[]} = {};
/**
* Gets all interfaces that have an @indexable tag
@@ -64,7 +65,6 @@ let inheritTagsMap: { [path: string]: CommentTag[]; } = {};
* @param projectReflection the project reflection from which to extract the indexable interfaces
*/
export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] {
let indexableInterfaces: DeclarationReflection[] = [];
if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) {
@@ -72,14 +72,14 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
}
// push all declaration reflections into one array
projectReflection.children.forEach((declarationReflection) => {
for (const declarationReflection of projectReflection.children) {
if (Array.isArray(declarationReflection.children)) {
indexableInterfaces = indexableInterfaces.concat(declarationReflection.children);
indexableInterfaces = [...indexableInterfaces, ...declarationReflection.children];
}
});
}
// filter all declaration reflections with an @indexable tag
indexableInterfaces = indexableInterfaces.filter((declarationReflection) => {
indexableInterfaces = indexableInterfaces.filter(declarationReflection => {
if (
typeof declarationReflection.comment === 'undefined' ||
typeof declarationReflection.comment.tags === 'undefined'
@@ -87,9 +87,11 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
return false;
}
return typeof declarationReflection.comment.tags.find((commentTag) => {
return commentTag.tagName === indexableTag;
}) !== 'undefined';
return (
typeof declarationReflection.comment.tags.find(commentTag => {
return commentTag.tagName === indexableTag;
}) !== 'undefined'
);
});
return indexableInterfaces;
@@ -104,8 +106,17 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
* @param object the object or name
* @param message the error message
*/
function composeErrorMessage(path: string, topTypeName: string, typeName: string, object: string, message: string) {
const error = `At "${topTypeName}::${path.substr(0, path.length - 1)}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
function composeErrorMessage(
path: string,
topTypeName: string,
typeName: string,
object: string,
message: string,
) {
const error = `At "${topTypeName}::${path.slice(
0,
Math.max(0, path.length - 1),
)}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
errors.push(error);
if (showErrors) {
// tslint:disable-next-line:no-floating-promises
@@ -120,9 +131,7 @@ function composeErrorMessage(path: string, topTypeName: string, typeName: string
* @param maxLength the maximum allowed length before it is clamped
*/
function trimString(value: string, maxLength: number): string {
return value.length > maxLength ?
`${value.substring(0, maxLength)}...` :
value;
return value.length > maxLength ? `${value.slice(0, Math.max(0, maxLength))}...` : value;
}
/**
@@ -137,18 +146,24 @@ function trimString(value: string, maxLength: number): string {
* @param path the current path to the object we are in
* @param tags any tags attached to the type
*/
function getReflectionGeneric(type: ReferenceType,
out: Map<string, ElasticsearchValue>,
topTypeName: string,
path: string,
tags: CommentTag[]): Map<string, ElasticsearchValue> {
if (typeof type.typeArguments !== 'undefined'
&& type.reflection instanceof DeclarationReflection
&& typeof type.reflection.typeParameters !== 'undefined') {
function getReflectionGeneric(
type: ReferenceType,
out: Map<string, ElasticsearchValue>,
topTypeName: string,
path: string,
tags: CommentTag[],
): Map<string, ElasticsearchValue> {
if (
typeof type.typeArguments !== 'undefined' &&
type.reflection instanceof DeclarationReflection &&
typeof type.reflection.typeParameters !== 'undefined'
) {
for (let i = 0; i < type.reflection.typeParameters.length; i++) {
if (i < type.typeArguments.length) {
out
.set(type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags));
out.set(
type.reflection.typeParameters[i].name,
handleType(type.typeArguments[i], out, topTypeName, path, tags),
);
} else {
// this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061
// we have no way to know the type here, so we have to use this.
@@ -157,8 +172,10 @@ function getReflectionGeneric(type: ReferenceType,
properties: {},
});
Logger.warn(`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
` in TypeDoc. It has been replaced by a dynamic type.`);
Logger.warn(
`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
` in TypeDoc. It has been replaced by a dynamic type.`,
);
}
}
}
@@ -177,15 +194,21 @@ function getReflectionGeneric(type: ReferenceType,
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleExternalType(ref: ReferenceType, generics: Map<string, ElasticsearchValue>,
path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
function handleExternalType(
ref: ReferenceType,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
tags: CommentTag[],
): ElasticsearchValue {
for (const premap of Object.keys(premaps)) {
if (premap === ref.name) {
return readFieldTags(premaps[premap], path, topTypeName, tags);
}
}
if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics
if (ref.name === 'Array') {
// basically an external type, but Array is quite common, especially with generics
if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') {
composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse');
@@ -194,12 +217,19 @@ function handleExternalType(ref: ReferenceType, generics: Map<string, Elasticsea
return readFieldTags(
handleType(
ref.typeArguments[0], getReflectionGeneric(
ref, new Map(generics), path, topTypeName, tags),
path, topTypeName, tags),
path, topTypeName, tags);
ref.typeArguments[0],
getReflectionGeneric(ref, new Map(generics), path, topTypeName, tags),
path,
topTypeName,
tags,
),
path,
topTypeName,
tags,
);
}
if (ref.name === '__type') { // empty object
if (ref.name === '__type') {
// empty object
return {
dynamic: 'strict',
properties: {},
@@ -220,18 +250,23 @@ function handleExternalType(ref: ReferenceType, generics: Map<string, Elasticsea
* @param topTypeName the name of the SCThingType
* @param inheritedTags the inherited tags
*/
function handleDeclarationReflection(decl: DeclarationReflection,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
inheritedTags?: CommentTag[]):
ElasticsearchValue {
function handleDeclarationReflection(
decl: DeclarationReflection,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
inheritedTags?: CommentTag[],
): ElasticsearchValue {
// check if we have an object referencing a generic
if (generics.has(decl.name)) { // if the object name is the same as the generic name
return readFieldTags(generics.get(decl.name) as ElasticsearchObject | ElasticsearchType, path, topTypeName,
decl.comment?.tags ?? []);
if (generics.has(decl.name)) {
// if the object name is the same as the generic name
return readFieldTags(
generics.get(decl.name) as ElasticsearchObject | ElasticsearchType,
path,
topTypeName,
decl.comment?.tags ?? [],
);
// use the value defined by the generic
}
// start the actual handling process
@@ -251,7 +286,9 @@ function handleDeclarationReflection(decl: DeclarationReflection,
template[decl.name] = {
mapping: handleType(
decl.indexSignature.type,
new Map(generics), path, topTypeName,
new Map(generics),
path,
topTypeName,
getCommentTags(decl.indexSignature, path, topTypeName),
),
match: '*',
@@ -270,21 +307,31 @@ function handleDeclarationReflection(decl: DeclarationReflection,
if (typeof decl.children !== 'undefined' && decl.children.length > 0) {
for (const child of decl.children) {
empty = false;
out.properties[child.name] =
handleDeclarationReflection(child, new Map(generics), `${path}${child.name}.`, topTypeName);
out.properties[child.name] = handleDeclarationReflection(
child,
new Map(generics),
`${path}${child.name}.`,
topTypeName,
);
}
} else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY
} else if (decl.type instanceof Type) {
// if the object is a type, so we are dealing with a PROPERTY
// get inherited tags
const tags = (inheritedTags ?? []).length > 0
? (typeof inheritedTags!.find(it => isTagType(it.tagName)) !== 'undefined'
? inheritedTags!
: [...inheritedTags ?? [], ...getCommentTags(decl, path, topTypeName)])
: getCommentTags(decl, path, topTypeName);
const tags =
(inheritedTags ?? []).length > 0
? typeof inheritedTags!.find(it => isTagType(it.tagName)) !== 'undefined'
? inheritedTags!
: [...(inheritedTags ?? []), ...getCommentTags(decl, path, topTypeName)]
: getCommentTags(decl, path, topTypeName);
return handleType(decl.type, new Map(generics), path, topTypeName, tags);
} else if (decl.kindString === 'Enumeration member') {
return readTypeTags(typeof decl.defaultValue, path, topTypeName,
getCommentTags(decl, path, topTypeName, inheritedTags));
return readTypeTags(
typeof decl.defaultValue,
path,
topTypeName,
getCommentTags(decl, path, topTypeName, inheritedTags),
);
}
if (empty) {
@@ -315,19 +362,33 @@ function getCommentTags(
return [];
}
let out: CommentTag[] = decl.comment instanceof Comment ?
typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : inheritedTags : inheritedTags;
if (decl.overwrites instanceof ReferenceType && decl.overwrites.reflection instanceof DeclarationReflection) {
let out: CommentTag[] =
decl.comment instanceof Comment
? typeof decl.comment.tags !== 'undefined'
? decl.comment.tags
: inheritedTags
: inheritedTags;
if (
decl.overwrites instanceof ReferenceType &&
decl.overwrites.reflection instanceof DeclarationReflection
) {
out = arrayPriorityJoin(
getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id), out);
getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id),
out,
);
}
if (decl.inheritedFrom instanceof ReferenceType && decl.inheritedFrom.reflection instanceof DeclarationReflection) {
if (
decl.inheritedFrom instanceof ReferenceType &&
decl.inheritedFrom.reflection instanceof DeclarationReflection
) {
out = arrayPriorityJoin(
getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id), out);
getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id),
out,
);
}
saveCommentTags(out, path, topTypeName);
const inheritTag = out.find(((value) => value.tagName === inheritTagsName));
const inheritTag = out.find(value => value.tagName === inheritTagsName);
if (typeof inheritTag !== 'undefined') {
out = arrayPriorityJoin(out, retrieveCommentTags(inheritTag.text.trim(), path, topTypeName));
}
@@ -343,8 +404,9 @@ function getCommentTags(
* @param topTypeName the name of the SCThingType
*/
function saveCommentTags(tags: CommentTag[], path: string, topTypeName: string) {
inheritTagsMap[`${topTypeName}::${path.substr(0, path.length - 1)}`] =
tags.filter(((value) => value.tagName !== 'see' && value.tagName !== inheritTagsName));
inheritTagsMap[`${topTypeName}::${path.slice(0, Math.max(0, path.length - 1))}`] = tags.filter(
value => value.tagName !== 'see' && value.tagName !== inheritTagsName,
);
}
/**
@@ -373,14 +435,14 @@ function retrieveCommentTags(path: string, currentPath: string, topTypeName: str
function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] {
const out: CommentTag[] = overrider;
originals.forEach((original) => {
const result = overrider.find((element) => original.tagName === element.tagName);
for (const original of originals) {
const result = overrider.find(element => original.tagName === element.tagName);
// no support for multiple tags with the same name
if (!(result instanceof CommentTag)) {
out.push(original);
}
});
}
return out;
}
@@ -397,11 +459,13 @@ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): Co
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleUnionType(type: UnionType,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
tags: CommentTag[]): ElasticsearchValue {
function handleUnionType(
type: UnionType,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
tags: CommentTag[],
): ElasticsearchValue {
const list: ElasticsearchValue[] = [];
for (const subType of type.types) {
@@ -421,8 +485,13 @@ function handleUnionType(type: UnionType,
return out;
}
composeErrorMessage(path, topTypeName, 'Union Type', stringify(list),
'Empty union type. This is likely not a user error.');
composeErrorMessage(
path,
topTypeName,
'Union Type',
stringify(list),
'Empty union type. This is likely not a user error.',
);
return {type: ElasticsearchDataType.parse_error};
}
@@ -436,37 +505,50 @@ function handleUnionType(type: UnionType,
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleType(type: Type, generics: Map<string, ElasticsearchValue>, path: string, topTypeName: string,
tags: CommentTag[]):
ElasticsearchValue {
function handleType(
type: Type,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
tags: CommentTag[],
): ElasticsearchValue {
// logger.log((type as any).name);
if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type
if (type instanceof ArrayType) {
// array is irrelevant in Elasticsearch, so just go with the element type
const esType = handleType(type.elementType, new Map(generics), path, topTypeName, tags);
// also merge tags of the array to the element type
// filter out the type tags lazily, this can lead to double messages for "Not implemented tag"
let newTags = tags;
if ('type' in esType) {
newTags = tags.filter((tag) => {
newTags = tags.filter(tag => {
return !(tag.tagName === esType.type);
});
}
return readFieldTags(esType, path, topTypeName, newTags);
}
if (type.type === 'stringLiteral') { // a string literal, usually for type
if (type.type === 'stringLiteral') {
// a string literal, usually for type
return readTypeTags(type.type, path, topTypeName, tags);
}
if (type instanceof IntrinsicType) { // the absolute default type, like strings
if (type instanceof IntrinsicType) {
// the absolute default type, like strings
return readTypeTags(type.name, path, topTypeName, tags);
}
if (type instanceof UnionType) { // the union type...
if (type instanceof UnionType) {
// the union type...
return handleUnionType(type, new Map(generics), path, topTypeName, tags);
}
if (type instanceof ReferenceType) {
if (typeof premaps[type.name] === 'undefined' && typeof type.reflection !== 'undefined') {
// there is really no way to make this typesafe, every element in DeclarationReflection is optional.
return handleDeclarationReflection(type.reflection as DeclarationReflection,
getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags);
return handleDeclarationReflection(
type.reflection as DeclarationReflection,
getReflectionGeneric(type, new Map(generics), path, topTypeName, tags),
path,
topTypeName,
tags,
);
}
return handleExternalType(type, new Map(generics), path, topTypeName, tags);
@@ -479,11 +561,14 @@ function handleType(type: Type, generics: Map<string, ElasticsearchValue>, path:
composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
return {type: ElasticsearchDataType.parse_error};
}
if (type instanceof ReflectionType) {
return readFieldTags(handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
path, topTypeName, tags);
return readFieldTags(
handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
path,
topTypeName,
tags,
);
}
composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type');
@@ -502,43 +587,42 @@ function addAggregatable(path: string, topTypeName: string, global: boolean) {
// push type.path and remove the '.' at the end of the path
if (global) {
const prop = path.slice(0, -1)
.split('.')
.pop() as string; // cannot be undefined
const property_ = path.slice(0, -1).split('.').pop() as string; // cannot be undefined
return (aggregations['@all'] as ESNestedAggregation).aggs[prop.split('.')
.pop() as string] = {
return ((aggregations['@all'] as ESNestedAggregation).aggs[property_.split('.').pop() as string] = {
terms: {
field: `${prop}.raw`,
field: `${property_}.raw`,
size: 1000,
},
};
});
}
const property = path.slice(0, -1);
return (aggregations[topTypeName] as ESNestedAggregation).aggs[property] = {
return ((aggregations[topTypeName] as ESNestedAggregation).aggs[property] = {
terms: {
field: `${property}.raw`,
size: 1000,
},
};
});
}
/**
* Reads all tags related to Elasticsearch fields from the fieldMap
*
* @param prev the previous ElasticsearchValue, for example and object
* @param previous the previous ElasticsearchValue, for example and object
* @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags tags attached to the value
* @param dataType the ElasticsearchDataType, for checking if a tag is a type tag
*/
function readFieldTags(prev: ElasticsearchValue,
path: string,
topTypeName: string,
tags: CommentTag[],
dataType?: string): ElasticsearchValue {
function readFieldTags(
previous: ElasticsearchValue,
path: string,
topTypeName: string,
tags: CommentTag[],
dataType?: string,
): ElasticsearchValue {
for (const tag of tags) {
if (tag.tagName === aggregatableTag) {
addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal);
@@ -546,31 +630,43 @@ function readFieldTags(prev: ElasticsearchValue,
if (!ignoredTagsList.includes(tag.tagName)) {
if (typeof fieldmap[tag.tagName] !== 'undefined') {
if (typeof prev.fields === 'undefined') {
if (typeof previous.fields === 'undefined') {
// create in case it doesn't exist
prev.fields = {};
previous.fields = {};
}
if (tag.text.trim() === '') {
// merge fields
prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default};
previous.fields = {...previous.fields, ...fieldmap[tag.tagName].default};
} else if (typeof fieldmap[tag.tagName][tag.text.trim()] !== 'undefined') {
// merge fields
prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
previous.fields = {...previous.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
} else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) {
// when there is an unidentified tag
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`);
composeErrorMessage(
path,
topTypeName,
'tag',
tag.tagName,
`Not implemented tag param "${tag.text.trim()}"`,
);
}
} else if (tag.tagName === filterableTagName) {
if (typeof prev.fields === 'undefined') {
prev.fields = {};
if (typeof previous.fields === 'undefined') {
previous.fields = {};
}
if ('type' in prev) {
const type = filterableMap[prev.type];
if ('type' in previous) {
const type = filterableMap[previous.type];
if (typeof type !== 'undefined') {
// merge fields
prev.fields = {...prev.fields, ...{raw: {type: type}}};
previous.fields = {...previous.fields, raw: {type: type}};
} else {
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented for ${prev.type}`);
composeErrorMessage(
path,
topTypeName,
'tag',
tag.tagName,
`Not implemented for ${previous.type}`,
);
}
} else {
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types');
@@ -581,7 +677,7 @@ function readFieldTags(prev: ElasticsearchValue,
}
}
return prev;
return previous;
}
/**
@@ -592,16 +688,30 @@ function readFieldTags(prev: ElasticsearchValue,
* @param topTypeName the name of the SCThingType
* @param tags tags attached to the value
*/
function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
function readTypeTags(
type: string,
path: string,
topTypeName: string,
tags: CommentTag[],
): ElasticsearchValue {
let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error};
if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap
if (typeof typemap[type] !== 'undefined') {
// first look if the value has a definition in the typemap
for (let i = tags.length - 1; i >= 0; i--) {
if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') {
if (
!ignoredTagsList.includes(tags[i].tagName) &&
typeof typemap[type][tags[i].tagName] !== 'undefined'
) {
// if we have a tag that indicates a type
if (out.type !== ElasticsearchDataType.parse_error) {
composeErrorMessage(path, topTypeName, 'type', type,
`Type conflict; "${typemap[type][tags[i].tagName]}" would override "${out.type}"`);
composeErrorMessage(
path,
topTypeName,
'type',
type,
`Type conflict; "${typemap[type][tags[i].tagName]}" would override "${out.type}"`,
);
out.type = ElasticsearchDataType.type_conflict;
continue;
}
@@ -617,7 +727,8 @@ function readTypeTags(type: string, path: string, topTypeName: string, tags: Com
return out;
}
if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types
if (dynamicTypes.includes(type)) {
// Elasticsearch dynamic type TODO: doesn't work for direct types
return {
dynamic: true,
properties: {},
@@ -666,15 +777,18 @@ function reset(resetInheritTags = true) {
* @param showErrorOutput whether to print all errors in the command line or not
* @param interfaceFilter only parse specific interfaces, this is for testing purposes
*/
export function generateTemplate(projectReflection: ProjectReflection,
ignoredTags: string[],
showErrorOutput = true,
interfaceFilter: string[] = []): MappingGenTemplate {
export function generateTemplate(
projectReflection: ProjectReflection,
ignoredTags: string[],
showErrorOutput = true,
interfaceFilter: string[] = [],
): MappingGenTemplate {
reset();
showErrors = showErrorOutput;
ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
// eslint-disable-next-line prefer-spread
ignoredTagsList.push.apply(ignoredTagsList, ignoredTags);
const indexableInterfaces = getAllIndexableInterfaces(projectReflection);
@@ -687,37 +801,42 @@ export function generateTemplate(projectReflection: ProjectReflection,
throw new Error('Interface needs at least some properties to be indexable');
}
const typeObject = _interface.children.find((declarationReflection) => {
const typeObject = _interface.children.find(declarationReflection => {
return declarationReflection.name === 'type';
});
if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') {
throw new Error('Interface needs a type to be indexable');
throw new TypeError('Interface needs a type to be indexable');
}
let typeName = 'INVALID_TYPE';
if (typeObject.type instanceof ReferenceType) {
if (typeObject.type.reflection instanceof DeclarationReflection
&& typeof typeObject.type.reflection.defaultValue === 'string') {
typeName = typeObject.type.reflection.defaultValue.replace('"', '')
.replace('"', '');
if (
typeObject.type.reflection instanceof DeclarationReflection &&
typeof typeObject.type.reflection.defaultValue === 'string'
) {
typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.');
void Logger.error(
'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
);
}
} else if (typeObject.type instanceof StringLiteralType) {
Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
typeName = typeObject.type.value;
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`);
void Logger.error(
`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
);
}
// init aggregation schema for type
aggregations[typeName] = {
aggs: {},
filter: {
type: {
value: typeName,
term: {
type: typeName,
},
},
};
@@ -732,45 +851,48 @@ export function generateTemplate(projectReflection: ProjectReflection,
throw new Error('Interface needs at least some properties to be indexable');
}
const typeObject = _interface.children.find((declarationReflection) => {
const typeObject = _interface.children.find(declarationReflection => {
return declarationReflection.name === 'type';
});
if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') {
throw new Error('Interface needs a type to be indexable');
throw new TypeError('Interface needs a type to be indexable');
}
let typeName = 'INVALID_TYPE';
if (typeObject.type instanceof ReferenceType) {
if (typeObject.type.reflection instanceof DeclarationReflection
&& typeof typeObject.type.reflection.defaultValue === 'string') {
typeName = typeObject.type.reflection.defaultValue.replace('"', '')
.replace('"', '');
if (
typeObject.type.reflection instanceof DeclarationReflection &&
typeof typeObject.type.reflection.defaultValue === 'string'
) {
typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.');
void Logger.error(
'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
);
}
} else if (typeObject.type instanceof StringLiteralType) {
Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
typeName = typeObject.type.value;
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`);
void Logger.error(
`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
);
}
// filter out
if (interfaceFilter.length !== 0) {
if (typeof interfaceFilter.find((it) => it === typeName) === 'undefined') {
continue;
}
if (interfaceFilter.length > 0 && typeof interfaceFilter.find(it => it === typeName) === 'undefined') {
continue;
}
// init aggregation schema for type
aggregations[typeName] = {
aggs: {},
filter: {
type: {
value: typeName,
term: {
type: typeName,
},
},
};
@@ -782,26 +904,21 @@ export function generateTemplate(projectReflection: ProjectReflection,
const templateName = `template_${typeNameWithoutSpaces}`;
out[templateName] = {
mappings: {
[typeName]: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject,
},
mappings: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject,
settings: settings,
template: `stapps_${typeNameWithoutSpaces}*`,
}
;
out[templateName].mappings[typeName].properties.creation_date = {
index_patterns: [`stapps_${typeNameWithoutSpaces}*`],
};
out[templateName].mappings.properties.creation_date = {
type: ElasticsearchDataType.date,
};
out[templateName].mappings[typeName].dynamic_templates = dynamicTemplates;
out[templateName].mappings.dynamic_templates = dynamicTemplates;
// Set some properties
out[templateName].mappings[typeName]._source = {
excludes: [
'creation_date',
],
out[templateName].mappings._source = {
excludes: ['creation_date'],
};
out[templateName].mappings[typeName].date_detection = false;
out[templateName].mappings.date_detection = false;
dynamicTemplates = [];

28
src/project-reflection.ts
View File

@@ -15,20 +15,20 @@
import {Logger} from '@openstapps/logger';
import {existsSync, PathLike} from 'fs';
import {platform} from 'os';
import {join, sep} from 'path';
import path from 'path';
import {Application, ProjectReflection} from 'typedoc';
import {ModuleKind, ScriptTarget} from 'typescript';
/**
* Get a project reflection from a path
*
* @param srcPath Path to get reflection from
* @param sourcePath Path to get reflection from
* @param excludeExternals Exclude external dependencies
*/
export function getProjectReflection(srcPath: PathLike, excludeExternals = true): ProjectReflection {
Logger.info(`Generating project reflection for ${srcPath.toString()}.`);
export function getProjectReflection(sourcePath: PathLike, excludeExternals = true): ProjectReflection {
Logger.info(`Generating project reflection for ${sourcePath.toString()}.`);
const tsconfigPath = getTsconfigPath(srcPath.toString());
const tsconfigPath = getTsconfigPath(sourcePath.toString());
// initialize new Typedoc application
const app = new Application();
@@ -39,12 +39,12 @@ export function getProjectReflection(srcPath: PathLike, excludeExternals = true)
includeDeclarations: true,
module: ModuleKind.CommonJS,
target: ScriptTarget.Latest,
tsconfig: join(tsconfigPath, 'tsconfig.json'),
tsconfig: path.join(tsconfigPath, 'tsconfig.json'),
});
let inputFilePath = srcPath;
let inputFilePath = sourcePath;
if (inputFilePath === tsconfigPath) {
inputFilePath = join(tsconfigPath, 'src');
inputFilePath = path.join(tsconfigPath, 'src');
}
// get input files
@@ -54,7 +54,7 @@ export function getProjectReflection(srcPath: PathLike, excludeExternals = true)
const result = app.convert(inputFiles);
if (typeof result === 'undefined') {
throw new Error('Project reflection could not be generated.');
throw new TypeError('Project reflection could not be generated.');
}
return result;
@@ -69,12 +69,10 @@ export function getTsconfigPath(startPath: string): string {
let tsconfigPath = startPath;
// see https://stackoverflow.com/questions/9652043/identifying-the-file-system-root-with-node-js
const root = (platform() === 'win32') ? process
.cwd()
.split(sep)[0] : '/';
const root = platform() === 'win32' ? process.cwd().split(path.sep)[0] : '/';
// repeat until a tsconfig.json is found
while (!existsSync(join(tsconfigPath, 'tsconfig.json'))) {
while (!existsSync(path.join(tsconfigPath, 'tsconfig.json'))) {
if (tsconfigPath === root) {
throw new Error(
`Reached file system root ${root} while searching for 'tsconfig.json' in ${startPath}!`,
@@ -82,9 +80,9 @@ export function getTsconfigPath(startPath: string): string {
}
// pop last directory
const tsconfigPathParts = tsconfigPath.split(sep);
const tsconfigPathParts = tsconfigPath.split(path.sep);
tsconfigPathParts.pop();
tsconfigPath = tsconfigPathParts.join(sep);
tsconfigPath = tsconfigPathParts.join(path.sep);
}
Logger.info(`Using 'tsconfig.json' from ${tsconfigPath}.`);

7
src/types/aggregation.d.ts vendored
View File

@@ -15,6 +15,7 @@
/**
* An elasticsearch bucket aggregation
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
*/
export interface AggregationSchema {
@@ -48,11 +49,11 @@ export interface ESAggTypeFilter {
/**
* The type of the object to find
*/
type: {
term: {
/**
* The name of the type
*/
value: string;
type: string;
};
}
@@ -63,7 +64,7 @@ export interface ESAggMatchAllFilter {
/**
* Filter that matches everything
*/
match_all: {};
match_all: object;
}
/**

22
src/types/mapping.d.ts vendored
View File

@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
@@ -219,7 +220,6 @@ export interface ElasticsearchGeoShape {
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/object.html
*/
export interface ElasticsearchObject {
/**
* Only for the top type
*/
@@ -227,9 +227,7 @@ export interface ElasticsearchObject {
/**
* Fields that should be excluded in the _source field
*/
excludes: [
'creation_date'
];
excludes: ['creation_date'];
};
/**
@@ -291,22 +289,20 @@ export interface ElasticsearchTemplateCollection {
* This is what you pass to Elasticsearch
*/
export interface ElasticsearchTemplate {
/**
* The name of the template, for referencing in Elasticsearch
*/
index_patterns: [string];
/**
* This is a pre-defined structure you should use for your mapping
*/
mappings: {
[typeName: string]: ElasticsearchObject;
};
mappings: ElasticsearchObject;
/**
* The settings for Elasticsearch
*/
settings: ElasticsearchSettings;
/**
* The name of the template, for referencing in Elasticsearch
*/
template: string;
}
/**
@@ -323,7 +319,7 @@ export interface ElasticsearchSettings {
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/analysis-analyzers.html
*/
analysis: {
analysis?: {
[name: string]: any;
};
}