/* * Copyright (C) 2019-2021 StApps * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, version 3. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ import { MappingDynamicTemplate, MappingObjectProperty, MappingProperty, } from '@elastic/elasticsearch/lib/api/types'; import merge from 'deepmerge'; import {stringify} from 'flatted'; import {DeclarationReflection, ProjectReflection, SignatureReflection} from 'typedoc'; import { ArrayType, Comment, CommentTag, IntrinsicType, ReferenceType, ReflectionType, StringLiteralType, Type, TypeParameterType, UnionType, } from 'typedoc/dist/lib/models'; import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap'; import {premaps} from './config/premap'; import {settings} from './config/settings'; import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap'; import type {AggregationSchema, ESNestedAggregation} from '../schema/aggregations'; import type {ElasticsearchTemplateCollection, MappingGenTemplate} from '../schema/mappings'; import * as console from 'console'; let dynamicTemplates: Record[] = []; let errors: string[] = []; let showErrors = true; let aggregations: AggregationSchema = {}; const indexableTag = 'indexable'; const aggregatableTag = 'aggregatable'; const aggregatableTagParameterGlobal = 'global'; const inheritTagsName = 'inherittags'; // clamp printed object to 1000 chars to keep error messages readable const maxErrorObjectChars = 1000; let ignoredTagsList = ['indexable', 'validatable', inheritTagsName]; let inheritTagsMap: {[path: string]: CommentTag[]} = {}; /** * Gets all interfaces that have an @indexable tag * @param projectReflection the project reflection from which to extract the indexable interfaces */ export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] { let indexableInterfaces: DeclarationReflection[] = []; if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) { throw new Error('No DeclarationReflections found. Please check your input path'); } // push all declaration reflections into one array for (const declarationReflection of projectReflection.children) { if (Array.isArray(declarationReflection.children)) { indexableInterfaces = [...indexableInterfaces, ...declarationReflection.children]; } } // filter all declaration reflections with an @indexable tag indexableInterfaces = indexableInterfaces.filter(declarationReflection => { if (declarationReflection.comment === undefined || declarationReflection.comment.tags === undefined) { return false; } return declarationReflection.comment.tags.some(commentTag => { return commentTag.tagName === indexableTag; }); }); return indexableInterfaces; } /** * Composes error messages, that are readable and contain a certain minimum of information * @param path the path where the error took place * @param topTypeName the name of the SCThingType * @param typeName the name of the object, with which something went wrong * @param object the object or name * @param message the error message */ function composeErrorMessage( path: string, topTypeName: string, typeName: string, object: string, message: string, ) { const error = `At "${topTypeName}::${path.slice( 0, Math.max(0, path.length - 1), )}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`; errors.push(error); if (showErrors) { // tslint:disable-next-line:no-floating-promises void console.error(error); } } /** * Trims a string to a readable size and appends "..." * @param value the string to trim * @param maxLength the maximum allowed length before it is clamped */ function trimString(value: string, maxLength: number): string { return value.length > maxLength ? `${value.slice(0, Math.max(0, maxLength))}...` : value; } /** * Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection * * Warning to future maintainers: The code for generics doesn't account for depth. when there is a new generic, it will * override the previous one, if there isn't, it will just continue passing it down. * @param type the ReferenceType of a DeclarationReflection * @param out the previous reflection, it then overrides all parameters or keeps old ones * @param topTypeName the name of the object, with which something went wrong * @param path the current path to the object we are in * @param tags any tags attached to the type */ function getReflectionGeneric( type: ReferenceType, out: Map, topTypeName: string, path: string, tags: CommentTag[], ): Map { if ( type.typeArguments !== undefined && type.reflection instanceof DeclarationReflection && type.reflection.typeParameters !== undefined ) { for (let i = 0; i < type.reflection.typeParameters.length; i++) { if (i < type.typeArguments.length) { out.set( type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags), ); } else { // this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061 // we have no way to know the type here, so we have to use this. out.set(type.reflection.typeParameters[i].name, { dynamic: true, properties: {}, }); console.warn( `Type "${type.name}": Defaults of generics (Foo) currently don't work due to a bug` + ` in TypeDoc. It has been replaced by a dynamic type.`, ); } } } return out; } /** * Handles a ReferenceType that has no value * * Most of the times that is an external type. * @param ref the ReferenceType * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function handleExternalType( ref: ReferenceType, generics: Map, path: string, topTypeName: string, tags: CommentTag[], ): MappingProperty { for (const premap of Object.keys(premaps)) { if (premap === ref.name) { return readFieldTags(premaps[premap], path, topTypeName, tags); } } if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics if (ref.typeArguments === undefined || ref.typeArguments[0] === undefined) { composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse'); return {type: PARSE_ERROR}; } return readFieldTags( handleType( ref.typeArguments[0], getReflectionGeneric(ref, new Map(generics), path, topTypeName, tags), path, topTypeName, tags, ), path, topTypeName, tags, ); } if (ref.name === '__type') { // empty object return { dynamic: 'strict', properties: {}, }; } composeErrorMessage(path, topTypeName, 'external type', ref.name, 'Missing pre-map'); return readFieldTags({type: MISSING_PREMAP}, path, topTypeName, tags); } /** * Handles an object * @param decl the DeclarationReflection of the object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param inheritedTags the inherited tags */ function handleDeclarationReflection( decl: DeclarationReflection, generics: Map, path: string, topTypeName: string, inheritedTags?: CommentTag[], ): MappingProperty { // check if we have an object referencing a generic if (generics.has(decl.name)) { // if the object name is the same as the generic name return readFieldTags(generics.get(decl.name)!, path, topTypeName, decl.comment?.tags ?? []); // use the value defined by the generic } // start the actual handling process const out: MappingProperty = { dynamic: 'strict', properties: {}, }; let empty = true; // first check if there are any index signatures, so for example `[name: string]: Foo` if (decl.indexSignature !== undefined) { out.dynamic = true; if (decl.indexSignature.type !== undefined) { empty = false; const template: Record = {}; template[decl.name] = { mapping: handleType( decl.indexSignature.type, new Map(generics), path, topTypeName, getCommentTags(decl.indexSignature, path, topTypeName), ), match: '*', match_mapping_type: '*', path_match: `${path}*`, }; dynamicTemplates.push(template); } } if (decl.kindString === 'Enumeration') { return readTypeTags('string', path, topTypeName, getCommentTags(decl, path, topTypeName, inheritedTags)); } // check all the children, so in this case we are dealing with an OBJECT if (decl.children !== undefined && decl.children.length > 0) { for (const child of decl.children) { empty = false; out.properties![child.name] = handleDeclarationReflection( child, new Map(generics), `${path}${child.name}.`, topTypeName, ); } } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY // get inherited tags const tags = (inheritedTags ?? []).length > 0 ? inheritedTags!.some(it => isTagType(it.tagName)) ? inheritedTags! : [...(inheritedTags ?? []), ...getCommentTags(decl, path, topTypeName)] : getCommentTags(decl, path, topTypeName); return handleType(decl.type, new Map(generics), path, topTypeName, tags); } else if (decl.kindString === 'Enumeration member') { return readTypeTags( typeof decl.defaultValue, path, topTypeName, getCommentTags(decl, path, topTypeName, inheritedTags), ); } if (empty) { composeErrorMessage(path, topTypeName, 'object', decl.name, 'Empty object'); } return readFieldTags(out, path, topTypeName, getCommentTags(decl, path, topTypeName)); } /** * Reads all comment tags, including inherited ones * @param decl the DeclarationReflection to read the tags from * @param path the path on which the comments lie * @param topTypeName the name of the SCThingType * @param inheritedTags any tags that might have been inherited by a parent * @param breakId the id of the previous reflection to prevent infinite recursion in some cases */ function getCommentTags( decl: DeclarationReflection | SignatureReflection, path: string, topTypeName: string, inheritedTags: CommentTag[] = [], // tslint:disable-next-line:no-unnecessary-initializer breakId: number | undefined = undefined, ): CommentTag[] { if (decl.id === breakId) { return []; } let out: CommentTag[] = decl.comment instanceof Comment ? decl.comment.tags !== undefined ? decl.comment.tags : inheritedTags : inheritedTags; if ( decl.overwrites instanceof ReferenceType && decl.overwrites.reflection instanceof DeclarationReflection ) { out = arrayPriorityJoin( getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id), out, ); } if ( decl.inheritedFrom instanceof ReferenceType && decl.inheritedFrom.reflection instanceof DeclarationReflection ) { out = arrayPriorityJoin( getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id), out, ); } saveCommentTags(out, path, topTypeName); const inheritTag = out.find(value => value.tagName === inheritTagsName); if (inheritTag !== undefined) { out = arrayPriorityJoin(out, retrieveCommentTags(inheritTag.text.trim(), path, topTypeName)); } return out; } /** * Saves all comment tags to the map * @param tags all tags to be saved (@see and @[inheritTags] will be stripped) * @param path the path of field * @param topTypeName the name of the SCThingType */ function saveCommentTags(tags: CommentTag[], path: string, topTypeName: string) { inheritTagsMap[`${topTypeName}::${path.slice(0, Math.max(0, path.length - 1))}`] = tags.filter( value => value.tagName !== 'see' && value.tagName !== inheritTagsName, ); } /** * Retrieves any saved tags * @param path the path to the original field * @param currentPath the current path to the object we are in * @param topTypeName the name of the SCThingType */ function retrieveCommentTags(path: string, currentPath: string, topTypeName: string): CommentTag[] { if (!(path in inheritTagsMap)) { composeErrorMessage(currentPath, topTypeName, path, 'Comment', 'Referenced path to tags does not exist!'); return []; } return inheritTagsMap[path]; } /** * Joins two arrays of CommentTags, but overrides all original CommentTags with the same tagName * @param originals the original array * @param overrider the array that should be appended and provide the override values */ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] { const out: CommentTag[] = overrider; for (const original of originals) { const result = overrider.find(element => original.tagName === element.tagName); // no support for multiple tags with the same name if (!(result instanceof CommentTag)) { out.push(original); } } return out; } /** * Handles UnionTypes * * Put into a separate function as it is a little bit more complex * Works fairly reliable, although there are issues with primitive union types, which don't work at all (And never will) * @param type the type object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function handleUnionType( type: UnionType, generics: Map, path: string, topTypeName: string, tags: CommentTag[], ): MappingProperty { const list: MappingProperty[] = []; for (const subType of type.types) { if (subType instanceof IntrinsicType && subType.name === 'undefined') { continue; } list.push(handleType(subType, new Map(generics), path, topTypeName, tags)); } if (list.length > 0) { let out = list[0]; for (const item of list) { out = merge(out, item); } return out; } composeErrorMessage( path, topTypeName, 'Union Type', stringify(list), 'Empty union type. This is likely not a user error.', ); return {type: PARSE_ERROR}; } /** * Serves as a kind of distributor for the different types, should not contain any specific code * @param type the type object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function handleType( type: Type, generics: Map, path: string, topTypeName: string, tags: CommentTag[], ): MappingProperty { // logger.log((type as any).name); if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type const esType = handleType(type.elementType, new Map(generics), path, topTypeName, tags); // also merge tags of the array to the element type // filter out the type tags lazily, this can lead to double messages for "Not implemented tag" let newTags = tags; if ('type' in esType) { newTags = tags.filter(tag => { return !(tag.tagName === esType.type); }); } return readFieldTags(esType, path, topTypeName, newTags); } if (type.type === 'stringLiteral') { // a string literal, usually for type return readTypeTags(type.type, path, topTypeName, tags); } if (type instanceof IntrinsicType) { // the absolute default type, like strings return readTypeTags(type.name, path, topTypeName, tags); } if (type instanceof UnionType) { // the union type... return handleUnionType(type, new Map(generics), path, topTypeName, tags); } if (type instanceof ReferenceType) { if (premaps[type.name] === undefined && type.reflection !== undefined) { // there is really no way to make this typesafe, every element in DeclarationReflection is optional. return handleDeclarationReflection( type.reflection as DeclarationReflection, getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags, ); } return handleExternalType(type, new Map(generics), path, topTypeName, tags); } if (type instanceof TypeParameterType) { // check if we have an object referencing a generic if (generics.has(type.name)) { return generics.get(type.name)!; } composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!'); return {type: PARSE_ERROR}; } if (type instanceof ReflectionType) { return readFieldTags( handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName), path, topTypeName, tags, ); } composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type'); return {type: PARSE_ERROR}; } /** * Adds an aggregatable to the aggregations list * @param path the current path * @param topTypeName the name of the top type * @param global whether the topTypeName will be used */ function addAggregatable(path: string, topTypeName: string, global: boolean) { // push type.path and remove the '.' at the end of the path if (global) { const property_ = path.slice(0, -1).split('.').pop() as string; // cannot be undefined return ((aggregations['@all'] as ESNestedAggregation).aggs[property_.split('.').pop() as string] = { terms: { field: `${property_}.raw`, size: 1000, }, }); } const property = path.slice(0, -1); return ((aggregations[topTypeName] as ESNestedAggregation).aggs[property] = { terms: { field: `${property}.raw`, size: 1000, }, }); } /** * Reads all tags related to Elasticsearch fields from the fieldMap * @param previous the previous ElasticsearchValue, for example and object * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags tags attached to the value * @param dataType the ElasticsearchDataType, for checking if a tag is a type tag */ function readFieldTags( previous: MappingProperty, path: string, topTypeName: string, tags: CommentTag[], dataType?: string, ): MappingProperty { for (const tag of tags) { if (tag.tagName === aggregatableTag) { addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal); } if (!ignoredTagsList.includes(tag.tagName)) { if (fieldmap[tag.tagName] !== undefined) { if (previous.fields === undefined) { // create in case it doesn't exist previous.fields = {}; } if (tag.text.trim() === '') { // merge fields previous.fields = {...previous.fields, ...fieldmap[tag.tagName].default}; } else if (fieldmap[tag.tagName][tag.text.trim()] !== undefined) { // merge fields previous.fields = {...previous.fields, ...fieldmap[tag.tagName][tag.text.trim()]}; } else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) { // when there is an unidentified tag composeErrorMessage( path, topTypeName, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`, ); } } else if (tag.tagName === filterableTagName) { if (previous.fields === undefined) { previous.fields = {}; } if ('type' in previous) { const type = filterableMap[previous.type!]; if (type !== undefined) { // merge fields previous.fields = {...previous.fields, raw: {type: type}}; } else { composeErrorMessage( path, topTypeName, 'tag', tag.tagName, `Not implemented for ${previous.type}`, ); } } else { composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types'); } } else if (dataType === undefined || typemap[dataType][tag.tagName] === undefined) { composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag`); } } } return previous; } /** * Reads all types related to Elasticsearch fields from the fieldMap * @param type the type of the value * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags tags attached to the value */ function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): MappingProperty { let out: MappingProperty = {type: PARSE_ERROR}; if (typemap[type] !== undefined) { // first look if the value has a definition in the typemap for (let i = tags.length - 1; i >= 0; i--) { if (!ignoredTagsList.includes(tags[i].tagName) && typemap[type][tags[i].tagName] !== undefined) { // if we have a tag that indicates a type if (out.type !== PARSE_ERROR) { composeErrorMessage( path, topTypeName, 'type', type, `Type conflict; "${typemap[type][tags[i].tagName]}" would override "${ (out as MappingProperty).type }"`, ); (out as MappingProperty).type = TYPE_CONFLICT; continue; } (out as MappingProperty).type = typemap[type][tags[i].tagName]; } } if (out.type === PARSE_ERROR) { (out as MappingProperty).type = typemap[type].default; } out = readFieldTags(out, path, topTypeName, tags, type); return out; } if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types return { dynamic: true, properties: {}, }; } composeErrorMessage(path, topTypeName, 'type', type, 'Not implemented type'); return out; } /** * Reset the state * * This is kind of a suboptimal solution and should be changed in the future. * https://gitlab.com/openstapps/core-tools/-/issues/49 * @param resetInheritTags whether inherited tags should be reset as well */ function reset(resetInheritTags = true) { errors = []; dynamicTemplates = []; aggregations = { '@all': { aggs: {}, filter: { match_all: {}, }, }, }; if (resetInheritTags) { inheritTagsMap = {}; } } /** * Takes a project reflection and generates an ElasticsearchTemplate from it * * Serves as the entry point for getting the mapping, so if you just want to get the mapping files for Elasticsearch, * you can do so by calling this function, `RETURNED_VALUE.template` contains the mapping in a fashion that is directly * readable by Elasticsearch. * @param projectReflection a reflection of the project you want to get the ES Mappings from * @param ignoredTags the tag names for which the error output should be suppressed * @param showErrorOutput whether to print all errors in the command line or not * @param interfaceFilter only parse specific interfaces, this is for testing purposes */ export function generateTemplate( projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true, interfaceFilter: string[] = [], ): MappingGenTemplate { reset(); showErrors = showErrorOutput; ignoredTagsList = ['indexable', 'validatable', inheritTagsName]; // eslint-disable-next-line prefer-spread ignoredTagsList.push.apply(ignoredTagsList, ignoredTags); const indexableInterfaces = getAllIndexableInterfaces(projectReflection); const out: ElasticsearchTemplateCollection = {}; for (const _interface of indexableInterfaces) { // TODO: lots of duplicate code, this all needs to be changed https://gitlab.com/openstapps/core-tools/-/issues/49 if (!Array.isArray(_interface.children) || _interface.children.length === 0) { throw new Error('Interface needs at least some properties to be indexable'); } const typeObject = _interface.children.find(declarationReflection => { return declarationReflection.name === 'type'; }); if (typeObject === undefined || typeObject.type === undefined) { throw new TypeError('Interface needs a type to be indexable'); } let typeName = 'INVALID_TYPE'; if (typeObject.type instanceof ReferenceType) { if ( typeObject.type.reflection instanceof DeclarationReflection && typeof typeObject.type.reflection.defaultValue === 'string' ) { typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', ''); } else { // tslint:disable-next-line:no-floating-promises void console.error( 'Your input files seem to be incorrect, or there is a major bug in the mapping generator.', ); } } else if (typeObject.type instanceof StringLiteralType) { console.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`); typeName = typeObject.type.value; } else { // tslint:disable-next-line:no-floating-promises void console.error( `The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`, ); } // init aggregation schema for type aggregations[typeName] = { aggs: {}, filter: { term: { type: typeName, }, }, }; handleDeclarationReflection(_interface, new Map(), '', typeName); } // second traversal reset(false); for (const _interface of indexableInterfaces) { if (!Array.isArray(_interface.children) || _interface.children.length === 0) { throw new Error('Interface needs at least some properties to be indexable'); } const typeObject = _interface.children.find(declarationReflection => { return declarationReflection.name === 'type'; }); if (typeObject === undefined || typeObject.type === undefined) { throw new TypeError('Interface needs a type to be indexable'); } let typeName = 'INVALID_TYPE'; if (typeObject.type instanceof ReferenceType) { if ( typeObject.type.reflection instanceof DeclarationReflection && typeof typeObject.type.reflection.defaultValue === 'string' ) { typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', ''); } else { // tslint:disable-next-line:no-floating-promises void console.error( 'Your input files seem to be incorrect, or there is a major bug in the mapping generator.', ); } } else if (typeObject.type instanceof StringLiteralType) { console.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`); typeName = typeObject.type.value; } else { // tslint:disable-next-line:no-floating-promises void console.error( `The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`, ); } // filter out if (interfaceFilter.length > 0 && !interfaceFilter.includes(typeName)) { continue; } // init aggregation schema for type aggregations[typeName] = { aggs: {}, filter: { term: { type: typeName, }, }, }; // eslint-disable-next-line unicorn/prefer-string-replace-all const typeNameWithoutSpaces = typeName.toLowerCase().replace(/\s/g, '_'); const templateName = `template_${typeNameWithoutSpaces}`; out[templateName] = { mappings: handleDeclarationReflection(_interface, new Map(), '', typeName) as MappingObjectProperty, settings: settings, index_patterns: [`stapps_${typeNameWithoutSpaces}*`], }; out[templateName].mappings!.properties!.creation_date = { type: 'date', }; out[templateName].mappings!.dynamic_templates = dynamicTemplates; // Set some properties out[templateName].mappings!._source = { excludes: ['creation_date'], }; out[templateName].mappings!.date_detection = false; dynamicTemplates = []; if (Object.keys((aggregations[typeName] as ESNestedAggregation).aggs).length === 0) { delete aggregations[typeName]; } } return {aggregations, mappings: out, errors}; }