/* * Copyright (C) 2019 StApps * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, version 3. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ import {Logger} from '@openstapps/logger'; import * as deepmerge from 'deepmerge'; import {stringify} from 'flatted'; import {DeclarationReflection, ProjectReflection} from 'typedoc'; import { ArrayType, Comment, CommentTag, IntrinsicType, ReferenceType, ReflectionType, StringLiteralType, Type, TypeParameterType, UnionType, } from 'typedoc/dist/lib/models'; import {AggregationSchema, ESNestedAggregation} from './mappings/aggregation-definitions'; import {fieldmap, filterableMap, filterableTagName} from './mappings/definitions/fieldmap'; import {premaps} from './mappings/definitions/premap'; import {settings} from './mappings/definitions/settings'; import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap'; import { ElasticsearchDynamicTemplate, ElasticsearchObject, ElasticsearchTemplateCollection, ElasticsearchType, ElasticsearchValue, } from './mappings/mapping-definitions'; let dynamicTemplates: ElasticsearchDynamicTemplate[] = []; let errors: string[] = []; let showErrors = true; let aggregations: AggregationSchema = {}; const indexableTag = 'indexable'; const aggregatableTag = 'aggregatable'; const aggregatableTagParameterGlobal = 'global'; let ignoredTagsList = ['indexable', 'validatable']; /** * Gets all interfaces that have an @indexable tag * * @param projectReflection the project reflection from which to extract the indexable interfaces */ export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] { let indexableInterfaces: DeclarationReflection[] = []; if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) { throw new Error('No DeclarationReflections found. Please check your input path'); } // push all declaration reflections into one array projectReflection.children.forEach((declarationReflection) => { if (Array.isArray(declarationReflection.children)) { indexableInterfaces = indexableInterfaces.concat(declarationReflection.children); } }); // filter all declaration reflections with an @indexable tag indexableInterfaces = indexableInterfaces.filter((declarationReflection) => { if ( typeof declarationReflection.comment === 'undefined' || typeof declarationReflection.comment.tags === 'undefined' ) { return false; } return typeof declarationReflection.comment.tags.find((commentTag) => { return commentTag.tagName === indexableTag; }) !== 'undefined'; }); return indexableInterfaces; } /** * Composes error messages, that are readable and contain a certain minumum of information * * @param path the path where the error took place * @param topTypeName the name of the SCThingType * @param typeName the name of the object, with which something went wrong * @param object the object or name * @param message the error message */ function composeErrorMessage(path: string, topTypeName: string, typeName: string, object: string, message: string) { const error = `At "${topTypeName}::${path.substr(0, path.length - 1)}" for ${typeName} "${object}": ${message}`; errors.push(error); if (showErrors) { // tslint:disable-next-line:no-floating-promises Logger.error(error); } } /** * Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection * * Warning to future maintainers: The code for generics doesn't account for depth. when there is a new generic, it will * override the previous one, if there isn't, it will just continue passing it down. * * @param type the ReferenceType of a DeclarationReflection * @param out the previous reflection, it then overrides all parameters or keeps old ones * @param path the current path to the object we are in * @param tags any tags attached to the type */ function getReflectionGeneric(type: ReferenceType, out: Map, topTypeName: string, path: string, tags: CommentTag[]): Map { if (typeof type.typeArguments !== 'undefined' && type.reflection instanceof DeclarationReflection && typeof type.reflection.typeParameters !== 'undefined') { for (let i = 0; i < type.reflection.typeParameters.length; i++) { if (i < type.typeArguments.length) { out .set(type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags)); } else { // this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061 // we have no way to know the type here, so we have to use this. out.set(type.reflection.typeParameters[i].name, { dynamic: true, properties: {}, }); Logger.warn(`Type "${type.name}": Defaults of generics (Foo) currently don't work due to a bug` + ` in TypeDoc. It has been replaced by a dynamic type.`); } } } return out; } /** * Handles a ReferenceType that has no value * * Most of the times that is an external type. * * @param ref the ReferenceType * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function handleExternalType(ref: ReferenceType, generics: Map, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { for (const premap in premaps) { if (premap === ref.name) { return readFieldTags(premaps[premap], path, topTypeName, tags); } } if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') { composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse'); return {type: ElasticsearchDataType.parse_error}; } return readFieldTags( handleType( ref.typeArguments[0], getReflectionGeneric( ref, new Map(generics), path, topTypeName, tags), path, topTypeName, tags), path, topTypeName, tags); } if (ref.name === '__type') { // empty object return { dynamic: 'strict', properties: {}, }; } composeErrorMessage(path, topTypeName, 'external type', ref.name, 'Missing pre-map'); return readFieldTags({type: ElasticsearchDataType.missing_premap}, path, topTypeName, tags); } /** * Handles an object * * @param decl the DeclarationReflection of the object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType */ function handleDeclarationReflection(decl: DeclarationReflection, generics: Map, path: string, topTypeName: string, inheritedTags?: CommentTag[]): ElasticsearchValue { // check if we have an object referencing a generic if (generics.has(decl.name)) { // if the object name is the same as the generic name return readFieldTags(generics.get(decl.name) as ElasticsearchObject | ElasticsearchType, path, topTypeName, typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []); // use the value defined by the generic } // start the actual handling process const out: ElasticsearchObject = { dynamic: 'strict', properties: {}, }; let empty = true; // first check if there are any index signatures, so for example `[name: string]: Foo` if (typeof decl.indexSignature !== 'undefined' && typeof decl.indexSignature.parameters !== 'undefined') { for (const param of decl.indexSignature.parameters) { empty = false; const template: ElasticsearchDynamicTemplate = {}; template[decl.name] = { mapping: handleDeclarationReflection(param as DeclarationReflection, new Map(generics), path, topTypeName), match: '*', match_mapping_type: '*', path_match: `${path}*`, }; dynamicTemplates.push(template); } } if (decl.kindString === 'Enumeration') { return readTypeTags('string', path, topTypeName, getCommentTags(decl, inheritedTags)); } // check all the children, so in this case we are dealing with an OBJECT if (typeof decl.children !== 'undefined' && decl.children.length > 0) { for (const child of decl.children) { empty = false; out.properties[child.name] = handleDeclarationReflection(child, new Map(generics), `${path}${child.name}.`, topTypeName); } } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY // get inherited tags return handleType(decl.type, new Map(generics), path, topTypeName, getCommentTags(decl)); } else if (decl.kindString === 'Enumeration member') { return readTypeTags(typeof decl.defaultValue, path, topTypeName, getCommentTags(decl, inheritedTags)); } if (empty) { composeErrorMessage(path, topTypeName, 'object', decl.name, 'Empty object'); } return readFieldTags(out, path, topTypeName, getCommentTags(decl)); } /** * Reads all comment tags, including inherited ones * * @param decl the DeclarationReflection to read the tags from * @param inheritedTags any tags that might have been inherited by a parent */ function getCommentTags(decl: DeclarationReflection, inheritedTags: CommentTag[] = []): CommentTag[] { let out: CommentTag[] = decl.comment instanceof Comment ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : inheritedTags : inheritedTags; if (decl.overwrites instanceof ReferenceType && decl.overwrites.reflection instanceof DeclarationReflection) { out = arrayPriorityJoin(out, getCommentTags(decl.overwrites.reflection)); } return out; } /** * Joins two arrays of CommentTags, but overrides all original CommentTags with the same tagName * * @param originals the original array * @param overrider the array that should be appended and provide the override values */ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] { const out: CommentTag[] = overrider; originals.forEach((original) => { const result = overrider.find((element) => { return original.tagName === element.tagName; }); // no support for multiple tags with the same name if (!(result instanceof CommentTag)) { out.push(original); } }); return out; } /** * Handles UnionTypes * * Put into a separate function as it is a little bit more complex * Works fairly reliable, although there are issues with primitive union types, which don't work at all (And never will) * * @param type the type object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function handleUnionType(type: UnionType, generics: Map, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { const list: ElasticsearchValue[] = []; for (const subType of type.types) { if (subType instanceof IntrinsicType && subType.name === 'undefined') { continue; } list.push(handleType(subType, new Map(generics), path, topTypeName, tags)); } if (list.length > 0) { let out = list[0]; for (const item of list) { out = deepmerge(out, item); } return out; } composeErrorMessage(path, topTypeName, 'Union Type', stringify(list), 'Empty union type. This is likely not a user error.'); return {type: ElasticsearchDataType.parse_error}; } /** * Serves as a kind of distributor for the different types, should not contain any specific code * * @param type the type object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function handleType(type: Type, generics: Map, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { // logger.log((type as any).name); if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type return handleType(type.elementType, new Map(generics), path, topTypeName, tags); } if (type.type === 'stringLiteral') { // a string literal, usually for type return readTypeTags(type.type, path, topTypeName, tags); } if (type instanceof IntrinsicType) { // the absolute default type, like strings return readTypeTags(type.name, path, topTypeName, tags); } if (type instanceof UnionType) { // the union type... return handleUnionType(type, new Map(generics), path, topTypeName, tags); } if (type instanceof ReferenceType) { if (typeof type.reflection !== 'undefined') { // there is really no way to make this typesafe, every element in DeclarationReflection is optional. return handleDeclarationReflection(type.reflection as DeclarationReflection, getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags); } return handleExternalType(type, new Map(generics), path, topTypeName, tags); } if (type instanceof TypeParameterType) { // check if we have an object referencing a generic if (generics.has(type.name)) { return generics.get(type.name) as ElasticsearchObject | ElasticsearchType; } composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!'); return {type: ElasticsearchDataType.parse_error}; } if (type instanceof ReflectionType) { return readFieldTags(handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName), path, topTypeName, tags); } composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type'); return {type: ElasticsearchDataType.parse_error}; } /** * Adds an aggregatable to the aggregations list * * @param path the current path * @param topTypeName the name of the top type * @param global whether the topTypeName will be used */ function addAggregatable(path: string, topTypeName: string, global: boolean) { // push type.path and remove the '.' at the end of the path const property = path.slice(0, -1) .split('.') .pop() as string; // cannot be undefined (aggregations[global ? '@all' : topTypeName] as ESNestedAggregation).aggs[property] = { terms: { field: `${property}.raw`, size: 1000, }, }; } /** * Reads all tags related to Elasticsearch fields from the fieldMap * * @param prev the previous ElasticsearchValue, for example and object * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags tags attached to the value * @param dataType the ElasticsearchDataType, for checking if a tag is a type tag */ function readFieldTags(prev: ElasticsearchValue, path: string, topTypeName: string, tags: CommentTag[], dataType?: string): ElasticsearchValue { for (const tag of tags) { if (tag.tagName === aggregatableTag) { addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal); } if (!ignoredTagsList.includes(tag.tagName)) { if (typeof fieldmap[tag.tagName] !== 'undefined') { if (typeof prev.fields === 'undefined') { // create in case it doesn't exist prev.fields = {}; } if (tag.text.trim() === '') { // merge fields prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}; } else if (typeof fieldmap[tag.tagName][tag.text.trim()] !== 'undefined') { // merge fields prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}; } else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) { // when there is an unidentified tag composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`); } } else if (tag.tagName === filterableTagName) { if (typeof prev.fields === 'undefined') { prev.fields = {}; } if ('type' in prev) { const type = filterableMap[prev.type]; if (typeof type !== 'undefined') { // merge fields prev.fields = {...prev.fields, ...{raw: {type: type}}}; } else { composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented for ${prev.type}`); } } else { composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types'); } } else if (typeof dataType === 'undefined' || typeof typemap[dataType][tag.tagName] === 'undefined') { composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag`); } } } return prev; } /** * Reads all types related to Elasticsearch fields from the fieldMap * * @param type the type of the value * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags tags attached to the value */ function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error}; if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap for (let i = tags.length - 1; i >= 0; i--) { if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') { // if we have a tag that indicates a type if (out.type !== ElasticsearchDataType.parse_error) { composeErrorMessage(path, topTypeName, 'type', type, `Type conflict; "${typemap[type][tags[i].tagName]}" would override "${out.type}"`); out.type = ElasticsearchDataType.type_conflict; continue; } out.type = typemap[type][tags[i].tagName]; } } if (out.type === ElasticsearchDataType.parse_error) { out.type = typemap[type].default; } out = readFieldTags(out, path, topTypeName, tags, type); return out; } if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types return { dynamic: true, properties: {}, }; } composeErrorMessage(path, topTypeName, 'type', type, 'Not implemented type'); return out; } /** * Takes a project reflection and generates an ElasticsearchTemplate from it * * Serves as the entry point for getting the mapping, so if you just want to get the mapping files for Elasticsearch, * you can do so by calling this function, `RETURNED_VALUE.template` contains the mapping in a fashion that is directly * readable by Elasticsearch. * * @param projectReflection a reflection of the project you want to get the ES Mappings from * @param ignoredTags the tag names for which the error output should be suppressed * @param showErrorOutput whether to print all errors in the command line or not */ export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true): // tslint:disable-next-line:completed-docs { aggregations: AggregationSchema; errors: string[]; mappings: ElasticsearchTemplateCollection; } { errors = []; aggregations = { '@all': { aggs: {}, filter: { match_all: {}, }, }, }; showErrors = showErrorOutput; ignoredTagsList = ['indexable', 'validatable']; ignoredTagsList.push.apply(ignoredTagsList, ignoredTags); const indexableInterfaces = getAllIndexableInterfaces(projectReflection); const out: ElasticsearchTemplateCollection = {}; for (const _interface of indexableInterfaces) { if (!Array.isArray(_interface.children) || _interface.children.length === 0) { throw new Error('Interface needs at least some properties to be indexable'); } const typeObject = _interface.children.find((declarationReflection) => { return declarationReflection.name === 'type'; }); if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') { throw new Error('Interface needs a type to be indexable'); } let typeName = 'INVALID_TYPE'; if (typeObject.type instanceof ReferenceType) { if (typeObject.type.reflection instanceof DeclarationReflection && typeof typeObject.type.reflection.defaultValue === 'string') { typeName = typeObject.type.reflection.defaultValue.replace('"', '') .replace('"', ''); } else { // tslint:disable-next-line:no-floating-promises Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.'); } } else if (typeObject.type instanceof StringLiteralType) { Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`); typeName = typeObject.type.value; } else { // tslint:disable-next-line:no-floating-promises Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`); } // init aggregation schema for type aggregations[typeName] = { aggs: {}, filter: { type: { value: typeName, }, }, }; let typeNameWithoutSpaces = typeName.toLowerCase(); while (typeNameWithoutSpaces.includes(' ')) { typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_'); } const templateName = `template_${typeNameWithoutSpaces}`; out[templateName] = { mappings: { [typeName]: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject, }, settings: settings, template: `stapps_${typeNameWithoutSpaces}*`, } ; out[templateName].mappings[typeName].properties.creation_date = { type: ElasticsearchDataType.date, }; out[templateName].mappings[typeName].dynamic_templates = dynamicTemplates; // Set some properties out[templateName].mappings[typeName]._source = { excludes: [ 'creation_date', ], }; out[templateName].mappings[typeName].date_detection = false; dynamicTemplates = []; if (Object.keys((aggregations[typeName] as ESNestedAggregation).aggs).length === 0) { delete aggregations[typeName]; } } return {aggregations, mappings: out, errors}; }