/* * Copyright (C) 2019 StApps * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, version 3. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ import {Logger} from '@openstapps/logger'; import * as deepmerge from 'deepmerge'; import {stringify} from 'flatted'; import {DeclarationReflection, ProjectReflection} from 'typedoc'; import { ArrayType, CommentTag, IntrinsicType, ReferenceType, ReflectionType, StringLiteralType, Type, TypeParameterType, UnionType, } from 'typedoc/dist/lib/models'; import {fieldmap, filterableMap, filterableTagName} from './mappings/definitions/fieldmap'; import {premaps} from './mappings/definitions/premap'; import {settings} from './mappings/definitions/settings'; import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap'; import { ElasticsearchDynamicTemplate, ElasticsearchObject, ElasticsearchTemplate, ElasticsearchValue, ReflectionGeneric, } from './mappings/mapping-definitions'; const dynamicTemplates: ElasticsearchDynamicTemplate[] = []; let errors: string[] = []; let showErrors = true; const indexableTag = 'indexable'; let ignoredTagsList = ['indexable', 'validatable']; /** * Gets all interfaces that have an @indexable tag * * @param projectReflection the project reflection from which to extract the indexable interfaces */ export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] { let indexableInterfaces: DeclarationReflection[] = []; if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) { throw new Error('No DeclarationReflections found. Please check your input path'); } // push all declaration reflections into one array projectReflection.children.forEach((declarationReflection) => { if (Array.isArray(declarationReflection.children)) { indexableInterfaces = indexableInterfaces.concat(declarationReflection.children); } }); // filter all declaration reflections with an @indexable tag indexableInterfaces = indexableInterfaces.filter((declarationReflection) => { if ( typeof declarationReflection.comment === 'undefined' || typeof declarationReflection.comment.tags === 'undefined' ) { return false; } return typeof declarationReflection.comment.tags.find((commentTag) => { return commentTag.tagName === indexableTag; }) !== 'undefined'; }); return indexableInterfaces; } /** * Composes error messages, that are readable and contain a certain minumum of information * * @param path the path where the error took place * @param typeName the name of the object, with which something went wrong * @param object the object or name * @param message the error message */ function composeErrorMessage(path: string, typeName: string, object: string, message: string) { const error = `At "${path.substr(0, path.length - 1)}" for ${typeName} "${object}": ${message}`; errors.push(error); if (showErrors) { // tslint:disable-next-line:no-floating-promises Logger.error(error); } } /** * Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection * * Warning to future maintainers: The code for generics doesn't account for depth. when there is a new generic, it will * override the previous one, if there isn't, it will just continue passing it down. * * @param type the ReferenceType of a DeclarationReflection * @param out the previous reflection, it then overrides all parameters or keeps old ones * @param path the current path to the object we are in */ function getReflectionGeneric(type: ReferenceType, out: ReflectionGeneric[], path: string): ReflectionGeneric[] { if (typeof type.typeArguments !== 'undefined' && type.reflection instanceof DeclarationReflection && typeof type.reflection.typeParameters !== 'undefined' && type.typeArguments.length === type.reflection.typeParameters.length) { for (let i = 0; i < type.typeArguments.length; i++) { let replaced = false; for (const old of out) { if (old.name === type.reflection.typeParameters[i].name) { old.value = handleType(type.typeArguments[i], out, path); replaced = true; } } if (!replaced) { out.push({ name: type.reflection.typeParameters[i].name, value: handleType(type.typeArguments[i], out, path), }); } } } return out; } /** * Handles a ReferenceType that has no value * * @param ref the ReferenceType * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param tags any tags attached to the type */ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[], path: string, tags?: CommentTag[]): ElasticsearchValue { for (const premap in premaps) { if (premap === ref.name) { return readFieldTags(premaps[premap], path, tags); } } if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') { composeErrorMessage(path, 'Array with generics', 'array', 'Failed to parse'); return {type: ElasticsearchDataType.parse_error}; } return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path), path), path, tags); } if (ref.name === '__type') { // empty object return { dynamic: 'strict', properties: {}, }; } composeErrorMessage(path, 'external type', ref.name, 'Missing pre-map'); return readFieldTags({type: ElasticsearchDataType.missing_premap}, path, tags); } /** * Handles an object * * @param decl the DeclarationReflection of the object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in */ function handleDeclarationReflection(decl: DeclarationReflection, generics: ReflectionGeneric[], path: string): ElasticsearchValue { // check if we have an object referencing a generic for (const gRefl of generics) { if (gRefl.name === decl.name) { // if the object name is the same as the generic name return readFieldTags(gRefl.value, path, typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); // use the value defined by the generic } } // start the actual handling process const out: ElasticsearchObject = { dynamic: 'strict', properties: {}, }; let empty = true; // first check if there are any index signatures, so for example `[name: string]: Foo` if (typeof decl.indexSignature !== 'undefined' && typeof decl.indexSignature.parameters !== 'undefined') { for (const param of decl.indexSignature.parameters) { empty = false; const template: ElasticsearchDynamicTemplate = {}; template[decl.name] = { mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path), match: '*', match_mapping_type: '*', path_match: `${path}*`, }; dynamicTemplates.push(template); } } // check all the children, so in this case we are dealing with an OBJECT if (typeof decl.children !== 'undefined' && decl.children.length > 0) { for (const child of decl.children) { empty = false; out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`); } } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY return handleType(decl.type, generics, path, typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); } else if (decl.kindString === 'Enumeration member') { return readTypeTags(typeof decl.defaultValue, path, typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); } if (empty) { composeErrorMessage(path, 'object', decl.name, 'Empty object'); } return readFieldTags(out, path, typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); } /** * Handles UnionTypes * * Put into a separate function as it is a little bit more complex * Works fairly reliable, although there are issues with primitive union types, which don't work at all (And never will) * * @param type the type object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in */ function handleUnionType(type: UnionType, generics: ReflectionGeneric[], path: string): ElasticsearchValue { const list: ElasticsearchValue[] = []; for (const subType of type.types) { if (subType instanceof IntrinsicType && subType.name === 'undefined') { continue; } list.push(handleType(subType, generics, path)); } if (list.length > 0) { let out = list[0]; for (const item of list) { out = deepmerge(out, item); } return out; } composeErrorMessage(path, 'Union Type', stringify(list), 'Empty union type. This is likely not a user error.'); return {type: ElasticsearchDataType.parse_error}; } /** * Serves as a kind of distributor for the different types, should not contain any specific code * * @param type the type object * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param tags any tags attached to the type */ function handleType(type: Type, generics: ReflectionGeneric[], path: string, tags?: CommentTag[]): ElasticsearchValue { // logger.log((type as any).name); if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type return handleType(type.elementType, generics, path, tags); } if (type.type === 'stringLiteral') { // a string literal, usually for type return readTypeTags(type.type, path, tags); } if (type instanceof IntrinsicType) { // the absolute default type, like strings return readTypeTags(type.name, path, tags); } if (type instanceof UnionType) { // the union type... return handleUnionType(type, generics, path); } if (type instanceof ReferenceType) { if (typeof type.reflection !== 'undefined') { // there is really no way to make this typesafe, every element in DeclarationReflection is optional. return handleDeclarationReflection(type.reflection as DeclarationReflection, getReflectionGeneric(type, generics, path), path); } return handleRefWithoutReflection(type, generics, path, tags); } if (type instanceof TypeParameterType) { // check if we have an object referencing a generic for (const gRefl of generics) { if (gRefl.name === type.name) { // if the object name is the same as the generic name return gRefl.value; // use the value defined by the generic } } composeErrorMessage(path, 'Generic', type.name, 'Missing reflection, please report!'); return {type: ElasticsearchDataType.parse_error}; } if (type instanceof ReflectionType) { return readFieldTags(handleDeclarationReflection(type.declaration, generics, path), path, tags); } composeErrorMessage(path, 'type', stringify(type), 'Not implemented type'); return {type: ElasticsearchDataType.parse_error}; } /** * Reads all tags related to Elasticsearch fields from the fieldMap * * @param prev the previous ElasticsearchValue, for example and object * @param path the current path to the object we are in * @param tags tags attached to the value */ function readFieldTags(prev: ElasticsearchValue, path: string, tags?: CommentTag[]): ElasticsearchValue { if (typeof tags !== 'undefined') { for (const tag of tags) { if (!ignoredTagsList.includes(tag.tagName)) { if (typeof fieldmap[tag.tagName] !== 'undefined') { if (typeof prev.fields === 'undefined') { prev.fields = {}; } if (tag.text.trim() === '') { prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}; } else if (typeof fieldmap[tag.tagName][tag.text.trim()] !== 'undefined') { // merge the fields prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}; } else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) { composeErrorMessage(path, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`); } } else if (tag.tagName === filterableTagName) { if (typeof prev.fields === 'undefined') { prev.fields = {}; } if ('type' in prev) { const type = filterableMap[prev.type]; if (typeof type !== 'undefined') { prev.fields = {...prev.fields, ...{raw: {type: type}}}; } else { composeErrorMessage(path, 'tag', tag.tagName, `Not implemented for ${prev.type}`); } } else { composeErrorMessage(path, 'tag', tag.tagName, 'Not applicable for object types'); } } else { composeErrorMessage(path, 'tag', tag.tagName, `Not implemented tag`); } } } } return prev; } /** * Reads all types related to Elasticsearch fields from the fieldMap * * @param type the type of the value * @param path the current path to the object we are in * @param tags tags attached to the value */ function readTypeTags(type: string, path: string, tags?: CommentTag[]): ElasticsearchValue { let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error}; if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap if (typeof tags !== 'undefined') { // look if there are any tags for (let i = tags.length - 1; i >= 0; i--) { if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') { // if we have a tag that indicates a type if (out.type !== ElasticsearchDataType.parse_error) { composeErrorMessage(path, 'type', type, `Type conflict; "${typemap[type][tags[i].tagName]}" would` + ` override "${out.type}"`); out.type = ElasticsearchDataType.type_conflict; continue; } out.type = typemap[type][tags[i].tagName]; tags.splice(i, 1); // we need this so readFieldTags can process correctly } } } if (out.type === ElasticsearchDataType.parse_error) { out.type = typemap[type].default; } out = readFieldTags(out, path, tags); return out; } if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type return { dynamic: true, properties: {}, }; } composeErrorMessage(path, 'type', type, 'Not implemented type'); return out; } /** * Takes a project reflection and generates an ElasticsearchTemplate from it * * Serves as the entry point for getting the mapping, so if you just want to get the mapping files for Elasticsearch, * you can do so by calling this function, `RETURNED_VALUE.template` contains the mapping in a fashion that is directly * readable by Elasticsearch. * * @param projectReflection a reflection of the project you want to get the ES Mappings from * @param ignoredTags the tag names for which the error output should be suppressed * @param showErrorOutput whether to print all errors in the command line or not */ export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true): // tslint:disable-next-line:completed-docs { errors: string[]; template: ElasticsearchTemplate; } { errors = []; showErrors = showErrorOutput; ignoredTagsList = ['indexable', 'validatable']; ignoredTagsList.push.apply(ignoredTagsList, ignoredTags); const indexableInterfaces = getAllIndexableInterfaces(projectReflection); const out: ElasticsearchTemplate = { mappings: { _default_: { _source: { excludes: [ 'creation_date', ], }, date_detection: false, dynamic_templates: [], properties: {}, }, }, settings: settings, template: 'stapps_*', }; for (const _interface of indexableInterfaces) { if (!Array.isArray(_interface.children) || _interface.children.length === 0) { throw new Error('Interface needs at least some properties to be indexable'); } const typeObject = _interface.children.find((declarationReflection) => { return declarationReflection.name === 'type'; }); if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') { throw new Error('Interface needs a type to be indexable'); } let typeName = 'INVALID_TYPE'; if (typeObject.type instanceof ReferenceType) { if (typeObject.type.reflection instanceof DeclarationReflection && typeof typeObject.type.reflection.defaultValue === 'string') { typeName = typeObject.type.reflection.defaultValue.replace('"', '') .replace('"', ''); } else { // tslint:disable-next-line:no-floating-promises Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.'); } } else if (typeObject.type instanceof StringLiteralType) { Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`); typeName = typeObject.type.value; } else { // tslint:disable-next-line:no-floating-promises Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`); } out.mappings._default_.properties[typeName] = handleDeclarationReflection(_interface, [], '') as ElasticsearchObject; } out.mappings._default_.dynamic_templates = dynamicTemplates; return {template: out, errors}; }