diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 194c7df2..e34628f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,8 +65,8 @@ mapping: alias: elasticsearch script: - npm install @openstapps/core - - node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength" - - curl http://elasticsearch:9200/stapps --upload-file mapping.json -o response.json + - node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength,tjs-format" + - curl http://elasticsearch:9200/_template --upload-file mapping.json -o response.json - cat response.json - grep -q "\"acknowledged\":true" response.json # - curl --show-error --fail http://elasticsearch:9200/stapps --upload-file mapping.json diff --git a/src/mapping.ts b/src/mapping.ts index a5b6bdf7..acee776e 100644 --- a/src/mapping.ts +++ b/src/mapping.ts @@ -31,13 +31,13 @@ import { import {AggregationSchema, ESNestedAggregation} from './mappings/aggregation-definitions'; import {fieldmap, filterableMap, filterableTagName} from './mappings/definitions/fieldmap'; import {premaps} from './mappings/definitions/premap'; +import {settings} from './mappings/definitions/settings'; import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap'; import { ElasticsearchDynamicTemplate, ElasticsearchMappings, - ElasticsearchObject, + ElasticsearchObject, ElasticsearchType, ElasticsearchValue, - ReflectionGeneric, } from './mappings/mapping-definitions'; let dynamicTemplates: ElasticsearchDynamicTemplate[] = []; @@ -116,29 +116,29 @@ function composeErrorMessage(path: string, topTypeName: string, typeName: string * @param type the ReferenceType of a DeclarationReflection * @param out the previous reflection, it then overrides all parameters or keeps old ones * @param path the current path to the object we are in - * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ function getReflectionGeneric(type: ReferenceType, - out: ReflectionGeneric[], - path: string, topTypeName: string, tags: CommentTag[]): ReflectionGeneric[] { + out: Map, + topTypeName: string, + path: string, tags: CommentTag[]): Map { if (typeof type.typeArguments !== 'undefined' && type.reflection instanceof DeclarationReflection - && typeof type.reflection.typeParameters !== 'undefined' - && type.typeArguments.length === type.reflection.typeParameters.length) { - for (let i = 0; i < type.typeArguments.length; i++) { - let replaced = false; - for (const old of out) { - if (old.name === type.reflection.typeParameters[i].name) { - old.value = handleType(type.typeArguments[i], out, path, topTypeName, tags); - replaced = true; - } - } - if (!replaced) { - out.push({ - name: type.reflection.typeParameters[i].name, - value: handleType(type.typeArguments[i], out, path, topTypeName, tags), + && typeof type.reflection.typeParameters !== 'undefined') { + for (let i = 0; i < type.reflection.typeParameters.length; i++) { + if (i < type.typeArguments.length) { + out + .set(type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags)); + } else { + // this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061 + // we have no way to know the type here, so we have to use this. + out.set(type.reflection.typeParameters[i].name, { + dynamic: true, + properties: {}, }); + + Logger.warn(`Type "${type.name}": Defaults of generics (Foo) currently don't work due to a bug` + + ` in TypeDoc. It has been replaced by a dynamic type.`); } } } @@ -149,14 +149,16 @@ function getReflectionGeneric(type: ReferenceType, /** * Handles a ReferenceType that has no value * + * Most of the times that is an external type. + * * @param ref the ReferenceType * @param generics the generics from levels above, so we can use them without having access to the parent * @param path the current path to the object we are in * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ -function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[], - path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { +function handleExternalType(ref: ReferenceType, generics: Map, + path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { for (const premap in premaps) { if (premap === ref.name) { return readFieldTags(premaps[premap], path, topTypeName, tags); @@ -170,7 +172,8 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene return {type: ElasticsearchDataType.parse_error}; } - return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path, topTypeName, tags), + return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, new Map(generics), path, + topTypeName, tags), path, topTypeName, tags), path, topTypeName, tags); } @@ -195,17 +198,17 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene * @param topTypeName the name of the SCThingType */ function handleDeclarationReflection(decl: DeclarationReflection, - generics: ReflectionGeneric[], + generics: Map, path: string, topTypeName: string, inheritedTags?: CommentTag[]): ElasticsearchValue { // check if we have an object referencing a generic - for (const gRefl of generics) { - if (gRefl.name === decl.name) { // if the object name is the same as the generic name - return readFieldTags(gRefl.value, path, topTypeName, getCommentTags(decl)); - // use the value defined by the generic - } + if (generics.has(decl.name)) { // if the object name is the same as the generic name + return readFieldTags(generics.get(decl.name) as ElasticsearchObject | ElasticsearchType, path, topTypeName, + typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []); + // use the value defined by the generic + } // start the actual handling process @@ -221,7 +224,7 @@ function handleDeclarationReflection(decl: DeclarationReflection, empty = false; const template: ElasticsearchDynamicTemplate = {}; template[decl.name] = { - mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path, topTypeName), + mapping: handleDeclarationReflection(param as DeclarationReflection, new Map(generics), path, topTypeName), match: '*', match_mapping_type: '*', path_match: `${path}*`, @@ -238,11 +241,12 @@ function handleDeclarationReflection(decl: DeclarationReflection, if (typeof decl.children !== 'undefined' && decl.children.length > 0) { for (const child of decl.children) { empty = false; - out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`, topTypeName); + out.properties[child.name] = + handleDeclarationReflection(child, new Map(generics), `${path}${child.name}.`, topTypeName); } } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY // get inherited tags - return handleType(decl.type, generics, path, topTypeName, getCommentTags(decl)); + return handleType(decl.type, new Map(generics), path, topTypeName, getCommentTags(decl)); } else if (decl.kindString === 'Enumeration member') { return readTypeTags(typeof decl.defaultValue, path, topTypeName, getCommentTags(decl, inheritedTags)); } @@ -258,6 +262,7 @@ function handleDeclarationReflection(decl: DeclarationReflection, * Reads all comment tags, including inherited ones * * @param decl the DeclarationReflection to read the tags from + * @param inheritedTags any tags that might have been inherited by a parent */ function getCommentTags(decl: DeclarationReflection, inheritedTags: CommentTag[] = []): CommentTag[] { let out: CommentTag[] = decl.comment instanceof Comment ? @@ -305,7 +310,7 @@ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): Co * @param tags any tags attached to the type */ function handleUnionType(type: UnionType, - generics: ReflectionGeneric[], + generics: Map, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue { @@ -315,7 +320,7 @@ function handleUnionType(type: UnionType, if (subType instanceof IntrinsicType && subType.name === 'undefined') { continue; } - list.push(handleType(subType, generics, path, topTypeName, tags)); + list.push(handleType(subType, new Map(generics), path, topTypeName, tags)); } if (list.length > 0) { @@ -343,11 +348,12 @@ function handleUnionType(type: UnionType, * @param topTypeName the name of the SCThingType * @param tags any tags attached to the type */ -function handleType(type: Type, generics: ReflectionGeneric[], path: string, topTypeName: string, tags: CommentTag[]): +function handleType(type: Type, generics: Map, path: string, topTypeName: string, + tags: CommentTag[]): ElasticsearchValue { // logger.log((type as any).name); if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type - return handleType(type.elementType, generics, path, topTypeName, tags); + return handleType(type.elementType, new Map(generics), path, topTypeName, tags); } if (type.type === 'stringLiteral') { // a string literal, usually for type return readTypeTags(type.type, path, topTypeName, tags); @@ -356,23 +362,21 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top return readTypeTags(type.name, path, topTypeName, tags); } if (type instanceof UnionType) { // the union type... - return handleUnionType(type, generics, path, topTypeName, tags); + return handleUnionType(type, new Map(generics), path, topTypeName, tags); } if (type instanceof ReferenceType) { if (typeof type.reflection !== 'undefined') { // there is really no way to make this typesafe, every element in DeclarationReflection is optional. return handleDeclarationReflection(type.reflection as DeclarationReflection, - getReflectionGeneric(type, generics, path, topTypeName, tags), path, topTypeName, tags); + getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags); } - return handleRefWithoutReflection(type, generics, path, topTypeName, tags); + return handleExternalType(type, new Map(generics), path, topTypeName, tags); } if (type instanceof TypeParameterType) { // check if we have an object referencing a generic - for (const gRefl of generics) { - if (gRefl.name === type.name) { // if the object name is the same as the generic name - return gRefl.value; // use the value defined by the generic - } + if (generics.has(type.name)) { + return generics.get(type.name) as ElasticsearchObject | ElasticsearchType; } composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!'); @@ -380,7 +384,7 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top } if (type instanceof ReflectionType) { - return readFieldTags(handleDeclarationReflection(type.declaration, generics, path, topTypeName), + return readFieldTags(handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName), path, topTypeName, tags); } @@ -502,7 +506,7 @@ function readTypeTags(type: string, path: string, topTypeName: string, tags: Com return out; } - if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type + if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types return { dynamic: true, properties: {}, @@ -587,21 +591,33 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa }, }; - out[typeName] = - handleDeclarationReflection(_interface, [], '', typeName) as ElasticsearchObject; - out[typeName].properties.creation_date = { + let typeNameWithoutSpaces = typeName.toLowerCase(); + while (typeNameWithoutSpaces.includes(' ')) { + typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_'); + } + const templateName = `template_${typeNameWithoutSpaces}`; + + out[templateName] = { + mappings: { + [typeName]: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject, + }, + settings: settings, + template: `stapps_${typeNameWithoutSpaces}*`, + } + ; + out[templateName].mappings[typeName].properties.creation_date = { type: ElasticsearchDataType.date, }; - out[typeName].dynamic_templates = dynamicTemplates; + out[templateName].mappings[typeName].dynamic_templates = dynamicTemplates; // Set some properties - out[typeName]._source = { + out[templateName].mappings[typeName]._source = { excludes: [ 'creation_date', ], }; - out[typeName].date_detection = false; + out[templateName].mappings[typeName].date_detection = false; dynamicTemplates = []; diff --git a/src/mappings/definitions/premap.ts b/src/mappings/definitions/premap.ts index 6d637523..f7be4a18 100644 --- a/src/mappings/definitions/premap.ts +++ b/src/mappings/definitions/premap.ts @@ -12,15 +12,17 @@ * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ -import {Logger} from '@openstapps/logger'; -import {ElasticsearchPremap, ElasticsearchValue} from '../mapping-definitions'; +import {ElasticsearchPremap} from '../mapping-definitions'; import {ElasticsearchDataType} from './typemap'; export const premaps: ElasticsearchPremap = { CoordinateReferenceSystem: { - precision: '1m', - tree: 'quadtree', - type: ElasticsearchDataType.geo_shape, + dynamic: true, + properties: { + type: { + type: ElasticsearchDataType.keyword, + }, + }, }, LineString: { precision: '1m', @@ -50,27 +52,9 @@ export const premaps: ElasticsearchPremap = { type: ElasticsearchDataType.keyword, }, value: { - dynamic: true, - properties: {}, + // this is actually an 'any' type, however ES does not really support that. + type: ElasticsearchDataType.keyword, }, }, }, }; - -/** - * Gets an ElasticsearchValue for a name - * - * @param name the name of the premap - */ -export function getPremap(name: string): ElasticsearchValue { - for (const premap in premaps) { - if (premap === name) { - return premaps[premap]; - } - } - - // tslint:disable-next-line:no-floating-promises - Logger.error(`Missing pre-map for external type ${name}`); - - return {type: ElasticsearchDataType.missing_premap}; -} diff --git a/src/mappings/mapping-definitions.ts b/src/mappings/mapping-definitions.ts index ffe1ad28..97754c00 100644 --- a/src/mappings/mapping-definitions.ts +++ b/src/mappings/mapping-definitions.ts @@ -24,23 +24,6 @@ import {ElasticsearchDataType} from './definitions/typemap'; */ export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject | ElasticsearchGeoShape; -/** - * Used internally for saving a generic value contained in a reflection - */ -export interface ReflectionGeneric { - /** - * The name of the generic - * - * For example in `` the name would be 'A' - */ - name: string; - - /** - * The value of the generic - */ - value: ElasticsearchValue; -} - /** * The Typemap is used to get the corresponding ElasicsearchDataType for a name provided by the ProjectReflection */ @@ -273,7 +256,24 @@ export interface ElasticsearchObject { }; } -export type ElasticsearchMapping = ElasticsearchObject; +export interface ElasticsearchMapping { + /** + * The mappings of the index + */ + mappings: { + [name: string]: ElasticsearchObject; + }; + + /** + * The settings for the index + */ + settings: unknown; + + /** + * The name of the index + */ + template: string; +} // TODO: docs export interface ElasticsearchMappings {