Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-07 05:52:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: make mapping of generics work correctly

Browse Source 
fixes #27
This commit is contained in:
Wieland Schöbl
2019-11-01 14:47:02 +01:00
parent 47361d412a
commit 8f7201e2cf
4 changed files with 95 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitlab-ci.yml
View File

@@ -65,8 +65,8 @@ mapping:
alias: elasticsearch
script:
- npm install @openstapps/core
- node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength"
- curl http://elasticsearch:9200/stapps --upload-file mapping.json -o response.json
- node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength,tjs-format"
- curl http://elasticsearch:9200/_template --upload-file mapping.json -o response.json
- cat response.json
- grep -q "\"acknowledged\":true" response.json
# - curl --show-error --fail http://elasticsearch:9200/stapps --upload-file mapping.json

112
src/mapping.ts
View File

@@ -31,13 +31,13 @@ import {
import {AggregationSchema, ESNestedAggregation} from './mappings/aggregation-definitions';
import {fieldmap, filterableMap, filterableTagName} from './mappings/definitions/fieldmap';
import {premaps} from './mappings/definitions/premap';
import {settings} from './mappings/definitions/settings';
import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap';
import {
ElasticsearchDynamicTemplate,
ElasticsearchMappings,
ElasticsearchObject,
ElasticsearchObject, ElasticsearchType,
ElasticsearchValue,
ReflectionGeneric,
} from './mappings/mapping-definitions';
let dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
@@ -116,29 +116,29 @@ function composeErrorMessage(path: string, topTypeName: string, typeName: string
* @param type the ReferenceType of a DeclarationReflection
* @param out the previous reflection, it then overrides all parameters or keeps old ones
* @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function getReflectionGeneric(type: ReferenceType,
out: ReflectionGeneric[],
path: string, topTypeName: string, tags: CommentTag[]): ReflectionGeneric[] {
out: Map<string, ElasticsearchValue>,
topTypeName: string,
path: string, tags: CommentTag[]): Map<string, ElasticsearchValue> {
if (typeof type.typeArguments !== 'undefined'
&& type.reflection instanceof DeclarationReflection
&& typeof type.reflection.typeParameters !== 'undefined'
&& type.typeArguments.length === type.reflection.typeParameters.length) {
for (let i = 0; i < type.typeArguments.length; i++) {
let replaced = false;
for (const old of out) {
if (old.name === type.reflection.typeParameters[i].name) {
old.value = handleType(type.typeArguments[i], out, path, topTypeName, tags);
replaced = true;
}
}
if (!replaced) {
out.push({
name: type.reflection.typeParameters[i].name,
value: handleType(type.typeArguments[i], out, path, topTypeName, tags),
&& typeof type.reflection.typeParameters !== 'undefined') {
for (let i = 0; i < type.reflection.typeParameters.length; i++) {
if (i < type.typeArguments.length) {
out
.set(type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags));
} else {
// this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061
// we have no way to know the type here, so we have to use this.
out.set(type.reflection.typeParameters[i].name, {
dynamic: true,
properties: {},
});
Logger.warn(`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
` in TypeDoc. It has been replaced by a dynamic type.`);
}
}
}
@@ -149,13 +149,15 @@ function getReflectionGeneric(type: ReferenceType,
/**
* Handles a ReferenceType that has no value
*
* Most of the times that is an external type.
*
* @param ref the ReferenceType
* @param generics the generics from levels above, so we can use them without having access to the parent
* @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[],
function handleExternalType(ref: ReferenceType, generics: Map<string, ElasticsearchValue>,
path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
for (const premap in premaps) {
if (premap === ref.name) {
@@ -170,7 +172,8 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
return {type: ElasticsearchDataType.parse_error};
}
return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path, topTypeName, tags),
return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, new Map(generics), path,
topTypeName, tags),
path, topTypeName, tags),
path, topTypeName, tags);
}
@@ -195,17 +198,17 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
* @param topTypeName the name of the SCThingType
*/
function handleDeclarationReflection(decl: DeclarationReflection,
generics: ReflectionGeneric[],
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
inheritedTags?: CommentTag[]):
ElasticsearchValue {
// check if we have an object referencing a generic
for (const gRefl of generics) {
if (gRefl.name === decl.name) { // if the object name is the same as the generic name
return readFieldTags(gRefl.value, path, topTypeName, getCommentTags(decl));
if (generics.has(decl.name)) { // if the object name is the same as the generic name
return readFieldTags(generics.get(decl.name) as ElasticsearchObject | ElasticsearchType, path, topTypeName,
typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
// use the value defined by the generic
}
}
// start the actual handling process
@@ -221,7 +224,7 @@ function handleDeclarationReflection(decl: DeclarationReflection,
empty = false;
const template: ElasticsearchDynamicTemplate = {};
template[decl.name] = {
mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path, topTypeName),
mapping: handleDeclarationReflection(param as DeclarationReflection, new Map(generics), path, topTypeName),
match: '*',
match_mapping_type: '*',
path_match: `${path}*`,
@@ -238,11 +241,12 @@ function handleDeclarationReflection(decl: DeclarationReflection,
if (typeof decl.children !== 'undefined' && decl.children.length > 0) {
for (const child of decl.children) {
empty = false;
out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`, topTypeName);
out.properties[child.name] =
handleDeclarationReflection(child, new Map(generics), `${path}${child.name}.`, topTypeName);
}
} else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY
// get inherited tags
return handleType(decl.type, generics, path, topTypeName, getCommentTags(decl));
return handleType(decl.type, new Map(generics), path, topTypeName, getCommentTags(decl));
} else if (decl.kindString === 'Enumeration member') {
return readTypeTags(typeof decl.defaultValue, path, topTypeName, getCommentTags(decl, inheritedTags));
}
@@ -258,6 +262,7 @@ function handleDeclarationReflection(decl: DeclarationReflection,
* Reads all comment tags, including inherited ones
*
* @param decl the DeclarationReflection to read the tags from
* @param inheritedTags any tags that might have been inherited by a parent
*/
function getCommentTags(decl: DeclarationReflection, inheritedTags: CommentTag[] = []): CommentTag[] {
let out: CommentTag[] = decl.comment instanceof Comment ?
@@ -305,7 +310,7 @@ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): Co
* @param tags any tags attached to the type
*/
function handleUnionType(type: UnionType,
generics: ReflectionGeneric[],
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
tags: CommentTag[]): ElasticsearchValue {
@@ -315,7 +320,7 @@ function handleUnionType(type: UnionType,
if (subType instanceof IntrinsicType && subType.name === 'undefined') {
continue;
}
list.push(handleType(subType, generics, path, topTypeName, tags));
list.push(handleType(subType, new Map(generics), path, topTypeName, tags));
}
if (list.length > 0) {
@@ -343,11 +348,12 @@ function handleUnionType(type: UnionType,
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleType(type: Type, generics: ReflectionGeneric[], path: string, topTypeName: string, tags: CommentTag[]):
function handleType(type: Type, generics: Map<string, ElasticsearchValue>, path: string, topTypeName: string,
tags: CommentTag[]):
ElasticsearchValue {
// logger.log((type as any).name);
if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type
return handleType(type.elementType, generics, path, topTypeName, tags);
return handleType(type.elementType, new Map(generics), path, topTypeName, tags);
}
if (type.type === 'stringLiteral') { // a string literal, usually for type
return readTypeTags(type.type, path, topTypeName, tags);
@@ -356,23 +362,21 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top
return readTypeTags(type.name, path, topTypeName, tags);
}
if (type instanceof UnionType) { // the union type...
return handleUnionType(type, generics, path, topTypeName, tags);
return handleUnionType(type, new Map(generics), path, topTypeName, tags);
}
if (type instanceof ReferenceType) {
if (typeof type.reflection !== 'undefined') {
// there is really no way to make this typesafe, every element in DeclarationReflection is optional.
return handleDeclarationReflection(type.reflection as DeclarationReflection,
getReflectionGeneric(type, generics, path, topTypeName, tags), path, topTypeName, tags);
getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags);
}
return handleRefWithoutReflection(type, generics, path, topTypeName, tags);
return handleExternalType(type, new Map(generics), path, topTypeName, tags);
}
if (type instanceof TypeParameterType) {
// check if we have an object referencing a generic
for (const gRefl of generics) {
if (gRefl.name === type.name) { // if the object name is the same as the generic name
return gRefl.value; // use the value defined by the generic
}
if (generics.has(type.name)) {
return generics.get(type.name) as ElasticsearchObject | ElasticsearchType;
}
composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
@@ -380,7 +384,7 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, top
}
if (type instanceof ReflectionType) {
return readFieldTags(handleDeclarationReflection(type.declaration, generics, path, topTypeName),
return readFieldTags(handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
path, topTypeName, tags);
}
@@ -502,7 +506,7 @@ function readTypeTags(type: string, path: string, topTypeName: string, tags: Com
return out;
}
if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type
if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types
return {
dynamic: true,
properties: {},
@@ -587,21 +591,33 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
},
};
out[typeName] =
handleDeclarationReflection(_interface, [], '', typeName) as ElasticsearchObject;
out[typeName].properties.creation_date = {
let typeNameWithoutSpaces = typeName.toLowerCase();
while (typeNameWithoutSpaces.includes(' ')) {
typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_');
}
const templateName = `template_${typeNameWithoutSpaces}`;
out[templateName] = {
mappings: {
[typeName]: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject,
},
settings: settings,
template: `stapps_${typeNameWithoutSpaces}*`,
}
;
out[templateName].mappings[typeName].properties.creation_date = {
type: ElasticsearchDataType.date,
};
out[typeName].dynamic_templates = dynamicTemplates;
out[templateName].mappings[typeName].dynamic_templates = dynamicTemplates;
// Set some properties
out[typeName]._source = {
out[templateName].mappings[typeName]._source = {
excludes: [
'creation_date',
],
};
out[typeName].date_detection = false;
out[templateName].mappings[typeName].date_detection = false;
dynamicTemplates = [];

34
src/mappings/definitions/premap.ts
View File

@@ -12,15 +12,17 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from '@openstapps/logger';
import {ElasticsearchPremap, ElasticsearchValue} from '../mapping-definitions';
import {ElasticsearchPremap} from '../mapping-definitions';
import {ElasticsearchDataType} from './typemap';
export const premaps: ElasticsearchPremap = {
CoordinateReferenceSystem: {
precision: '1m',
tree: 'quadtree',
type: ElasticsearchDataType.geo_shape,
dynamic: true,
properties: {
type: {
type: ElasticsearchDataType.keyword,
},
},
},
LineString: {
precision: '1m',
@@ -50,27 +52,9 @@ export const premaps: ElasticsearchPremap = {
type: ElasticsearchDataType.keyword,
},
value: {
dynamic: true,
properties: {},
// this is actually an 'any' type, however ES does not really support that.
type: ElasticsearchDataType.keyword,
},
},
},
};
/**
* Gets an ElasticsearchValue for a name
*
* @param name the name of the premap
*/
export function getPremap(name: string): ElasticsearchValue {
for (const premap in premaps) {
if (premap === name) {
return premaps[premap];
}
}
// tslint:disable-next-line:no-floating-promises
Logger.error(`Missing pre-map for external type ${name}`);
return {type: ElasticsearchDataType.missing_premap};
}

36
src/mappings/mapping-definitions.ts
View File

@@ -24,23 +24,6 @@ import {ElasticsearchDataType} from './definitions/typemap';
*/
export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject | ElasticsearchGeoShape;
/**
* Used internally for saving a generic value contained in a reflection
*/
export interface ReflectionGeneric {
/**
* The name of the generic
*
* For example in `<number A>` the name would be 'A'
*/
name: string;
/**
* The value of the generic
*/
value: ElasticsearchValue;
}
/**
* The Typemap is used to get the corresponding ElasicsearchDataType for a name provided by the ProjectReflection
*/
@@ -273,7 +256,24 @@ export interface ElasticsearchObject {
};
}
export type ElasticsearchMapping = ElasticsearchObject;
export interface ElasticsearchMapping {
/**
* The mappings of the index
*/
mappings: {
[name: string]: ElasticsearchObject;
};
/**
* The settings for the index
*/
settings: unknown;
/**
* The name of the index
*/
template: string;
}
// TODO: docs
export interface ElasticsearchMappings {