Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-22 09:32:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: generate aggreations from annotations in the core

Browse Source
This commit is contained in:
Wieland Schöbl
2019-09-18 13:07:32 +02:00
parent 84acd7f3ab
commit 18ad651286
1 changed files with 68 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
src/mapping.ts
View File

@@ -41,7 +41,10 @@ const dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
let errors: string[] = []; let errors: string[] = [];
let showErrors = true; let showErrors = true;
let aggregatablePaths: string[] = [];
const indexableTag = 'indexable'; const indexableTag = 'indexable';
const aggregatableTag = 'aggregatable';
let ignoredTagsList = ['indexable', 'validatable']; let ignoredTagsList = ['indexable', 'validatable'];
@@ -86,12 +89,13 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
* Composes error messages, that are readable and contain a certain minumum of information * Composes error messages, that are readable and contain a certain minumum of information
* *
* @param path the path where the error took place * @param path the path where the error took place
* @param topTypeName the name of the SCThingType
* @param typeName the name of the object, with which something went wrong * @param typeName the name of the object, with which something went wrong
* @param object the object or name * @param object the object or name
* @param message the error message * @param message the error message
*/ */
function composeErrorMessage(path: string, typeName: string, object: string, message: string) { function composeErrorMessage(path: string, topTypeName: string, typeName: string, object: string, message: string) {
const error = `At "${path.substr(0, path.length - 1)}" for ${typeName} "${object}": ${message}`; const error = `At "${topTypeName}::${path.substr(0, path.length - 1)}" for ${typeName} "${object}": ${message}`;
errors.push(error); errors.push(error);
if (showErrors) { if (showErrors) {
// tslint:disable-next-line:no-floating-promises // tslint:disable-next-line:no-floating-promises
@@ -108,11 +112,12 @@ function composeErrorMessage(path: string, typeName: string, object: string, mes
* @param type the ReferenceType of a DeclarationReflection * @param type the ReferenceType of a DeclarationReflection
* @param out the previous reflection, it then overrides all parameters or keeps old ones * @param out the previous reflection, it then overrides all parameters or keeps old ones
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type * @param tags any tags attached to the type
*/ */
function getReflectionGeneric(type: ReferenceType, function getReflectionGeneric(type: ReferenceType,
out: ReflectionGeneric[], out: ReflectionGeneric[],
path: string, tags: CommentTag[]): ReflectionGeneric[] { path: string, topTypeName: string, tags: CommentTag[]): ReflectionGeneric[] {
if (typeof type.typeArguments !== 'undefined' if (typeof type.typeArguments !== 'undefined'
&& type.reflection instanceof DeclarationReflection && type.reflection instanceof DeclarationReflection
&& typeof type.reflection.typeParameters !== 'undefined' && typeof type.reflection.typeParameters !== 'undefined'
@@ -121,14 +126,14 @@ function getReflectionGeneric(type: ReferenceType,
let replaced = false; let replaced = false;
for (const old of out) { for (const old of out) {
if (old.name === type.reflection.typeParameters[i].name) { if (old.name === type.reflection.typeParameters[i].name) {
old.value = handleType(type.typeArguments[i], out, path, tags); old.value = handleType(type.typeArguments[i], out, path, topTypeName, tags);
replaced = true; replaced = true;
} }
} }
if (!replaced) { if (!replaced) {
out.push({ out.push({
name: type.reflection.typeParameters[i].name, name: type.reflection.typeParameters[i].name,
value: handleType(type.typeArguments[i], out, path, tags), value: handleType(type.typeArguments[i], out, path, topTypeName, tags),
}); });
} }
} }
@@ -143,25 +148,27 @@ function getReflectionGeneric(type: ReferenceType,
* @param ref the ReferenceType * @param ref the ReferenceType
* @param generics the generics from levels above, so we can use them without having access to the parent * @param generics the generics from levels above, so we can use them without having access to the parent
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type * @param tags any tags attached to the type
*/ */
function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[], function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[],
path: string, tags: CommentTag[]): ElasticsearchValue { path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
for (const premap in premaps) { for (const premap in premaps) {
if (premap === ref.name) { if (premap === ref.name) {
return readFieldTags(premaps[premap], path, tags); return readFieldTags(premaps[premap], path, topTypeName, tags);
} }
} }
if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics
if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') { if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') {
composeErrorMessage(path, 'Array with generics', 'array', 'Failed to parse'); composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse');
return {type: ElasticsearchDataType.parse_error}; return {type: ElasticsearchDataType.parse_error};
} }
return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path, tags), path, tags), return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path, topTypeName, tags),
path, tags); path, topTypeName, tags),
path, topTypeName, tags);
} }
if (ref.name === '__type') { // empty object if (ref.name === '__type') { // empty object
return { return {
@@ -170,9 +177,9 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
}; };
} }
composeErrorMessage(path, 'external type', ref.name, 'Missing pre-map'); composeErrorMessage(path, topTypeName, 'external type', ref.name, 'Missing pre-map');
return readFieldTags({type: ElasticsearchDataType.missing_premap}, path, tags); return readFieldTags({type: ElasticsearchDataType.missing_premap}, path, topTypeName, tags);
} }
/** /**
@@ -181,15 +188,17 @@ function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGene
* @param decl the DeclarationReflection of the object * @param decl the DeclarationReflection of the object
* @param generics the generics from levels above, so we can use them without having access to the parent * @param generics the generics from levels above, so we can use them without having access to the parent
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
*/ */
function handleDeclarationReflection(decl: DeclarationReflection, function handleDeclarationReflection(decl: DeclarationReflection,
generics: ReflectionGeneric[], generics: ReflectionGeneric[],
path: string): path: string,
topTypeName: string):
ElasticsearchValue { ElasticsearchValue {
// check if we have an object referencing a generic // check if we have an object referencing a generic
for (const gRefl of generics) { for (const gRefl of generics) {
if (gRefl.name === decl.name) { // if the object name is the same as the generic name if (gRefl.name === decl.name) { // if the object name is the same as the generic name
return readFieldTags(gRefl.value, path, return readFieldTags(gRefl.value, path, topTypeName,
typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []); typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
// use the value defined by the generic // use the value defined by the generic
} }
@@ -208,7 +217,7 @@ function handleDeclarationReflection(decl: DeclarationReflection,
empty = false; empty = false;
const template: ElasticsearchDynamicTemplate = {}; const template: ElasticsearchDynamicTemplate = {};
template[decl.name] = { template[decl.name] = {
mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path), mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path, topTypeName),
match: '*', match: '*',
match_mapping_type: '*', match_mapping_type: '*',
path_match: `${path}*`, path_match: `${path}*`,
@@ -221,21 +230,21 @@ function handleDeclarationReflection(decl: DeclarationReflection,
if (typeof decl.children !== 'undefined' && decl.children.length > 0) { if (typeof decl.children !== 'undefined' && decl.children.length > 0) {
for (const child of decl.children) { for (const child of decl.children) {
empty = false; empty = false;
out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`); out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`, topTypeName);
} }
} else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY
return handleType(decl.type, generics, path, return handleType(decl.type, generics, path, topTypeName,
typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []); typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
} else if (decl.kindString === 'Enumeration member') { } else if (decl.kindString === 'Enumeration member') {
return readTypeTags(typeof decl.defaultValue, path, return readTypeTags(typeof decl.defaultValue, path, topTypeName,
typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []); typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
} }
if (empty) { if (empty) {
composeErrorMessage(path, 'object', decl.name, 'Empty object'); composeErrorMessage(path, topTypeName, 'object', decl.name, 'Empty object');
} }
return readFieldTags(out, path, return readFieldTags(out, path, topTypeName,
typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []); typeof decl.comment !== 'undefined' ? typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : [] : []);
} }
@@ -248,11 +257,13 @@ function handleDeclarationReflection(decl: DeclarationReflection,
* @param type the type object * @param type the type object
* @param generics the generics from levels above, so we can use them without having access to the parent * @param generics the generics from levels above, so we can use them without having access to the parent
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type * @param tags any tags attached to the type
*/ */
function handleUnionType(type: UnionType, function handleUnionType(type: UnionType,
generics: ReflectionGeneric[], generics: ReflectionGeneric[],
path: string, path: string,
topTypeName: string,
tags: CommentTag[]): ElasticsearchValue { tags: CommentTag[]): ElasticsearchValue {
const list: ElasticsearchValue[] = []; const list: ElasticsearchValue[] = [];
@@ -260,7 +271,7 @@ function handleUnionType(type: UnionType,
if (subType instanceof IntrinsicType && subType.name === 'undefined') { if (subType instanceof IntrinsicType && subType.name === 'undefined') {
continue; continue;
} }
list.push(handleType(subType, generics, path, tags)); list.push(handleType(subType, generics, path, topTypeName, tags));
} }
if (list.length > 0) { if (list.length > 0) {
@@ -273,7 +284,8 @@ function handleUnionType(type: UnionType,
return out; return out;
} }
composeErrorMessage(path, 'Union Type', stringify(list), 'Empty union type. This is likely not a user error.'); composeErrorMessage(path, topTypeName, 'Union Type', stringify(list),
'Empty union type. This is likely not a user error.');
return {type: ElasticsearchDataType.parse_error}; return {type: ElasticsearchDataType.parse_error};
} }
@@ -284,30 +296,32 @@ function handleUnionType(type: UnionType,
* @param type the type object * @param type the type object
* @param generics the generics from levels above, so we can use them without having access to the parent * @param generics the generics from levels above, so we can use them without having access to the parent
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type * @param tags any tags attached to the type
*/ */
function handleType(type: Type, generics: ReflectionGeneric[], path: string, tags: CommentTag[]): ElasticsearchValue { function handleType(type: Type, generics: ReflectionGeneric[], path: string, topTypeName: string, tags: CommentTag[]):
ElasticsearchValue {
// logger.log((type as any).name); // logger.log((type as any).name);
if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type
return handleType(type.elementType, generics, path, tags); return handleType(type.elementType, generics, path, topTypeName, tags);
} }
if (type.type === 'stringLiteral') { // a string literal, usually for type if (type.type === 'stringLiteral') { // a string literal, usually for type
return readTypeTags(type.type, path, tags); return readTypeTags(type.type, path, topTypeName, tags);
} }
if (type instanceof IntrinsicType) { // the absolute default type, like strings if (type instanceof IntrinsicType) { // the absolute default type, like strings
return readTypeTags(type.name, path, tags); return readTypeTags(type.name, path, topTypeName, tags);
} }
if (type instanceof UnionType) { // the union type... if (type instanceof UnionType) { // the union type...
return handleUnionType(type, generics, path, tags); return handleUnionType(type, generics, path, topTypeName, tags);
} }
if (type instanceof ReferenceType) { if (type instanceof ReferenceType) {
if (typeof type.reflection !== 'undefined') { if (typeof type.reflection !== 'undefined') {
// there is really no way to make this typesafe, every element in DeclarationReflection is optional. // there is really no way to make this typesafe, every element in DeclarationReflection is optional.
return handleDeclarationReflection(type.reflection as DeclarationReflection, return handleDeclarationReflection(type.reflection as DeclarationReflection,
getReflectionGeneric(type, generics, path, tags), path); getReflectionGeneric(type, generics, path, topTypeName, tags), path, topTypeName);
} }
return handleRefWithoutReflection(type, generics, path, tags); return handleRefWithoutReflection(type, generics, path, topTypeName, tags);
} }
if (type instanceof TypeParameterType) { if (type instanceof TypeParameterType) {
// check if we have an object referencing a generic // check if we have an object referencing a generic
@@ -316,16 +330,17 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, tag
return gRefl.value; // use the value defined by the generic return gRefl.value; // use the value defined by the generic
} }
} }
composeErrorMessage(path, 'Generic', type.name, 'Missing reflection, please report!'); composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
return {type: ElasticsearchDataType.parse_error}; return {type: ElasticsearchDataType.parse_error};
} }
if (type instanceof ReflectionType) { if (type instanceof ReflectionType) {
return readFieldTags(handleDeclarationReflection(type.declaration, generics, path), path, tags); return readFieldTags(handleDeclarationReflection(type.declaration, generics, path, topTypeName),
path, topTypeName, tags);
} }
composeErrorMessage(path, 'type', stringify(type), 'Not implemented type'); composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type');
return {type: ElasticsearchDataType.parse_error}; return {type: ElasticsearchDataType.parse_error};
} }
@@ -335,14 +350,21 @@ function handleType(type: Type, generics: ReflectionGeneric[], path: string, tag
* *
* @param prev the previous ElasticsearchValue, for example and object * @param prev the previous ElasticsearchValue, for example and object
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags tags attached to the value * @param tags tags attached to the value
* @param dataType the ElasticsearchDataType, for checking if a tag is a type tag * @param dataType the ElasticsearchDataType, for checking if a tag is a type tag
*/ */
function readFieldTags(prev: ElasticsearchValue, function readFieldTags(prev: ElasticsearchValue,
path: string, path: string,
topTypeName: string,
tags: CommentTag[], tags: CommentTag[],
dataType?: string): ElasticsearchValue { dataType?: string): ElasticsearchValue {
for (const tag of tags) { for (const tag of tags) {
if (tag.tagName === aggregatableTag) {
// push type.path and remove the '.' at the end of the path
aggregatablePaths.push(`${topTypeName}.${path.slice(0, -1)}`);
}
if (!ignoredTagsList.includes(tag.tagName)) { if (!ignoredTagsList.includes(tag.tagName)) {
if (typeof fieldmap[tag.tagName] !== 'undefined') { if (typeof fieldmap[tag.tagName] !== 'undefined') {
if (typeof prev.fields === 'undefined') { if (typeof prev.fields === 'undefined') {
@@ -357,7 +379,7 @@ function readFieldTags(prev: ElasticsearchValue,
prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}; prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
} else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) { } else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) {
// when there is an unidentified tag // when there is an unidentified tag
composeErrorMessage(path, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`); composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`);
} }
} else if (tag.tagName === filterableTagName) { } else if (tag.tagName === filterableTagName) {
if (typeof prev.fields === 'undefined') { if (typeof prev.fields === 'undefined') {
@@ -369,13 +391,13 @@ function readFieldTags(prev: ElasticsearchValue,
// merge fields // merge fields
prev.fields = {...prev.fields, ...{raw: {type: type}}}; prev.fields = {...prev.fields, ...{raw: {type: type}}};
} else { } else {
composeErrorMessage(path, 'tag', tag.tagName, `Not implemented for ${prev.type}`); composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented for ${prev.type}`);
} }
} else { } else {
composeErrorMessage(path, 'tag', tag.tagName, 'Not applicable for object types'); composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types');
} }
} else if (typeof dataType === 'undefined' || typeof typemap[dataType][tag.tagName] === 'undefined') { } else if (typeof dataType === 'undefined' || typeof typemap[dataType][tag.tagName] === 'undefined') {
composeErrorMessage(path, 'tag', tag.tagName, `Not implemented tag`); composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag`);
} }
} }
} }
@@ -388,9 +410,10 @@ function readFieldTags(prev: ElasticsearchValue,
* *
* @param type the type of the value * @param type the type of the value
* @param path the current path to the object we are in * @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags tags attached to the value * @param tags tags attached to the value
*/ */
function readTypeTags(type: string, path: string, tags: CommentTag[]): ElasticsearchValue { function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error}; let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error};
if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap
@@ -398,8 +421,8 @@ function readTypeTags(type: string, path: string, tags: CommentTag[]): Elasticse
if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') { if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') {
// if we have a tag that indicates a type // if we have a tag that indicates a type
if (out.type !== ElasticsearchDataType.parse_error) { if (out.type !== ElasticsearchDataType.parse_error) {
composeErrorMessage(path, 'type', type, `Type conflict; "${typemap[type][tags[i].tagName]}" would` + composeErrorMessage(path, topTypeName, 'type', type,
` override "${out.type}"`); `Type conflict; "${typemap[type][tags[i].tagName]}" would override "${out.type}"`);
out.type = ElasticsearchDataType.type_conflict; out.type = ElasticsearchDataType.type_conflict;
continue; continue;
} }
@@ -411,7 +434,7 @@ function readTypeTags(type: string, path: string, tags: CommentTag[]): Elasticse
out.type = typemap[type].default; out.type = typemap[type].default;
} }
out = readFieldTags(out, path, tags, type); out = readFieldTags(out, path, topTypeName, tags, type);
return out; return out;
} }
@@ -422,7 +445,7 @@ function readTypeTags(type: string, path: string, tags: CommentTag[]): Elasticse
}; };
} }
composeErrorMessage(path, 'type', type, 'Not implemented type'); composeErrorMessage(path, topTypeName, 'type', type, 'Not implemented type');
return out; return out;
} }
@@ -440,8 +463,9 @@ function readTypeTags(type: string, path: string, tags: CommentTag[]): Elasticse
*/ */
export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true): export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true):
// tslint:disable-next-line:completed-docs // tslint:disable-next-line:completed-docs
{ errors: string[]; template: ElasticsearchTemplate; } { { aggregations: string[]; errors: string[]; template: ElasticsearchTemplate; } {
errors = []; errors = [];
aggregatablePaths = [];
showErrors = showErrorOutput; showErrors = showErrorOutput;
ignoredTagsList = ['indexable', 'validatable']; ignoredTagsList = ['indexable', 'validatable'];
@@ -498,10 +522,10 @@ export function generateTemplate(projectReflection: ProjectReflection, ignoredTa
} }
out.mappings._default_.properties[typeName] = out.mappings._default_.properties[typeName] =
handleDeclarationReflection(_interface, [], '') as ElasticsearchObject; handleDeclarationReflection(_interface, [], '', typeName) as ElasticsearchObject;
} }
out.mappings._default_.dynamic_templates = dynamicTemplates; out.mappings._default_.dynamic_templates = dynamicTemplates;
return {template: out, errors}; return {aggregations: aggregatablePaths, template: out, errors};
} }