Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 00:52:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'es-mapping-generator/master'

Browse Source
This commit is contained in:
Thea Schöbl
2023-05-24 13:29:50 +02:00
parent c8703ec66b e76ffe9371
commit b819dba17f
62 changed files with 2163 additions and 1681 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
packages/es-mapping-generator/src/cli.ts
View File

@@ -1,3 +1,4 @@
/* eslint-disable unicorn/no-null */
/*
* Copyright (C) 2018-2021 StApps
* This program is free software: you can redistribute it and/or modify it
@@ -16,7 +17,7 @@ import {Logger} from '@openstapps/logger';
import {Command} from 'commander';
import {readFileSync, writeFileSync} from 'fs';
import got from 'got';
import {resolve} from 'path';
import path from 'path';
import {exit} from 'process';
import {generateTemplate} from './mapping';
import {getProjectReflection} from './project-reflection';
@@ -33,11 +34,8 @@ process.on('unhandledRejection', async (reason: unknown) => {
const commander = new Command('openstapps-core-tools');
commander
.version(JSON.parse(
readFileSync(resolve(__dirname, '..', 'package.json'))
.toString(),
).version);
// eslint-disable-next-line unicorn/prefer-module
commander.version(JSON.parse(readFileSync(path.resolve(__dirname, '..', 'package.json')).toString()).version);
commander
.command('mapping <relativeSrcPath>')
@@ -45,9 +43,9 @@ commander
.option('-i, --ignoredTags <ignoredTags>', 'Ignored Tags (comma-separated)')
.option('-a, --aggPath <relativeAggregationPath>', 'Aggregations Path')
.option('-e, --errorPath <relativeErrorPath>', 'Error Path')
.action(async (relativeSrcPath, options) => {
.action(async (relativeSourcePath, options) => {
// get absolute paths
const srcPath = resolve(relativeSrcPath);
const sourcePath = path.resolve(relativeSourcePath);
let ignoredTagsList: string[] = [];
if (typeof options.ignoredTags === 'string') {
@@ -55,10 +53,10 @@ commander
}
// get project reflection
const projectReflection = getProjectReflection(srcPath);
const projectReflection = getProjectReflection(sourcePath);
const result = generateTemplate(projectReflection, ignoredTagsList, true);
if (result.errors.length !== 0) {
if (result.errors.length > 0) {
await Logger.error('Mapping generated with errors!');
} else {
Logger.ok('Mapping generated without errors!');
@@ -66,22 +64,22 @@ commander
// write documentation to file
if (typeof options.aggPath !== 'undefined') {
const aggPath = resolve(options.aggPath);
const aggPath = path.resolve(options.aggPath);
// tslint:disable-next-line:no-magic-numbers
writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
Logger.ok(`Elasticsearch aggregations written to ${aggPath}.`);
}
if (typeof options.mappingPath !== 'undefined') {
const mappingPath = resolve(options.mappingPath);
const mappingPath = path.resolve(options.mappingPath);
// tslint:disable-next-line:no-magic-numbers
writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
Logger.ok(`Elasticsearch mappings written to ${mappingPath}.`);
}
if (typeof options.errorPath !== 'undefined') {
const errPath = resolve(options.errorPath);
const errorPath = path.resolve(options.errorPath);
// tslint:disable-next-line:no-magic-numbers
writeFileSync(errPath, JSON.stringify(result.errors, null, 2));
Logger.ok(`Mapping errors written to ${errPath}.`);
writeFileSync(errorPath, JSON.stringify(result.errors, null, 2));
Logger.ok(`Mapping errors written to ${errorPath}.`);
} else if (result.errors.length > 0) {
for (const error of result.errors) {
await Logger.error(error);
@@ -93,12 +91,13 @@ commander
commander
.command('put-es-templates <srcPath> <esAddress> [ignoredTags]')
.action(async (relativeSrcPath, esAddress) => {
.action(async (relativeSourcePath, esAddress) => {
// get absolute paths
const srcPath = resolve(relativeSrcPath);
const sourcePath = path.resolve(relativeSourcePath);
// get project reflection
const templates = require(srcPath) as ElasticsearchTemplateCollection;
// eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module
const templates = require(sourcePath) as ElasticsearchTemplateCollection;
for (const template in templates) {
if (!templates.hasOwnProperty(template)) {
@@ -111,7 +110,9 @@ commander
const HTTP_STATUS_OK = 200;
if (response.statusCode !== HTTP_STATUS_OK) {
await Logger.error(`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`);
await Logger.error(
`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`,
);
exit(-1);
}
}

42
packages/es-mapping-generator/src/config/fieldmap.ts
View File

@@ -12,38 +12,34 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ElasticsearchFieldmap, ElasticsearchFilterableMap} from '../types/mapping';
import {ElasticsearchDataType} from './typemap';
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
import {ElasticsearchFieldmap, SimpleType} from '../types/mapping';
export enum analyzers {
ducet_sort = 'ducet_sort',
search_german = 'search_german',
}
const ducetSort = {
type: 'icu_collation_keyword',
language: 'de',
country: 'DE',
variant: '@collation=phonebook',
};
const keyword: MappingProperty['type'] = 'keyword';
export const fieldmap: ElasticsearchFieldmap = {
aggregatable: {
default: {
raw: {
ignore_above: 10000,
type: ElasticsearchDataType.keyword,
ignore_above: 10_000,
type: keyword,
},
},
ignore: ['global'],
},
sortable: {
default: {
sort: {
analyzer: analyzers.ducet_sort,
fielddata: true,
type: ElasticsearchDataType.text,
},
sort: ducetSort,
},
ducet: {
sort: {
analyzer: analyzers.ducet_sort,
fielddata: true,
type: ElasticsearchDataType.text,
},
sort: ducetSort,
},
ignore: ['price'],
},
@@ -51,9 +47,9 @@ export const fieldmap: ElasticsearchFieldmap = {
export const filterableTagName = 'filterable';
export const filterableMap: ElasticsearchFilterableMap = {
date: ElasticsearchDataType.keyword,
keyword: ElasticsearchDataType.keyword,
text: ElasticsearchDataType.keyword,
integer: ElasticsearchDataType.integer,
export const filterableMap: Record<string, SimpleType> = {
date: 'keyword',
keyword: 'keyword',
text: 'keyword',
integer: 'integer',
};

41
packages/es-mapping-generator/src/config/premap.ts
View File

@@ -12,57 +12,52 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ElasticsearchPremap} from '../types/mapping';
import {ElasticsearchDataType} from './typemap';
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
export const premaps: ElasticsearchPremap = {
CoordinateReferenceSystem: {
export const premaps: Record<string, MappingProperty> = {
'CoordinateReferenceSystem': {
dynamic: true,
properties: {
type: {
type: ElasticsearchDataType.keyword,
type: 'keyword',
},
},
},
LineString: {
precision: '1m',
tree: 'quadtree',
type: ElasticsearchDataType.geo_shape,
'LineString': {
type: 'geo_shape',
},
Point: {
'Point': {
properties: {
type: {
type: ElasticsearchDataType.keyword,
type: 'keyword',
},
coordinates: {
type: ElasticsearchDataType.geo_point,
type: 'geo_point',
},
},
dynamic: 'strict',
},
Polygon: {
precision: '1m',
tree: 'quadtree',
type: ElasticsearchDataType.geo_shape,
'Polygon': {
type: 'geo_shape',
},
SCISO8601DateRange: {
type: ElasticsearchDataType.date_range,
'SCISO8601DateRange': {
type: 'date_range',
},
'jsonpatch.OpPatch': {
dynamic: 'strict',
properties: {
from: {
type: ElasticsearchDataType.keyword,
type: 'keyword',
},
op: {
type: ElasticsearchDataType.keyword,
type: 'keyword',
},
path: {
type: ElasticsearchDataType.keyword,
type: 'keyword',
},
value: {
// this is actually an 'any' type, however ES does not really support that.
type: ElasticsearchDataType.keyword,
// this is actually an 'any' type; however, ES does not really support that.
type: 'keyword',
},
},
},

55
packages/es-mapping-generator/src/config/settings.ts
View File

@@ -12,54 +12,11 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ElasticsearchSettings} from '../types/mapping';
import {IndicesPutTemplateRequest} from '@elastic/elasticsearch/lib/api/types';
export const settings: ElasticsearchSettings = {
analysis: {
analyzer: {
ducet_sort: {
filter: [
'german_phonebook',
],
tokenizer: 'keyword',
type: 'custom',
},
search_german: {
filter: [
'lowercase',
'german_stop',
'german_stemmer',
],
tokenizer: 'stapps_ngram',
type: 'custom',
},
},
filter: {
german_phonebook: {
country: 'DE',
language: 'de',
type: 'icu_collation',
variant: '@collation=phonebook',
},
german_stemmer: {
language: 'german',
type: 'stemmer',
},
german_stop: {
stopwords: '_german_',
type: 'stop',
},
},
tokenizer: {
stapps_ngram: {
max_gram: 7,
min_gram: 4,
type: 'ngram',
},
},
},
'mapping.total_fields.limit': 10000,
max_result_window: 30000,
number_of_replicas: 0,
number_of_shards: 1,
export const settings: IndicesPutTemplateRequest['settings'] = {
'mapping.total_fields.limit': 10_000,
'max_result_window': 30_000,
'number_of_replicas': 0,
'number_of_shards': 1,
};

57
packages/es-mapping-generator/src/config/typemap.ts
View File

@@ -12,67 +12,46 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {MappingFloatNumberProperty} from '@elastic/elasticsearch/lib/api/types';
import {ElasticsearchTypemap} from '../types/mapping';
export enum ElasticsearchDataType {
missing_premap = 'MISSING_PREMAP',
parse_error = 'PARSE_ERROR',
type_conflict = 'TYPE_CONFLICT',
text = 'text',
keyword = 'keyword',
date = 'date',
// long = 'long',
// double = 'double',
float = 'float',
boolean = 'boolean',
ip = 'ip',
integer = 'integer',
object = 'object',
nested = 'nested',
geo_point = 'geo_point',
geo_shape = 'geo_shape',
completion = 'completion',
date_range = 'date_range',
// integer_range = 'integer_range',
// float_range = 'float_range',
// long_range = 'long_range',
// double_range = 'double_range',
// ip_range = 'ip_range',
}
export const PARSE_ERROR = 'PARSE_ERROR' as MappingFloatNumberProperty['type'];
export const MISSING_PREMAP = 'MISSING_PREMAP' as MappingFloatNumberProperty['type'];
export const TYPE_CONFLICT = 'TYPE_CONFLICT' as MappingFloatNumberProperty['type'];
export const typemap: ElasticsearchTypemap = {
boolean: {
default: ElasticsearchDataType.boolean,
default: 'boolean',
},
false: {
default: ElasticsearchDataType.boolean,
default: 'boolean',
},
number: {
default: ElasticsearchDataType.integer,
float: ElasticsearchDataType.float,
integer: ElasticsearchDataType.integer,
date: ElasticsearchDataType.date,
default: 'integer',
float: 'float',
integer: 'integer',
date: 'date',
},
string: {
default: ElasticsearchDataType.text,
keyword: ElasticsearchDataType.keyword,
text: ElasticsearchDataType.text,
date: ElasticsearchDataType.date,
default: 'text',
keyword: 'keyword',
text: 'text',
date: 'date',
},
stringLiteral: {
default: ElasticsearchDataType.keyword,
default: 'keyword',
},
true: {
default: ElasticsearchDataType.boolean,
default: 'boolean',
},
};
/**
* If the string is a tag type
*/
export function isTagType(str: string): boolean {
export function isTagType(string_: string): boolean {
for (const key in typemap) {
if (typemap.hasOwnProperty(key) && typeof typemap[key][str] !== 'undefined') {
if (typemap.hasOwnProperty(key) && typeof typemap[key][string_] !== 'undefined') {
return true;
}
}

481
packages/es-mapping-generator/src/mapping.ts
View File

@@ -12,6 +12,11 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
MappingDynamicTemplate,
MappingObjectProperty,
MappingProperty,
} from '@elastic/elasticsearch/lib/api/types';
import {Logger} from '@openstapps/logger';
import merge from 'deepmerge';
import {stringify} from 'flatted';
@@ -31,17 +36,11 @@ import {
import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap';
import {premaps} from './config/premap';
import {settings} from './config/settings';
import {dynamicTypes, ElasticsearchDataType, isTagType, typemap} from './config/typemap';
import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap';
import {AggregationSchema, ESNestedAggregation} from './types/aggregation';
import {
ElasticsearchDynamicTemplate,
ElasticsearchObject,
ElasticsearchTemplateCollection,
ElasticsearchType,
ElasticsearchValue, MappingGenTemplate,
} from './types/mapping';
import {ElasticsearchTemplateCollection, MappingGenTemplate} from './types/mapping';
let dynamicTemplates: ElasticsearchDynamicTemplate[] = [];
let dynamicTemplates: Record<string, MappingDynamicTemplate>[] = [];
let errors: string[] = [];
let showErrors = true;
@@ -56,7 +55,7 @@ const inheritTagsName = 'inherittags';
const maxErrorObjectChars = 1000;
let ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
let inheritTagsMap: { [path: string]: CommentTag[]; } = {};
let inheritTagsMap: {[path: string]: CommentTag[]} = {};
/**
* Gets all interfaces that have an @indexable tag
@@ -64,7 +63,6 @@ let inheritTagsMap: { [path: string]: CommentTag[]; } = {};
* @param projectReflection the project reflection from which to extract the indexable interfaces
*/
export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] {
let indexableInterfaces: DeclarationReflection[] = [];
if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) {
@@ -72,14 +70,14 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
}
// push all declaration reflections into one array
projectReflection.children.forEach((declarationReflection) => {
for (const declarationReflection of projectReflection.children) {
if (Array.isArray(declarationReflection.children)) {
indexableInterfaces = indexableInterfaces.concat(declarationReflection.children);
indexableInterfaces = [...indexableInterfaces, ...declarationReflection.children];
}
});
}
// filter all declaration reflections with an @indexable tag
indexableInterfaces = indexableInterfaces.filter((declarationReflection) => {
indexableInterfaces = indexableInterfaces.filter(declarationReflection => {
if (
typeof declarationReflection.comment === 'undefined' ||
typeof declarationReflection.comment.tags === 'undefined'
@@ -87,9 +85,11 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
return false;
}
return typeof declarationReflection.comment.tags.find((commentTag) => {
return commentTag.tagName === indexableTag;
}) !== 'undefined';
return (
typeof declarationReflection.comment.tags.find(commentTag => {
return commentTag.tagName === indexableTag;
}) !== 'undefined'
);
});
return indexableInterfaces;
@@ -104,8 +104,17 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
* @param object the object or name
* @param message the error message
*/
function composeErrorMessage(path: string, topTypeName: string, typeName: string, object: string, message: string) {
const error = `At "${topTypeName}::${path.substr(0, path.length - 1)}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
function composeErrorMessage(
path: string,
topTypeName: string,
typeName: string,
object: string,
message: string,
) {
const error = `At "${topTypeName}::${path.slice(
0,
Math.max(0, path.length - 1),
)}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
errors.push(error);
if (showErrors) {
// tslint:disable-next-line:no-floating-promises
@@ -120,9 +129,7 @@ function composeErrorMessage(path: string, topTypeName: string, typeName: string
* @param maxLength the maximum allowed length before it is clamped
*/
function trimString(value: string, maxLength: number): string {
return value.length > maxLength ?
`${value.substring(0, maxLength)}...` :
value;
return value.length > maxLength ? `${value.slice(0, Math.max(0, maxLength))}...` : value;
}
/**
@@ -137,18 +144,24 @@ function trimString(value: string, maxLength: number): string {
* @param path the current path to the object we are in
* @param tags any tags attached to the type
*/
function getReflectionGeneric(type: ReferenceType,
out: Map<string, ElasticsearchValue>,
topTypeName: string,
path: string,
tags: CommentTag[]): Map<string, ElasticsearchValue> {
if (typeof type.typeArguments !== 'undefined'
&& type.reflection instanceof DeclarationReflection
&& typeof type.reflection.typeParameters !== 'undefined') {
function getReflectionGeneric(
type: ReferenceType,
out: Map<string, MappingProperty>,
topTypeName: string,
path: string,
tags: CommentTag[],
): Map<string, MappingProperty> {
if (
typeof type.typeArguments !== 'undefined' &&
type.reflection instanceof DeclarationReflection &&
typeof type.reflection.typeParameters !== 'undefined'
) {
for (let i = 0; i < type.reflection.typeParameters.length; i++) {
if (i < type.typeArguments.length) {
out
.set(type.reflection.typeParameters[i].name, handleType(type.typeArguments[i], out, topTypeName, path, tags));
out.set(
type.reflection.typeParameters[i].name,
handleType(type.typeArguments[i], out, topTypeName, path, tags),
);
} else {
// this can happen due to a bug in TypeDoc https://github.com/TypeStrong/typedoc/issues/1061
// we have no way to know the type here, so we have to use this.
@@ -157,8 +170,10 @@ function getReflectionGeneric(type: ReferenceType,
properties: {},
});
Logger.warn(`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
` in TypeDoc. It has been replaced by a dynamic type.`);
Logger.warn(
`Type "${type.name}": Defaults of generics (Foo<T = any>) currently don't work due to a bug` +
` in TypeDoc. It has been replaced by a dynamic type.`,
);
}
}
}
@@ -177,29 +192,42 @@ function getReflectionGeneric(type: ReferenceType,
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleExternalType(ref: ReferenceType, generics: Map<string, ElasticsearchValue>,
path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
function handleExternalType(
ref: ReferenceType,
generics: Map<string, MappingProperty>,
path: string,
topTypeName: string,
tags: CommentTag[],
): MappingProperty {
for (const premap of Object.keys(premaps)) {
if (premap === ref.name) {
return readFieldTags(premaps[premap], path, topTypeName, tags);
}
}
if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics
if (ref.name === 'Array') {
// basically an external type, but Array is quite common, especially with generics
if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') {
composeErrorMessage(path, topTypeName, 'Array with generics', 'array', 'Failed to parse');
return {type: ElasticsearchDataType.parse_error};
return {type: PARSE_ERROR};
}
return readFieldTags(
handleType(
ref.typeArguments[0], getReflectionGeneric(
ref, new Map(generics), path, topTypeName, tags),
path, topTypeName, tags),
path, topTypeName, tags);
ref.typeArguments[0],
getReflectionGeneric(ref, new Map(generics), path, topTypeName, tags),
path,
topTypeName,
tags,
),
path,
topTypeName,
tags,
);
}
if (ref.name === '__type') { // empty object
if (ref.name === '__type') {
// empty object
return {
dynamic: 'strict',
properties: {},
@@ -208,7 +236,7 @@ function handleExternalType(ref: ReferenceType, generics: Map<string, Elasticsea
composeErrorMessage(path, topTypeName, 'external type', ref.name, 'Missing pre-map');
return readFieldTags({type: ElasticsearchDataType.missing_premap}, path, topTypeName, tags);
return readFieldTags({type: MISSING_PREMAP}, path, topTypeName, tags);
}
/**
@@ -220,22 +248,22 @@ function handleExternalType(ref: ReferenceType, generics: Map<string, Elasticsea
* @param topTypeName the name of the SCThingType
* @param inheritedTags the inherited tags
*/
function handleDeclarationReflection(decl: DeclarationReflection,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
inheritedTags?: CommentTag[]):
ElasticsearchValue {
function handleDeclarationReflection(
decl: DeclarationReflection,
generics: Map<string, MappingProperty>,
path: string,
topTypeName: string,
inheritedTags?: CommentTag[],
): MappingProperty {
// check if we have an object referencing a generic
if (generics.has(decl.name)) { // if the object name is the same as the generic name
return readFieldTags(generics.get(decl.name) as ElasticsearchObject | ElasticsearchType, path, topTypeName,
decl.comment?.tags ?? []);
if (generics.has(decl.name)) {
// if the object name is the same as the generic name
return readFieldTags(generics.get(decl.name)!, path, topTypeName, decl.comment?.tags ?? []);
// use the value defined by the generic
}
// start the actual handling process
const out: ElasticsearchObject = {
const out: MappingProperty = {
dynamic: 'strict',
properties: {},
};
@@ -247,11 +275,13 @@ function handleDeclarationReflection(decl: DeclarationReflection,
if (typeof decl.indexSignature.type !== 'undefined') {
empty = false;
const template: ElasticsearchDynamicTemplate = {};
const template: Record<string, MappingDynamicTemplate> = {};
template[decl.name] = {
mapping: handleType(
decl.indexSignature.type,
new Map(generics), path, topTypeName,
new Map(generics),
path,
topTypeName,
getCommentTags(decl.indexSignature, path, topTypeName),
),
match: '*',
@@ -270,21 +300,31 @@ function handleDeclarationReflection(decl: DeclarationReflection,
if (typeof decl.children !== 'undefined' && decl.children.length > 0) {
for (const child of decl.children) {
empty = false;
out.properties[child.name] =
handleDeclarationReflection(child, new Map(generics), `${path}${child.name}.`, topTypeName);
out.properties![child.name] = handleDeclarationReflection(
child,
new Map(generics),
`${path}${child.name}.`,
topTypeName,
);
}
} else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY
} else if (decl.type instanceof Type) {
// if the object is a type, so we are dealing with a PROPERTY
// get inherited tags
const tags = (inheritedTags ?? []).length > 0
? (typeof inheritedTags!.find(it => isTagType(it.tagName)) !== 'undefined'
? inheritedTags!
: [...inheritedTags ?? [], ...getCommentTags(decl, path, topTypeName)])
: getCommentTags(decl, path, topTypeName);
const tags =
(inheritedTags ?? []).length > 0
? typeof inheritedTags!.find(it => isTagType(it.tagName)) !== 'undefined'
? inheritedTags!
: [...(inheritedTags ?? []), ...getCommentTags(decl, path, topTypeName)]
: getCommentTags(decl, path, topTypeName);
return handleType(decl.type, new Map(generics), path, topTypeName, tags);
} else if (decl.kindString === 'Enumeration member') {
return readTypeTags(typeof decl.defaultValue, path, topTypeName,
getCommentTags(decl, path, topTypeName, inheritedTags));
return readTypeTags(
typeof decl.defaultValue,
path,
topTypeName,
getCommentTags(decl, path, topTypeName, inheritedTags),
);
}
if (empty) {
@@ -315,19 +355,33 @@ function getCommentTags(
return [];
}
let out: CommentTag[] = decl.comment instanceof Comment ?
typeof decl.comment.tags !== 'undefined' ? decl.comment.tags : inheritedTags : inheritedTags;
if (decl.overwrites instanceof ReferenceType && decl.overwrites.reflection instanceof DeclarationReflection) {
let out: CommentTag[] =
decl.comment instanceof Comment
? typeof decl.comment.tags !== 'undefined'
? decl.comment.tags
: inheritedTags
: inheritedTags;
if (
decl.overwrites instanceof ReferenceType &&
decl.overwrites.reflection instanceof DeclarationReflection
) {
out = arrayPriorityJoin(
getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id), out);
getCommentTags(decl.overwrites.reflection, path, topTypeName, inheritedTags, decl.id),
out,
);
}
if (decl.inheritedFrom instanceof ReferenceType && decl.inheritedFrom.reflection instanceof DeclarationReflection) {
if (
decl.inheritedFrom instanceof ReferenceType &&
decl.inheritedFrom.reflection instanceof DeclarationReflection
) {
out = arrayPriorityJoin(
getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id), out);
getCommentTags(decl.inheritedFrom.reflection, path, topTypeName, inheritedTags, decl.id),
out,
);
}
saveCommentTags(out, path, topTypeName);
const inheritTag = out.find(((value) => value.tagName === inheritTagsName));
const inheritTag = out.find(value => value.tagName === inheritTagsName);
if (typeof inheritTag !== 'undefined') {
out = arrayPriorityJoin(out, retrieveCommentTags(inheritTag.text.trim(), path, topTypeName));
}
@@ -343,8 +397,9 @@ function getCommentTags(
* @param topTypeName the name of the SCThingType
*/
function saveCommentTags(tags: CommentTag[], path: string, topTypeName: string) {
inheritTagsMap[`${topTypeName}::${path.substr(0, path.length - 1)}`] =
tags.filter(((value) => value.tagName !== 'see' && value.tagName !== inheritTagsName));
inheritTagsMap[`${topTypeName}::${path.slice(0, Math.max(0, path.length - 1))}`] = tags.filter(
value => value.tagName !== 'see' && value.tagName !== inheritTagsName,
);
}
/**
@@ -373,14 +428,14 @@ function retrieveCommentTags(path: string, currentPath: string, topTypeName: str
function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): CommentTag[] {
const out: CommentTag[] = overrider;
originals.forEach((original) => {
const result = overrider.find((element) => original.tagName === element.tagName);
for (const original of originals) {
const result = overrider.find(element => original.tagName === element.tagName);
// no support for multiple tags with the same name
if (!(result instanceof CommentTag)) {
out.push(original);
}
});
}
return out;
}
@@ -397,12 +452,14 @@ function arrayPriorityJoin(originals: CommentTag[], overrider: CommentTag[]): Co
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleUnionType(type: UnionType,
generics: Map<string, ElasticsearchValue>,
path: string,
topTypeName: string,
tags: CommentTag[]): ElasticsearchValue {
const list: ElasticsearchValue[] = [];
function handleUnionType(
type: UnionType,
generics: Map<string, MappingProperty>,
path: string,
topTypeName: string,
tags: CommentTag[],
): MappingProperty {
const list: MappingProperty[] = [];
for (const subType of type.types) {
if (subType instanceof IntrinsicType && subType.name === 'undefined') {
@@ -415,16 +472,21 @@ function handleUnionType(type: UnionType,
let out = list[0];
for (const item of list) {
out = merge<ElasticsearchValue>(out, item);
out = merge<MappingProperty>(out, item);
}
return out;
}
composeErrorMessage(path, topTypeName, 'Union Type', stringify(list),
'Empty union type. This is likely not a user error.');
composeErrorMessage(
path,
topTypeName,
'Union Type',
stringify(list),
'Empty union type. This is likely not a user error.',
);
return {type: ElasticsearchDataType.parse_error};
return {type: PARSE_ERROR};
}
/**
@@ -436,37 +498,50 @@ function handleUnionType(type: UnionType,
* @param topTypeName the name of the SCThingType
* @param tags any tags attached to the type
*/
function handleType(type: Type, generics: Map<string, ElasticsearchValue>, path: string, topTypeName: string,
tags: CommentTag[]):
ElasticsearchValue {
function handleType(
type: Type,
generics: Map<string, MappingProperty>,
path: string,
topTypeName: string,
tags: CommentTag[],
): MappingProperty {
// logger.log((type as any).name);
if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type
if (type instanceof ArrayType) {
// array is irrelevant in Elasticsearch, so just go with the element type
const esType = handleType(type.elementType, new Map(generics), path, topTypeName, tags);
// also merge tags of the array to the element type
// filter out the type tags lazily, this can lead to double messages for "Not implemented tag"
let newTags = tags;
if ('type' in esType) {
newTags = tags.filter((tag) => {
newTags = tags.filter(tag => {
return !(tag.tagName === esType.type);
});
}
return readFieldTags(esType, path, topTypeName, newTags);
}
if (type.type === 'stringLiteral') { // a string literal, usually for type
if (type.type === 'stringLiteral') {
// a string literal, usually for type
return readTypeTags(type.type, path, topTypeName, tags);
}
if (type instanceof IntrinsicType) { // the absolute default type, like strings
if (type instanceof IntrinsicType) {
// the absolute default type, like strings
return readTypeTags(type.name, path, topTypeName, tags);
}
if (type instanceof UnionType) { // the union type...
if (type instanceof UnionType) {
// the union type...
return handleUnionType(type, new Map(generics), path, topTypeName, tags);
}
if (type instanceof ReferenceType) {
if (typeof premaps[type.name] === 'undefined' && typeof type.reflection !== 'undefined') {
// there is really no way to make this typesafe, every element in DeclarationReflection is optional.
return handleDeclarationReflection(type.reflection as DeclarationReflection,
getReflectionGeneric(type, new Map(generics), path, topTypeName, tags), path, topTypeName, tags);
return handleDeclarationReflection(
type.reflection as DeclarationReflection,
getReflectionGeneric(type, new Map(generics), path, topTypeName, tags),
path,
topTypeName,
tags,
);
}
return handleExternalType(type, new Map(generics), path, topTypeName, tags);
@@ -474,21 +549,24 @@ function handleType(type: Type, generics: Map<string, ElasticsearchValue>, path:
if (type instanceof TypeParameterType) {
// check if we have an object referencing a generic
if (generics.has(type.name)) {
return generics.get(type.name) as ElasticsearchObject | ElasticsearchType;
return generics.get(type.name)!;
}
composeErrorMessage(path, topTypeName, 'Generic', type.name, 'Missing reflection, please report!');
return {type: ElasticsearchDataType.parse_error};
return {type: PARSE_ERROR};
}
if (type instanceof ReflectionType) {
return readFieldTags(handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
path, topTypeName, tags);
return readFieldTags(
handleDeclarationReflection(type.declaration, new Map(generics), path, topTypeName),
path,
topTypeName,
tags,
);
}
composeErrorMessage(path, topTypeName, 'type', stringify(type), 'Not implemented type');
return {type: ElasticsearchDataType.parse_error};
return {type: PARSE_ERROR};
}
/**
@@ -502,43 +580,42 @@ function addAggregatable(path: string, topTypeName: string, global: boolean) {
// push type.path and remove the '.' at the end of the path
if (global) {
const prop = path.slice(0, -1)
.split('.')
.pop() as string; // cannot be undefined
const property_ = path.slice(0, -1).split('.').pop() as string; // cannot be undefined
return (aggregations['@all'] as ESNestedAggregation).aggs[prop.split('.')
.pop() as string] = {
return ((aggregations['@all'] as ESNestedAggregation).aggs[property_.split('.').pop() as string] = {
terms: {
field: `${prop}.raw`,
field: `${property_}.raw`,
size: 1000,
},
};
});
}
const property = path.slice(0, -1);
return (aggregations[topTypeName] as ESNestedAggregation).aggs[property] = {
return ((aggregations[topTypeName] as ESNestedAggregation).aggs[property] = {
terms: {
field: `${property}.raw`,
size: 1000,
},
};
});
}
/**
* Reads all tags related to Elasticsearch fields from the fieldMap
*
* @param prev the previous ElasticsearchValue, for example and object
* @param previous the previous ElasticsearchValue, for example and object
* @param path the current path to the object we are in
* @param topTypeName the name of the SCThingType
* @param tags tags attached to the value
* @param dataType the ElasticsearchDataType, for checking if a tag is a type tag
*/
function readFieldTags(prev: ElasticsearchValue,
path: string,
topTypeName: string,
tags: CommentTag[],
dataType?: string): ElasticsearchValue {
function readFieldTags(
previous: MappingProperty,
path: string,
topTypeName: string,
tags: CommentTag[],
dataType?: string,
): MappingProperty {
for (const tag of tags) {
if (tag.tagName === aggregatableTag) {
addAggregatable(path, topTypeName, tag.text.trim() === aggregatableTagParameterGlobal);
@@ -546,31 +623,43 @@ function readFieldTags(prev: ElasticsearchValue,
if (!ignoredTagsList.includes(tag.tagName)) {
if (typeof fieldmap[tag.tagName] !== 'undefined') {
if (typeof prev.fields === 'undefined') {
if (typeof previous.fields === 'undefined') {
// create in case it doesn't exist
prev.fields = {};
previous.fields = {};
}
if (tag.text.trim() === '') {
// merge fields
prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default};
previous.fields = {...previous.fields, ...fieldmap[tag.tagName].default};
} else if (typeof fieldmap[tag.tagName][tag.text.trim()] !== 'undefined') {
// merge fields
prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
previous.fields = {...previous.fields, ...fieldmap[tag.tagName][tag.text.trim()]};
} else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) {
// when there is an unidentified tag
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`);
composeErrorMessage(
path,
topTypeName,
'tag',
tag.tagName,
`Not implemented tag param "${tag.text.trim()}"`,
);
}
} else if (tag.tagName === filterableTagName) {
if (typeof prev.fields === 'undefined') {
prev.fields = {};
if (typeof previous.fields === 'undefined') {
previous.fields = {};
}
if ('type' in prev) {
const type = filterableMap[prev.type];
if ('type' in previous) {
const type = filterableMap[previous.type!];
if (typeof type !== 'undefined') {
// merge fields
prev.fields = {...prev.fields, ...{raw: {type: type}}};
previous.fields = {...previous.fields, raw: {type: type}};
} else {
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, `Not implemented for ${prev.type}`);
composeErrorMessage(
path,
topTypeName,
'tag',
tag.tagName,
`Not implemented for ${previous.type}`,
);
}
} else {
composeErrorMessage(path, topTypeName, 'tag', tag.tagName, 'Not applicable for object types');
@@ -581,7 +670,7 @@ function readFieldTags(prev: ElasticsearchValue,
}
}
return prev;
return previous;
}
/**
@@ -592,32 +681,44 @@ function readFieldTags(prev: ElasticsearchValue,
* @param topTypeName the name of the SCThingType
* @param tags tags attached to the value
*/
function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error};
function readTypeTags(type: string, path: string, topTypeName: string, tags: CommentTag[]): MappingProperty {
let out: MappingProperty = {type: PARSE_ERROR};
if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap
if (typeof typemap[type] !== 'undefined') {
// first look if the value has a definition in the typemap
for (let i = tags.length - 1; i >= 0; i--) {
if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') {
if (
!ignoredTagsList.includes(tags[i].tagName) &&
typeof typemap[type][tags[i].tagName] !== 'undefined'
) {
// if we have a tag that indicates a type
if (out.type !== ElasticsearchDataType.parse_error) {
composeErrorMessage(path, topTypeName, 'type', type,
`Type conflict; "${typemap[type][tags[i].tagName]}" would override "${out.type}"`);
out.type = ElasticsearchDataType.type_conflict;
if (out.type !== PARSE_ERROR) {
composeErrorMessage(
path,
topTypeName,
'type',
type,
`Type conflict; "${typemap[type][tags[i].tagName]}" would override "${
(out as MappingProperty).type
}"`,
);
(out as MappingProperty).type = TYPE_CONFLICT;
continue;
}
out.type = typemap[type][tags[i].tagName];
(out as MappingProperty).type = typemap[type][tags[i].tagName];
}
}
if (out.type === ElasticsearchDataType.parse_error) {
out.type = typemap[type].default;
if (out.type === PARSE_ERROR) {
(out as MappingProperty).type = typemap[type].default;
}
out = readFieldTags(out, path, topTypeName, tags, type);
return out;
}
if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type TODO: doesn't work for direct types
if (dynamicTypes.includes(type)) {
// Elasticsearch dynamic type TODO: doesn't work for direct types
return {
dynamic: true,
properties: {},
@@ -666,15 +767,18 @@ function reset(resetInheritTags = true) {
* @param showErrorOutput whether to print all errors in the command line or not
* @param interfaceFilter only parse specific interfaces, this is for testing purposes
*/
export function generateTemplate(projectReflection: ProjectReflection,
ignoredTags: string[],
showErrorOutput = true,
interfaceFilter: string[] = []): MappingGenTemplate {
export function generateTemplate(
projectReflection: ProjectReflection,
ignoredTags: string[],
showErrorOutput = true,
interfaceFilter: string[] = [],
): MappingGenTemplate {
reset();
showErrors = showErrorOutput;
ignoredTagsList = ['indexable', 'validatable', inheritTagsName];
// eslint-disable-next-line prefer-spread
ignoredTagsList.push.apply(ignoredTagsList, ignoredTags);
const indexableInterfaces = getAllIndexableInterfaces(projectReflection);
@@ -687,37 +791,42 @@ export function generateTemplate(projectReflection: ProjectReflection,
throw new Error('Interface needs at least some properties to be indexable');
}
const typeObject = _interface.children.find((declarationReflection) => {
const typeObject = _interface.children.find(declarationReflection => {
return declarationReflection.name === 'type';
});
if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') {
throw new Error('Interface needs a type to be indexable');
throw new TypeError('Interface needs a type to be indexable');
}
let typeName = 'INVALID_TYPE';
if (typeObject.type instanceof ReferenceType) {
if (typeObject.type.reflection instanceof DeclarationReflection
&& typeof typeObject.type.reflection.defaultValue === 'string') {
typeName = typeObject.type.reflection.defaultValue.replace('"', '')
.replace('"', '');
if (
typeObject.type.reflection instanceof DeclarationReflection &&
typeof typeObject.type.reflection.defaultValue === 'string'
) {
typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.');
void Logger.error(
'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
);
}
} else if (typeObject.type instanceof StringLiteralType) {
Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
typeName = typeObject.type.value;
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`);
void Logger.error(
`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
);
}
// init aggregation schema for type
aggregations[typeName] = {
aggs: {},
filter: {
type: {
value: typeName,
term: {
type: typeName,
},
},
};
@@ -732,45 +841,48 @@ export function generateTemplate(projectReflection: ProjectReflection,
throw new Error('Interface needs at least some properties to be indexable');
}
const typeObject = _interface.children.find((declarationReflection) => {
const typeObject = _interface.children.find(declarationReflection => {
return declarationReflection.name === 'type';
});
if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') {
throw new Error('Interface needs a type to be indexable');
throw new TypeError('Interface needs a type to be indexable');
}
let typeName = 'INVALID_TYPE';
if (typeObject.type instanceof ReferenceType) {
if (typeObject.type.reflection instanceof DeclarationReflection
&& typeof typeObject.type.reflection.defaultValue === 'string') {
typeName = typeObject.type.reflection.defaultValue.replace('"', '')
.replace('"', '');
if (
typeObject.type.reflection instanceof DeclarationReflection &&
typeof typeObject.type.reflection.defaultValue === 'string'
) {
typeName = typeObject.type.reflection.defaultValue.replace('"', '').replace('"', '');
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.');
void Logger.error(
'Your input files seem to be incorrect, or there is a major bug in the mapping generator.',
);
}
} else if (typeObject.type instanceof StringLiteralType) {
Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
typeName = typeObject.type.value;
} else {
// tslint:disable-next-line:no-floating-promises
void Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`);
void Logger.error(
`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`,
);
}
// filter out
if (interfaceFilter.length !== 0) {
if (typeof interfaceFilter.find((it) => it === typeName) === 'undefined') {
continue;
}
if (interfaceFilter.length > 0 && typeof interfaceFilter.find(it => it === typeName) === 'undefined') {
continue;
}
// init aggregation schema for type
aggregations[typeName] = {
aggs: {},
filter: {
type: {
value: typeName,
term: {
type: typeName,
},
},
};
@@ -782,26 +894,21 @@ export function generateTemplate(projectReflection: ProjectReflection,
const templateName = `template_${typeNameWithoutSpaces}`;
out[templateName] = {
mappings: {
[typeName]: handleDeclarationReflection(_interface, new Map(), '', typeName) as ElasticsearchObject,
},
mappings: handleDeclarationReflection(_interface, new Map(), '', typeName) as MappingObjectProperty,
settings: settings,
template: `stapps_${typeNameWithoutSpaces}*`,
}
;
out[templateName].mappings[typeName].properties.creation_date = {
type: ElasticsearchDataType.date,
index_patterns: [`stapps_${typeNameWithoutSpaces}*`],
};
out[templateName].mappings!.properties!.creation_date = {
type: 'date',
};
out[templateName].mappings[typeName].dynamic_templates = dynamicTemplates;
out[templateName].mappings!.dynamic_templates = dynamicTemplates;
// Set some properties
out[templateName].mappings[typeName]._source = {
excludes: [
'creation_date',
],
out[templateName].mappings!._source = {
excludes: ['creation_date'],
};
out[templateName].mappings[typeName].date_detection = false;
out[templateName].mappings!.date_detection = false;
dynamicTemplates = [];

30
packages/es-mapping-generator/src/project-reflection.ts
View File

@@ -15,36 +15,36 @@
import {Logger} from '@openstapps/logger';
import {existsSync, PathLike} from 'fs';
import {platform} from 'os';
import {join, sep} from 'path';
import path from 'path';
import {Application, ProjectReflection} from 'typedoc';
import {ModuleKind, ScriptTarget} from 'typescript';
/**
* Get a project reflection from a path
*
* @param srcPath Path to get reflection from
* @param sourcePath Path to get reflection from
* @param excludeExternals Exclude external dependencies
*/
export function getProjectReflection(srcPath: PathLike, excludeExternals = true): ProjectReflection {
Logger.info(`Generating project reflection for ${srcPath.toString()}.`);
export function getProjectReflection(sourcePath: PathLike, excludeExternals = true): ProjectReflection {
Logger.info(`Generating project reflection for ${sourcePath.toString()}.`);
const tsconfigPath = getTsconfigPath(srcPath.toString());
const tsconfigPath = getTsconfigPath(sourcePath.toString());
// initialize new Typedoc application
const app = new Application();
app.bootstrap({
excludeExternals: excludeExternals,
ignoreCompilerErrors: false, // TODO: true
ignoreCompilerErrors: true,
includeDeclarations: true,
module: ModuleKind.CommonJS,
target: ScriptTarget.Latest,
tsconfig: join(tsconfigPath, 'tsconfig.json'),
tsconfig: path.join(tsconfigPath, 'tsconfig.json'),
});
let inputFilePath = srcPath;
let inputFilePath = sourcePath;
if (inputFilePath === tsconfigPath) {
inputFilePath = join(tsconfigPath, 'src');
inputFilePath = path.join(tsconfigPath, 'src');
}
// get input files
@@ -54,7 +54,7 @@ export function getProjectReflection(srcPath: PathLike, excludeExternals = true)
const result = app.convert(inputFiles);
if (typeof result === 'undefined') {
throw new Error('Project reflection could not be generated.');
throw new TypeError('Project reflection could not be generated.');
}
return result;
@@ -69,12 +69,10 @@ export function getTsconfigPath(startPath: string): string {
let tsconfigPath = startPath;
// see https://stackoverflow.com/questions/9652043/identifying-the-file-system-root-with-node-js
const root = (platform() === 'win32') ? process
.cwd()
.split(sep)[0] : '/';
const root = platform() === 'win32' ? process.cwd().split(path.sep)[0] : '/';
// repeat until a tsconfig.json is found
while (!existsSync(join(tsconfigPath, 'tsconfig.json'))) {
while (!existsSync(path.join(tsconfigPath, 'tsconfig.json'))) {
if (tsconfigPath === root) {
throw new Error(
`Reached file system root ${root} while searching for 'tsconfig.json' in ${startPath}!`,
@@ -82,9 +80,9 @@ export function getTsconfigPath(startPath: string): string {
}
// pop last directory
const tsconfigPathParts = tsconfigPath.split(sep);
const tsconfigPathParts = tsconfigPath.split(path.sep);
tsconfigPathParts.pop();
tsconfigPath = tsconfigPathParts.join(sep);
tsconfigPath = tsconfigPathParts.join(path.sep);
}
Logger.info(`Using 'tsconfig.json' from ${tsconfigPath}.`);

81
packages/es-mapping-generator/src/types/aggregation.d.ts vendored
View File

@@ -1,81 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* An elasticsearch bucket aggregation
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
*/
export interface AggregationSchema {
[aggregationName: string]: ESTermsFilter | ESNestedAggregation;
}
/**
* An elasticsearch terms filter
*/
export interface ESTermsFilter {
/**
* Terms filter definition
*/
terms: {
/**
* Field to apply filter to
*/
field: string;
/**
* Number of results
*/
size?: number;
};
}
/**
* Filter that filters by name of the the field type
*/
export interface ESAggTypeFilter {
/**
* The type of the object to find
*/
type: {
/**
* The name of the type
*/
value: string;
};
}
/**
* Filter that matches everything
*/
export interface ESAggMatchAllFilter {
/**
* Filter that matches everything
*/
match_all: {};
}
/**
* For nested aggregations
*/
export interface ESNestedAggregation {
/**
* Possible nested Aggregations
*/
aggs: AggregationSchema;
/**
* Possible filter for types
*/
filter: ESAggTypeFilter | ESAggMatchAllFilter;
}

0
packages/es-mapping-generator/src/types/aggregation.ts Normal file
View File

345
packages/es-mapping-generator/src/types/mapping.d.ts vendored
View File

@@ -1,345 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ElasticsearchDataType} from '../config/typemap';
import {AggregationSchema} from './aggregation';
// tslint:disable:no-any
/**
* Template output of the mapping generation
*/
export interface MappingGenTemplate {
/**
* All generated aggregations
*/
aggregations: AggregationSchema;
/**
* All errors that occurred
*/
errors: string[];
/**
* All mappings that were generated
*/
mappings: ElasticsearchTemplateCollection;
}
/**
* ElasticsearchValue can be either a type or an object.
*
* Both are composed similarly, and can be the value of a property
* of an Elasticsearch Object.
*/
export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject | ElasticsearchGeoShape;
/**
* The Typemap is used to get the corresponding ElasticsearchDataType for a name provided by the ProjectReflection
*/
export interface ElasticsearchTypemap {
/**
* The `stringLiteral` type must always be provided
*/
stringLiteral: {
/**
* The default can be chosen freely, but must be provided
*/
default: ElasticsearchDataType;
};
/**
* The name of the JS type, so for `number` it would be number
*/
[name: string]: {
/**
* The default ElasticsearchDataType that should be used, if no tag or only not implemented tags are found
*/
default: ElasticsearchDataType;
/**
* The name of the tag, so for `@integer` it would be `integer`
*/
[name: string]: ElasticsearchDataType;
};
}
/**
* The representation of a `DynamicTemplate` in Elasticsearch
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/dynamic-templates.html
*/
export interface ElasticsearchDynamicTemplate {
/**
* The name of the dynamicTemplate
*/
[name: string]: {
/**
* The mapping of the template
*/
mapping: ElasticsearchValue;
/**
* With automatic mapping, we use `path_match` more or less out of convenience and because it is least error-prone
*
* This also means that match should match all ("*") interface names (because we provide the exact path of the
* interface)
*/
match: '*';
/**
* With automatic mapping, we use `path_match` more or less out of convenience and because it is least error-prone
*
* This also means that match_mapping_type should match all ("*") names (because we provide the exact path of the
* interface)
*/
match_mapping_type: '*';
/**
* With automatic mapping, we use `path_match` more or less out of convenience and because it is least error-prone
*/
path_match: string;
};
}
export interface ElasticsearchFilterableMap {
[name: string]: ElasticsearchDataType;
}
/**
* The Fieldmap contains all tag names for fields and the corresponding fields
*
* The Fieldmap works in a similar fashion to the Typemap
*/
export interface ElasticsearchFieldmap {
/**
* The name of the tag, so for `@sortable` it would be `sortable`
*/
[name: string]: {
/**
* The default value if no parameter is provided
*/
default: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
/**
* The tag parameters that will be ignored
*
* Some tag parameters might not be important for your implementation, so you can add their names here to not get
* any errors. The `default` will be used in that case.
*/
ignore: string[];
/**
* The parameters of the tag, so for `@sortable ducet` it would be `ducet`
*/
[name: string]: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
};
}
/**
* A primitive data type
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-types.html
*/
export interface ElasticsearchType {
/**
* Fields for a type
*
* The fields are optional, they are used for things like sorting, which is not needed for every single type.
*/
fields?: {
[name: string]: any;
};
/**
* The type as an ElasticsearchDataType
*/
type: ElasticsearchDataType;
}
/**
* A GeoShape data type
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/geo-shape.html
*/
export interface ElasticsearchGeoShape {
/**
* Does not exist; here for TypeScript compiler
*/
fields?: undefined;
/**
* This parameter may be used instead of tree_levels to set an appropriate value for the tree_levels parameter.
*
* The value specifies the desired precision and Elasticsearch will calculate the best tree_levels value to honor
* this precision. The value should be a number followed by an optional distance unit. Valid distance units include:
* in, inch, yd, yard, mi, miles, km, kilometers, m,meters, cm,centimeters, mm, millimeters.
*/
precision: string;
/**
* Name of the PrefixTree implementation to be used: geohash for GeohashPrefixTree and quadtree for QuadPrefixTree.
*/
tree: 'quadtree' | 'geohash';
/**
* The type of the object, obviously geo_shape
*/
type: ElasticsearchDataType.geo_shape;
}
/**
* An object data type
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/object.html
*/
export interface ElasticsearchObject {
/**
* Only for the top type
*/
_source?: {
/**
* Fields that should be excluded in the _source field
*/
excludes: [
'creation_date'
];
};
/**
* Whether the creation date should be set automatically
*/
date_detection?: boolean;
/**
* If the object is a dynamic
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/dynamic.html
* The default should be `'strict'`
*/
dynamic: true | false | 'strict';
/**
* dynamic_templates for an object
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/dynamic-templates.html
* This is a more complex topic, before touching this you should really know what you are doing.
*/
dynamic_templates?: ElasticsearchDynamicTemplate[];
/**
* Fields for a type
*
* The fields are optional, they are used for things like sorting, which is not needed for every single type.
*/
fields?: {
[name: string]: any;
};
/**
* Any properties of the object
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/properties.html
*/
properties: {
/**
* Each property can be any Elasticsearch value
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-types.html
*/
[name: string]: ElasticsearchValue;
};
}
/**
* A collection of Elasticsearch Templates
*/
export interface ElasticsearchTemplateCollection {
[indexName: string]: ElasticsearchTemplate;
}
/**
* An Elasticsearch template
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html
* This is what you pass to Elasticsearch
*/
export interface ElasticsearchTemplate {
/**
* This is a pre-defined structure you should use for your mapping
*/
mappings: {
[typeName: string]: ElasticsearchObject;
};
/**
* The settings for Elasticsearch
*/
settings: ElasticsearchSettings;
/**
* The name of the template, for referencing in Elasticsearch
*/
template: string;
}
/**
* A representation of ElasticsearchSettings used in Mappings
*/
export interface ElasticsearchSettings {
/**
* The settings
*/
[name: string]: any;
/**
* This is where any analyzers go
*
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/analysis-analyzers.html
*/
analysis: {
[name: string]: any;
};
}
/**
* A premap for a specific value in a ProjectReflection
*
* This is meant to be used for external types. To aid performance, you usually should not include external libs in the
* ProjectReflection. This means that there is no way the generator can generate a mapping for it, so you can use the
* premaps to map out a type manually.
*/
export interface ElasticsearchPremap {
/**
* The name of the type with the corresponding map
*
* So for `const a: B` the name would be `B`
*/
[name: string]: ElasticsearchValue;
}

119
packages/es-mapping-generator/src/types/mapping.ts Normal file
View File

@@ -0,0 +1,119 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types';
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {AggregationSchema} from './aggregation';
/**
* Template output of the mapping generation
*/
export interface MappingGenTemplate {
/**
* All generated aggregations
*/
aggregations: AggregationSchema;
/**
* All errors that occurred
*/
errors: string[];
/**
* All mappings that were generated
*/
mappings: ElasticsearchTemplateCollection;
}
export type SimpleType = MappingProperty['type'] &
('keyword' | 'float' | 'boolean' | 'date' | 'integer' | 'text');
/**
* The Typemap is used to get the corresponding ElasticsearchDataType for a name provided by the ProjectReflection
*/
export interface ElasticsearchTypemap {
/**
* The `stringLiteral` type must always be provided
*/
stringLiteral: {
/**
* The default can be chosen freely, but must be provided
*/
default: SimpleType;
};
/**
* The name of the JS type, so for `number` it would be number
*/
[name: string]: {
/**
* The default ElasticsearchDataType that should be used, if no tag or only not implemented tags are found
*/
default: SimpleType;
/**
* The name of the tag, so for `@integer` it would be `integer`
*/
[name: string]: SimpleType;
};
}
/**
* The Fieldmap contains all tag names for fields and the corresponding fields
*
* The Fieldmap works in a similar fashion to the Typemap
*/
export interface ElasticsearchFieldmap {
/**
* The name of the tag, so for `@sortable` it would be `sortable`
*/
[name: string]: {
/**
* The default value if no parameter is provided
*/
default: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
/**
* The tag parameters that will be ignored
*
* Some tag parameters might not be important for your implementation, so you can add their names here to not get
* any errors. The `default` will be used in that case.
*/
ignore: string[];
/**
* The parameters of the tag, so for `@sortable ducet` it would be `ducet`
*/
[name: string]: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
};
}
/**
* A collection of Elasticsearch Templates
*/
export type ElasticsearchTemplateCollection = Record<string, Omit<IndicesPutTemplateRequest, 'name'>>;