Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 00:52:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: migrate es mapping types from es-mapping-generator to .d.ts next to generated mappings

Browse Source
This commit is contained in:
Thea Schöbl
2023-06-30 12:01:38 +02:00
parent 1aaf85b444
commit 0a7e6af141
19 changed files with 354 additions and 435 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
packages/es-mapping-generator/src/app.ts
View File

@@ -13,13 +13,10 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Command} from 'commander';
import {mkdirSync, readFileSync, writeFileSync} from 'fs';
import got from 'got';
import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'fs';
import path from 'path';
import {exit} from 'process';
import {generateTemplate} from './mapping';
import {getProjectReflection} from './project-reflection';
import {ElasticsearchTemplateCollection} from './types/mapping';
// handle unhandled promise rejections
process.on('unhandledRejection', async (reason: unknown) => {
@@ -66,13 +63,22 @@ commander
mkdirSync(path.dirname(aggPath), {recursive: true});
// tslint:disable-next-line:no-magic-numbers
writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
copyFileSync(
// eslint-disable-next-line unicorn/prefer-module
require.resolve('../schema/aggregations.d.ts'),
path.join(path.dirname(aggPath), 'aggregations.json.d.ts'),
);
console.log(`Elasticsearch aggregations written to ${aggPath}.`);
}
if (options.mappingPath !== undefined) {
const mappingPath = path.resolve(options.mappingPath);
mkdirSync(path.dirname(mappingPath), {recursive: true});
// tslint:disable-next-line:no-magic-numbers
writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
copyFileSync(
// eslint-disable-next-line unicorn/prefer-module
require.resolve('../schema/mappings.d.ts'),
path.join(path.dirname(mappingPath), 'mappings.json.d.ts'),
);
console.log(`Elasticsearch mappings written to ${mappingPath}.`);
}
if (options.errorPath !== undefined) {
@@ -90,35 +96,4 @@ commander
}
});
commander
.command('put-es-templates <srcPath> <esAddress> [ignoredTags]')
.action(async (relativeSourcePath, esAddress) => {
// get absolute paths
const sourcePath = path.resolve(relativeSourcePath);
// get project reflection
// eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module
const templates = require(sourcePath) as ElasticsearchTemplateCollection;
for (const template in templates) {
if (!templates.hasOwnProperty(template)) {
continue;
}
const response = await got.put(`${esAddress}_template/${template}`, {
json: templates[template],
});
const HTTP_STATUS_OK = 200;
if (response.statusCode !== HTTP_STATUS_OK) {
await console.error(
`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`,
);
exit(-1);
}
}
console.log(`Templates accepted by Elasticsearch.`);
});
commander.parse(process.argv);

2
packages/es-mapping-generator/src/config/fieldmap.ts
View File

@@ -13,7 +13,7 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
import {ElasticsearchFieldmap, SimpleType} from '../types/mapping.js';
import type {ElasticsearchFieldmap, SimpleType} from '../../schema/mappings.js';
const ducetSort = {
type: 'icu_collation_keyword',

2
packages/es-mapping-generator/src/config/typemap.ts
View File

@@ -13,7 +13,7 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {MappingFloatNumberProperty} from '@elastic/elasticsearch/lib/api/types';
import {ElasticsearchTypemap} from '../types/mapping';
import type {ElasticsearchTypemap} from '../../schema/mappings';
export const PARSE_ERROR = 'PARSE_ERROR' as MappingFloatNumberProperty['type'];
export const MISSING_PREMAP = 'MISSING_PREMAP' as MappingFloatNumberProperty['type'];

3
packages/es-mapping-generator/src/index.ts
View File

@@ -5,6 +5,3 @@ export * from './config/premap';
export * from './config/fieldmap';
export * from './config/settings';
export * from './config/typemap';
export * from './types/mapping';
export * from './types/aggregation';

10
packages/es-mapping-generator/src/mapping.ts
View File

@@ -36,8 +36,8 @@ import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap';
import {premaps} from './config/premap';
import {settings} from './config/settings';
import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap';
import {AggregationSchema, ESNestedAggregation} from './types/aggregation';
import {ElasticsearchTemplateCollection, MappingGenTemplate} from './types/mapping';
import type {AggregationSchema, ESNestedAggregation} from '../schema/aggregations';
import type {ElasticsearchTemplateCollection, MappingGenTemplate} from '../schema/mappings';
import * as console from 'console';
let dynamicTemplates: Record<string, MappingDynamicTemplate>[] = [];
@@ -862,10 +862,8 @@ export function generateTemplate(
},
};
let typeNameWithoutSpaces = typeName.toLowerCase();
while (typeNameWithoutSpaces.includes(' ')) {
typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_');
}
// eslint-disable-next-line unicorn/prefer-string-replace-all
const typeNameWithoutSpaces = typeName.toLowerCase().replace(/\s/g, '_');
const templateName = `template_${typeNameWithoutSpaces}`;
out[templateName] = {

81
packages/es-mapping-generator/src/types/aggregation.ts
View File

@@ -1,81 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* An elasticsearch bucket aggregation
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
*/
export interface AggregationSchema {
[aggregationName: string]: ESTermsFilter | ESNestedAggregation;
}
/**
* An elasticsearch terms filter
*/
export interface ESTermsFilter {
/**
* Terms filter definition
*/
terms: {
/**
* Field to apply filter to
*/
field: string;
/**
* Number of results
*/
size?: number;
};
}
/**
* Filter that filters by name of the the field type
*/
export interface ESAggTypeFilter {
/**
* The type of the object to find
*/
term: {
/**
* The name of the type
*/
type: string;
};
}
/**
* Filter that matches everything
*/
export interface ESAggMatchAllFilter {
/**
* Filter that matches everything
*/
match_all: object;
}
/**
* For nested aggregations
*/
export interface ESNestedAggregation {
/**
* Possible nested Aggregations
*/
aggs: AggregationSchema;
/**
* Possible filter for types
*/
filter: ESAggTypeFilter | ESAggMatchAllFilter;
}

119
packages/es-mapping-generator/src/types/mapping.ts
View File

@@ -1,119 +0,0 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types';
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {AggregationSchema} from './aggregation';
/**
* Template output of the mapping generation
*/
export interface MappingGenTemplate {
/**
* All generated aggregations
*/
aggregations: AggregationSchema;
/**
* All errors that occurred
*/
errors: string[];
/**
* All mappings that were generated
*/
mappings: ElasticsearchTemplateCollection;
}
export type SimpleType = MappingProperty['type'] &
('keyword' | 'float' | 'boolean' | 'date' | 'integer' | 'text');
/**
* The Typemap is used to get the corresponding ElasticsearchDataType for a name provided by the ProjectReflection
*/
export interface ElasticsearchTypemap {
/**
* The `stringLiteral` type must always be provided
*/
stringLiteral: {
/**
* The default can be chosen freely, but must be provided
*/
default: SimpleType;
};
/**
* The name of the JS type, so for `number` it would be number
*/
[name: string]: {
/**
* The default ElasticsearchDataType that should be used, if no tag or only not implemented tags are found
*/
default: SimpleType;
/**
* The name of the tag, so for `@integer` it would be `integer`
*/
[name: string]: SimpleType;
};
}
/**
* The Fieldmap contains all tag names for fields and the corresponding fields
*
* The Fieldmap works in a similar fashion to the Typemap
*/
export interface ElasticsearchFieldmap {
/**
* The name of the tag, so for `@sortable` it would be `sortable`
*/
[name: string]: {
/**
* The default value if no parameter is provided
*/
default: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
/**
* The tag parameters that will be ignored
*
* Some tag parameters might not be important for your implementation, so you can add their names here to not get
* any errors. The `default` will be used in that case.
*/
ignore: string[];
/**
* The parameters of the tag, so for `@sortable ducet` it would be `ducet`
*/
[name: string]: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
};
}
/**
* A collection of Elasticsearch Templates
*/
export type ElasticsearchTemplateCollection = Record<string, Omit<IndicesPutTemplateRequest, 'name'>>;