refactor: migrate es mapping types from es-mapping-generator to .d.ts next to generated mappings

This commit is contained in:
2023-06-30 12:01:38 +02:00
parent 1aaf85b444
commit 0a7e6af141
19 changed files with 354 additions and 435 deletions

View File

@@ -13,6 +13,7 @@
"files": [
"app.js",
"lib",
"schema",
"README.md"
],
"scripts": {
@@ -28,7 +29,6 @@
"commander": "10.0.0",
"deepmerge": "4.3.1",
"flatted": "3.2.7",
"got": "11.8.6",
"typedoc": "0.18.0",
"typescript": "3.8.3"
},

View File

@@ -12,6 +12,10 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare module 'aggregations.json' {
const value: AggregationSchema;
export default value;
}
/**
* An elasticsearch bucket aggregation

View File

@@ -14,7 +14,12 @@ import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {AggregationSchema} from './aggregation';
import {AggregationSchema} from './aggregations';
declare module 'mappings.json' {
const value: ElasticsearchTemplateCollection;
export default value;
}
/**
* Template output of the mapping generation

View File

@@ -13,13 +13,10 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Command} from 'commander';
import {mkdirSync, readFileSync, writeFileSync} from 'fs';
import got from 'got';
import {copyFileSync, mkdirSync, readFileSync, writeFileSync} from 'fs';
import path from 'path';
import {exit} from 'process';
import {generateTemplate} from './mapping';
import {getProjectReflection} from './project-reflection';
import {ElasticsearchTemplateCollection} from './types/mapping';
// handle unhandled promise rejections
process.on('unhandledRejection', async (reason: unknown) => {
@@ -66,13 +63,22 @@ commander
mkdirSync(path.dirname(aggPath), {recursive: true});
// tslint:disable-next-line:no-magic-numbers
writeFileSync(aggPath, JSON.stringify(result.aggregations, null, 2));
copyFileSync(
// eslint-disable-next-line unicorn/prefer-module
require.resolve('../schema/aggregations.d.ts'),
path.join(path.dirname(aggPath), 'aggregations.json.d.ts'),
);
console.log(`Elasticsearch aggregations written to ${aggPath}.`);
}
if (options.mappingPath !== undefined) {
const mappingPath = path.resolve(options.mappingPath);
mkdirSync(path.dirname(mappingPath), {recursive: true});
// tslint:disable-next-line:no-magic-numbers
writeFileSync(mappingPath, JSON.stringify(result.mappings, null, 2));
copyFileSync(
// eslint-disable-next-line unicorn/prefer-module
require.resolve('../schema/mappings.d.ts'),
path.join(path.dirname(mappingPath), 'mappings.json.d.ts'),
);
console.log(`Elasticsearch mappings written to ${mappingPath}.`);
}
if (options.errorPath !== undefined) {
@@ -90,35 +96,4 @@ commander
}
});
commander
.command('put-es-templates <srcPath> <esAddress> [ignoredTags]')
.action(async (relativeSourcePath, esAddress) => {
// get absolute paths
const sourcePath = path.resolve(relativeSourcePath);
// get project reflection
// eslint-disable-next-line @typescript-eslint/no-var-requires,unicorn/prefer-module
const templates = require(sourcePath) as ElasticsearchTemplateCollection;
for (const template in templates) {
if (!templates.hasOwnProperty(template)) {
continue;
}
const response = await got.put(`${esAddress}_template/${template}`, {
json: templates[template],
});
const HTTP_STATUS_OK = 200;
if (response.statusCode !== HTTP_STATUS_OK) {
await console.error(
`Template for "${template}" failed in Elasticsearch:\n${JSON.stringify(response.body)}`,
);
exit(-1);
}
}
console.log(`Templates accepted by Elasticsearch.`);
});
commander.parse(process.argv);

View File

@@ -13,7 +13,7 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {MappingProperty} from '@elastic/elasticsearch/lib/api/types';
import {ElasticsearchFieldmap, SimpleType} from '../types/mapping.js';
import type {ElasticsearchFieldmap, SimpleType} from '../../schema/mappings.js';
const ducetSort = {
type: 'icu_collation_keyword',

View File

@@ -13,7 +13,7 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {MappingFloatNumberProperty} from '@elastic/elasticsearch/lib/api/types';
import {ElasticsearchTypemap} from '../types/mapping';
import type {ElasticsearchTypemap} from '../../schema/mappings';
export const PARSE_ERROR = 'PARSE_ERROR' as MappingFloatNumberProperty['type'];
export const MISSING_PREMAP = 'MISSING_PREMAP' as MappingFloatNumberProperty['type'];

View File

@@ -5,6 +5,3 @@ export * from './config/premap';
export * from './config/fieldmap';
export * from './config/settings';
export * from './config/typemap';
export * from './types/mapping';
export * from './types/aggregation';

View File

@@ -36,8 +36,8 @@ import {fieldmap, filterableMap, filterableTagName} from './config/fieldmap';
import {premaps} from './config/premap';
import {settings} from './config/settings';
import {dynamicTypes, isTagType, MISSING_PREMAP, PARSE_ERROR, TYPE_CONFLICT, typemap} from './config/typemap';
import {AggregationSchema, ESNestedAggregation} from './types/aggregation';
import {ElasticsearchTemplateCollection, MappingGenTemplate} from './types/mapping';
import type {AggregationSchema, ESNestedAggregation} from '../schema/aggregations';
import type {ElasticsearchTemplateCollection, MappingGenTemplate} from '../schema/mappings';
import * as console from 'console';
let dynamicTemplates: Record<string, MappingDynamicTemplate>[] = [];
@@ -862,10 +862,8 @@ export function generateTemplate(
},
};
let typeNameWithoutSpaces = typeName.toLowerCase();
while (typeNameWithoutSpaces.includes(' ')) {
typeNameWithoutSpaces = typeNameWithoutSpaces.replace(' ', '_');
}
// eslint-disable-next-line unicorn/prefer-string-replace-all
const typeNameWithoutSpaces = typeName.toLowerCase().replace(/\s/g, '_');
const templateName = `template_${typeNameWithoutSpaces}`;
out[templateName] = {

View File

@@ -12,18 +12,13 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
generateTemplate,
getProjectReflection,
AggregationSchema,
ESNestedAggregation,
ElasticsearchTemplateCollection,
settings,
} from '../../src';
import {generateTemplate, getProjectReflection, settings} from '../../src';
import path from 'path';
import {expect} from 'chai';
import {ProjectReflection} from 'typedoc';
import {MapAggTestOptions, MinimalMappingDescription} from './map-agg-test-options';
import type {AggregationSchema, ESNestedAggregation} from '../../schema/aggregations';
import type {ElasticsearchTemplateCollection} from '../../schema/mappings';
export class MapAggTest {
mapping_model_path!: string;