Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 09:03:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: more fixes

Browse Source
This commit is contained in:
Thea Schöbl
2023-11-01 16:10:12 +01:00
parent 4bdd4b20d0
commit 62d5ea4275
23 changed files with 80 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/es-mapping-generator/package.json
View File

@@ -24,6 +24,8 @@
"test": "c8 mocha"
},
"dependencies": {
"ajv": "8.12.0",
"ajv-formats": "2.1.1",
"@elastic/elasticsearch": "8.10.0",
"@openstapps/json-schema-generator": "workspace:*",
"@openstapps/tsup-plugin": "workspace:*",

85
packages/es-mapping-generator/schema/aggregations.d.ts vendored
View File

@@ -1,85 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare module 'aggregations.json' {
const value: AggregationSchema;
export default value;
}
/**
* An elasticsearch bucket aggregation
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
*/
export interface AggregationSchema {
[aggregationName: string]: ESTermsFilter | ESNestedAggregation;
}
/**
* An elasticsearch terms filter
*/
export interface ESTermsFilter {
/**
* Terms filter definition
*/
terms: {
/**
* Field to apply filter to
*/
field: string;
/**
* Number of results
*/
size?: number;
};
}
/**
* Filter that filters by name of the the field type
*/
export interface ESAggTypeFilter {
/**
* The type of the object to find
*/
term: {
/**
* The name of the type
*/
type: string;
};
}
/**
* Filter that matches everything
*/
export interface ESAggMatchAllFilter {
/**
* Filter that matches everything
*/
match_all: object;
}
/**
* For nested aggregations
*/
export interface ESNestedAggregation {
/**
* Possible nested Aggregations
*/
aggs: AggregationSchema;
/**
* Possible filter for types
*/
filter: ESAggTypeFilter | ESAggMatchAllFilter;
}

127
packages/es-mapping-generator/schema/mappings.d.ts vendored
View File

@@ -1,124 +1,9 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types';
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {AggregationSchema} from './aggregations';
import {IndicesPutTemplateRequest, SearchRequest} from '@elastic/elasticsearch/lib/api/types.js';
declare module 'mappings.json' {
const value: ElasticsearchTemplateCollection;
export default value;
export interface ElasticsearchConfig {
mappings: Record<string, IndicesPutTemplateRequest>;
search: Partial<SearchRequest>;
}
/**
* Template output of the mapping generation
*/
export interface MappingGenTemplate {
/**
* All generated aggregations
*/
aggregations: AggregationSchema;
/**
* All errors that occurred
*/
errors: string[];
/**
* All mappings that were generated
*/
mappings: ElasticsearchTemplateCollection;
}
export type SimpleType = MappingProperty['type'] &
('keyword' | 'float' | 'boolean' | 'date' | 'integer' | 'text');
/**
* The Typemap is used to get the corresponding ElasticsearchDataType for a name provided by the ProjectReflection
*/
export interface ElasticsearchTypemap {
/**
* The `stringLiteral` type must always be provided
*/
stringLiteral: {
/**
* The default can be chosen freely, but must be provided
*/
default: SimpleType;
};
/**
* The name of the JS type, so for `number` it would be number
*/
[name: string]: {
/**
* The default ElasticsearchDataType that should be used, if no tag or only not implemented tags are found
*/
default: SimpleType;
/**
* The name of the tag, so for `@integer` it would be `integer`
*/
[name: string]: SimpleType;
};
}
/**
* The Fieldmap contains all tag names for fields and the corresponding fields
*
* The Fieldmap works in a similar fashion to the Typemap
*/
export interface ElasticsearchFieldmap {
/**
* The name of the tag, so for `@sortable` it would be `sortable`
*/
[name: string]: {
/**
* The default value if no parameter is provided
*/
default: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
/**
* The tag parameters that will be ignored
*
* Some tag parameters might not be important for your implementation, so you can add their names here to not get
* any errors. The `default` will be used in that case.
*/
ignore: string[];
/**
* The parameters of the tag, so for `@sortable ducet` it would be `ducet`
*/
[name: string]: {
/**
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}`
*
* We could also have used `default: any`, but this adds slightly more improved type-safety.
*/
[name: string]: any;
};
};
}
/**
* A collection of Elasticsearch Templates
*/
export type ElasticsearchTemplateCollection = Record<string, Omit<IndicesPutTemplateRequest, 'name'>>;
declare const elasticsearchConfig: ElasticsearchConfig;
export default elasticsearchConfig;

2
packages/es-mapping-generator/src/generator/context.ts
View File

@@ -58,7 +58,7 @@ export class Context {
this.propertyPath,
new Map(),
);
const result = transformDefinition(derivedContext, definition);
const result = transformDefinition(derivedContext, definition!);
referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});

8
packages/es-mapping-generator/src/generator/index.ts
View File

@@ -3,6 +3,7 @@ import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
import {MappingGenerator} from './mapping-generator.js';
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
import {ElasticsearchConfig} from '../../schema/mappings.js';
export interface GeneratorOptions {
/**
@@ -24,15 +25,16 @@ export interface GeneratorOptions {
/**
* Fully transform a project
*/
export function transformProject(project: JSONSchema7) {
export function transformProject(project: JSONSchema7): ElasticsearchConfig {
const context = new MappingGenerator(project, OPTIONS);
const results = [];
const results: Record<string, IndicesPutTemplateRequest> = {};
for (const name in project.definitions) {
const definition = project.definitions[name];
if (typeof definition !== 'object' || !getTags(definition).has(INDEXABLE_TAG_NAME)) continue;
results.push(context.buildTemplate(name));
const [type, template] = context.buildTemplate(name);
results[type] = template;
}
return {
mappings: results,

4
packages/es-mapping-generator/src/generator/mapping-generator.ts
View File

@@ -41,7 +41,7 @@ export class MappingGenerator {
);
}
buildTemplate(name: string): IndicesPutTemplateRequest {
buildTemplate(name: string): [string, IndicesPutTemplateRequest] {
const thingType = ((this.project.definitions![name] as JSONSchema7).properties!.type as JSONSchema7)
.const;
if (typeof thingType !== 'string') {
@@ -71,6 +71,6 @@ export class MappingGenerator {
}
}
return request;
return [thingType, request];
}
}

9
packages/es-mapping-generator/src/index.ts
View File

@@ -1,5 +1,13 @@
import {transformProject} from './generator/index.js';
import {SchemaConsumer} from '@openstapps/json-schema-generator';
import {readFile} from 'fs/promises';
import path from 'path';
import {fileURLToPath} from 'url';
const mappingTypes = await readFile(
path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'mappings.d.ts'),
'utf8',
);
/**
* JSON Schema Generator plugin for Elasticsearch Mappings
@@ -10,6 +18,7 @@ export function elasticsearchMappingGenerator(fileName: string): [string, Schema
function (schema) {
return {
[fileName]: JSON.stringify(transformProject(schema)),
[`${fileName}.d.ts`]: mappingTypes,
};
},
];