Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 09:03:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: use core supplied mappings

Browse Source
This commit is contained in:
Wieland Schöbl
2021-09-03 15:17:15 +00:00
committed by Rainer Killinger
parent 614a1b1e9b
commit 43a89ec4f2
22 changed files with 1622 additions and 1762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/storage/elasticsearch/aggregations.ts
View File

@@ -14,44 +14,31 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCFacet, SCThingType} from '@openstapps/core';
import {readFileSync} from 'fs';
import {aggregations} from './templating';
import {AggregationResponse} from './types/elasticsearch';
import {
AggregationResponse,
AggregationSchema,
isBucketAggregation,
isESAggMatchAllFilter,
isESNestedAggregation,
isESTermsFilter,
isNestedAggregation,
} from './common';
import {aggregationsPath} from './templating';
/**
* Builds the aggregation
* @returns a schema to tell elasticsearch which aggregations to collect
*/
export function buildAggregations(): AggregationSchema {
return JSON.parse((readFileSync(aggregationsPath, 'utf8')).toString());
}
} from './types/guards';
/**
* Parses elasticsearch aggregations (response from es) to facets for the app
* @param aggregationSchema - aggregation-schema for elasticsearch
* @param aggregations - aggregations response from elasticsearch
* @param aggregationResponse - aggregations response from elasticsearch
*/
export function parseAggregations(
aggregationSchema: AggregationSchema,
aggregations: AggregationResponse): SCFacet[] {
export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
const facets: SCFacet[] = [];
// get all names of the types an aggregation is on
for (const typeName in aggregationSchema) {
if (aggregationSchema.hasOwnProperty(typeName) && aggregations.hasOwnProperty(typeName)) {
for (const typeName in aggregations) {
if (aggregations.hasOwnProperty(typeName) && aggregationResponse.hasOwnProperty(typeName)) {
// the type object from the schema
const type = aggregationSchema[typeName];
const type = aggregations[typeName];
// the "real" type object from the response
const realType = aggregations[typeName];
const realType = aggregationResponse[typeName];
// both conditions must apply, else we have an error somewhere
if (isESNestedAggregation(type) && isNestedAggregation(realType)) {

52
src/storage/elasticsearch/elasticsearch.ts
View File

@@ -33,17 +33,16 @@ import moment from 'moment';
import {MailQueue} from '../../notification/mail-queue';
import {Bulk} from '../bulk-storage';
import {Database} from '../database';
import {buildAggregations, parseAggregations} from './aggregations';
import {parseAggregations} from './aggregations';
import * as Monitoring from './monitoring';
import {buildQuery, buildSort} from './query';
import {aggregations, putTemplate} from './templating';
import {
AggregationResponse,
AggregationSchema,
ElasticsearchConfig, ElasticsearchObject,
ElasticsearchQueryDisMaxConfig,
ElasticsearchQueryQueryStringConfig,
} from './common';
import * as Monitoring from './monitoring';
import {buildQuery, buildSort} from './query';
import {checkESTemplate, putTemplate} from './templating';
} from './types/elasticsearch';
/**
* Matches index names such as stapps_<type>_<source>_<random suffix>
@@ -60,11 +59,6 @@ export class Elasticsearch implements Database {
*/
static readonly INDEX_UID_LENGTH = 8;
/**
* Holds aggregations
*/
aggregationsSchema: AggregationSchema;
/**
* Holds a map of all elasticsearch indices that are available to search
*/
@@ -207,11 +201,6 @@ export class Elasticsearch implements Database {
this.aliasMap = {};
this.ready = false;
checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ?
process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false);
this.aggregationsSchema = buildAggregations();
this.mailQueue = mailQueue;
}
@@ -221,6 +210,8 @@ export class Elasticsearch implements Database {
private async getAliasMap() {
// delay after which alias map will be fetched again
const RETRY_INTERVAL = 5000;
// maximum number of retries
const RETRY_COUNT = 3;
// create a list of old indices that are not in use
const oldIndicesToDelete: string[] = [];
@@ -233,17 +224,24 @@ export class Elasticsearch implements Database {
[K in SCThingType]: unknown
};
};
};
} | undefined;
try {
aliases = (await this.client.indices.getAlias({})).body;
} catch (error) {
await Logger.error('Failed getting alias map:', error);
setTimeout(async () => {
return this.getAliasMap();
}, RETRY_INTERVAL); // retry after a delay
for(const retry of [...Array(RETRY_COUNT)].map((_, i) => i+1)) {
if (typeof aliases !== 'undefined') {
break;
}
try {
const aliasResponse = await this.client.indices.getAlias({});
aliases = aliasResponse.body;
} catch (error) {
Logger.warn('Failed getting alias map:', error);
Logger.warn(`Retrying in ${RETRY_INTERVAL} milliseconds. (${retry} of ${RETRY_COUNT})`);
await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL));
}
}
return;
if (typeof aliases === 'undefined') {
throw Error(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
}
for (const index in aliases) {
@@ -566,7 +564,7 @@ export class Elasticsearch implements Database {
const searchRequest: RequestParams.Search = {
body: {
aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes)
aggs: aggregations,
query: buildQuery(params, this.config, esConfig),
},
from: params.from,
@@ -603,7 +601,7 @@ export class Elasticsearch implements Database {
// read the aggregations from elasticsearch and parse them to facets by our configuration
if (typeof response.body.aggregations !== 'undefined') {
facets = parseAggregations(this.aggregationsSchema, response.body.aggregations as AggregationResponse);
facets = parseAggregations(response.body.aggregations as AggregationResponse);
}
return {

2
src/storage/elasticsearch/query.ts
View File

@@ -45,7 +45,7 @@ import {
ESTermFilter,
ESTypeFilter,
ScriptSort,
} from './common';
} from './types/elasticsearch';
/**
* Escapes any reserved character that would otherwise not be accepted by Elasticsearch

2
src/storage/elasticsearch/templates/.gitignore vendored
View File

@@ -1,2 +0,0 @@
*.json
*.txt

84
src/storage/elasticsearch/templating.ts
View File

@@ -15,77 +15,18 @@
*/
import {Client} from '@elastic/elasticsearch';
import {SCThingType} from '@openstapps/core';
import {getProjectReflection} from '@openstapps/core-tools/lib/common';
import {generateTemplate} from '@openstapps/core-tools/lib/mapping';
import {Logger} from '@openstapps/logger';
import {existsSync, writeFileSync} from 'fs';
import {readFile} from 'fs-extra';
// tslint:disable-next-line:no-implicit-dependencies
import {AggregationSchema} from '@openstapps/es-mapping-generator/src/types/aggregation';
// tslint:disable-next-line:no-implicit-dependencies
import {ElasticsearchTemplateCollection} from '@openstapps/es-mapping-generator/src/types/mapping';
import {readFileSync} from 'fs';
import {resolve} from 'path';
import sanitize = require('sanitize-filename');
import {configFile, coreVersion} from '../../common';
const dirPath = resolve('src', 'storage', 'elasticsearch', 'templates');
export const aggregationsPath = resolve(dirPath, sanitize(`${coreVersion}-aggregations.json`, {replacement: '-'}));
const templateErrorPath = resolve(dirPath, sanitize(`${coreVersion}-template-[type].error.json`, {replacement: '-'}));
const aggregationsErrorPath = resolve(dirPath, sanitize(`${coreVersion}-aggregations.error.json`, {replacement: '-'}));
const errorReportPath = resolve(dirPath, sanitize(`${coreVersion}-error-report.txt`, {replacement: '-'}));
const mappingsPath = resolve('node_modules', '@openstapps', 'core', 'lib','mappings');
/**
* Check if the correct template exists
*/
export function checkESTemplate(forceUpdate: boolean) {
// as the forced mapping update is only meant for development, print a warning if it is enabled
if (forceUpdate) {
Logger.warn('CAUTION: Force update of the mapping files is enabled. This causes the backend to ignore' +
' existing mapping files on start.');
}
// we don't exactly know which files are there, so we just check if the aggregations exist
// for the current core version
if (forceUpdate || !existsSync(aggregationsPath)) {
Logger.info(`No mapping for Core version ${coreVersion} found, starting automatic mapping generation. ` +
`This may take a while.`);
const map = generateTemplate(getProjectReflection(resolve('node_modules', '@openstapps', 'core', 'src')),
configFile.backend.mappingIgnoredTags, false);
export const mappings = JSON.parse(readFileSync(resolve(mappingsPath, 'mappings.json'), 'utf-8')) as ElasticsearchTemplateCollection;
export const aggregations = JSON.parse(readFileSync(resolve(mappingsPath, 'aggregations.json'), 'utf-8')) as AggregationSchema;
if (map.errors.length > 0) {
for (const type of Object.keys(map.mappings)) {
writeFileSync(getTemplatePath(Object.keys(map.mappings[type].mappings)[0] as SCThingType, true),
// tslint:disable-next-line:no-magic-numbers
JSON.stringify(map.mappings[type], null, 2));
}
// tslint:disable-next-line:no-magic-numbers
writeFileSync(aggregationsErrorPath, JSON.stringify(map.aggregations, null, 2));
writeFileSync(errorReportPath, `ERROR REPORT FOR CORE VERSION ${coreVersion}\n${map.errors.join('\n')}`);
void Logger.error(`There were errors while generating the template, and the backend cannot continue. A list of ` +
`all errors can be found at ${errorReportPath}. To resolve this` +
` issue by hand you can go to "${templateErrorPath}" and "${aggregationsErrorPath}", then correct the issues` +
` manually and move the files to the template paths and "${aggregationsPath}" respectively.`);
process.exit(1);
} else {
Logger.ok('Mapping files were generated successfully.');
for (const type of Object.keys(map.mappings)) {
writeFileSync(getTemplatePath(Object.keys(map.mappings[type].mappings)[0] as SCThingType, false),
// tslint:disable-next-line:no-magic-numbers
JSON.stringify(map.mappings[type], null, 2));
}
writeFileSync(aggregationsPath, JSON.stringify(map.aggregations));
}
} else {
Logger.info(`Using existing mappings for core version ${coreVersion}`);
}
}
/**
* Generates the path to the template of an SCThingType
*
* @param type the type for the path
* @param error whether an error occurred in the file
*/
function getTemplatePath(type: SCThingType, error = false): string {
return resolve(dirPath, sanitize(`${coreVersion}-template-${type}${error ? '.error' : ''}.json`, {replacement: '-'}));
}
/**
* Re-applies all interfaces for every type
@@ -107,13 +48,10 @@ export async function refreshAllTemplates(client: Client) {
* @param client An elasticsearch client to use
*/
export async function putTemplate(client: Client, type: SCThingType) {
let out = type.toLowerCase();
while (out.includes(' ')) {
out = out.replace(' ', '_');
}
const sanitizedType = `template_${type.replace(/\s/g, '_')}`;
return client.indices.putTemplate({
body: JSON.parse((await readFile(getTemplatePath(type), 'utf8')).toString()),
name: `template_${out}`,
body: mappings[sanitizedType],
name: sanitizedType,
});
}

58
src/storage/elasticsearch/common.ts → src/storage/elasticsearch/types/elasticsearch.ts
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019 StApps
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
@@ -13,15 +13,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCThingType} from '@openstapps/core';
import {SCThing} from '@openstapps/core';
import {
ESAggMatchAllFilter,
ESAggTypeFilter, ESNestedAggregation,
ESTermsFilter,
} from '@openstapps/core-tools/lib/mappings/aggregation-definitions';
import {SCThing, SCThingType} from '@openstapps/core';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {NameList} from 'elasticsearch';
// tslint:disable-next-line:no-implicit-dependencies
@@ -67,14 +60,6 @@ export interface BucketAggregation {
doc_count?: number;
}
/**
* Checks if the type is a BucketAggregation
* @param agg the type to check
*/
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
return typeof agg !== 'number';
}
/**
* An aggregation that contains more aggregations nested inside
*/
@@ -90,21 +75,6 @@ export interface NestedAggregation {
[name: string]: BucketAggregation | number;
}
/**
* Checks if the type is a NestedAggregation
* @param agg the type to check
*/
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
return typeof (agg as BucketAggregation).buckets === 'undefined';
}
/**
* An elasticsearch bucket aggregation
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
*/
export interface AggregationSchema {
[aggregationName: string]: ESTermsFilter | ESNestedAggregation;
}
/**
* A configuration for using the Dis Max Query
@@ -348,30 +318,6 @@ export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<n
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
/**
* Checks if the parameter is of type ESTermsFilter
* @param agg the value to check
*/
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
return typeof (agg as ESTermsFilter).terms !== 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
* @param agg the value to check
*/
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
}
/**
* Checks if the parameter is of type
*
* @param filter the filter to narrow the type of
*/
export function isESAggMatchAllFilter(filter: ESAggTypeFilter | ESAggMatchAllFilter): filter is ESAggMatchAllFilter {
return filter.hasOwnProperty('match_all');
}
/**
* An elasticsearch type filter

64
src/storage/elasticsearch/types/guards.ts Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
ESAggMatchAllFilter,
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter,
// tslint:disable-next-line:no-implicit-dependencies we're just using the types here
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {BucketAggregation, NestedAggregation} from './elasticsearch';
/**
* Checks if the type is a BucketAggregation
* @param agg the type to check
*/
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
return typeof agg !== 'number';
}
/**
* Checks if the type is a NestedAggregation
* @param agg the type to check
*/
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
return typeof (agg as BucketAggregation).buckets === 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
* @param agg the value to check
*/
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
return typeof (agg as ESTermsFilter).terms !== 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
* @param agg the value to check
*/
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
}
/**
* Checks if the parameter is of type
*
* @param filter the filter to narrow the type of
*/
export function isESAggMatchAllFilter(filter: ESAggTypeFilter | ESAggMatchAllFilter): filter is ESAggMatchAllFilter {
return filter.hasOwnProperty('match_all');
}