Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 17:12:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: update to of elasticsearch 8.4

Browse Source
This commit is contained in:
Thea Schöbl
2023-04-28 12:43:31 +00:00
committed by Rainer Killinger
parent 515a6eeea5
commit c9b83b5d71
42 changed files with 1843 additions and 2242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/storage/elasticsearch/aggregations.ts
View File

@@ -13,73 +13,45 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCFacet, SCThingType} from '@openstapps/core';
import {aggregations} from './templating';
import {AggregationResponse} from './types/elasticsearch';
import {
isBucketAggregation,
isESAggMatchAllFilter,
isESNestedAggregation,
isESTermsFilter,
isNestedAggregation,
} from './types/guards';
AggregateName,
AggregationsAggregate,
AggregationsFiltersAggregate,
AggregationsMultiTermsBucket,
} from '@elastic/elasticsearch/lib/api/types';
import {SCFacet, SCThingType} from '@openstapps/core';
/**
* Parses elasticsearch aggregations (response from es) to facets for the app
*
* @param aggregationResponse - aggregations response from elasticsearch
*/
export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
export function parseAggregations(
aggregationResponse: Record<AggregateName, AggregationsAggregate>,
): SCFacet[] {
const facets: SCFacet[] = [];
// get all names of the types an aggregation is on
for (const typeName in aggregations) {
if (aggregations.hasOwnProperty(typeName) && aggregationResponse.hasOwnProperty(typeName)) {
// the type object from the schema
const type = aggregations[typeName];
// the "real" type object from the response
const realType = aggregationResponse[typeName];
for (const aggregateName in aggregationResponse) {
const aggregation = aggregationResponse[aggregateName] as AggregationsMultiTermsBucket;
const type = aggregateName === '@all' ? {} : {onlyOnType: aggregateName as SCThingType};
// both conditions must apply, else we have an error somewhere
if (isESNestedAggregation(type) && isNestedAggregation(realType)) {
for (const fieldName in type.aggs) {
if (type.aggs.hasOwnProperty(fieldName) && realType.hasOwnProperty(fieldName)) {
// the field object from the schema
const field = type.aggs[fieldName];
// the "real" field object from the response
const realField = realType[fieldName];
for (const field in aggregation) {
const fieldAggregate = aggregation[field] as AggregationsFiltersAggregate;
if (typeof fieldAggregate !== 'object') continue;
// this should always be true in theory...
if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) {
const facet: SCFacet = {
buckets: realField.buckets.map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key,
};
}),
field: fieldName,
};
// if it's not for all types then create the appropriate field and set the type name
if (!isESAggMatchAllFilter(type.filter)) {
facet.onlyOnType = type.filter.type.value as SCThingType;
}
facets.push(facet);
}
}
}
// the last part here means that it is a bucket aggregation
} else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) {
facets.push({
buckets: realType.buckets.map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key,
};
}),
field: typeName,
});
}
const buckets = Object.values(fieldAggregate.buckets).map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key as string,
};
});
if (buckets.length === 0) continue;
facets.push({
buckets,
field,
...type,
});
}
}

401
src/storage/elasticsearch/elasticsearch.ts
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019 StApps
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
@@ -13,58 +13,47 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch';
import {Client, events} from '@elastic/elasticsearch';
import {
SCBulkResponse,
SCConfigFile,
SCFacet,
SCSearchQuery,
SCSearchResponse,
SCThings,
SCThingType,
SCUuid,
} from '@openstapps/core';
AggregateName,
AggregationsMultiTermsBucket,
IndicesGetAliasResponse,
IndicesUpdateAliasesAction,
SearchHit,
SearchResponse,
} from '@elastic/elasticsearch/lib/api/types';
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
import moment from 'moment';
import {MailQueue} from '../../notification/mail-queue';
import {Bulk} from '../bulk-storage';
import {Database} from '../database';
import {parseAggregations} from './aggregations';
import * as Monitoring from './monitoring';
import {buildQuery, buildSort} from './query';
import {buildQuery} from './query/query';
import {buildSort} from './query/sort';
import {aggregations, putTemplate} from './templating';
import {
AggregationResponse,
ElasticsearchConfig,
ElasticsearchObject,
ElasticsearchQueryDisMaxConfig,
ElasticsearchQueryQueryStringConfig,
} from './types/elasticsearch';
/**
* Matches index names such as stapps_<type>_<source>_<random suffix>
*/
const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
} from './types/elasticsearch-config';
import {ALL_INDICES_QUERY, getThingIndexName, parseIndexName, VALID_INDEX_REGEX} from './util';
import {removeInvalidAliasChars} from './util/alias';
import {noUndefined} from './util/no-undefined';
import {retryCatch, RetryOptions} from './util/retry';
/**
* A database interface for elasticsearch
*/
export class Elasticsearch implements Database {
/**
* Length of the index UID used for generation of its name
*/
static readonly INDEX_UID_LENGTH = 8;
/**
* Holds a map of all elasticsearch indices that are available to search
*/
aliasMap: {
// each scType has a alias which can contain multiple sources
// each scType has an alias which can contain multiple sources
[scType: string]: {
// each source is assigned a index name in elasticsearch
// each source is assigned an index name in elasticsearch
[source: string]: string;
};
};
@@ -97,89 +86,11 @@ export class Elasticsearch implements Database {
return 'http://localhost:9200';
}
/**
* Gets the index name in elasticsearch for one SCThingType
*
* @param type SCThingType of data in the index
* @param source source of data in the index
* @param bulk bulk process which created this index
*/
static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) {
let out = type.toLowerCase();
while (out.includes(' ')) {
out = out.replace(' ', '_');
}
return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`;
}
/**
* Provides the index UID (for its name) from the bulk UID
*
* @param uid Bulk UID
*/
static getIndexUID(uid: SCUuid) {
return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH));
}
/**
* Generates a string which matches all indices
*/
static getListOfAllIndices(): string {
// map each SC type in upper camel case
return 'stapps_*_*_*';
}
/**
* Checks for invalid character in alias names and removes them
*
* @param alias The alias name
* @param uid The UID of the current bulk (for debugging purposes)
*/
static removeAliasChars(alias: string, uid: string | undefined): string {
let formattedAlias = alias;
// spaces are included in some types, replace them with underscores
if (formattedAlias.includes(' ')) {
formattedAlias = formattedAlias.trim();
formattedAlias = formattedAlias.split(' ').join('_');
}
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
if (formattedAlias.includes(value)) {
formattedAlias = formattedAlias.replace(value, '');
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
having the same alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
for (const value of ['-', '_', '+']) {
if (formattedAlias.charAt(0) === value) {
formattedAlias = formattedAlias.slice(1);
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
if (formattedAlias === '.' || formattedAlias === '..') {
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
return 'alias_placeholder';
}
if (formattedAlias.includes(':')) {
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
Elasticsearch versions!`);
}
return formattedAlias;
}
/**
* Create a new interface for elasticsearch
*
* @param config an assembled config file
* @param mailQueue a mailqueue for monitoring
* @param mailQueue a mail queue for monitoring
*/
constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) {
if (
@@ -192,7 +103,7 @@ export class Elasticsearch implements Database {
this.client = new Client({
node: Elasticsearch.getElasticsearchUrl(),
});
this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse<unknown>) => {
this.client.diagnostic.on(events.REQUEST, async (error: Error | null, result: unknown) => {
if (error !== null) {
await Logger.error(error);
}
@@ -210,73 +121,40 @@ export class Elasticsearch implements Database {
/**
* Gets a map which contains each alias and all indices that are associated with each alias
*/
private async getAliasMap() {
// delay after which alias map will be fetched again
const RETRY_INTERVAL = 5000;
// maximum number of retries
const RETRY_COUNT = 3;
// create a list of old indices that are not in use
const oldIndicesToDelete: string[] = [];
let aliases:
| {
[index: string]: {
/**
* Aliases of an index
*/
aliases: {
[K in SCThingType]: unknown;
};
};
}
| undefined;
for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) {
if (typeof aliases !== 'undefined') {
break;
}
try {
const aliasResponse = await this.client.indices.getAlias({});
aliases = aliasResponse.body;
} catch (error) {
private async getAliasMap(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}) {
const aliasResponse = await retryCatch({
maxRetries: 10,
retryInterval: 2000,
doAction: () => this.client.indices.getAlias(),
onFailedAttempt: (attempt, error, {maxRetries, retryInterval}) => {
Logger.warn('Failed getting alias map:', error);
Logger.warn(`Retrying in ${RETRY_INTERVAL} milliseconds. (${retry} of ${RETRY_COUNT})`);
await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL));
}
}
Logger.warn(`Retrying in ${retryInterval} milliseconds. (${attempt} of ${maxRetries})`);
},
onFail: ({maxRetries}) => {
throw new TypeError(`Failed to retrieve alias map after ${maxRetries} attempts!`);
},
...retryOptions,
});
if (typeof aliases === 'undefined') {
throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
}
const aliases = Object.entries(aliasResponse)
.filter(([index]) => !index.startsWith('.'))
.map(([index, alias]) => ({
index,
alias,
...parseIndexName(index),
}));
for (const index in aliases) {
if (aliases.hasOwnProperty(index)) {
const matches = indexRegex.exec(index);
if (matches !== null) {
const type = matches[1];
const source = matches[2];
// check if there is an alias for the current index
// check that alias equals type
const hasAlias = type in aliases[index].aliases;
if (hasAlias) {
if (typeof this.aliasMap[type] === 'undefined') {
this.aliasMap[type] = {};
}
this.aliasMap[type][source] = index;
} else {
oldIndicesToDelete.push(index);
}
}
}
for (const {type, index, source} of aliases.filter(({type, alias}) => type in alias.aliases)) {
this.aliasMap[type] = this.aliasMap[type] || {};
this.aliasMap[type][source] = index;
}
this.ready = true;
// delete old indices that are not used in any alias
if (oldIndicesToDelete.length > 0) {
const unusedIndices = aliases.filter(({type, alias}) => !(type in alias.aliases)).map(({index}) => index);
if (unusedIndices.length > 0) {
await this.client.indices.delete({
index: oldIndicesToDelete,
index: unusedIndices,
});
Logger.warn(`Deleted old indices: oldIndicesToDelete`);
}
@@ -291,8 +169,8 @@ export class Elasticsearch implements Database {
* @param uid an UID to use for the search
* @returns an elasticsearch object containing the thing
*/
private async getObject(uid: SCUuid): Promise<ElasticsearchObject<SCThings> | undefined> {
const searchResponse: ApiResponse<SearchResponse<SCThings>> = await this.client.search({
private async getObject(uid: SCUuid): Promise<SearchHit<SCThings> | undefined> {
const searchResponse = await this.client.search<SCThings>({
body: {
query: {
term: {
@@ -303,43 +181,44 @@ export class Elasticsearch implements Database {
},
},
from: 0,
index: Elasticsearch.getListOfAllIndices(),
index: ALL_INDICES_QUERY,
size: 1,
});
// return data from response
return searchResponse.body.hits.hits[0];
return searchResponse.hits.hits[0];
}
/**
* Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
*
* @param bulk the bulk process that was created
*/
public async bulkCreated(bulk: Bulk): Promise<void> {
// if our es instance is not ready yet, we cannot serve this request
private async prepareBulkWrite(bulk: Bulk): Promise<{index: string; alias: string}> {
if (!this.ready) {
throw new Error('No connection to elasticsearch established yet.');
}
// index name for elasticsearch
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
// there already is an index with this type and source. We will index the new one and switch the alias to it
// the old one is deleted
const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
const index = getThingIndexName(bulk.type, bulk.source, bulk);
const alias = removeInvalidAliasChars(bulk.type, bulk.uid);
if (typeof this.aliasMap[alias] === 'undefined') {
this.aliasMap[alias] = {};
}
if (!indexRegex.test(index)) {
if (!VALID_INDEX_REGEX.test(index)) {
throw new Error(
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
);
}
return {index, alias};
}
/**
* Should be called, when a new bulk was created. Creates a new index and applies the mapping to the index
*
* @param bulk the bulk process that was created
*/
public async bulkCreated(bulk: Bulk): Promise<void> {
const {index} = await this.prepareBulkWrite(bulk);
// re-apply the index template before each new bulk operation
await putTemplate(this.client, bulk.type);
await this.client.indices.create({
@@ -355,8 +234,7 @@ export class Elasticsearch implements Database {
* @param bulk the bulk process that is expired
*/
public async bulkExpired(bulk: Bulk): Promise<void> {
// index name for elasticsearch
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
const index: string = getThingIndexName(bulk.type, bulk.source, bulk);
Logger.info('Bulk expired. Deleting index', index);
@@ -375,31 +253,11 @@ export class Elasticsearch implements Database {
* @param bulk the new bulk process that should replace the old one with same type and source
*/
public async bulkUpdated(bulk: Bulk): Promise<void> {
// if our es instance is not ready yet, we cannot serve this request
if (!this.ready) {
throw new Error('No connection to elasticsearch established yet.');
}
const {index, alias} = await this.prepareBulkWrite(bulk);
// index name for elasticsearch
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
// alias for the indices
const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
if (typeof this.aliasMap[alias] === 'undefined') {
this.aliasMap[alias] = {};
}
if (!indexRegex.test(index)) {
throw new Error(
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
);
}
// create the new index if it does not exists
// create the new index if it does not exist
// eslint-disable-next-line unicorn/no-await-expression-member
if (!(await this.client.indices.exists({index})).body) {
if (!(await this.client.indices.exists({index}))) {
// re-apply the index template before each new bulk operation
await putTemplate(this.client, bulk.type);
await this.client.indices.create({
@@ -412,7 +270,7 @@ export class Elasticsearch implements Database {
// add our new index to the alias
// this was type safe with @types/elasticsearch, the new package however provides no type definitions
const actions: IndicesUpdateAliasesParamsAction[] = [
const actions: IndicesUpdateAliasesAction[] = [
{
add: {index: index, alias: alias},
},
@@ -427,16 +285,10 @@ export class Elasticsearch implements Database {
}
// refresh the index (fsync changes)
await this.client.indices.refresh({
index: index,
});
await this.client.indices.refresh({index});
// execute our alias actions
await this.client.indices.updateAliases({
body: {
actions,
},
});
await this.client.indices.updateAliases({actions});
// swap the index in our aliasMap
this.aliasMap[alias][bulk.source] = index;
@@ -457,7 +309,7 @@ export class Elasticsearch implements Database {
public async get(uid: SCUuid): Promise<SCThings> {
const object = await this.getObject(uid);
if (typeof object === 'undefined') {
if (typeof object?._source === 'undefined') {
throw new TypeError('Item not found.');
}
@@ -467,7 +319,7 @@ export class Elasticsearch implements Database {
/**
* Initialize the elasticsearch database (call all needed methods)
*/
public async init(): Promise<void> {
public async init(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}): Promise<void> {
const monitoringConfiguration = this.config.internal.monitoring;
if (typeof monitoringConfiguration !== 'undefined') {
@@ -480,7 +332,7 @@ export class Elasticsearch implements Database {
await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
}
return this.getAliasMap();
return this.getAliasMap(retryOptions);
}
/**
@@ -490,7 +342,7 @@ export class Elasticsearch implements Database {
* @param bulk the bulk process which item belongs to
*/
public async post(object: SCThings, bulk: Bulk): Promise<void> {
const object_: SCThings & {creation_date: string} = {
const thing: SCThings & {creation_date: string} = {
...object,
creation_date: moment().format(),
};
@@ -499,7 +351,7 @@ export class Elasticsearch implements Database {
// check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid)
if (typeof item !== 'undefined') {
const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk);
const indexOfNew = getThingIndexName(thing.type, bulk.source, bulk);
const oldIndex = item._index;
// new item doesn't replace the old one
@@ -509,22 +361,23 @@ export class Elasticsearch implements Database {
) {
throw new Error(
// eslint-disable-next-line unicorn/no-null
`Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`,
`Object "${thing.uid}" already exists. Object was: ${JSON.stringify(thing, null, 2)}`,
);
}
}
// regular bulk update (item gets replaced when bulk is updated)
const searchResponse = await this.client.create({
body: object_,
id: object_.uid,
index: Elasticsearch.getIndex(object_.type, bulk.source, bulk),
const searchResponse = await this.client.create<SCThings>({
document: thing,
id: thing.uid,
index: getThingIndexName(thing.type, bulk.source, bulk),
timeout: '90s',
type: object_.type,
});
if (!searchResponse.body.created) {
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`);
if (searchResponse.result !== 'created') {
throw new Error(
`Object creation Error (${searchResponse.result}: Instance was: ${JSON.stringify(thing)}`,
);
}
}
@@ -543,7 +396,6 @@ export class Elasticsearch implements Database {
},
id: object.uid,
index: item._index,
type: object.type.toLowerCase(),
});
return;
@@ -562,65 +414,46 @@ export class Elasticsearch implements Database {
throw new TypeError('Database is undefined. You have to configure the query build');
}
// create elasticsearch configuration out of data from database configuration
const esConfig: ElasticsearchConfig = {
name: this.config.internal.database.name as 'elasticsearch',
version: this.config.internal.database.version as string,
};
if (typeof this.config.internal.database.query !== 'undefined') {
esConfig.query = this.config.internal.database.query as
query: this.config.internal.database.query as
| ElasticsearchQueryDisMaxConfig
| ElasticsearchQueryQueryStringConfig;
}
| ElasticsearchQueryQueryStringConfig
| undefined,
};
const searchRequest: RequestParams.Search = {
body: {
aggs: aggregations,
query: buildQuery(parameters, this.config, esConfig),
},
const query = {
aggs: aggregations,
query: buildQuery(parameters, this.config, esConfig),
from: parameters.from,
index: Elasticsearch.getListOfAllIndices(),
index: ALL_INDICES_QUERY,
size: parameters.size,
sort: typeof parameters.sort !== 'undefined' ? buildSort(parameters.sort) : undefined,
};
if (typeof parameters.sort !== 'undefined') {
searchRequest.body.sort = buildSort(parameters.sort);
}
// perform the search against elasticsearch
const response: ApiResponse<SearchResponse<SCThings>> = await this.client.search(searchRequest);
// gather pagination information
const pagination = {
count: response.body.hits.hits.length,
offset: typeof parameters.from === 'number' ? parameters.from : 0,
total: response.body.hits.total,
};
// gather statistics about this search
const stats = {
time: response.body.took,
};
// we only directly return the _source documents
// elasticsearch provides much more information, the user shouldn't see
const data = response.body.hits.hits.map(hit => {
return hit._source; // SCThing
});
let facets: SCFacet[] = [];
// read the aggregations from elasticsearch and parse them to facets by our configuration
if (typeof response.body.aggregations !== 'undefined') {
facets = parseAggregations(response.body.aggregations as AggregationResponse);
}
const response: SearchResponse<SCThings> = await this.client.search(query);
return {
data,
facets,
pagination,
stats,
data: response.hits.hits
.map(hit => {
// we only directly return the _source documents
// elasticsearch provides much more information, the user shouldn't see
return hit._source;
})
.filter(noUndefined),
facets:
typeof response.aggregations !== 'undefined'
? parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>)
: [],
pagination: {
count: response.hits.hits.length,
offset: typeof parameters.from === 'number' ? parameters.from : 0,
total:
typeof response.hits.total === 'number' ? response.hits.total : response.hits.total?.value ?? 0,
},
stats: {
time: response.took,
},
};
}
}

13
src/storage/elasticsearch/monitoring.ts
View File

@@ -13,7 +13,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch';
import {Client} from '@elastic/elasticsearch';
import {SearchRequest} from '@elastic/elasticsearch/lib/api/types';
import {
SCMonitoringConfiguration,
SCMonitoringLogAction,
@@ -23,9 +24,6 @@ import {
SCThings,
} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
import {SearchResponse} from 'elasticsearch';
import cron from 'node-cron';
import {MailQueue} from '../../notification/mail-queue';
@@ -131,12 +129,11 @@ export async function setUp(
cron.schedule(trigger.executionTime, async () => {
// execute watch (search->condition->action)
const result: ApiResponse<SearchResponse<SCThings>> = await esClient.search(
watcher.query as RequestParams.Search,
);
const result = await esClient.search<SCThings>(watcher.query as SearchRequest);
// check conditions
const total = result.body.hits.total;
const total =
typeof result.hits.total === 'number' ? result.hits.total : result.hits.total?.value ?? -1;
for (const condition of watcher.conditions) {
if (conditionFails(condition, total)) {

501
src/storage/elasticsearch/query.ts
View File

@@ -1,501 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
SCBackendConfigurationSearchBoostingContext,
SCBackendConfigurationSearchBoostingType,
SCConfigFile,
SCSearchBooleanFilter,
SCSearchContext,
SCSearchFilter,
SCSearchQuery,
SCSearchSort,
SCSportCoursePriceGroup,
SCThingsField,
} from '@openstapps/core';
import {
ElasticsearchConfig,
ESBooleanFilter,
ESBooleanFilterArguments,
ESDateRange,
ESDateRangeFilter,
ESFunctionScoreQuery,
ESFunctionScoreQueryFunction,
ESGenericRange,
ESGenericSort,
ESGeoBoundingBoxFilter,
ESGeoDistanceFilter,
ESGeoDistanceFilterArguments,
ESGeoDistanceSort,
ESGeoDistanceSortArguments,
ESGeoShapeFilter,
ESNumericRangeFilter,
ESRangeFilter,
ESTermFilter,
ESTypeFilter,
ScriptSort,
} from './types/elasticsearch';
/**
* Escapes any reserved character that would otherwise not be accepted by Elasticsearch
*
* Elasticsearch as the following reserved characters:
* + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
* It is possible to use all, with the exception of < and >, of them by escaping them with a \
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*
* @param string_ the string to escape the characters from
*/
function escapeESReservedCharacters(string_: string): string {
return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
}
/**
* Builds a boolean filter. Returns an elasticsearch boolean filter
*
* @param booleanFilter a search boolean filter for the retrieval of the data
* @returns elasticsearch boolean arguments object
*/
export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<unknown> {
const result: ESBooleanFilterArguments<unknown> = {
minimum_should_match: 0,
must: [],
must_not: [],
should: [],
};
if (booleanFilter.arguments.operation === 'and') {
result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
}
if (booleanFilter.arguments.operation === 'or') {
result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
result.minimum_should_match = 1;
}
if (booleanFilter.arguments.operation === 'not') {
result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
}
return result;
}
/**
* Converts Array of Filters to elasticsearch query-syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildFilter(
filter: SCSearchFilter,
):
| ESTermFilter
| ESGeoDistanceFilter
| ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter>
| ESGeoShapeFilter
| ESBooleanFilter<unknown>
| ESRangeFilter {
switch (filter.type) {
case 'value':
return Array.isArray(filter.arguments.value)
? {
terms: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
}
: {
term: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
};
case 'availability':
const scope = filter.arguments.scope?.charAt(0) ?? 's';
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
return {
range: {
[filter.arguments.field]: {
gte: `${time}/${scope}`,
lt: `${time}+1${scope}/${scope}`,
relation: 'intersects',
},
},
};
case 'distance':
const geoObject: ESGeoDistanceFilterArguments = {
distance: `${filter.arguments.distance}m`,
[`${filter.arguments.field}.point.coordinates`]: {
lat: filter.arguments.position[1],
lon: filter.arguments.position[0],
},
};
return {
geo_distance: geoObject,
};
case 'boolean':
return {
bool: buildBooleanFilter(filter),
};
case 'numeric range':
const numericRangeObject: ESGenericRange<number> = {
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
const numericRangeFilter: ESNumericRangeFilter = {range: {}};
numericRangeFilter.range[filter.arguments.field] = numericRangeObject;
return numericRangeFilter;
case 'date range':
const dateRangeObject: ESDateRange = {
format: filter.arguments.format,
time_zone: filter.arguments.timeZone,
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
const dateRangeFilter: ESDateRangeFilter = {range: {}};
dateRangeFilter.range[filter.arguments.field] = dateRangeObject;
return dateRangeFilter;
case 'geo':
// TODO: on ES upgrade, use just geo_shape filters
const geoShapeFilter: ESGeoShapeFilter = {
geo_shape: {
/**
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
*/
// @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type.
ignore_unmapped: true,
[`${filter.arguments.field}.polygon`]: {
shape: filter.arguments.shape,
relation: filter.arguments.spatialRelation,
},
},
};
if (
(typeof filter.arguments.spatialRelation === 'undefined' ||
filter.arguments.spatialRelation === 'intersects') &&
filter.arguments.shape.type === 'envelope'
) {
return {
bool: {
minimum_should_match: 1,
should: [
geoShapeFilter,
{
geo_bounding_box: {
/**
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
*/
ignore_unmapped: true,
[`${filter.arguments.field}.point.coordinates`]: {
top_left: filter.arguments.shape.coordinates[0],
bottom_right: filter.arguments.shape.coordinates[1],
},
},
},
],
},
};
}
return geoShapeFilter;
}
}
/**
* Builds scoring functions from boosting config
*
* @param boostings Backend boosting configuration for contexts and types
* @param context The context of the app from where the search was initiated
*/
function buildFunctions(
boostings: SCBackendConfigurationSearchBoostingContext,
context: SCSearchContext | undefined,
): ESFunctionScoreQueryFunction[] {
// default context
let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes(
boostings['default' as SCSearchContext],
);
if (typeof context !== 'undefined' && context !== 'default') {
// specific context provided, extend default context with additional boosts
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
}
return functions;
}
/**
* Creates boost functions for all type boost configurations
*
* @param boostingTypes Array of type boosting configurations
*/
function buildFunctionsForBoostingTypes(
boostingTypes: SCBackendConfigurationSearchBoostingType[],
): ESFunctionScoreQueryFunction[] {
const functions: ESFunctionScoreQueryFunction[] = [];
for (const boostingForOneSCType of boostingTypes) {
const typeFilter: ESTypeFilter = {
type: {
value: boostingForOneSCType.type,
},
};
functions.push({
filter: typeFilter,
weight: boostingForOneSCType.factor,
});
if (typeof boostingForOneSCType.fields !== 'undefined') {
const fields = boostingForOneSCType.fields;
for (const fieldName in boostingForOneSCType.fields) {
if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
const boostingForOneField = fields[fieldName];
for (const value in boostingForOneField) {
if (boostingForOneField.hasOwnProperty(value)) {
const factor = boostingForOneField[value];
// build term filter
const termFilter: ESTermFilter = {
term: {},
};
termFilter.term[`${fieldName}.raw`] = value;
functions.push({
filter: {
bool: {
must: [typeFilter, termFilter],
should: [],
},
},
weight: factor,
});
}
}
}
}
}
}
return functions;
}
/**
* Builds body for Elasticsearch requests
*
* @param parameters Parameters for querying the backend
* @param defaultConfig Default configuration of the backend
* @param elasticsearchConfig Elasticsearch configuration
* @returns ElasticsearchQuery (body of a search-request)
*/
export function buildQuery(
parameters: SCSearchQuery,
defaultConfig: SCConfigFile,
elasticsearchConfig: ElasticsearchConfig,
): ESFunctionScoreQuery {
// if config provides an minMatch parameter we use query_string instead of match query
let query;
if (typeof elasticsearchConfig.query === 'undefined') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: '90%',
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
};
} else if (elasticsearchConfig.query.queryType === 'query_string') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
};
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
if (parameters.query !== '*') {
query = {
dis_max: {
boost: 1.2,
queries: [
{
match: {
name: {
boost: elasticsearchConfig.query.matchBoosting,
cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
fuzziness: elasticsearchConfig.query.fuzziness,
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
},
},
{
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query:
typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
},
],
tie_breaker: elasticsearchConfig.query.tieBreaker,
},
};
}
} else {
throw new Error(
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
);
}
const functionScoreQuery: ESFunctionScoreQuery = {
function_score: {
functions: buildFunctions(defaultConfig.internal.boostings, parameters.context),
query: {
bool: {
minimum_should_match: 0, // if we have no should, nothing can match
must: [],
should: [],
},
},
score_mode: 'multiply',
},
};
const mustMatch = functionScoreQuery.function_score.query.bool.must;
if (Array.isArray(mustMatch)) {
if (typeof query !== 'undefined') {
mustMatch.push(query);
}
if (typeof parameters.filter !== 'undefined') {
mustMatch.push(buildFilter(parameters.filter));
}
}
return functionScoreQuery;
}
/**
* converts query to
*
* @param sorts Sorting rules to apply to the data that is being queried
* @returns an array of sort queries
*/
export function buildSort(sorts: SCSearchSort[]): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
return sorts.map(sort => {
switch (sort.type) {
case 'generic':
const esGenericSort: ESGenericSort = {};
esGenericSort[sort.arguments.field] = sort.order;
return esGenericSort;
case 'ducet':
const esDucetSort: ESGenericSort = {};
esDucetSort[`${sort.arguments.field}.sort`] = sort.order;
return esDucetSort;
case 'distance':
const arguments_: ESGeoDistanceSortArguments = {
mode: 'avg',
order: sort.order,
unit: 'm',
};
arguments_[`${sort.arguments.field}.point.coordinates`] = {
lat: sort.arguments.position[1],
lon: sort.arguments.position[0],
};
return {
_geo_distance: arguments_,
};
case 'price':
return {
_script: {
order: sort.order,
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
type: 'number' as const,
},
};
}
});
}
/**
* Provides a script for sorting search results by prices
*
* @param universityRole User group which consumes university services
* @param field Field in which wanted offers with prices are located
*/
export function buildPriceSortScript(
universityRole: keyof SCSportCoursePriceGroup,
field: SCThingsField,
): string {
return `
// initialize the sort value with the maximum
double price = Double.MAX_VALUE;
// if we have any offers
if (params._source.containsKey('${field}')) {
// iterate through all offers
for (offer in params._source.${field}) {
// if this offer contains a role specific price
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
// if the role specific price is smaller than the cheapest we found
if (offer.prices.${universityRole} < price) {
// set the role specific price as cheapest for now
price = offer.prices.${universityRole};
}
} else { // we have no role specific price for our role in this offer
// if the default price of this offer is lower than the cheapest we found
if (offer.price < price) {
// set this price as the cheapest
price = offer.price;
}
}
}
}
// return cheapest price for our role
return price;
`;
}

75
src/storage/elasticsearch/query/boost/boost-functions.ts Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCBackendConfigurationSearchBoostingType} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Creates boost functions for all type boost configurations
*
* @param boostingTypes Array of type boosting configurations
*/
export function buildFunctionsForBoostingTypes(
boostingTypes: SCBackendConfigurationSearchBoostingType[],
): QueryDslFunctionScoreContainer[] {
const functions: QueryDslFunctionScoreContainer[] = [];
for (const boostingForOneSCType of boostingTypes) {
const typeFilter: QueryDslSpecificQueryContainer<'term'> = {
term: {
type: boostingForOneSCType.type,
},
};
functions.push({
filter: typeFilter,
weight: boostingForOneSCType.factor,
});
if (typeof boostingForOneSCType.fields !== 'undefined') {
const fields = boostingForOneSCType.fields;
for (const fieldName in boostingForOneSCType.fields) {
if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
const boostingForOneField = fields[fieldName];
for (const value in boostingForOneField) {
if (boostingForOneField.hasOwnProperty(value)) {
const factor = boostingForOneField[value];
// build term filter
const termFilter: QueryDslSpecificQueryContainer<'term'> = {
term: {},
};
termFilter.term[`${fieldName}.raw`] = value;
functions.push({
filter: {
bool: {
must: [typeFilter, termFilter],
should: [],
},
},
weight: factor,
});
}
}
}
}
}
}
return functions;
}

38
src/storage/elasticsearch/query/boost/scoring-functions.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCBackendConfigurationSearchBoostingContext, SCSearchContext} from '@openstapps/core';
import {buildFunctionsForBoostingTypes} from './boost-functions';
/**
* Builds scoring functions from boosting config
*
* @param boostings Backend boosting configuration for contexts and types
* @param context The context of the app from where the search was initiated
*/
export function buildScoringFunctions(
boostings: SCBackendConfigurationSearchBoostingContext,
context: SCSearchContext | undefined,
): QueryDslFunctionScoreContainer[] {
// default context
let functions = buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]);
if (typeof context !== 'undefined' && context !== 'default') {
// specific context provided, extend default context with additional boosts
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
}
return functions;
}

47
src/storage/elasticsearch/query/filter.ts Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchFilter} from '@openstapps/core';
import {buildBooleanFilter} from './filters/boolean';
import {buildAvailabilityFilter} from './filters/availability';
import {buildDateRangeFilter} from './filters/date-range';
import {buildDistanceFilter} from './filters/distance';
import {buildGeoFilter} from './filters/geo';
import {buildNumericRangeFilter} from './filters/numeric-range';
import {buildValueFilter} from './filters/value';
/**
* Converts Array of Filters to elasticsearch query-syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildFilter(filter: SCSearchFilter): QueryDslQueryContainer {
switch (filter.type) {
case 'value':
return buildValueFilter(filter);
case 'availability':
return buildAvailabilityFilter(filter);
case 'distance':
return buildDistanceFilter(filter);
case 'boolean':
return buildBooleanFilter(filter);
case 'numeric range':
return buildNumericRangeFilter(filter);
case 'date range':
return buildDateRangeFilter(filter);
case 'geo':
return buildGeoFilter(filter);
}
}

38
src/storage/elasticsearch/query/filters/availability.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCSearchAvailabilityFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts an availability filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildAvailabilityFilter(
filter: SCSearchAvailabilityFilter,
): QueryDslSpecificQueryContainer<'range'> {
const scope = filter.arguments.scope?.charAt(0) ?? 's';
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
return {
range: {
[filter.arguments.field]: {
gte: `${time}/${scope}`,
lt: `${time}+1${scope}/${scope}`,
relation: 'intersects',
},
},
};
}

49
src/storage/elasticsearch/query/filters/boolean.ts Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchBooleanFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
import {buildFilter} from '../filter';
/**
* Converts a boolean filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildBooleanFilter(filter: SCSearchBooleanFilter): QueryDslSpecificQueryContainer<'bool'> {
const result: QueryDslBoolQuery = {
minimum_should_match: 0,
must: [],
must_not: [],
should: [],
};
if (filter.arguments.operation === 'and') {
result.must = filter.arguments.filters.map(it => buildFilter(it));
}
if (filter.arguments.operation === 'or') {
result.should = filter.arguments.filters.map(it => buildFilter(it));
result.minimum_should_match = 1;
}
if (filter.arguments.operation === 'not') {
result.must_not = filter.arguments.filters.map(it => buildFilter(it));
}
return {
bool: result,
};
}

48
src/storage/elasticsearch/query/filters/date-range.ts Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchDateRangeFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a date range filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildDateRangeFilter(
filter: SCSearchDateRangeFilter,
): QueryDslSpecificQueryContainer<'range'> {
const dateRangeObject: QueryDslDateRangeQuery = {
format: filter.arguments.format,
time_zone: filter.arguments.timeZone,
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
return {
range: {
[filter.arguments.field]: dateRangeObject,
},
};
}

38
src/storage/elasticsearch/query/filters/distance.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchDistanceFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a distance filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildDistanceFilter(
filter: SCSearchDistanceFilter,
): QueryDslSpecificQueryContainer<'geo_distance'> {
const geoObject: QueryDslGeoDistanceQuery = {
distance: `${filter.arguments.distance}m`,
[`${filter.arguments.field}.point.coordinates`]: {
lat: filter.arguments.position[1],
lon: filter.arguments.position[0],
},
};
return {
geo_distance: geoObject,
};
}

33
src/storage/elasticsearch/query/filters/geo.ts Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCGeoFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a geo filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> {
return {
geo_shape: {
ignore_unmapped: true,
[`${filter.arguments.field}.polygon`]: {
shape: filter.arguments.shape,
relation: filter.arguments.spatialRelation,
},
},
};
}

47
src/storage/elasticsearch/query/filters/numeric-range.ts Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchNumericRangeFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a numeric range filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildNumericRangeFilter(
filter: SCSearchNumericRangeFilter,
): QueryDslSpecificQueryContainer<'range'> {
const numericRangeObject: QueryDslNumberRangeQuery = {
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
return {
range: {
[filter.arguments.field]: numericRangeObject,
},
};
}

37
src/storage/elasticsearch/query/filters/value.ts Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCSearchValueFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a value filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildValueFilter(
filter: SCSearchValueFilter,
): QueryDslSpecificQueryContainer<'term'> | QueryDslSpecificQueryContainer<'terms'> {
return Array.isArray(filter.arguments.value)
? {
terms: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
}
: {
term: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
};
}

114
src/storage/elasticsearch/query/query.ts Normal file
View File

@@ -0,0 +1,114 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCConfigFile, SCSearchQuery} from '@openstapps/core';
import {ElasticsearchConfig} from '../types/elasticsearch-config';
import {buildFilter} from './filter';
import {buildScoringFunctions} from './boost/scoring-functions';
/**
* Builds body for Elasticsearch requests
*
* @param parameters Parameters for querying the backend
* @param defaultConfig Default configuration of the backend
* @param elasticsearchConfig Elasticsearch configuration
* @returns ElasticsearchQuery (body of a search-request)
*/
export function buildQuery(
parameters: SCSearchQuery,
defaultConfig: SCConfigFile,
elasticsearchConfig: ElasticsearchConfig,
): QueryDslQueryContainer {
// if config provides an minMatch parameter we use query_string instead of match query
let query;
if (typeof elasticsearchConfig.query === 'undefined') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: '90%',
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
};
} else if (elasticsearchConfig.query.queryType === 'query_string') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
};
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
if (typeof parameters.query === 'string' && parameters.query !== '*') {
query = {
dis_max: {
boost: 1.2,
queries: [
{
match: {
name: {
boost: elasticsearchConfig.query.matchBoosting,
fuzziness: elasticsearchConfig.query.fuzziness,
query: parameters.query,
},
},
},
{
query_string: {
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: parameters.query,
},
},
],
tie_breaker: elasticsearchConfig.query.tieBreaker,
},
};
}
} else {
throw new Error(
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
);
}
const functionScoreQuery: QueryDslQueryContainer = {
function_score: {
functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
query: {
bool: {
minimum_should_match: 0, // if we have no should, nothing can match
must: [],
should: [],
},
},
score_mode: 'multiply',
},
};
const mustMatch = functionScoreQuery.function_score?.query?.bool?.must;
if (Array.isArray(mustMatch)) {
if (typeof query !== 'undefined') {
mustMatch.push(query);
}
if (typeof parameters.filter !== 'undefined') {
mustMatch.push(buildFilter(parameters.filter));
}
}
return functionScoreQuery;
}

41
src/storage/elasticsearch/query/sort.ts Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Sort} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchSort} from '@openstapps/core';
import {buildDistanceSort} from './sort/distance';
import {buildDucetSort} from './sort/ducet';
import {buildGenericSort} from './sort/generic';
import {buildPriceSort} from './sort/price';
/**
* converts query to
*
* @param sorts Sorting rules to apply to the data that is being queried
* @returns an array of sort queries
*/
export function buildSort(sorts: SCSearchSort[]): Sort {
return sorts.map(sort => {
switch (sort.type) {
case 'generic':
return buildGenericSort(sort);
case 'ducet':
return buildDucetSort(sort);
case 'distance':
return buildDistanceSort(sort);
case 'price':
return buildPriceSort(sort);
}
});
}

35
src/storage/elasticsearch/query/sort/distance.ts Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCDistanceSort} from '@openstapps/core';
/**
* Converts a distance sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildDistanceSort(sort: SCDistanceSort): SortOptions {
return {
_geo_distance: {
mode: 'avg',
order: sort.order,
unit: 'm',
[`${sort.arguments.field}.point.coordinates`]: {
lat: sort.arguments.position[1],
lon: sort.arguments.position[0],
},
},
};
}

27
src/storage/elasticsearch/query/sort/ducet.ts Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCDucetSort} from '@openstapps/core';
/**
* Converts a ducet sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildDucetSort(sort: SCDucetSort): SortOptions {
return {
[`${sort.arguments.field}.sort`]: sort.order,
};
}

27
src/storage/elasticsearch/query/sort/generic.ts Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCGenericSort} from '@openstapps/core';
/**
* Converts a generic sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildGenericSort(sort: SCGenericSort): SortOptions {
return {
[sort.arguments.field]: sort.order,
};
}

71
src/storage/elasticsearch/query/sort/price.ts Normal file
View File

@@ -0,0 +1,71 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
/**
* Converts a price sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildPriceSort(sort: SCPriceSort): SortOptions {
return {
_script: {
order: sort.order,
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
type: 'number' as const,
},
};
}
/**
* Provides a script for sorting search results by prices
*
* @param universityRole User group which consumes university services
* @param field Field in which wanted offers with prices are located
*/
export function buildPriceSortScript(
universityRole: keyof SCSportCoursePriceGroup,
field: SCThingsField,
): string {
return `
// initialize the sort value with the maximum
double price = Double.MAX_VALUE;
// if we have any offers
if (params._source.containsKey('${field}')) {
// iterate through all offers
for (offer in params._source.${field}) {
// if this offer contains a role specific price
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
// if the role specific price is smaller than the cheapest we found
if (offer.prices.${universityRole} < price) {
// set the role specific price as cheapest for now
price = offer.prices.${universityRole};
}
} else { // we have no role specific price for our role in this offer
// if the default price of this offer is lower than the cheapest we found
if (offer.price < price) {
// set this price as the cheapest
price = offer.price;
}
}
}
}
// return cheapest price for our role
return price;
`;
}

11
src/storage/elasticsearch/templating.ts
View File

@@ -29,17 +29,6 @@ export const aggregations = JSON.parse(
readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
) as AggregationSchema;
/**
* Re-applies all interfaces for every type
*
* @param client An elasticsearch client to use
*/
export async function refreshAllTemplates(client: Client) {
for (const type of Object.values(SCThingType)) {
await putTemplate(client, type as SCThingType);
}
}
/**
* Prepares all indices
*

121
src/storage/elasticsearch/types/elasticsearch-config.ts Normal file
View File

@@ -0,0 +1,121 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* A configuration for using the Dis Max Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryDisMaxConfig {
/**
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
*/
cutoffFrequency: number;
/**
* The maximum allowed Levenshtein Edit Distance (or number of edits)
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
*/
fuzziness: number | string;
/**
* Increase the importance (relevance score) of a field
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
*/
matchBoosting: number;
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'dis_max' which is a union of its subqueries
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
*/
queryType: 'dis_max';
/**
* Changes behavior of default calculation of the score when multiple results match
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
*/
tieBreaker: number;
}
/**
* A configuration for using Query String Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryQueryStringConfig {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*/
queryType: 'query_string';
}
/**
* An config file for the elasticsearch database interface
*
* The config file extends the SCConfig file by further defining how the database property
*/
export interface ElasticsearchConfigFile {
/**
* Configuration that is not visible to clients
*/
internal: {
/**
* Database configuration
*/
database: ElasticsearchConfig;
};
}
/**
* An elasticsearch configuration
*/
export interface ElasticsearchConfig {
/**
* Name of the database
*/
name: 'elasticsearch';
/**
* Configuration for using queries
*/
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
/**
* Version of the used elasticsearch
*/
version: string;
}

605
src/storage/elasticsearch/types/elasticsearch.ts
View File

@@ -1,605 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCThing, SCThingType} from '@openstapps/core';
// we only have the @types package because some things type definitions are still missing from the official
import {NameList} from 'elasticsearch';
import {Polygon, Position} from 'geojson';
/**
* An elasticsearch aggregation bucket
*/
interface Bucket {
/**
* Number of documents in the aggregation bucket
*/
doc_count: number;
/**
* Text representing the documents in the bucket
*/
key: string;
}
/**
* An elasticsearch aggregation response
*/
export interface AggregationResponse {
/**
* The individual aggregations
*/
[field: string]: BucketAggregation | NestedAggregation;
}
/**
* An elasticsearch bucket aggregation
*/
export interface BucketAggregation {
/**
* Buckets in an aggregation
*/
buckets: Bucket[];
/**
* Number of documents in an aggregation
*/
doc_count?: number;
}
/**
* An aggregation that contains more aggregations nested inside
*/
export interface NestedAggregation {
/**
* Number of documents in an aggregation
*/
doc_count: number;
/**
* Any nested responses
*/
[name: string]: BucketAggregation | number;
}
/**
* A configuration for using the Dis Max Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryDisMaxConfig {
/**
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
*/
cutoffFrequency: number;
/**
* The maximum allowed Levenshtein Edit Distance (or number of edits)
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
*/
fuzziness: number | string;
/**
* Increase the importance (relevance score) of a field
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
*/
matchBoosting: number;
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'dis_max' which is a union of its subqueries
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
*/
queryType: 'dis_max';
/**
* Changes behavior of default calculation of the score when multiple results match
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
*/
tieBreaker: number;
}
/**
* A configuration for using Query String Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryQueryStringConfig {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*/
queryType: 'query_string';
}
/**
* A hit in an elasticsearch search result
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
*/
export interface ElasticsearchObject<T extends SCThing> {
/**
* Unique identifier of a document (object)
*/
_id: string;
/**
* The index to which the document belongs
*/
_index: string;
/**
* Relevancy of the document to a query
*/
_score: number;
/**
* The original JSON representing the body of the document
*/
_source: T;
/**
* The document's mapping type
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
*/
_type: string;
/**
* Version of the document
*/
_version?: number;
/**
* Used to index the same field in different ways for different purposes
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
*/
fields?: NameList;
/**
* Used to highlight search results on one or more fields
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
highlight?: any;
/**
* Used in when nested/children documents match the query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
inner_hits?: any;
/**
* Queries that matched for documents in results
*/
matched_queries?: string[];
/**
* Sorting definition
*/
sort?: string[];
}
/**
* An config file for the elasticsearch database interface
*
* The config file extends the SCConfig file by further defining how the database property
*/
export interface ElasticsearchConfigFile {
/**
* Configuration that is not visible to clients
*/
internal: {
/**
* Database configuration
*/
database: ElasticsearchConfig;
};
}
/**
* An elasticsearch configuration
*/
export interface ElasticsearchConfig {
/**
* Name of the database
*/
name: 'elasticsearch';
/**
* Configuration for using queries
*/
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
/**
* Version of the used elasticsearch
*/
version: string;
}
/**
* An elasticsearch term filter
*/
export type ESTermFilter =
| {
/**
* Definition of a term to match
*/
term: {
[fieldName: string]: string;
};
}
| {
/**
* Definition of terms to match (or)
*/
terms: {
[fieldName: string]: string[];
};
};
export interface ESGenericRange<T> {
/**
* Greater than field
*/
gt?: T;
/**
* Greater or equal than field
*/
gte?: T;
/**
* Less than field
*/
lt?: T;
/**
* Less or equal than field
*/
lte?: T;
/**
* Relation of the range to a range field
*
* Intersects: Both ranges intersect
* Contains: Search range contains field range
* Within: Field range contains search range
*/
relation?: 'intersects' | 'within' | 'contains';
}
interface ESGenericRangeFilter<G, T extends ESGenericRange<G>> {
/**
* Range filter definition
*/
range: {
[fieldName: string]: T;
};
}
export interface ESDateRange extends ESGenericRange<string> {
/**
* Optional date format override
*/
format?: string;
/**
* Optional timezone specifier
*/
time_zone?: string;
}
export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<number>>;
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
/**
* An elasticsearch type filter
*/
export interface ESTypeFilter {
/**
* Type filter definition
*/
type: {
/**
* Type name (SCThingType) to filter with
*/
value: SCThingType;
};
}
/**
* Filter arguments for an elasticsearch geo distance filter
*/
export interface ESGeoDistanceFilterArguments {
/**
* The radius of the circle centred on the specified location
*/
distance: string;
[fieldName: string]:
| {
/**
* Latitude
*/
lat: number;
/**
* Longitude
*/
lon: number;
}
| string;
}
/**
* An elasticsearch geo distance filter
*/
export interface ESGeoDistanceFilter {
/**
* @see ESGeoDistanceFilterArguments
*/
geo_distance: ESGeoDistanceFilterArguments;
}
/**
* A rectangular geo shape, representing the top-left and bottom-right corners
*
* This is an extension of the Geojson type
* http://geojson.org/geojson-spec.html
*/
export interface ESEnvelope {
/**
* The top-left and bottom-right corners of the bounding box
*/
coordinates: [Position, Position];
/**
* The type of the geometry
*/
type: 'envelope';
}
/**
* An Elasticsearch geo bounding box filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
*/
export interface ESGeoBoundingBoxFilter {
/**
* An Elasticsearch geo bounding box filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
*/
geo_bounding_box: {
[fieldName: string]: {
/**
* Geo Shape
*/
bottom_right: Position;
/**
* Geo Shape
*/
top_left: Position;
};
};
}
/**
* An Elasticsearch geo shape filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html
*/
export interface ESGeoShapeFilter {
geo_shape: {
[fieldName: string]: {
/**
* Relation of the two shapes
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_spatial_relations
*/
relation?: 'intersects' | 'disjoint' | 'within' | 'contains';
/**
* Geo Shape
*/
shape: Polygon | ESEnvelope;
};
};
}
/**
* Filter arguments for an elasticsearch boolean filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
*/
export interface ESBooleanFilterArguments<T> {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minimum_should_match?: number;
/**
* The clause (query) must appear in matching documents and will contribute to the score.
*/
must?: T[];
/**
* The clause (query) must not appear in the matching documents.
*/
must_not?: T[];
/**
* The clause (query) should appear in the matching document.
*/
should?: T[];
}
/**
* An elasticsearch boolean filter
*/
export interface ESBooleanFilter<T> {
/**
* @see ESBooleanFilterArguments
*/
bool: ESBooleanFilterArguments<T>;
}
/**
* An elasticsearch function score query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
*/
export interface ESFunctionScoreQuery {
/**
* Function score definition
*/
function_score: {
/**
* Functions that compute score for query results (documents)
*
* @see ESFunctionScoreQueryFunction
*/
functions: ESFunctionScoreQueryFunction[];
/**
* @see ESBooleanFilter
*/
query: ESBooleanFilter<unknown>;
/**
* Specifies how the computed scores are combined
*/
score_mode: 'multiply';
};
}
/**
* An function for an elasticsearch functions score query
*/
export interface ESFunctionScoreQueryFunction {
/**
* Function is applied only if a document matches the given filtering query
*/
filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;
/**
* Weight (importance) of the filter
*/
weight: number;
}
/**
* An elasticsearch generic sort
*/
export interface ESGenericSort {
[field: string]: string;
}
/**
* Sort arguments for an elasticsearch geo distance sort
*/
export interface ESGeoDistanceSortArguments {
/**
* What value to pick for sorting
*/
mode: 'avg' | 'max' | 'median' | 'min';
/**
* Order
*/
order: 'asc' | 'desc';
/**
* Value unit
*/
unit: 'm';
[field: string]:
| {
/**
* Latitude
*/
lat: number;
/**
* Longitude
*/
lon: number;
}
| string;
}
/**
* An elasticsearch geo distance sort
*/
export interface ESGeoDistanceSort {
/**
* @see ESGeoDistanceFilterArguments
*/
_geo_distance: ESGeoDistanceSortArguments;
}
/**
* An elasticsearch script sort
*/
export interface ScriptSort {
/**
* A script
*/
_script: {
/**
* Order
*/
order: 'asc' | 'desc';
/**
* The custom script used for sorting
*/
script: string;
/**
* What type is being sorted
*/
type: 'number' | 'string';
};
}

69
src/storage/elasticsearch/types/guards.ts
View File

@@ -1,69 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
ESAggMatchAllFilter,
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter,
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {BucketAggregation, NestedAggregation} from './elasticsearch';
/**
* Checks if the type is a BucketAggregation
*
* @param agg the type to check
*/
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
return typeof agg !== 'number';
}
/**
* Checks if the type is a NestedAggregation
*
* @param agg the type to check
*/
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
return typeof (agg as BucketAggregation).buckets === 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
*
* @param agg the value to check
*/
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
return typeof (agg as ESTermsFilter).terms !== 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
*
* @param agg the value to check
*/
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
}
/**
* Checks if the parameter is of type
*
* @param filter the filter to narrow the type of
*/
export function isESAggMatchAllFilter(
filter: ESAggTypeFilter | ESAggMatchAllFilter,
): filter is ESAggMatchAllFilter {
return filter.hasOwnProperty('match_all');
}

20
src/storage/elasticsearch/types/util.ts Normal file
View File

@@ -0,0 +1,20 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
export type QueryDslSpecificQueryContainer<T extends keyof QueryDslQueryContainer> = Required<
Pick<QueryDslQueryContainer, T>
>;

60
src/storage/elasticsearch/util/alias.ts Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from '@openstapps/logger';
/**
* Checks for invalid character in alias names and removes them
*
* @param alias The alias name
* @param uid The UID of the current bulk (for debugging purposes)
*/
export function removeInvalidAliasChars(alias: string, uid: string | undefined): string {
let formattedAlias = alias;
// spaces are included in some types, replace them with underscores
if (formattedAlias.includes(' ')) {
formattedAlias = formattedAlias.trim();
formattedAlias = formattedAlias.split(' ').join('_');
}
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
if (formattedAlias.includes(value)) {
formattedAlias = formattedAlias.replace(value, '');
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
having the same alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
for (const value of ['-', '_', '+']) {
if (formattedAlias.charAt(0) === value) {
formattedAlias = formattedAlias.slice(1);
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
if (formattedAlias === '.' || formattedAlias === '..') {
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
return 'alias_placeholder';
}
if (formattedAlias.includes(':')) {
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
Elasticsearch versions!`);
}
return formattedAlias;
}

63
src/storage/elasticsearch/util/index.ts Normal file
View File

@@ -0,0 +1,63 @@
import {SCBulkResponse, SCThingType, SCUuid} from '@openstapps/core';
/**
* Length of the index UID used for generation of its name
*/
export const INDEX_UID_LENGTH = 8;
/**
* A string which matches all indices
*/
export const ALL_INDICES_QUERY = 'stapps_*_*_*';
/**
* Matches index names such as stapps_<type>_<source>_<random suffix>
*/
export const VALID_INDEX_REGEX = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
export interface ParsedIndexName {
type: SCThingType;
source: string;
randomSuffix: string;
}
/**
*
*/
export function parseIndexName(index: string): ParsedIndexName {
const match = VALID_INDEX_REGEX.exec(index);
if (!match) {
throw new SyntaxError(`Invalid index name ${index}!`);
}
return {
type: match[1] as SCThingType,
source: match[2],
randomSuffix: match[3],
};
}
/**
* Gets the index name in elasticsearch for one SCThingType
*
* @param type SCThingType of data in the index
* @param source source of data in the index
* @param bulk bulk process which created this index
*/
export function getThingIndexName(type: SCThingType, source: string, bulk: SCBulkResponse) {
let out = type.toLowerCase();
while (out.includes(' ')) {
out = out.replace(' ', '_');
}
return `stapps_${out}_${source}_${getIndexUID(bulk.uid)}`;
}
/**
* Provides the index UID (for its name) from the bulk UID
*
* @param uid Bulk UID
*/
export function getIndexUID(uid: SCUuid) {
return uid.slice(0, Math.max(0, INDEX_UID_LENGTH));
}

21
src/storage/elasticsearch/util/no-undefined.ts Normal file
View File

@@ -0,0 +1,21 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* Type guard for filter functions
*/
export function noUndefined<T>(item: T | undefined): item is T {
return typeof item !== 'undefined';
}

38
src/storage/elasticsearch/util/retry.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
export interface RetryOptions<T> {
maxRetries: number;
retryInterval: number;
doAction: () => Promise<T>;
onFailedAttempt: (attempt: number, error: unknown, options: RetryOptions<T>) => void;
onFail: (options: RetryOptions<T>) => never;
}
/**
* Retries a throwing function at a set interval, until a maximum amount of attempts
*/
export async function retryCatch<T>(options: RetryOptions<T>): Promise<T> {
for (let attempt = 0; attempt < options.maxRetries; attempt++) {
try {
return await options.doAction();
} catch (error) {
options.onFailedAttempt(attempt, error, options);
await new Promise(resolve => setTimeout(resolve, options.retryInterval));
}
}
options.onFail(options);
}