Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2025-12-11 08:46:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'backend/master'

Browse Source
This commit is contained in:
Thea Schöbl
2023-05-23 14:26:05 +02:00
parent 857e829e8b 1ea219b1ef
commit f69edc6393
45 changed files with 2879 additions and 5706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/backend/.dockerignore Normal file
View File

@@ -0,0 +1,4 @@
.idea/
.git/
.vscode/
Dockerfile

12
backend/backend/.gitlab-ci.yml
View File

@@ -46,7 +46,7 @@ integration:
services:
- docker:dind
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --abort-on-container-exit --exit-code-from apicli
tags:
- gitlab-org-docker
@@ -110,7 +110,7 @@ ci:
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
- docker push $IMAGETAG_BASE
except:
@@ -124,22 +124,22 @@ ci:
.publish_version_template: &publish_version_template
script:
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
- export VERSION=$(echo -n "$CI_BUILD_REF_NAME" | cut -c 2-)
- export VERSION=$(echo -n "$CI_COMMIT_REF_NAME" | cut -c 2-)
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
- export IMAGETAG_VERSION=$IMAGETAG_BASE:$VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_VERSION -t $IMAGETAG_CORE_VERSION .
- docker push $IMAGETAG_BASE
.publish_branch_template: &publish_branch_template
script:
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_BUILD_REF_NAME
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
- docker push $IMAGETAG_BASE

16
backend/backend/CHANGELOG.md
View File

@@ -1,3 +1,19 @@
# [1.0.0](https://gitlab.com/openstapps/backend/compare/v0.6.0...v1.0.0) (2023-05-08)
### Bug Fixes
* openapi docs generation ([4ebe44a](https://gitlab.com/openstapps/backend/commit/4ebe44a5a7a1b7bfd0aa5b84d47d4056d3068ffe))
* rename deprecated Gitlab CI variables ([3471591](https://gitlab.com/openstapps/backend/commit/3471591a7d458df70447c8dac91f96f3c83e763c))
* semster boosting ([515a6ee](https://gitlab.com/openstapps/backend/commit/515a6eeea56305a37510d99b9f84a6b118b66f8a))
### Features
* update to of elasticsearch 8.4 ([c9b83b5](https://gitlab.com/openstapps/backend/commit/c9b83b5d71610f82bd1d99e837e29ad445758aea))
# [0.6.0](https://gitlab.com/openstapps/backend/compare/v0.5.0...v0.6.0) (2023-01-30)

7
backend/backend/README.md
View File

@@ -18,9 +18,10 @@ you with everything you need to run this backend.
# Local usage for development purposes
## Requirements
* Elasticsearch (5.6)
* Elasticsearch (8.4)
- [ICU analysis plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html)
- OR Docker
* Node.js (~14) / NPM
* Docker
### Startup Behaviour
@@ -34,7 +35,7 @@ with the backend. To save you some work we provide a
[docker image](https://gitlab.com/openstapps/database) which
only needs to be executed to work with the backend.
Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:master`
Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:latest`
Elasticsearch should be running at port 9200 now. If you have problems with
getting elasticsearch to work, have a look in the

6
backend/backend/config/elasticsearch.ts
View File

@@ -1,6 +1,6 @@
// tslint:disable:no-default-export
// tslint:disable:no-magic-numbers
import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch';
import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch-config';
/**
* This is the default configuration for elasticsearch (a database)
@@ -19,13 +19,13 @@ const config: ElasticsearchConfigFile = {
internal: {
database: {
name: 'elasticsearch',
version: '5.6',
version: '8.4',
query: {
minMatch: '75%',
queryType: 'dis_max',
matchBoosting: 1.3,
fuzziness: 'AUTO',
cutoffFrequency: 0.0,
cutoffFrequency: 0,
tieBreaker: 0,
},
},

2
backend/backend/integration-test.yml
View File

@@ -15,7 +15,7 @@ services:
elasticsearch:
ports:
- "9200:9200"
image: "registry.gitlab.com/openstapps/database:master"
image: "registry.gitlab.com/openstapps/database:latest"
apicli:
image: "registry.gitlab.com/openstapps/api/cli:latest"

5177
backend/backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

32
backend/backend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@openstapps/backend",
"version": "0.6.0",
"version": "1.0.0",
"description": "A reference implementation for a StApps backend",
"license": "AGPL-3.0-only",
"author": "André Bierlein <andre.mt.bierlein@gmail.com>",
@@ -19,7 +19,7 @@
"changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0 && git add CHANGELOG.md",
"check-configuration": "openstapps-configuration",
"compile": "rimraf lib && tsc && prepend lib/cli.js '#!/usr/bin/env node\n'",
"documentation": "typedoc --includeVersion --out docs --readme README.md --listInvalidSymbolLinks --entryPointStrategy expand src && openstapps-core-tools openapi ./node_modules/@openstapps/core/lib ./docs/openapi && redoc-cli bundle docs/openapi/openapi.json -o docs/openapi/index.html",
"documentation": "typedoc --includeVersion --out docs --readme README.md --listInvalidSymbolLinks --entryPointStrategy expand src && openstapps-core-tools openapi ./node_modules/@openstapps/core/lib ./docs/openapi && openapi build-docs docs/openapi/openapi.json -o docs/openapi/index.html",
"version": "npm run changelog",
"prepublishOnly": "npm ci && npm run build",
"preversion": "npm run prepublishOnly",
@@ -27,17 +27,18 @@
"start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js",
"start-debug": "STAPPS_LOG_LEVEL=31 NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js --require ts-node/register",
"test": "npm run test-unit && npm run test-integration",
"test-unit": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
"test-unit": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
"test-integration": "docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --build --abort-on-container-exit --exit-code-from apicli",
"lint": "eslint -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/",
"lint:fix": "eslint --fix -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/"
},
"dependencies": {
"@elastic/elasticsearch": "5.6.22",
"@openstapps/core": "0.74.0",
"@elastic/elasticsearch": "8.4.0",
"@openstapps/core": "1.0.1",
"@openstapps/core-tools": "0.34.0",
"@openstapps/logger": "1.1.1",
"@types/node": "14.18.36",
"@redocly/cli": "1.0.0-beta.125",
"@types/node": "14.18.43",
"config": "3.3.9",
"cors": "2.8.5",
"express": "4.18.2",
@@ -46,26 +47,25 @@
"got": "11.8.6",
"moment": "2.29.4",
"morgan": "1.10.0",
"nock": "13.3.0",
"nock": "13.3.1",
"node-cache": "5.1.2",
"node-cron": "3.0.2",
"nodemailer": "6.9.1",
"prom-client": "14.1.1",
"prom-client": "14.2.0",
"promise-queue": "2.2.5",
"ts-node": "10.9.1",
"uuid": "8.3.2"
},
"devDependencies": {
"@openstapps/configuration": "0.34.0",
"@openstapps/es-mapping-generator": "0.4.0",
"@openstapps/es-mapping-generator": "0.6.0",
"@openstapps/eslint-config": "1.1.0",
"@testdeck/mocha": "0.3.3",
"@types/chai": "4.3.4",
"@types/chai-as-promised": "7.1.5",
"@types/config": "3.3.0",
"@types/cors": "2.8.13",
"@types/elasticsearch": "5.0.40",
"@types/express": "4.17.16",
"@types/express": "4.17.17",
"@types/geojson": "1.0.6",
"@types/mocha": "10.0.1",
"@types/morgan": "1.9.4",
@@ -80,9 +80,10 @@
"chai": "4.3.7",
"chai-as-promised": "7.1.1",
"conventional-changelog-cli": "2.2.2",
"eslint": "8.33.0",
"eslint-config-prettier": "8.6.0",
"eslint-plugin-jsdoc": "39.7.4",
"cross-env": "7.0.3",
"eslint": "8.39.0",
"eslint-config-prettier": "8.8.0",
"eslint-plugin-jsdoc": "39.9.1",
"eslint-plugin-prettier": "4.2.1",
"eslint-plugin-unicorn": "43.0.2",
"get-port": "5.1.1",
@@ -90,8 +91,7 @@
"mocked-env": "1.3.5",
"nyc": "15.1.0",
"prepend-file-cli": "1.0.6",
"prettier": "2.8.3",
"redoc-cli": "0.13.20",
"prettier": "2.8.8",
"rimraf": "3.0.2",
"sinon": "14.0.2",
"sinon-express-mock": "2.2.1",

6
backend/backend/src/app.ts
View File

@@ -64,6 +64,7 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
}),
);
/* istanbul ignore if */
if (process.env.PROMETHEUS_MIDDLEWARE === 'true') {
app.use(getPrometheusMiddleware());
}
@@ -142,7 +143,10 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
});
// validate config file
await validator.addSchemas(path.join('node_modules', '@openstapps', 'core', 'lib', 'schema'));
await validator.addSchemas(
// eslint-disable-next-line unicorn/prefer-module
path.join(path.dirname(require.resolve('@openstapps/core/package.json')), 'lib', 'schema'),
);
// validate the config file
const configValidation = validator.validate(configFile, 'SCConfigFile');

84
backend/backend/src/storage/elasticsearch/aggregations.ts
View File

@@ -13,73 +13,45 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCFacet, SCThingType} from '@openstapps/core';
import {aggregations} from './templating';
import {AggregationResponse} from './types/elasticsearch';
import {
isBucketAggregation,
isESAggMatchAllFilter,
isESNestedAggregation,
isESTermsFilter,
isNestedAggregation,
} from './types/guards';
AggregateName,
AggregationsAggregate,
AggregationsFiltersAggregate,
AggregationsMultiTermsBucket,
} from '@elastic/elasticsearch/lib/api/types';
import {SCFacet, SCThingType} from '@openstapps/core';
/**
* Parses elasticsearch aggregations (response from es) to facets for the app
*
* @param aggregationResponse - aggregations response from elasticsearch
*/
export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
export function parseAggregations(
aggregationResponse: Record<AggregateName, AggregationsAggregate>,
): SCFacet[] {
const facets: SCFacet[] = [];
// get all names of the types an aggregation is on
for (const typeName in aggregations) {
if (aggregations.hasOwnProperty(typeName) && aggregationResponse.hasOwnProperty(typeName)) {
// the type object from the schema
const type = aggregations[typeName];
// the "real" type object from the response
const realType = aggregationResponse[typeName];
for (const aggregateName in aggregationResponse) {
const aggregation = aggregationResponse[aggregateName] as AggregationsMultiTermsBucket;
const type = aggregateName === '@all' ? {} : {onlyOnType: aggregateName as SCThingType};
// both conditions must apply, else we have an error somewhere
if (isESNestedAggregation(type) && isNestedAggregation(realType)) {
for (const fieldName in type.aggs) {
if (type.aggs.hasOwnProperty(fieldName) && realType.hasOwnProperty(fieldName)) {
// the field object from the schema
const field = type.aggs[fieldName];
// the "real" field object from the response
const realField = realType[fieldName];
for (const field in aggregation) {
const fieldAggregate = aggregation[field] as AggregationsFiltersAggregate;
if (typeof fieldAggregate !== 'object') continue;
// this should always be true in theory...
if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) {
const facet: SCFacet = {
buckets: realField.buckets.map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key,
};
}),
field: fieldName,
};
// if it's not for all types then create the appropriate field and set the type name
if (!isESAggMatchAllFilter(type.filter)) {
facet.onlyOnType = type.filter.type.value as SCThingType;
}
facets.push(facet);
}
}
}
// the last part here means that it is a bucket aggregation
} else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) {
facets.push({
buckets: realType.buckets.map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key,
};
}),
field: typeName,
});
}
const buckets = Object.values(fieldAggregate.buckets).map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key as string,
};
});
if (buckets.length === 0) continue;
facets.push({
buckets,
field,
...type,
});
}
}

401
backend/backend/src/storage/elasticsearch/elasticsearch.ts
View File

@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019 StApps
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
@@ -13,58 +13,47 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch';
import {Client, events} from '@elastic/elasticsearch';
import {
SCBulkResponse,
SCConfigFile,
SCFacet,
SCSearchQuery,
SCSearchResponse,
SCThings,
SCThingType,
SCUuid,
} from '@openstapps/core';
AggregateName,
AggregationsMultiTermsBucket,
IndicesGetAliasResponse,
IndicesUpdateAliasesAction,
SearchHit,
SearchResponse,
} from '@elastic/elasticsearch/lib/api/types';
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
import moment from 'moment';
import {MailQueue} from '../../notification/mail-queue';
import {Bulk} from '../bulk-storage';
import {Database} from '../database';
import {parseAggregations} from './aggregations';
import * as Monitoring from './monitoring';
import {buildQuery, buildSort} from './query';
import {buildQuery} from './query/query';
import {buildSort} from './query/sort';
import {aggregations, putTemplate} from './templating';
import {
AggregationResponse,
ElasticsearchConfig,
ElasticsearchObject,
ElasticsearchQueryDisMaxConfig,
ElasticsearchQueryQueryStringConfig,
} from './types/elasticsearch';
/**
* Matches index names such as stapps_<type>_<source>_<random suffix>
*/
const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
} from './types/elasticsearch-config';
import {ALL_INDICES_QUERY, getThingIndexName, parseIndexName, VALID_INDEX_REGEX} from './util';
import {removeInvalidAliasChars} from './util/alias';
import {noUndefined} from './util/no-undefined';
import {retryCatch, RetryOptions} from './util/retry';
/**
* A database interface for elasticsearch
*/
export class Elasticsearch implements Database {
/**
* Length of the index UID used for generation of its name
*/
static readonly INDEX_UID_LENGTH = 8;
/**
* Holds a map of all elasticsearch indices that are available to search
*/
aliasMap: {
// each scType has a alias which can contain multiple sources
// each scType has an alias which can contain multiple sources
[scType: string]: {
// each source is assigned a index name in elasticsearch
// each source is assigned an index name in elasticsearch
[source: string]: string;
};
};
@@ -97,89 +86,11 @@ export class Elasticsearch implements Database {
return 'http://localhost:9200';
}
/**
* Gets the index name in elasticsearch for one SCThingType
*
* @param type SCThingType of data in the index
* @param source source of data in the index
* @param bulk bulk process which created this index
*/
static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) {
let out = type.toLowerCase();
while (out.includes(' ')) {
out = out.replace(' ', '_');
}
return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`;
}
/**
* Provides the index UID (for its name) from the bulk UID
*
* @param uid Bulk UID
*/
static getIndexUID(uid: SCUuid) {
return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH));
}
/**
* Generates a string which matches all indices
*/
static getListOfAllIndices(): string {
// map each SC type in upper camel case
return 'stapps_*_*_*';
}
/**
* Checks for invalid character in alias names and removes them
*
* @param alias The alias name
* @param uid The UID of the current bulk (for debugging purposes)
*/
static removeAliasChars(alias: string, uid: string | undefined): string {
let formattedAlias = alias;
// spaces are included in some types, replace them with underscores
if (formattedAlias.includes(' ')) {
formattedAlias = formattedAlias.trim();
formattedAlias = formattedAlias.split(' ').join('_');
}
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
if (formattedAlias.includes(value)) {
formattedAlias = formattedAlias.replace(value, '');
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
having the same alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
for (const value of ['-', '_', '+']) {
if (formattedAlias.charAt(0) === value) {
formattedAlias = formattedAlias.slice(1);
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
if (formattedAlias === '.' || formattedAlias === '..') {
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
return 'alias_placeholder';
}
if (formattedAlias.includes(':')) {
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
Elasticsearch versions!`);
}
return formattedAlias;
}
/**
* Create a new interface for elasticsearch
*
* @param config an assembled config file
* @param mailQueue a mailqueue for monitoring
* @param mailQueue a mail queue for monitoring
*/
constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) {
if (
@@ -192,7 +103,7 @@ export class Elasticsearch implements Database {
this.client = new Client({
node: Elasticsearch.getElasticsearchUrl(),
});
this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse<unknown>) => {
this.client.diagnostic.on(events.REQUEST, async (error: Error | null, result: unknown) => {
if (error !== null) {
await Logger.error(error);
}
@@ -210,73 +121,40 @@ export class Elasticsearch implements Database {
/**
* Gets a map which contains each alias and all indices that are associated with each alias
*/
private async getAliasMap() {
// delay after which alias map will be fetched again
const RETRY_INTERVAL = 5000;
// maximum number of retries
const RETRY_COUNT = 3;
// create a list of old indices that are not in use
const oldIndicesToDelete: string[] = [];
let aliases:
| {
[index: string]: {
/**
* Aliases of an index
*/
aliases: {
[K in SCThingType]: unknown;
};
};
}
| undefined;
for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) {
if (typeof aliases !== 'undefined') {
break;
}
try {
const aliasResponse = await this.client.indices.getAlias({});
aliases = aliasResponse.body;
} catch (error) {
private async getAliasMap(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}) {
const aliasResponse = await retryCatch({
maxRetries: 10,
retryInterval: 2000,
doAction: () => this.client.indices.getAlias(),
onFailedAttempt: (attempt, error, {maxRetries, retryInterval}) => {
Logger.warn('Failed getting alias map:', error);
Logger.warn(`Retrying in ${RETRY_INTERVAL} milliseconds. (${retry} of ${RETRY_COUNT})`);
await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL));
}
}
Logger.warn(`Retrying in ${retryInterval} milliseconds. (${attempt} of ${maxRetries})`);
},
onFail: ({maxRetries}) => {
throw new TypeError(`Failed to retrieve alias map after ${maxRetries} attempts!`);
},
...retryOptions,
});
if (typeof aliases === 'undefined') {
throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
}
const aliases = Object.entries(aliasResponse)
.filter(([index]) => !index.startsWith('.'))
.map(([index, alias]) => ({
index,
alias,
...parseIndexName(index),
}));
for (const index in aliases) {
if (aliases.hasOwnProperty(index)) {
const matches = indexRegex.exec(index);
if (matches !== null) {
const type = matches[1];
const source = matches[2];
// check if there is an alias for the current index
// check that alias equals type
const hasAlias = type in aliases[index].aliases;
if (hasAlias) {
if (typeof this.aliasMap[type] === 'undefined') {
this.aliasMap[type] = {};
}
this.aliasMap[type][source] = index;
} else {
oldIndicesToDelete.push(index);
}
}
}
for (const {type, index, source} of aliases.filter(({type, alias}) => type in alias.aliases)) {
this.aliasMap[type] = this.aliasMap[type] || {};
this.aliasMap[type][source] = index;
}
this.ready = true;
// delete old indices that are not used in any alias
if (oldIndicesToDelete.length > 0) {
const unusedIndices = aliases.filter(({type, alias}) => !(type in alias.aliases)).map(({index}) => index);
if (unusedIndices.length > 0) {
await this.client.indices.delete({
index: oldIndicesToDelete,
index: unusedIndices,
});
Logger.warn(`Deleted old indices: oldIndicesToDelete`);
}
@@ -291,8 +169,8 @@ export class Elasticsearch implements Database {
* @param uid an UID to use for the search
* @returns an elasticsearch object containing the thing
*/
private async getObject(uid: SCUuid): Promise<ElasticsearchObject<SCThings> | undefined> {
const searchResponse: ApiResponse<SearchResponse<SCThings>> = await this.client.search({
private async getObject(uid: SCUuid): Promise<SearchHit<SCThings> | undefined> {
const searchResponse = await this.client.search<SCThings>({
body: {
query: {
term: {
@@ -303,43 +181,44 @@ export class Elasticsearch implements Database {
},
},
from: 0,
index: Elasticsearch.getListOfAllIndices(),
index: ALL_INDICES_QUERY,
size: 1,
});
// return data from response
return searchResponse.body.hits.hits[0];
return searchResponse.hits.hits[0];
}
/**
* Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
*
* @param bulk the bulk process that was created
*/
public async bulkCreated(bulk: Bulk): Promise<void> {
// if our es instance is not ready yet, we cannot serve this request
private async prepareBulkWrite(bulk: Bulk): Promise<{index: string; alias: string}> {
if (!this.ready) {
throw new Error('No connection to elasticsearch established yet.');
}
// index name for elasticsearch
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
// there already is an index with this type and source. We will index the new one and switch the alias to it
// the old one is deleted
const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
const index = getThingIndexName(bulk.type, bulk.source, bulk);
const alias = removeInvalidAliasChars(bulk.type, bulk.uid);
if (typeof this.aliasMap[alias] === 'undefined') {
this.aliasMap[alias] = {};
}
if (!indexRegex.test(index)) {
if (!VALID_INDEX_REGEX.test(index)) {
throw new Error(
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
);
}
return {index, alias};
}
/**
* Should be called, when a new bulk was created. Creates a new index and applies the mapping to the index
*
* @param bulk the bulk process that was created
*/
public async bulkCreated(bulk: Bulk): Promise<void> {
const {index} = await this.prepareBulkWrite(bulk);
// re-apply the index template before each new bulk operation
await putTemplate(this.client, bulk.type);
await this.client.indices.create({
@@ -355,8 +234,7 @@ export class Elasticsearch implements Database {
* @param bulk the bulk process that is expired
*/
public async bulkExpired(bulk: Bulk): Promise<void> {
// index name for elasticsearch
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
const index: string = getThingIndexName(bulk.type, bulk.source, bulk);
Logger.info('Bulk expired. Deleting index', index);
@@ -375,31 +253,11 @@ export class Elasticsearch implements Database {
* @param bulk the new bulk process that should replace the old one with same type and source
*/
public async bulkUpdated(bulk: Bulk): Promise<void> {
// if our es instance is not ready yet, we cannot serve this request
if (!this.ready) {
throw new Error('No connection to elasticsearch established yet.');
}
const {index, alias} = await this.prepareBulkWrite(bulk);
// index name for elasticsearch
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
// alias for the indices
const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
if (typeof this.aliasMap[alias] === 'undefined') {
this.aliasMap[alias] = {};
}
if (!indexRegex.test(index)) {
throw new Error(
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
);
}
// create the new index if it does not exists
// create the new index if it does not exist
// eslint-disable-next-line unicorn/no-await-expression-member
if (!(await this.client.indices.exists({index})).body) {
if (!(await this.client.indices.exists({index}))) {
// re-apply the index template before each new bulk operation
await putTemplate(this.client, bulk.type);
await this.client.indices.create({
@@ -412,7 +270,7 @@ export class Elasticsearch implements Database {
// add our new index to the alias
// this was type safe with @types/elasticsearch, the new package however provides no type definitions
const actions: IndicesUpdateAliasesParamsAction[] = [
const actions: IndicesUpdateAliasesAction[] = [
{
add: {index: index, alias: alias},
},
@@ -427,16 +285,10 @@ export class Elasticsearch implements Database {
}
// refresh the index (fsync changes)
await this.client.indices.refresh({
index: index,
});
await this.client.indices.refresh({index});
// execute our alias actions
await this.client.indices.updateAliases({
body: {
actions,
},
});
await this.client.indices.updateAliases({actions});
// swap the index in our aliasMap
this.aliasMap[alias][bulk.source] = index;
@@ -457,7 +309,7 @@ export class Elasticsearch implements Database {
public async get(uid: SCUuid): Promise<SCThings> {
const object = await this.getObject(uid);
if (typeof object === 'undefined') {
if (typeof object?._source === 'undefined') {
throw new TypeError('Item not found.');
}
@@ -467,7 +319,7 @@ export class Elasticsearch implements Database {
/**
* Initialize the elasticsearch database (call all needed methods)
*/
public async init(): Promise<void> {
public async init(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}): Promise<void> {
const monitoringConfiguration = this.config.internal.monitoring;
if (typeof monitoringConfiguration !== 'undefined') {
@@ -480,7 +332,7 @@ export class Elasticsearch implements Database {
await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
}
return this.getAliasMap();
return this.getAliasMap(retryOptions);
}
/**
@@ -490,7 +342,7 @@ export class Elasticsearch implements Database {
* @param bulk the bulk process which item belongs to
*/
public async post(object: SCThings, bulk: Bulk): Promise<void> {
const object_: SCThings & {creation_date: string} = {
const thing: SCThings & {creation_date: string} = {
...object,
creation_date: moment().format(),
};
@@ -499,7 +351,7 @@ export class Elasticsearch implements Database {
// check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid)
if (typeof item !== 'undefined') {
const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk);
const indexOfNew = getThingIndexName(thing.type, bulk.source, bulk);
const oldIndex = item._index;
// new item doesn't replace the old one
@@ -509,22 +361,23 @@ export class Elasticsearch implements Database {
) {
throw new Error(
// eslint-disable-next-line unicorn/no-null
`Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`,
`Object "${thing.uid}" already exists. Object was: ${JSON.stringify(thing, null, 2)}`,
);
}
}
// regular bulk update (item gets replaced when bulk is updated)
const searchResponse = await this.client.create({
body: object_,
id: object_.uid,
index: Elasticsearch.getIndex(object_.type, bulk.source, bulk),
const searchResponse = await this.client.create<SCThings>({
document: thing,
id: thing.uid,
index: getThingIndexName(thing.type, bulk.source, bulk),
timeout: '90s',
type: object_.type,
});
if (!searchResponse.body.created) {
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`);
if (searchResponse.result !== 'created') {
throw new Error(
`Object creation Error (${searchResponse.result}: Instance was: ${JSON.stringify(thing)}`,
);
}
}
@@ -543,7 +396,6 @@ export class Elasticsearch implements Database {
},
id: object.uid,
index: item._index,
type: object.type.toLowerCase(),
});
return;
@@ -562,65 +414,46 @@ export class Elasticsearch implements Database {
throw new TypeError('Database is undefined. You have to configure the query build');
}
// create elasticsearch configuration out of data from database configuration
const esConfig: ElasticsearchConfig = {
name: this.config.internal.database.name as 'elasticsearch',
version: this.config.internal.database.version as string,
};
if (typeof this.config.internal.database.query !== 'undefined') {
esConfig.query = this.config.internal.database.query as
query: this.config.internal.database.query as
| ElasticsearchQueryDisMaxConfig
| ElasticsearchQueryQueryStringConfig;
}
| ElasticsearchQueryQueryStringConfig
| undefined,
};
const searchRequest: RequestParams.Search = {
body: {
aggs: aggregations,
query: buildQuery(parameters, this.config, esConfig),
},
const query = {
aggs: aggregations,
query: buildQuery(parameters, this.config, esConfig),
from: parameters.from,
index: Elasticsearch.getListOfAllIndices(),
index: ALL_INDICES_QUERY,
size: parameters.size,
sort: typeof parameters.sort !== 'undefined' ? buildSort(parameters.sort) : undefined,
};
if (typeof parameters.sort !== 'undefined') {
searchRequest.body.sort = buildSort(parameters.sort);
}
// perform the search against elasticsearch
const response: ApiResponse<SearchResponse<SCThings>> = await this.client.search(searchRequest);
// gather pagination information
const pagination = {
count: response.body.hits.hits.length,
offset: typeof parameters.from === 'number' ? parameters.from : 0,
total: response.body.hits.total,
};
// gather statistics about this search
const stats = {
time: response.body.took,
};
// we only directly return the _source documents
// elasticsearch provides much more information, the user shouldn't see
const data = response.body.hits.hits.map(hit => {
return hit._source; // SCThing
});
let facets: SCFacet[] = [];
// read the aggregations from elasticsearch and parse them to facets by our configuration
if (typeof response.body.aggregations !== 'undefined') {
facets = parseAggregations(response.body.aggregations as AggregationResponse);
}
const response: SearchResponse<SCThings> = await this.client.search(query);
return {
data,
facets,
pagination,
stats,
data: response.hits.hits
.map(hit => {
// we only directly return the _source documents
// elasticsearch provides much more information, the user shouldn't see
return hit._source;
})
.filter(noUndefined),
facets:
typeof response.aggregations !== 'undefined'
? parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>)
: [],
pagination: {
count: response.hits.hits.length,
offset: typeof parameters.from === 'number' ? parameters.from : 0,
total:
typeof response.hits.total === 'number' ? response.hits.total : response.hits.total?.value ?? 0,
},
stats: {
time: response.took,
},
};
}
}

13
backend/backend/src/storage/elasticsearch/monitoring.ts
View File

@@ -13,7 +13,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch';
import {Client} from '@elastic/elasticsearch';
import {SearchRequest} from '@elastic/elasticsearch/lib/api/types';
import {
SCMonitoringConfiguration,
SCMonitoringLogAction,
@@ -23,9 +24,6 @@ import {
SCThings,
} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
import {SearchResponse} from 'elasticsearch';
import cron from 'node-cron';
import {MailQueue} from '../../notification/mail-queue';
@@ -131,12 +129,11 @@ export async function setUp(
cron.schedule(trigger.executionTime, async () => {
// execute watch (search->condition->action)
const result: ApiResponse<SearchResponse<SCThings>> = await esClient.search(
watcher.query as RequestParams.Search,
);
const result = await esClient.search<SCThings>(watcher.query as SearchRequest);
// check conditions
const total = result.body.hits.total;
const total =
typeof result.hits.total === 'number' ? result.hits.total : result.hits.total?.value ?? -1;
for (const condition of watcher.conditions) {
if (conditionFails(condition, total)) {

501
backend/backend/src/storage/elasticsearch/query.ts
View File

@@ -1,501 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
SCBackendConfigurationSearchBoostingContext,
SCBackendConfigurationSearchBoostingType,
SCConfigFile,
SCSearchBooleanFilter,
SCSearchContext,
SCSearchFilter,
SCSearchQuery,
SCSearchSort,
SCSportCoursePriceGroup,
SCThingsField,
} from '@openstapps/core';
import {
ElasticsearchConfig,
ESBooleanFilter,
ESBooleanFilterArguments,
ESDateRange,
ESDateRangeFilter,
ESFunctionScoreQuery,
ESFunctionScoreQueryFunction,
ESGenericRange,
ESGenericSort,
ESGeoBoundingBoxFilter,
ESGeoDistanceFilter,
ESGeoDistanceFilterArguments,
ESGeoDistanceSort,
ESGeoDistanceSortArguments,
ESGeoShapeFilter,
ESNumericRangeFilter,
ESRangeFilter,
ESTermFilter,
ESTypeFilter,
ScriptSort,
} from './types/elasticsearch';
/**
* Escapes any reserved character that would otherwise not be accepted by Elasticsearch
*
* Elasticsearch as the following reserved characters:
* + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
* It is possible to use all, with the exception of < and >, of them by escaping them with a \
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*
* @param string_ the string to escape the characters from
*/
function escapeESReservedCharacters(string_: string): string {
return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
}
/**
* Builds a boolean filter. Returns an elasticsearch boolean filter
*
* @param booleanFilter a search boolean filter for the retrieval of the data
* @returns elasticsearch boolean arguments object
*/
export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<unknown> {
const result: ESBooleanFilterArguments<unknown> = {
minimum_should_match: 0,
must: [],
must_not: [],
should: [],
};
if (booleanFilter.arguments.operation === 'and') {
result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
}
if (booleanFilter.arguments.operation === 'or') {
result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
result.minimum_should_match = 1;
}
if (booleanFilter.arguments.operation === 'not') {
result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
}
return result;
}
/**
* Converts Array of Filters to elasticsearch query-syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildFilter(
filter: SCSearchFilter,
):
| ESTermFilter
| ESGeoDistanceFilter
| ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter>
| ESGeoShapeFilter
| ESBooleanFilter<unknown>
| ESRangeFilter {
switch (filter.type) {
case 'value':
return Array.isArray(filter.arguments.value)
? {
terms: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
}
: {
term: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
};
case 'availability':
const scope = filter.arguments.scope?.charAt(0) ?? 's';
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
return {
range: {
[filter.arguments.field]: {
gte: `${time}/${scope}`,
lt: `${time}+1${scope}/${scope}`,
relation: 'intersects',
},
},
};
case 'distance':
const geoObject: ESGeoDistanceFilterArguments = {
distance: `${filter.arguments.distance}m`,
[`${filter.arguments.field}.point.coordinates`]: {
lat: filter.arguments.position[1],
lon: filter.arguments.position[0],
},
};
return {
geo_distance: geoObject,
};
case 'boolean':
return {
bool: buildBooleanFilter(filter),
};
case 'numeric range':
const numericRangeObject: ESGenericRange<number> = {
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
const numericRangeFilter: ESNumericRangeFilter = {range: {}};
numericRangeFilter.range[filter.arguments.field] = numericRangeObject;
return numericRangeFilter;
case 'date range':
const dateRangeObject: ESDateRange = {
format: filter.arguments.format,
time_zone: filter.arguments.timeZone,
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
const dateRangeFilter: ESDateRangeFilter = {range: {}};
dateRangeFilter.range[filter.arguments.field] = dateRangeObject;
return dateRangeFilter;
case 'geo':
// TODO: on ES upgrade, use just geo_shape filters
const geoShapeFilter: ESGeoShapeFilter = {
geo_shape: {
/**
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
*/
// @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type.
ignore_unmapped: true,
[`${filter.arguments.field}.polygon`]: {
shape: filter.arguments.shape,
relation: filter.arguments.spatialRelation,
},
},
};
if (
(typeof filter.arguments.spatialRelation === 'undefined' ||
filter.arguments.spatialRelation === 'intersects') &&
filter.arguments.shape.type === 'envelope'
) {
return {
bool: {
minimum_should_match: 1,
should: [
geoShapeFilter,
{
geo_bounding_box: {
/**
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
*/
ignore_unmapped: true,
[`${filter.arguments.field}.point.coordinates`]: {
top_left: filter.arguments.shape.coordinates[0],
bottom_right: filter.arguments.shape.coordinates[1],
},
},
},
],
},
};
}
return geoShapeFilter;
}
}
/**
* Builds scoring functions from boosting config
*
* @param boostings Backend boosting configuration for contexts and types
* @param context The context of the app from where the search was initiated
*/
function buildFunctions(
boostings: SCBackendConfigurationSearchBoostingContext,
context: SCSearchContext | undefined,
): ESFunctionScoreQueryFunction[] {
// default context
let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes(
boostings['default' as SCSearchContext],
);
if (typeof context !== 'undefined' && context !== 'default') {
// specific context provided, extend default context with additional boosts
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
}
return functions;
}
/**
* Creates boost functions for all type boost configurations
*
* @param boostingTypes Array of type boosting configurations
*/
function buildFunctionsForBoostingTypes(
boostingTypes: SCBackendConfigurationSearchBoostingType[],
): ESFunctionScoreQueryFunction[] {
const functions: ESFunctionScoreQueryFunction[] = [];
for (const boostingForOneSCType of boostingTypes) {
const typeFilter: ESTypeFilter = {
type: {
value: boostingForOneSCType.type,
},
};
functions.push({
filter: typeFilter,
weight: boostingForOneSCType.factor,
});
if (typeof boostingForOneSCType.fields !== 'undefined') {
const fields = boostingForOneSCType.fields;
for (const fieldName in boostingForOneSCType.fields) {
if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
const boostingForOneField = fields[fieldName];
for (const value in boostingForOneField) {
if (boostingForOneField.hasOwnProperty(value)) {
const factor = boostingForOneField[value];
// build term filter
const termFilter: ESTermFilter = {
term: {},
};
termFilter.term[`${fieldName}.raw`] = value;
functions.push({
filter: {
bool: {
must: [typeFilter, termFilter],
should: [],
},
},
weight: factor,
});
}
}
}
}
}
}
return functions;
}
/**
* Builds body for Elasticsearch requests
*
* @param parameters Parameters for querying the backend
* @param defaultConfig Default configuration of the backend
* @param elasticsearchConfig Elasticsearch configuration
* @returns ElasticsearchQuery (body of a search-request)
*/
export function buildQuery(
parameters: SCSearchQuery,
defaultConfig: SCConfigFile,
elasticsearchConfig: ElasticsearchConfig,
): ESFunctionScoreQuery {
// if config provides an minMatch parameter we use query_string instead of match query
let query;
if (typeof elasticsearchConfig.query === 'undefined') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: '90%',
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
};
} else if (elasticsearchConfig.query.queryType === 'query_string') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
};
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
if (parameters.query !== '*') {
query = {
dis_max: {
boost: 1.2,
queries: [
{
match: {
name: {
boost: elasticsearchConfig.query.matchBoosting,
cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
fuzziness: elasticsearchConfig.query.fuzziness,
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
},
},
{
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query:
typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
},
],
tie_breaker: elasticsearchConfig.query.tieBreaker,
},
};
}
} else {
throw new Error(
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
);
}
const functionScoreQuery: ESFunctionScoreQuery = {
function_score: {
functions: buildFunctions(defaultConfig.internal.boostings, parameters.context),
query: {
bool: {
minimum_should_match: 0, // if we have no should, nothing can match
must: [],
should: [],
},
},
score_mode: 'multiply',
},
};
const mustMatch = functionScoreQuery.function_score.query.bool.must;
if (Array.isArray(mustMatch)) {
if (typeof query !== 'undefined') {
mustMatch.push(query);
}
if (typeof parameters.filter !== 'undefined') {
mustMatch.push(buildFilter(parameters.filter));
}
}
return functionScoreQuery;
}
/**
* converts query to
*
* @param sorts Sorting rules to apply to the data that is being queried
* @returns an array of sort queries
*/
export function buildSort(sorts: SCSearchSort[]): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
return sorts.map(sort => {
switch (sort.type) {
case 'generic':
const esGenericSort: ESGenericSort = {};
esGenericSort[sort.arguments.field] = sort.order;
return esGenericSort;
case 'ducet':
const esDucetSort: ESGenericSort = {};
esDucetSort[`${sort.arguments.field}.sort`] = sort.order;
return esDucetSort;
case 'distance':
const arguments_: ESGeoDistanceSortArguments = {
mode: 'avg',
order: sort.order,
unit: 'm',
};
arguments_[`${sort.arguments.field}.point.coordinates`] = {
lat: sort.arguments.position[1],
lon: sort.arguments.position[0],
};
return {
_geo_distance: arguments_,
};
case 'price':
return {
_script: {
order: sort.order,
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
type: 'number' as const,
},
};
}
});
}
/**
* Provides a script for sorting search results by prices
*
* @param universityRole User group which consumes university services
* @param field Field in which wanted offers with prices are located
*/
export function buildPriceSortScript(
universityRole: keyof SCSportCoursePriceGroup,
field: SCThingsField,
): string {
return `
// initialize the sort value with the maximum
double price = Double.MAX_VALUE;
// if we have any offers
if (params._source.containsKey('${field}')) {
// iterate through all offers
for (offer in params._source.${field}) {
// if this offer contains a role specific price
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
// if the role specific price is smaller than the cheapest we found
if (offer.prices.${universityRole} < price) {
// set the role specific price as cheapest for now
price = offer.prices.${universityRole};
}
} else { // we have no role specific price for our role in this offer
// if the default price of this offer is lower than the cheapest we found
if (offer.price < price) {
// set this price as the cheapest
price = offer.price;
}
}
}
}
// return cheapest price for our role
return price;
`;
}

75
backend/backend/src/storage/elasticsearch/query/boost/boost-functions.ts Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCBackendConfigurationSearchBoostingType} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Creates boost functions for all type boost configurations
*
* @param boostingTypes Array of type boosting configurations
*/
export function buildFunctionsForBoostingTypes(
boostingTypes: SCBackendConfigurationSearchBoostingType[],
): QueryDslFunctionScoreContainer[] {
const functions: QueryDslFunctionScoreContainer[] = [];
for (const boostingForOneSCType of boostingTypes) {
const typeFilter: QueryDslSpecificQueryContainer<'term'> = {
term: {
type: boostingForOneSCType.type,
},
};
functions.push({
filter: typeFilter,
weight: boostingForOneSCType.factor,
});
if (typeof boostingForOneSCType.fields !== 'undefined') {
const fields = boostingForOneSCType.fields;
for (const fieldName in boostingForOneSCType.fields) {
if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
const boostingForOneField = fields[fieldName];
for (const value in boostingForOneField) {
if (boostingForOneField.hasOwnProperty(value)) {
const factor = boostingForOneField[value];
// build term filter
const termFilter: QueryDslSpecificQueryContainer<'term'> = {
term: {},
};
termFilter.term[`${fieldName}.raw`] = value;
functions.push({
filter: {
bool: {
must: [typeFilter, termFilter],
should: [],
},
},
weight: factor,
});
}
}
}
}
}
}
return functions;
}

38
backend/backend/src/storage/elasticsearch/query/boost/scoring-functions.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCBackendConfigurationSearchBoostingContext, SCSearchContext} from '@openstapps/core';
import {buildFunctionsForBoostingTypes} from './boost-functions';
/**
* Builds scoring functions from boosting config
*
* @param boostings Backend boosting configuration for contexts and types
* @param context The context of the app from where the search was initiated
*/
export function buildScoringFunctions(
boostings: SCBackendConfigurationSearchBoostingContext,
context: SCSearchContext | undefined,
): QueryDslFunctionScoreContainer[] {
// default context
let functions = buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]);
if (typeof context !== 'undefined' && context !== 'default') {
// specific context provided, extend default context with additional boosts
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
}
return functions;
}

47
backend/backend/src/storage/elasticsearch/query/filter.ts Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchFilter} from '@openstapps/core';
import {buildBooleanFilter} from './filters/boolean';
import {buildAvailabilityFilter} from './filters/availability';
import {buildDateRangeFilter} from './filters/date-range';
import {buildDistanceFilter} from './filters/distance';
import {buildGeoFilter} from './filters/geo';
import {buildNumericRangeFilter} from './filters/numeric-range';
import {buildValueFilter} from './filters/value';
/**
* Converts Array of Filters to elasticsearch query-syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildFilter(filter: SCSearchFilter): QueryDslQueryContainer {
switch (filter.type) {
case 'value':
return buildValueFilter(filter);
case 'availability':
return buildAvailabilityFilter(filter);
case 'distance':
return buildDistanceFilter(filter);
case 'boolean':
return buildBooleanFilter(filter);
case 'numeric range':
return buildNumericRangeFilter(filter);
case 'date range':
return buildDateRangeFilter(filter);
case 'geo':
return buildGeoFilter(filter);
}
}

38
backend/backend/src/storage/elasticsearch/query/filters/availability.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCSearchAvailabilityFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts an availability filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildAvailabilityFilter(
filter: SCSearchAvailabilityFilter,
): QueryDslSpecificQueryContainer<'range'> {
const scope = filter.arguments.scope?.charAt(0) ?? 's';
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
return {
range: {
[filter.arguments.field]: {
gte: `${time}/${scope}`,
lt: `${time}+1${scope}/${scope}`,
relation: 'intersects',
},
},
};
}

49
backend/backend/src/storage/elasticsearch/query/filters/boolean.ts Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchBooleanFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
import {buildFilter} from '../filter';
/**
* Converts a boolean filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildBooleanFilter(filter: SCSearchBooleanFilter): QueryDslSpecificQueryContainer<'bool'> {
const result: QueryDslBoolQuery = {
minimum_should_match: 0,
must: [],
must_not: [],
should: [],
};
if (filter.arguments.operation === 'and') {
result.must = filter.arguments.filters.map(it => buildFilter(it));
}
if (filter.arguments.operation === 'or') {
result.should = filter.arguments.filters.map(it => buildFilter(it));
result.minimum_should_match = 1;
}
if (filter.arguments.operation === 'not') {
result.must_not = filter.arguments.filters.map(it => buildFilter(it));
}
return {
bool: result,
};
}

48
backend/backend/src/storage/elasticsearch/query/filters/date-range.ts Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchDateRangeFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a date range filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildDateRangeFilter(
filter: SCSearchDateRangeFilter,
): QueryDslSpecificQueryContainer<'range'> {
const dateRangeObject: QueryDslDateRangeQuery = {
format: filter.arguments.format,
time_zone: filter.arguments.timeZone,
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
return {
range: {
[filter.arguments.field]: dateRangeObject,
},
};
}

38
backend/backend/src/storage/elasticsearch/query/filters/distance.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchDistanceFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a distance filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildDistanceFilter(
filter: SCSearchDistanceFilter,
): QueryDslSpecificQueryContainer<'geo_distance'> {
const geoObject: QueryDslGeoDistanceQuery = {
distance: `${filter.arguments.distance}m`,
[`${filter.arguments.field}.point.coordinates`]: {
lat: filter.arguments.position[1],
lon: filter.arguments.position[0],
},
};
return {
geo_distance: geoObject,
};
}

33
backend/backend/src/storage/elasticsearch/query/filters/geo.ts Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCGeoFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a geo filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> {
return {
geo_shape: {
ignore_unmapped: true,
[`${filter.arguments.field}.polygon`]: {
shape: filter.arguments.shape,
relation: filter.arguments.spatialRelation,
},
},
};
}

47
backend/backend/src/storage/elasticsearch/query/filters/numeric-range.ts Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchNumericRangeFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a numeric range filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildNumericRangeFilter(
filter: SCSearchNumericRangeFilter,
): QueryDslSpecificQueryContainer<'range'> {
const numericRangeObject: QueryDslNumberRangeQuery = {
relation: filter.arguments.relation,
};
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
}
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
}
return {
range: {
[filter.arguments.field]: numericRangeObject,
},
};
}

37
backend/backend/src/storage/elasticsearch/query/filters/value.ts Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCSearchValueFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util';
/**
* Converts a value filter to elasticsearch syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildValueFilter(
filter: SCSearchValueFilter,
): QueryDslSpecificQueryContainer<'term'> | QueryDslSpecificQueryContainer<'terms'> {
return Array.isArray(filter.arguments.value)
? {
terms: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
}
: {
term: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
};
}

114
backend/backend/src/storage/elasticsearch/query/query.ts Normal file
View File

@@ -0,0 +1,114 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
import {SCConfigFile, SCSearchQuery} from '@openstapps/core';
import {ElasticsearchConfig} from '../types/elasticsearch-config';
import {buildFilter} from './filter';
import {buildScoringFunctions} from './boost/scoring-functions';
/**
* Builds body for Elasticsearch requests
*
* @param parameters Parameters for querying the backend
* @param defaultConfig Default configuration of the backend
* @param elasticsearchConfig Elasticsearch configuration
* @returns ElasticsearchQuery (body of a search-request)
*/
export function buildQuery(
parameters: SCSearchQuery,
defaultConfig: SCConfigFile,
elasticsearchConfig: ElasticsearchConfig,
): QueryDslQueryContainer {
// if config provides an minMatch parameter we use query_string instead of match query
let query;
if (typeof elasticsearchConfig.query === 'undefined') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: '90%',
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
};
} else if (elasticsearchConfig.query.queryType === 'query_string') {
query = {
query_string: {
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
};
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
if (typeof parameters.query === 'string' && parameters.query !== '*') {
query = {
dis_max: {
boost: 1.2,
queries: [
{
match: {
name: {
boost: elasticsearchConfig.query.matchBoosting,
fuzziness: elasticsearchConfig.query.fuzziness,
query: parameters.query,
},
},
},
{
query_string: {
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: parameters.query,
},
},
],
tie_breaker: elasticsearchConfig.query.tieBreaker,
},
};
}
} else {
throw new Error(
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
);
}
const functionScoreQuery: QueryDslQueryContainer = {
function_score: {
functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
query: {
bool: {
minimum_should_match: 0, // if we have no should, nothing can match
must: [],
should: [],
},
},
score_mode: 'multiply',
},
};
const mustMatch = functionScoreQuery.function_score?.query?.bool?.must;
if (Array.isArray(mustMatch)) {
if (typeof query !== 'undefined') {
mustMatch.push(query);
}
if (typeof parameters.filter !== 'undefined') {
mustMatch.push(buildFilter(parameters.filter));
}
}
return functionScoreQuery;
}

41
backend/backend/src/storage/elasticsearch/query/sort.ts Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Sort} from '@elastic/elasticsearch/lib/api/types';
import {SCSearchSort} from '@openstapps/core';
import {buildDistanceSort} from './sort/distance';
import {buildDucetSort} from './sort/ducet';
import {buildGenericSort} from './sort/generic';
import {buildPriceSort} from './sort/price';
/**
* converts query to
*
* @param sorts Sorting rules to apply to the data that is being queried
* @returns an array of sort queries
*/
export function buildSort(sorts: SCSearchSort[]): Sort {
return sorts.map(sort => {
switch (sort.type) {
case 'generic':
return buildGenericSort(sort);
case 'ducet':
return buildDucetSort(sort);
case 'distance':
return buildDistanceSort(sort);
case 'price':
return buildPriceSort(sort);
}
});
}

35
backend/backend/src/storage/elasticsearch/query/sort/distance.ts Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCDistanceSort} from '@openstapps/core';
/**
* Converts a distance sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildDistanceSort(sort: SCDistanceSort): SortOptions {
return {
_geo_distance: {
mode: 'avg',
order: sort.order,
unit: 'm',
[`${sort.arguments.field}.point.coordinates`]: {
lat: sort.arguments.position[1],
lon: sort.arguments.position[0],
},
},
};
}

27
backend/backend/src/storage/elasticsearch/query/sort/ducet.ts Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCDucetSort} from '@openstapps/core';
/**
* Converts a ducet sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildDucetSort(sort: SCDucetSort): SortOptions {
return {
[`${sort.arguments.field}.sort`]: sort.order,
};
}

27
backend/backend/src/storage/elasticsearch/query/sort/generic.ts Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCGenericSort} from '@openstapps/core';
/**
* Converts a generic sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildGenericSort(sort: SCGenericSort): SortOptions {
return {
[sort.arguments.field]: sort.order,
};
}

71
backend/backend/src/storage/elasticsearch/query/sort/price.ts Normal file
View File

@@ -0,0 +1,71 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
/**
* Converts a price sort to elasticsearch syntax
*
* @param sort A sorting definition
*/
export function buildPriceSort(sort: SCPriceSort): SortOptions {
return {
_script: {
order: sort.order,
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
type: 'number' as const,
},
};
}
/**
* Provides a script for sorting search results by prices
*
* @param universityRole User group which consumes university services
* @param field Field in which wanted offers with prices are located
*/
export function buildPriceSortScript(
universityRole: keyof SCSportCoursePriceGroup,
field: SCThingsField,
): string {
return `
// initialize the sort value with the maximum
double price = Double.MAX_VALUE;
// if we have any offers
if (params._source.containsKey('${field}')) {
// iterate through all offers
for (offer in params._source.${field}) {
// if this offer contains a role specific price
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
// if the role specific price is smaller than the cheapest we found
if (offer.prices.${universityRole} < price) {
// set the role specific price as cheapest for now
price = offer.prices.${universityRole};
}
} else { // we have no role specific price for our role in this offer
// if the default price of this offer is lower than the cheapest we found
if (offer.price < price) {
// set this price as the cheapest
price = offer.price;
}
}
}
}
// return cheapest price for our role
return price;
`;
}

11
backend/backend/src/storage/elasticsearch/templating.ts
View File

@@ -29,17 +29,6 @@ export const aggregations = JSON.parse(
readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
) as AggregationSchema;
/**
* Re-applies all interfaces for every type
*
* @param client An elasticsearch client to use
*/
export async function refreshAllTemplates(client: Client) {
for (const type of Object.values(SCThingType)) {
await putTemplate(client, type as SCThingType);
}
}
/**
* Prepares all indices
*

121
backend/backend/src/storage/elasticsearch/types/elasticsearch-config.ts Normal file
View File

@@ -0,0 +1,121 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* A configuration for using the Dis Max Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryDisMaxConfig {
/**
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
*/
cutoffFrequency: number;
/**
* The maximum allowed Levenshtein Edit Distance (or number of edits)
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
*/
fuzziness: number | string;
/**
* Increase the importance (relevance score) of a field
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
*/
matchBoosting: number;
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'dis_max' which is a union of its subqueries
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
*/
queryType: 'dis_max';
/**
* Changes behavior of default calculation of the score when multiple results match
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
*/
tieBreaker: number;
}
/**
* A configuration for using Query String Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryQueryStringConfig {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*/
queryType: 'query_string';
}
/**
* An config file for the elasticsearch database interface
*
* The config file extends the SCConfig file by further defining how the database property
*/
export interface ElasticsearchConfigFile {
/**
* Configuration that is not visible to clients
*/
internal: {
/**
* Database configuration
*/
database: ElasticsearchConfig;
};
}
/**
* An elasticsearch configuration
*/
export interface ElasticsearchConfig {
/**
* Name of the database
*/
name: 'elasticsearch';
/**
* Configuration for using queries
*/
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
/**
* Version of the used elasticsearch
*/
version: string;
}

605
backend/backend/src/storage/elasticsearch/types/elasticsearch.ts
View File

@@ -1,605 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCThing, SCThingType} from '@openstapps/core';
// we only have the @types package because some things type definitions are still missing from the official
import {NameList} from 'elasticsearch';
import {Polygon, Position} from 'geojson';
/**
* An elasticsearch aggregation bucket
*/
interface Bucket {
/**
* Number of documents in the aggregation bucket
*/
doc_count: number;
/**
* Text representing the documents in the bucket
*/
key: string;
}
/**
* An elasticsearch aggregation response
*/
export interface AggregationResponse {
/**
* The individual aggregations
*/
[field: string]: BucketAggregation | NestedAggregation;
}
/**
* An elasticsearch bucket aggregation
*/
export interface BucketAggregation {
/**
* Buckets in an aggregation
*/
buckets: Bucket[];
/**
* Number of documents in an aggregation
*/
doc_count?: number;
}
/**
* An aggregation that contains more aggregations nested inside
*/
export interface NestedAggregation {
/**
* Number of documents in an aggregation
*/
doc_count: number;
/**
* Any nested responses
*/
[name: string]: BucketAggregation | number;
}
/**
* A configuration for using the Dis Max Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryDisMaxConfig {
/**
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
*/
cutoffFrequency: number;
/**
* The maximum allowed Levenshtein Edit Distance (or number of edits)
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
*/
fuzziness: number | string;
/**
* Increase the importance (relevance score) of a field
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
*/
matchBoosting: number;
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'dis_max' which is a union of its subqueries
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
*/
queryType: 'dis_max';
/**
* Changes behavior of default calculation of the score when multiple results match
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
*/
tieBreaker: number;
}
/**
* A configuration for using Query String Query
*
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
* explanation of what the parameters mean
*/
export interface ElasticsearchQueryQueryStringConfig {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*/
queryType: 'query_string';
}
/**
* A hit in an elasticsearch search result
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
*/
export interface ElasticsearchObject<T extends SCThing> {
/**
* Unique identifier of a document (object)
*/
_id: string;
/**
* The index to which the document belongs
*/
_index: string;
/**
* Relevancy of the document to a query
*/
_score: number;
/**
* The original JSON representing the body of the document
*/
_source: T;
/**
* The document's mapping type
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
*/
_type: string;
/**
* Version of the document
*/
_version?: number;
/**
* Used to index the same field in different ways for different purposes
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
*/
fields?: NameList;
/**
* Used to highlight search results on one or more fields
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
highlight?: any;
/**
* Used in when nested/children documents match the query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
inner_hits?: any;
/**
* Queries that matched for documents in results
*/
matched_queries?: string[];
/**
* Sorting definition
*/
sort?: string[];
}
/**
* An config file for the elasticsearch database interface
*
* The config file extends the SCConfig file by further defining how the database property
*/
export interface ElasticsearchConfigFile {
/**
* Configuration that is not visible to clients
*/
internal: {
/**
* Database configuration
*/
database: ElasticsearchConfig;
};
}
/**
* An elasticsearch configuration
*/
export interface ElasticsearchConfig {
/**
* Name of the database
*/
name: 'elasticsearch';
/**
* Configuration for using queries
*/
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
/**
* Version of the used elasticsearch
*/
version: string;
}
/**
* An elasticsearch term filter
*/
export type ESTermFilter =
| {
/**
* Definition of a term to match
*/
term: {
[fieldName: string]: string;
};
}
| {
/**
* Definition of terms to match (or)
*/
terms: {
[fieldName: string]: string[];
};
};
export interface ESGenericRange<T> {
/**
* Greater than field
*/
gt?: T;
/**
* Greater or equal than field
*/
gte?: T;
/**
* Less than field
*/
lt?: T;
/**
* Less or equal than field
*/
lte?: T;
/**
* Relation of the range to a range field
*
* Intersects: Both ranges intersect
* Contains: Search range contains field range
* Within: Field range contains search range
*/
relation?: 'intersects' | 'within' | 'contains';
}
interface ESGenericRangeFilter<G, T extends ESGenericRange<G>> {
/**
* Range filter definition
*/
range: {
[fieldName: string]: T;
};
}
export interface ESDateRange extends ESGenericRange<string> {
/**
* Optional date format override
*/
format?: string;
/**
* Optional timezone specifier
*/
time_zone?: string;
}
export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<number>>;
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
/**
* An elasticsearch type filter
*/
export interface ESTypeFilter {
/**
* Type filter definition
*/
type: {
/**
* Type name (SCThingType) to filter with
*/
value: SCThingType;
};
}
/**
* Filter arguments for an elasticsearch geo distance filter
*/
export interface ESGeoDistanceFilterArguments {
/**
* The radius of the circle centred on the specified location
*/
distance: string;
[fieldName: string]:
| {
/**
* Latitude
*/
lat: number;
/**
* Longitude
*/
lon: number;
}
| string;
}
/**
* An elasticsearch geo distance filter
*/
export interface ESGeoDistanceFilter {
/**
* @see ESGeoDistanceFilterArguments
*/
geo_distance: ESGeoDistanceFilterArguments;
}
/**
* A rectangular geo shape, representing the top-left and bottom-right corners
*
* This is an extension of the Geojson type
* http://geojson.org/geojson-spec.html
*/
export interface ESEnvelope {
/**
* The top-left and bottom-right corners of the bounding box
*/
coordinates: [Position, Position];
/**
* The type of the geometry
*/
type: 'envelope';
}
/**
* An Elasticsearch geo bounding box filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
*/
export interface ESGeoBoundingBoxFilter {
/**
* An Elasticsearch geo bounding box filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
*/
geo_bounding_box: {
[fieldName: string]: {
/**
* Geo Shape
*/
bottom_right: Position;
/**
* Geo Shape
*/
top_left: Position;
};
};
}
/**
* An Elasticsearch geo shape filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html
*/
export interface ESGeoShapeFilter {
geo_shape: {
[fieldName: string]: {
/**
* Relation of the two shapes
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_spatial_relations
*/
relation?: 'intersects' | 'disjoint' | 'within' | 'contains';
/**
* Geo Shape
*/
shape: Polygon | ESEnvelope;
};
};
}
/**
* Filter arguments for an elasticsearch boolean filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
*/
export interface ESBooleanFilterArguments<T> {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minimum_should_match?: number;
/**
* The clause (query) must appear in matching documents and will contribute to the score.
*/
must?: T[];
/**
* The clause (query) must not appear in the matching documents.
*/
must_not?: T[];
/**
* The clause (query) should appear in the matching document.
*/
should?: T[];
}
/**
* An elasticsearch boolean filter
*/
export interface ESBooleanFilter<T> {
/**
* @see ESBooleanFilterArguments
*/
bool: ESBooleanFilterArguments<T>;
}
/**
* An elasticsearch function score query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
*/
export interface ESFunctionScoreQuery {
/**
* Function score definition
*/
function_score: {
/**
* Functions that compute score for query results (documents)
*
* @see ESFunctionScoreQueryFunction
*/
functions: ESFunctionScoreQueryFunction[];
/**
* @see ESBooleanFilter
*/
query: ESBooleanFilter<unknown>;
/**
* Specifies how the computed scores are combined
*/
score_mode: 'multiply';
};
}
/**
* An function for an elasticsearch functions score query
*/
export interface ESFunctionScoreQueryFunction {
/**
* Function is applied only if a document matches the given filtering query
*/
filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;
/**
* Weight (importance) of the filter
*/
weight: number;
}
/**
* An elasticsearch generic sort
*/
export interface ESGenericSort {
[field: string]: string;
}
/**
* Sort arguments for an elasticsearch geo distance sort
*/
export interface ESGeoDistanceSortArguments {
/**
* What value to pick for sorting
*/
mode: 'avg' | 'max' | 'median' | 'min';
/**
* Order
*/
order: 'asc' | 'desc';
/**
* Value unit
*/
unit: 'm';
[field: string]:
| {
/**
* Latitude
*/
lat: number;
/**
* Longitude
*/
lon: number;
}
| string;
}
/**
* An elasticsearch geo distance sort
*/
export interface ESGeoDistanceSort {
/**
* @see ESGeoDistanceFilterArguments
*/
_geo_distance: ESGeoDistanceSortArguments;
}
/**
* An elasticsearch script sort
*/
export interface ScriptSort {
/**
* A script
*/
_script: {
/**
* Order
*/
order: 'asc' | 'desc';
/**
* The custom script used for sorting
*/
script: string;
/**
* What type is being sorted
*/
type: 'number' | 'string';
};
}

69
backend/backend/src/storage/elasticsearch/types/guards.ts
View File

@@ -1,69 +0,0 @@
/*
* Copyright (C) 2019-2021 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
ESAggMatchAllFilter,
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter,
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {BucketAggregation, NestedAggregation} from './elasticsearch';
/**
* Checks if the type is a BucketAggregation
*
* @param agg the type to check
*/
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
return typeof agg !== 'number';
}
/**
* Checks if the type is a NestedAggregation
*
* @param agg the type to check
*/
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
return typeof (agg as BucketAggregation).buckets === 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
*
* @param agg the value to check
*/
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
return typeof (agg as ESTermsFilter).terms !== 'undefined';
}
/**
* Checks if the parameter is of type ESTermsFilter
*
* @param agg the value to check
*/
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
}
/**
* Checks if the parameter is of type
*
* @param filter the filter to narrow the type of
*/
export function isESAggMatchAllFilter(
filter: ESAggTypeFilter | ESAggMatchAllFilter,
): filter is ESAggMatchAllFilter {
return filter.hasOwnProperty('match_all');
}

20
backend/backend/src/storage/elasticsearch/types/util.ts Normal file
View File

@@ -0,0 +1,20 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
export type QueryDslSpecificQueryContainer<T extends keyof QueryDslQueryContainer> = Required<
Pick<QueryDslQueryContainer, T>
>;

60
backend/backend/src/storage/elasticsearch/util/alias.ts Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Logger} from '@openstapps/logger';
/**
* Checks for invalid character in alias names and removes them
*
* @param alias The alias name
* @param uid The UID of the current bulk (for debugging purposes)
*/
export function removeInvalidAliasChars(alias: string, uid: string | undefined): string {
let formattedAlias = alias;
// spaces are included in some types, replace them with underscores
if (formattedAlias.includes(' ')) {
formattedAlias = formattedAlias.trim();
formattedAlias = formattedAlias.split(' ').join('_');
}
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
if (formattedAlias.includes(value)) {
formattedAlias = formattedAlias.replace(value, '');
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
having the same alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
for (const value of ['-', '_', '+']) {
if (formattedAlias.charAt(0) === value) {
formattedAlias = formattedAlias.slice(1);
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
}
if (formattedAlias === '.' || formattedAlias === '..') {
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
return 'alias_placeholder';
}
if (formattedAlias.includes(':')) {
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
Elasticsearch versions!`);
}
return formattedAlias;
}

63
backend/backend/src/storage/elasticsearch/util/index.ts Normal file
View File

@@ -0,0 +1,63 @@
import {SCBulkResponse, SCThingType, SCUuid} from '@openstapps/core';
/**
* Length of the index UID used for generation of its name
*/
export const INDEX_UID_LENGTH = 8;
/**
* A string which matches all indices
*/
export const ALL_INDICES_QUERY = 'stapps_*_*_*';
/**
* Matches index names such as stapps_<type>_<source>_<random suffix>
*/
export const VALID_INDEX_REGEX = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
export interface ParsedIndexName {
type: SCThingType;
source: string;
randomSuffix: string;
}
/**
*
*/
export function parseIndexName(index: string): ParsedIndexName {
const match = VALID_INDEX_REGEX.exec(index);
if (!match) {
throw new SyntaxError(`Invalid index name ${index}!`);
}
return {
type: match[1] as SCThingType,
source: match[2],
randomSuffix: match[3],
};
}
/**
* Gets the index name in elasticsearch for one SCThingType
*
* @param type SCThingType of data in the index
* @param source source of data in the index
* @param bulk bulk process which created this index
*/
export function getThingIndexName(type: SCThingType, source: string, bulk: SCBulkResponse) {
let out = type.toLowerCase();
while (out.includes(' ')) {
out = out.replace(' ', '_');
}
return `stapps_${out}_${source}_${getIndexUID(bulk.uid)}`;
}
/**
* Provides the index UID (for its name) from the bulk UID
*
* @param uid Bulk UID
*/
export function getIndexUID(uid: SCUuid) {
return uid.slice(0, Math.max(0, INDEX_UID_LENGTH));
}

21
backend/backend/src/storage/elasticsearch/util/no-undefined.ts Normal file
View File

@@ -0,0 +1,21 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* Type guard for filter functions
*/
export function noUndefined<T>(item: T | undefined): item is T {
return typeof item !== 'undefined';
}

38
backend/backend/src/storage/elasticsearch/util/retry.ts Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright (C) 2022 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
export interface RetryOptions<T> {
maxRetries: number;
retryInterval: number;
doAction: () => Promise<T>;
onFailedAttempt: (attempt: number, error: unknown, options: RetryOptions<T>) => void;
onFail: (options: RetryOptions<T>) => never;
}
/**
* Retries a throwing function at a set interval, until a maximum amount of attempts
*/
export async function retryCatch<T>(options: RetryOptions<T>): Promise<T> {
for (let attempt = 0; attempt < options.maxRetries; attempt++) {
try {
return await options.doAction();
} catch (error) {
options.onFailedAttempt(attempt, error, options);
await new Promise(resolve => setTimeout(resolve, options.retryInterval));
}
}
options.onFail(options);
}

5
backend/backend/test/common.ts
View File

@@ -16,6 +16,7 @@
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCThingType, SCUuid} from '@openstapps/core';
import {Express} from 'express';
import moment from 'moment';
import {getIndexUID} from '../src/storage/elasticsearch/util';
import {configureApp} from '../src/app';
import express from 'express';
import http from 'http';
@@ -24,7 +25,6 @@ import {MailQueue} from '../src/notification/mail-queue';
import {Bulk, BulkStorage} from '../src/storage/bulk-storage';
import getPort from 'get-port';
import {Database} from '../src/storage/database';
import {Elasticsearch} from '../src/storage/elasticsearch/elasticsearch';
import {v4} from 'uuid';
/**
@@ -147,5 +147,4 @@ export const getTransport = (verified: boolean) => {
};
};
export const getIndex = (uid?: string) =>
`stapps_footype_foosource_${uid ?? Elasticsearch.getIndexUID(v4())}`;
export const getIndex = (uid?: string) => `stapps_footype_foosource_${uid ?? getIndexUID(v4())}`;

67
backend/backend/test/storage/elasticsearch/aggregations.spec.ts
View File

@@ -13,13 +13,13 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {AggregateName, AggregationsMultiTermsBucket} from '@elastic/elasticsearch/lib/api/types';
import {SCFacet, SCThingType} from '@openstapps/core';
import {expect} from 'chai';
import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch';
describe('Aggregations', function () {
const aggregations: AggregationResponse = {
const aggregations: Record<AggregateName, Partial<AggregationsMultiTermsBucket>> = {
'catalog': {
'doc_count': 4,
'superCatalogs.categories': {
@@ -76,14 +76,6 @@ describe('Aggregations', function () {
buckets: [],
},
},
'fooType': {
buckets: [
{
doc_count: 321,
key: 'foo',
},
],
},
'@all': {
doc_count: 17,
type: {
@@ -102,33 +94,6 @@ describe('Aggregations', function () {
};
const expectedFacets: SCFacet[] = [
{
buckets: [
{
count: 13,
key: 'person',
},
{
count: 4,
key: 'catalog',
},
],
field: 'type',
},
{
buckets: [
{
count: 8,
key: 'foobar',
},
{
count: 2,
key: 'bar',
},
],
field: 'categories',
onlyOnType: SCThingType.AcademicEvent,
},
{
buckets: [
{
@@ -153,7 +118,33 @@ describe('Aggregations', function () {
field: 'categories',
onlyOnType: SCThingType.Catalog,
},
// no fooType as it doesn't appear in the aggregation schema
{
buckets: [
{
count: 8,
key: 'foobar',
},
{
count: 2,
key: 'bar',
},
],
field: 'categories',
onlyOnType: SCThingType.AcademicEvent,
},
{
buckets: [
{
count: 13,
key: 'person',
},
{
count: 4,
key: 'catalog',
},
],
field: 'type',
},
];
it('should parse the aggregations providing the appropriate facets', function () {

90
backend/backend/test/storage/elasticsearch/common.spec.ts
View File

@@ -1,90 +0,0 @@
/*
* Copyright (C) 2020 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
ESAggMatchAllFilter,
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter,
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {expect} from 'chai';
import {
isNestedAggregation,
isBucketAggregation,
isESTermsFilter,
isESAggMatchAllFilter,
isESNestedAggregation,
} from '../../../lib/storage/elasticsearch/types/guards';
import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';
describe('Common', function () {
const bucketAggregation: BucketAggregation = {buckets: []};
const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
describe('isBucketAggregation', function () {
it('should be false for a number', function () {
expect(isBucketAggregation(123)).to.be.false;
});
it('should be true for a bucket aggregation', function () {
expect(isBucketAggregation(bucketAggregation)).to.be.true;
});
});
describe('isNestedAggregation', function () {
it('should be false for a bucket aggregation', function () {
expect(isNestedAggregation(bucketAggregation)).to.be.false;
});
it('should be true for a nested aggregation', function () {
const nestedAggregation: NestedAggregation = {doc_count: 123};
expect(isNestedAggregation(nestedAggregation)).to.be.true;
});
});
describe('isESTermsFilter', function () {
it('should be false for an elasticsearch nested aggregation', function () {
expect(isESTermsFilter(esNestedAggregation)).to.be.false;
});
it('should be true for an elasticsearch terms filter', function () {
expect(isESTermsFilter(esTermsFilter)).to.be.true;
});
});
describe('isESNestedAggregation', function () {
it('should be false for an elasticsearch terms filter', function () {
expect(isESNestedAggregation(esTermsFilter)).to.be.false;
});
it('should be true for an elasticsearch nested aggregation', function () {
expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
});
});
describe('isESAggMatchAllFilter', function () {
it('should be false for an elasticsearch aggregation type filter', function () {
const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
});
it('should be true for an elasticsearch aggregation match all filter', function () {
const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
});
});
});

250
backend/backend/test/storage/elasticsearch/elasticsearch.spec.ts
View File

@@ -14,7 +14,14 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client} from '@elastic/elasticsearch';
import {Client, Diagnostic} from '@elastic/elasticsearch';
import Indices from '@elastic/elasticsearch/lib/api/api/indices';
import {
CreateResponse,
SearchHit,
SearchResponse,
SortCombinations,
} from '@elastic/elasticsearch/lib/api/types';
import {
SCBook,
SCBulkResponse,
@@ -30,22 +37,32 @@ import {Logger} from '@openstapps/logger';
import {SMTP} from '@openstapps/logger/lib/smtp';
import {expect, use} from 'chai';
import chaiAsPromised from 'chai-as-promised';
import {SearchResponse} from 'elasticsearch';
import mockedEnv from 'mocked-env';
import sinon from 'sinon';
import {ALL_INDICES_QUERY, parseIndexName} from '../../../src/storage/elasticsearch/util';
import * as queryModule from '../../../src/storage/elasticsearch/query/query';
import * as sortModule from '../../../src/storage/elasticsearch/query/sort';
import sinon, {SinonStub} from 'sinon';
import {getIndexUID, getThingIndexName, INDEX_UID_LENGTH} from '../../../src/storage/elasticsearch/util';
import * as utilModule from '../../../src/storage/elasticsearch/util';
import {removeInvalidAliasChars} from '../../../src/storage/elasticsearch/util/alias';
import {configFile} from '../../../src/common';
import {MailQueue} from '../../../src/notification/mail-queue';
import {aggregations} from '../../../src/storage/elasticsearch/templating';
import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch';
import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
import * as query from '../../../src/storage/elasticsearch/query';
import * as templating from '../../../src/storage/elasticsearch/templating';
import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common';
import fs from 'fs';
use(chaiAsPromised);
/**
*
*/
function searchResponse<T>(...hits: SearchHit<T>[]): SearchResponse<T> {
return {hits: {hits}, took: 0, timed_out: false, _shards: {total: 1, failed: 0, successful: 1}};
}
describe('Elasticsearch', function () {
// increase timeout for the suite
this.timeout(DEFAULT_TEST_TIMEOUT);
@@ -83,6 +100,14 @@ describe('Elasticsearch', function () {
});
});
describe('getAliasMap', function () {
it('should fail after retries', async function () {
const es = new Elasticsearch(configFile);
sandbox.stub(es.client.indices, 'getAlias').throws();
await expect(es.init({maxRetries: 1, retryInterval: 10})).to.be.rejected;
});
});
describe('getIndex (including getIndexUID)', function () {
const type = 'foo bar type';
const source = 'foo_source';
@@ -95,59 +120,63 @@ describe('Elasticsearch', function () {
};
it('should provide index UID from the provided UID', function () {
const indexUID = Elasticsearch.getIndexUID(bulk.uid);
const indexUID = getIndexUID(bulk.uid);
expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
expect(indexUID.length).to.be.equal(INDEX_UID_LENGTH);
// test starting and ending character
expect(indexUID[0]).to.be.equal(bulk.uid[0]);
expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[INDEX_UID_LENGTH - 1]);
});
it('should provide index name from the provided data', function () {
expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
expect(getThingIndexName(type as SCThingType, source, bulk)).to.be.equal(
`stapps_${type.split(' ').join('_')}_${source}_${getIndexUID(bulk.uid)}`,
);
});
it('should reject invalid index names', function () {
expect(() => parseIndexName(':)')).to.throw(SyntaxError);
});
});
describe('removeAliasChars', function () {
it('should remove spaces from both ends', function () {
expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should replace inner spaces with underscores', function () {
expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
expect(removeInvalidAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
});
it('should remove invalid characters', function () {
expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should remove invalid starting characters', function () {
expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should replace with a placeholder in case of invalid alias', function () {
expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
expect(removeInvalidAliasChars('.', 'bulk-uid')).to.contain('placeholder');
expect(removeInvalidAliasChars('..', 'bulk-uid')).to.contain('placeholder');
});
it('should work with common cases', function () {
expect(
Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
removeInvalidAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
expect(
Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
expect(removeInvalidAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid')).to.be.equal(
'THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG',
);
});
it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
const sandbox = sinon.createSandbox();
const loggerWarnStub = sandbox.stub(Logger, 'warn');
expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
expect(removeInvalidAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
expect(loggerWarnStub.called).to.be.true;
});
});
@@ -182,7 +211,7 @@ describe('Elasticsearch', function () {
it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
const error = new Error('Foo Error');
const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
sandbox.stub(Client.prototype, 'on').yields(error);
sandbox.stub(Diagnostic.prototype, 'on').yields(error);
new Elasticsearch(configFile);
@@ -192,7 +221,7 @@ describe('Elasticsearch', function () {
it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
const fakeResponse = {foo: 'bar'};
const loggerLogStub = sandbox.stub(Logger, 'log');
sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
sandbox.stub(Diagnostic.prototype, 'on').yields(null, fakeResponse);
new Elasticsearch(configFile);
expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
@@ -254,26 +283,24 @@ describe('Elasticsearch', function () {
describe('Operations with bundle/index', async function () {
const sandbox = sinon.createSandbox();
let es: Elasticsearch;
let createStub: SinonStub;
let deleteStub: SinonStub;
let refreshStub: SinonStub;
let updateAliasesStub: SinonStub;
let existsStub: SinonStub;
const oldIndex = 'stapps_footype_foosource_oldindex';
beforeEach(function () {
sandbox
.stub(Indices.prototype, 'getAlias')
.resolves({[oldIndex]: {aliases: {[SCThingType.Book]: {}}}} as any);
sandbox.stub(Indices.prototype, 'putTemplate').resolves({} as any);
createStub = sandbox.stub(Indices.prototype, 'create').resolves({} as any);
deleteStub = sandbox.stub(Indices.prototype, 'delete').resolves({} as any);
existsStub = sandbox.stub(Indices.prototype, 'exists').resolves({} as any);
refreshStub = sandbox.stub(Indices.prototype, 'refresh').resolves({} as any);
updateAliasesStub = sandbox.stub(Indices.prototype, 'updateAliases').resolves({} as any);
es = new Elasticsearch(configFile);
es.client.indices = {
// @ts-expect-error not assignable
getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
// @ts-expect-error not assignable
putTemplate: () => Promise.resolve({}),
// @ts-expect-error not assignable
create: () => Promise.resolve({}),
// @ts-expect-error not assignable
delete: () => Promise.resolve({}),
// @ts-expect-error not assignable
exists: () => Promise.resolve({}),
// @ts-expect-error not assignable
refresh: () => Promise.resolve({}),
// @ts-expect-error not assignable
updateAliases: () => Promise.resolve({}),
};
});
afterEach(function () {
@@ -286,8 +313,8 @@ describe('Elasticsearch', function () {
});
it('should reject (throw an error) if the index name is not valid', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`);
sandbox.createStubInstance(Client, {});
sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex}`);
await es.init();
return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
@@ -295,9 +322,8 @@ describe('Elasticsearch', function () {
it('should create a new index', async function () {
const index = getIndex();
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
const putTemplateStub = sandbox.stub(templating, 'putTemplate');
const createStub = sandbox.stub(es.client.indices, 'create');
await es.init();
await es.bulkCreated(bulk);
@@ -313,21 +339,19 @@ describe('Elasticsearch', function () {
sandbox.restore();
});
it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
await es.bulkExpired({...bulk, state: 'in progress'});
expect(clientDeleteStub.called).to.be.true;
expect(deleteStub.called).to.be.true;
});
it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
await es.bulkExpired({...bulk, state: 'done'});
expect(clientDeleteStub.called).to.be.false;
expect(deleteStub.called).to.be.false;
});
});
@@ -337,13 +361,23 @@ describe('Elasticsearch', function () {
});
it('should reject if the index name is not valid', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`);
sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex()}`);
sandbox.createStubInstance(Client, {});
await es.init();
return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
});
it("should create templates if index doesn't exist", async function () {
await es.init();
existsStub.resolves(false);
const putTemplateSpy = sandbox.spy(templating, 'putTemplate');
await es.bulkUpdated(bulk);
expect(createStub.called).to.be.true;
expect(putTemplateSpy.called).to.be.true;
});
it('should create a new index', async function () {
const index = getIndex();
const expectedRefreshActions = [
@@ -354,15 +388,12 @@ describe('Elasticsearch', function () {
remove: {index: oldIndex, alias: SCThingType.Book},
},
];
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
sandbox.stub(es, 'aliasMap').value({
[SCThingType.Book]: {
[bulk.source]: oldIndex,
},
});
const refreshStub = sandbox.stub(es.client.indices, 'refresh');
const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
const deleteStub = sandbox.stub(es.client.indices, 'delete');
sandbox.stub(templating, 'putTemplate');
await es.init();
@@ -371,9 +402,7 @@ describe('Elasticsearch', function () {
expect(refreshStub.calledWith({index})).to.be.true;
expect(
updateAliasesStub.calledWith({
body: {
actions: expectedRefreshActions,
},
actions: expectedRefreshActions,
}),
).to.be.true;
expect(deleteStub.called).to.be.true;
@@ -394,20 +423,19 @@ describe('Elasticsearch', function () {
});
it('should reject if object is not found', async function () {
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
return expect(es.get('123')).to.rejectedWith('found');
});
it('should provide the thing if object is found', async function () {
const foundObject: ElasticsearchObject<SCMessage> = {
const foundObject: SearchHit<SCMessage> = {
_id: '',
_index: '',
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
sandbox.stub(es.client, 'search').resolves(searchResponse(foundObject));
return expect(await es.get('123')).to.be.eql(message);
});
@@ -428,56 +456,54 @@ describe('Elasticsearch', function () {
it('should not post if the object already exists in an index which will not be rolled over', async function () {
const index = getIndex();
const oldIndex = index.replace('foosource', 'barsource');
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: oldIndex,
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
return expect(es.post(object._source!, bulk)).to.rejectedWith('exist');
});
it('should not reject if the object already exists but in an index which will be rolled over', async function () {
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
// return index name with different generated UID (see getIndex method)
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist');
return expect(es.post(object._source!, bulk)).to.not.rejectedWith('exist');
});
it('should reject if there is an object creation error on the elasticsearch side', async function () {
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'create').resolves({body: {created: false}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse);
return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
});
it('should create a new object', async function () {
let caughtParameter: any;
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
// @ts-expect-error call
const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
caughtParameter = parameter;
return Promise.resolve({body: {created: true}});
return Promise.resolve({result: 'created'});
});
await es.post(message as SCMessage, bulk);
expect(createStub.called).to.be.true;
expect(caughtParameter.body).to.be.eql({
expect(caughtParameter.document).to.be.eql({
...message,
creation_date: caughtParameter.body.creation_date,
creation_date: caughtParameter.document.creation_date,
});
});
});
@@ -493,29 +519,27 @@ describe('Elasticsearch', function () {
sandbox.restore();
});
it('should reject to put if the object does not already exist', async function () {
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
return expect(es.put(object._source)).to.rejectedWith('exist');
return expect(es.put(object._source!)).to.rejectedWith('exist');
});
// noinspection JSUnusedLocalSymbols
it('should update the object if it already exists', async function () {
let caughtParameter: any;
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
// @ts-expect-error unused
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
@@ -523,7 +547,7 @@ describe('Elasticsearch', function () {
return Promise.resolve({body: {created: true}});
});
await es.put(object._source);
await es.put(object._source!);
expect(caughtParameter.body.doc).to.be.eql(object._source);
});
@@ -532,18 +556,16 @@ describe('Elasticsearch', function () {
describe('search', async function () {
let es: Elasticsearch;
const sandbox = sinon.createSandbox();
const objectMessage: ElasticsearchObject<SCMessage> = {
const objectMessage: SearchHit<SCMessage> = {
_id: '123',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
const objectBook: ElasticsearchObject<SCBook> = {
const objectBook: SearchHit<SCBook> = {
_id: '321',
_index: getIndex(),
_score: 0,
_type: '',
_source: book as SCBook,
};
const fakeEsAggregations = {
@@ -565,26 +587,16 @@ describe('Elasticsearch', function () {
},
},
};
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
body: {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
hits: [objectMessage, objectBook],
total: 123,
},
aggregations: fakeEsAggregations,
const fakeSearchResponse: SearchResponse<SCThings> = {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
hits: {
hits: [objectMessage, objectBook],
total: 123,
},
headers: {},
// @ts-expect-error not assignable
meta: {},
// @ts-expect-error not assignable
statusCode: {},
// @ts-expect-error not assignable
warnings: {},
aggregations: fakeEsAggregations,
};
let searchStub: sinon.SinonStub;
before(function () {
@@ -625,9 +637,9 @@ describe('Elasticsearch', function () {
const {pagination} = await es.search({from});
expect(pagination).to.be.eql({
count: fakeSearchResponse.body!.hits.hits.length,
count: fakeSearchResponse.hits.hits.length,
offset: from,
total: fakeSearchResponse.body!.hits.total,
total: fakeSearchResponse.hits.total,
});
});
@@ -659,22 +671,20 @@ describe('Elasticsearch', function () {
},
},
};
const fakeResponse = {foo: 'bar'};
const fakeResponse = {foo: 'bar'} as SortCombinations;
const fakeBuildSortResponse = [fakeResponse];
// @ts-expect-error not assignable
sandbox.stub(query, 'buildQuery').returns(fakeResponse);
sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
sandbox.stub(queryModule, 'buildQuery').returns(fakeResponse);
sandbox.stub(sortModule, 'buildSort').returns(fakeBuildSortResponse);
await es.search(parameters);
sandbox.assert.calledWithMatch(searchStub, {
body: {
aggs: aggregations,
query: fakeResponse,
sort: fakeBuildSortResponse,
},
aggs: aggregations,
query: fakeResponse,
sort: fakeBuildSortResponse,
from: parameters.from,
index: Elasticsearch.getListOfAllIndices(),
index: ALL_INDICES_QUERY,
size: parameters.size,
});
});

22
backend/backend/test/storage/elasticsearch/monitoring.spec.ts
View File

@@ -14,7 +14,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client} from '@elastic/elasticsearch';
import {Client} from '@elastic/elasticsearch';
import {SearchResponse} from '@elastic/elasticsearch/lib/api/types';
import {
SCMonitoringConfiguration,
SCMonitoringLogAction,
@@ -23,7 +24,6 @@ import {
SCThings,
} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
import {SearchResponse} from 'elasticsearch';
import {MailQueue} from '../../../src/notification/mail-queue';
import {setUp} from '../../../src/storage/elasticsearch/monitoring';
@@ -111,16 +111,14 @@ describe('Monitoring', async function () {
});
it('should log errors where conditions failed', async function () {
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
body: {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
total: 123,
},
const fakeSearchResponse: SearchResponse<SCThings> = {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
total: 123,
},
};
const fakeClient = new Client({node: 'http://foohost:9200'});

113
backend/backend/test/storage/elasticsearch/query.spec.ts
View File

@@ -25,25 +25,14 @@ import {
SCThingType,
} from '@openstapps/core';
import {expect} from 'chai';
import {
ESDateRangeFilter,
ESRangeFilter,
ESNumericRangeFilter,
ElasticsearchConfig,
ESBooleanFilter,
ESGenericSort,
ESGeoDistanceFilter,
ESGeoDistanceSort,
ESTermFilter,
ScriptSort,
} from '../../../src/storage/elasticsearch/types/elasticsearch';
import {buildFilter} from '../../../src/storage/elasticsearch/query/filter';
import {buildBooleanFilter} from '../../../src/storage/elasticsearch/query/filters/boolean';
import {buildQuery} from '../../../src/storage/elasticsearch/query/query';
import {buildSort} from '../../../src/storage/elasticsearch/query/sort';
import {ElasticsearchConfig} from '../../../src/storage/elasticsearch/types/elasticsearch-config';
import {QueryDslSpecificQueryContainer} from '../../../src/storage/elasticsearch/types/util';
import {configFile} from '../../../src/common';
import {
buildBooleanFilter,
buildFilter,
buildQuery,
buildSort,
} from '../../../src/storage/elasticsearch/query';
import {SortCombinations} from '@elastic/elasticsearch/lib/api/types';
describe('Query', function () {
describe('buildBooleanFilter', function () {
@@ -74,7 +63,7 @@ describe('Query', function () {
or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
};
const expectedEsFilters: Array<ESTermFilter> = [
const expectedEsFilters: Array<QueryDslSpecificQueryContainer<'term'>> = [
{
term: {
'type.raw': 'catalog',
@@ -88,20 +77,20 @@ describe('Query', function () {
];
it('should create appropriate elasticsearch "and" filter argument', function () {
const {must} = buildBooleanFilter(booleanFilters.and);
const {must} = buildBooleanFilter(booleanFilters.and).bool;
expect(must).to.be.eql(expectedEsFilters);
});
it('should create appropriate elasticsearch "or" filter argument', function () {
const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or).bool;
expect(should).to.be.eql(expectedEsFilters);
expect(minimum_should_match).to.be.equal(1);
});
it('should create appropriate elasticsearch "not" filter argument', function () {
const {must_not} = buildBooleanFilter(booleanFilters.not);
const {must_not} = buildBooleanFilter(booleanFilters.not).bool;
expect(must_not).to.be.eql(expectedEsFilters);
});
@@ -196,6 +185,10 @@ describe('Query', function () {
expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
});
it('should accept other search contexts', function () {
expect(buildQuery({context: 'place', ...parameters}, config, esConfig)).to.be.an('object');
});
});
describe('buildFilter', function () {
@@ -267,7 +260,7 @@ describe('Query', function () {
it('should build value filter', function () {
const filter = buildFilter(searchFilters.value);
const expectedFilter: ESTermFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'term'> = {
term: {
'type.raw': SCThingType.Dish,
},
@@ -279,7 +272,7 @@ describe('Query', function () {
it('should build numeric range filters', function () {
for (const upperMode of ['inclusive', 'exclusive', null]) {
for (const lowerMode of ['inclusive', 'exclusive', null]) {
const expectedFilter: ESNumericRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
price: {
relation: undefined,
@@ -304,7 +297,7 @@ describe('Query', function () {
mode: bound as 'inclusive' | 'exclusive',
limit: out,
};
expectedFilter.range.price[
expectedFilter.range.price![
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
] = out;
}
@@ -312,7 +305,7 @@ describe('Query', function () {
setBound('upperBound', upperMode);
setBound('lowerBound', lowerMode);
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'term'>;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
// @ts-expect-error implicit any
@@ -330,7 +323,7 @@ describe('Query', function () {
it('should build date range filters', function () {
for (const upperMode of ['inclusive', 'exclusive', null]) {
for (const lowerMode of ['inclusive', 'exclusive', null]) {
const expectedFilter: ESDateRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
price: {
format: 'thisIsADummyFormat',
@@ -359,7 +352,7 @@ describe('Query', function () {
mode: bound as 'inclusive' | 'exclusive',
limit: out,
};
expectedFilter.range.price[
expectedFilter.range.price![
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
] = out;
}
@@ -367,7 +360,7 @@ describe('Query', function () {
setBound('upperBound', upperMode);
setBound('lowerBound', lowerMode);
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'range'>;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
// @ts-expect-error implicit any
@@ -394,7 +387,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: `test||/${scope}`,
@@ -415,7 +408,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: 'test||/s',
@@ -436,7 +429,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: `test||/d`,
@@ -456,7 +449,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: `now/d`,
@@ -470,7 +463,7 @@ describe('Query', function () {
it('should build distance filter', function () {
const filter = buildFilter(searchFilters.distance);
const expectedFilter: ESGeoDistanceFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'geo_distance'> = {
geo_distance: {
'distance': '1000m',
'geo.point.coordinates': {
@@ -486,34 +479,18 @@ describe('Query', function () {
it('should build geo filter for shapes and points', function () {
const filter = buildFilter(searchFilters.geoPoint);
const expectedFilter = {
bool: {
minimum_should_match: 1,
should: [
{
geo_shape: {
'geo.polygon': {
relation: undefined,
shape: {
type: 'envelope',
coordinates: [
[50.123, 8.123],
[50.123, 8.123],
],
},
},
'ignore_unmapped': true,
},
geo_shape: {
'geo.polygon': {
relation: undefined,
shape: {
type: 'envelope',
coordinates: [
[50.123, 8.123],
[50.123, 8.123],
],
},
{
geo_bounding_box: {
'geo.point.coordinates': {
bottom_right: [50.123, 8.123],
top_left: [50.123, 8.123],
},
'ignore_unmapped': true,
},
},
],
},
'ignore_unmapped': true,
},
};
@@ -543,7 +520,7 @@ describe('Query', function () {
it('should build boolean filter', function () {
const filter = buildFilter(searchFilters.boolean);
const expectedFilter: ESBooleanFilter<any> = {
const expectedFilter: QueryDslSpecificQueryContainer<'bool'> = {
bool: {
minimum_should_match: 0,
must: [
@@ -604,8 +581,8 @@ describe('Query', function () {
},
},
];
let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
let sorts: SortCombinations[] = [];
const expectedSorts: {[key: string]: SortCombinations} = {
ducet: {
'name.sort': 'desc',
},
@@ -632,7 +609,7 @@ describe('Query', function () {
},
};
before(function () {
sorts = buildSort(searchSCSearchSort);
sorts = buildSort(searchSCSearchSort) as SortCombinations[];
});
it('should build ducet sort', function () {
@@ -649,10 +626,10 @@ describe('Query', function () {
it('should build price sort', function () {
const priceSortNoScript = {
...sorts[3],
...(sorts[3] as any),
_script: {
...(sorts[3] as ScriptSort)._script,
script: (expectedSorts.price as ScriptSort)._script.script,
...(sorts[3] as any)._script,
script: (expectedSorts.price as any)._script.script,
},
};
expect(priceSortNoScript).to.be.eql(expectedSorts.price);

4
backend/backend/tsconfig.json
View File

@@ -2,7 +2,9 @@
"extends": "./node_modules/@openstapps/configuration/tsconfig.json",
"compilerOptions": {
"resolveJsonModule": true,
"useUnknownInCatchVariables": false
"skipLibCheck": true,
"useUnknownInCatchVariables": false,
"lib": ["ES2020"]
},
"exclude": [
"./config/",