feat: update to of elasticsearch 8.4

README.md

@@ -18,9 +18,10 @@ you with everything you need to run this backend.
 
 # Local usage for development purposes
 ## Requirements
-* Elasticsearch (5.6)
+* Elasticsearch (8.4)
+  - [ICU analysis plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html)
+  - OR Docker
 * Node.js (~14) / NPM
-* Docker
 
 ### Startup Behaviour
 
@@ -34,7 +35,7 @@ with the backend. To save you some work we provide a
 [docker image](https://gitlab.com/openstapps/database) which
 only needs to be executed to work with the backend.
 
-Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:master`
+Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:latest`
 
 Elasticsearch should be running at port 9200 now. If you have problems with
 getting elasticsearch to work, have a look in the

config/elasticsearch.ts

@@ -1,6 +1,6 @@
 // tslint:disable:no-default-export
 // tslint:disable:no-magic-numbers
-import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch';
+import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch-config';
 
 /**
  * This is the default configuration for elasticsearch (a database)
@@ -19,13 +19,13 @@ const config: ElasticsearchConfigFile = {
   internal: {
     database: {
       name: 'elasticsearch',
-      version: '5.6',
+      version: '8.4',
       query: {
         minMatch: '75%',
         queryType: 'dis_max',
         matchBoosting: 1.3,
         fuzziness: 'AUTO',
-        cutoffFrequency: 0.0,
+        cutoffFrequency: 0,
         tieBreaker: 0,
       },
     },

integration-test.yml

@@ -15,7 +15,7 @@ services:
   elasticsearch:
     ports:
       - "9200:9200"
-    image: "registry.gitlab.com/openstapps/database:master"
+    image: "registry.gitlab.com/openstapps/database:latest"
 
   apicli:
     image: "registry.gitlab.com/openstapps/api/cli:latest"

package.json

@@ -27,14 +27,14 @@
     "start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js",
     "start-debug": "STAPPS_LOG_LEVEL=31 NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js --require ts-node/register",
     "test": "npm run test-unit && npm run test-integration",
-    "test-unit": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
+    "test-unit": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
     "test-integration": "docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --build --abort-on-container-exit --exit-code-from apicli",
     "lint": "eslint -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/",
     "lint:fix": "eslint --fix -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "5.6.22",
+    "@elastic/elasticsearch": "8.4.0",
-    "@openstapps/core": "0.74.0",
+    "@openstapps/core": "1.0.1",
     "@openstapps/core-tools": "0.34.0",
     "@openstapps/logger": "1.1.1",
     "@types/node": "14.18.36",
@@ -57,14 +57,13 @@
   },
   "devDependencies": {
     "@openstapps/configuration": "0.34.0",
-    "@openstapps/es-mapping-generator": "0.4.0",
+    "@openstapps/es-mapping-generator": "0.6.0",
     "@openstapps/eslint-config": "1.1.0",
     "@testdeck/mocha": "0.3.3",
     "@types/chai": "4.3.4",
     "@types/chai-as-promised": "7.1.5",
     "@types/config": "3.3.0",
     "@types/cors": "2.8.13",
-    "@types/elasticsearch": "5.0.40",
     "@types/express": "4.17.16",
     "@types/geojson": "1.0.6",
     "@types/mocha": "10.0.1",
@@ -80,6 +79,7 @@
     "chai": "4.3.7",
     "chai-as-promised": "7.1.1",
     "conventional-changelog-cli": "2.2.2",
+    "cross-env": "7.0.3",
     "eslint": "8.33.0",
     "eslint-config-prettier": "8.6.0",
     "eslint-plugin-jsdoc": "39.7.4",

src/app.ts

@@ -64,6 +64,7 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
     }),
   );
 
+  /* istanbul ignore if */
   if (process.env.PROMETHEUS_MIDDLEWARE === 'true') {
     app.use(getPrometheusMiddleware());
   }
@@ -142,7 +143,10 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
   });
 
   // validate config file
-  await validator.addSchemas(path.join('node_modules', '@openstapps', 'core', 'lib', 'schema'));
+  await validator.addSchemas(
+    // eslint-disable-next-line unicorn/prefer-module
+    path.join(path.dirname(require.resolve('@openstapps/core/package.json')), 'lib', 'schema'),
+  );
 
   // validate the config file
   const configValidation = validator.validate(configFile, 'SCConfigFile');

src/storage/elasticsearch/aggregations.ts

@@ -13,73 +13,45 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {SCFacet, SCThingType} from '@openstapps/core';
-import {aggregations} from './templating';
-import {AggregationResponse} from './types/elasticsearch';
 import {
-  isBucketAggregation,
+  AggregateName,
-  isESAggMatchAllFilter,
+  AggregationsAggregate,
-  isESNestedAggregation,
+  AggregationsFiltersAggregate,
-  isESTermsFilter,
+  AggregationsMultiTermsBucket,
-  isNestedAggregation,
+} from '@elastic/elasticsearch/lib/api/types';
-} from './types/guards';
+import {SCFacet, SCThingType} from '@openstapps/core';
 
 /**
  * Parses elasticsearch aggregations (response from es) to facets for the app
  *
  * @param aggregationResponse - aggregations response from elasticsearch
  */
-export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
+export function parseAggregations(
+  aggregationResponse: Record<AggregateName, AggregationsAggregate>,
+): SCFacet[] {
   const facets: SCFacet[] = [];
 
-  // get all names of the types an aggregation is on
+  for (const aggregateName in aggregationResponse) {
-  for (const typeName in aggregations) {
+    const aggregation = aggregationResponse[aggregateName] as AggregationsMultiTermsBucket;
-    if (aggregations.hasOwnProperty(typeName) && aggregationResponse.hasOwnProperty(typeName)) {
+    const type = aggregateName === '@all' ? {} : {onlyOnType: aggregateName as SCThingType};
-      // the type object from the schema
-      const type = aggregations[typeName];
-      // the "real" type object from the response
-      const realType = aggregationResponse[typeName];
 
-      // both conditions must apply, else we have an error somewhere
+    for (const field in aggregation) {
-      if (isESNestedAggregation(type) && isNestedAggregation(realType)) {
+      const fieldAggregate = aggregation[field] as AggregationsFiltersAggregate;
-        for (const fieldName in type.aggs) {
+      if (typeof fieldAggregate !== 'object') continue;
-          if (type.aggs.hasOwnProperty(fieldName) && realType.hasOwnProperty(fieldName)) {
-            // the field object from the schema
-            const field = type.aggs[fieldName];
-            // the "real" field object from the response
-            const realField = realType[fieldName];
 
-            // this should always be true in theory...
+      const buckets = Object.values(fieldAggregate.buckets).map(bucket => {
-            if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) {
-              const facet: SCFacet = {
-                buckets: realField.buckets.map(bucket => {
         return {
           count: bucket.doc_count,
-                    key: bucket.key,
+          key: bucket.key as string,
         };
-                }),
-                field: fieldName,
-              };
-              // if it's not for all types then create the appropriate field and set the type name
-              if (!isESAggMatchAllFilter(type.filter)) {
-                facet.onlyOnType = type.filter.type.value as SCThingType;
-              }
-              facets.push(facet);
-            }
-          }
-        }
-        // the last part here means that it is a bucket aggregation
-      } else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) {
-        facets.push({
-          buckets: realType.buckets.map(bucket => {
-            return {
-              count: bucket.doc_count,
-              key: bucket.key,
-            };
-          }),
-          field: typeName,
       });
-      }
+      if (buckets.length === 0) continue;
+
+      facets.push({
+        buckets,
+        field,
+        ...type,
+      });
     }
   }
 

src/storage/elasticsearch/elasticsearch.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019 StApps
+ * Copyright (C) 2022 StApps
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
  * published by the Free Software Foundation, either version 3 of the
@@ -13,58 +13,47 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch';
+import {Client, events} from '@elastic/elasticsearch';
 import {
-  SCBulkResponse,
+  AggregateName,
-  SCConfigFile,
+  AggregationsMultiTermsBucket,
-  SCFacet,
+  IndicesGetAliasResponse,
-  SCSearchQuery,
+  IndicesUpdateAliasesAction,
-  SCSearchResponse,
+  SearchHit,
-  SCThings,
+  SearchResponse,
-  SCThingType,
+} from '@elastic/elasticsearch/lib/api/types';
-  SCUuid,
+import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
-} from '@openstapps/core';
 import {Logger} from '@openstapps/logger';
-// we only have the @types package because some things type definitions are still missing from the official
-// @elastic/elasticsearch package
-import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
 import moment from 'moment';
 import {MailQueue} from '../../notification/mail-queue';
 import {Bulk} from '../bulk-storage';
 import {Database} from '../database';
 import {parseAggregations} from './aggregations';
 import * as Monitoring from './monitoring';
-import {buildQuery, buildSort} from './query';
+import {buildQuery} from './query/query';
+import {buildSort} from './query/sort';
 import {aggregations, putTemplate} from './templating';
 import {
-  AggregationResponse,
   ElasticsearchConfig,
-  ElasticsearchObject,
   ElasticsearchQueryDisMaxConfig,
   ElasticsearchQueryQueryStringConfig,
-} from './types/elasticsearch';
+} from './types/elasticsearch-config';
-
+import {ALL_INDICES_QUERY, getThingIndexName, parseIndexName, VALID_INDEX_REGEX} from './util';
-/**
+import {removeInvalidAliasChars} from './util/alias';
- * Matches index names such as stapps_<type>_<source>_<random suffix>
+import {noUndefined} from './util/no-undefined';
- */
+import {retryCatch, RetryOptions} from './util/retry';
-const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
 
 /**
  * A database interface for elasticsearch
  */
 export class Elasticsearch implements Database {
-  /**
-   * Length of the index UID used for generation of its name
-   */
-  static readonly INDEX_UID_LENGTH = 8;
-
   /**
    * Holds a map of all elasticsearch indices that are available to search
    */
   aliasMap: {
-    // each scType has a alias which can contain multiple sources
+    // each scType has an alias which can contain multiple sources
     [scType: string]: {
-      // each source is assigned a index name in elasticsearch
+      // each source is assigned an index name in elasticsearch
       [source: string]: string;
     };
   };
@@ -97,84 +86,6 @@ export class Elasticsearch implements Database {
     return 'http://localhost:9200';
   }
 
-  /**
-   * Gets the index name in elasticsearch for one SCThingType
-   *
-   * @param type SCThingType of data in the index
-   * @param source source of data in the index
-   * @param bulk bulk process which created this index
-   */
-  static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) {
-    let out = type.toLowerCase();
-    while (out.includes(' ')) {
-      out = out.replace(' ', '_');
-    }
-
-    return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`;
-  }
-
-  /**
-   * Provides the index UID (for its name) from the bulk UID
-   *
-   * @param uid Bulk UID
-   */
-  static getIndexUID(uid: SCUuid) {
-    return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH));
-  }
-
-  /**
-   * Generates a string which matches all indices
-   */
-  static getListOfAllIndices(): string {
-    // map each SC type in upper camel case
-    return 'stapps_*_*_*';
-  }
-
-  /**
-   * Checks for invalid character in alias names and removes them
-   *
-   * @param alias The alias name
-   * @param uid   The UID of the current bulk (for debugging purposes)
-   */
-  static removeAliasChars(alias: string, uid: string | undefined): string {
-    let formattedAlias = alias;
-
-    // spaces are included in some types, replace them with underscores
-    if (formattedAlias.includes(' ')) {
-      formattedAlias = formattedAlias.trim();
-      formattedAlias = formattedAlias.split(' ').join('_');
-    }
-    // List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
-    for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
-      if (formattedAlias.includes(value)) {
-        formattedAlias = formattedAlias.replace(value, '');
-        Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
-        having the same alias despite having different types, as invalid characters are removed automatically.
-        New alias name is "${formattedAlias}."`);
-      }
-    }
-    for (const value of ['-', '_', '+']) {
-      if (formattedAlias.charAt(0) === value) {
-        formattedAlias = formattedAlias.slice(1);
-        Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
-        alias despite having different types, as invalid characters are removed automatically.
-        New alias name is "${formattedAlias}."`);
-      }
-    }
-    if (formattedAlias === '.' || formattedAlias === '..') {
-      Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
-      another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
-
-      return 'alias_placeholder';
-    }
-    if (formattedAlias.includes(':')) {
-      Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
-      Elasticsearch versions!`);
-    }
-
-    return formattedAlias;
-  }
-
   /**
    * Create a new interface for elasticsearch
    *
@@ -192,7 +103,7 @@ export class Elasticsearch implements Database {
     this.client = new Client({
       node: Elasticsearch.getElasticsearchUrl(),
     });
-    this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse<unknown>) => {
+    this.client.diagnostic.on(events.REQUEST, async (error: Error | null, result: unknown) => {
       if (error !== null) {
         await Logger.error(error);
       }
@@ -210,73 +121,40 @@ export class Elasticsearch implements Database {
   /**
    * Gets a map which contains each alias and all indices that are associated with each alias
    */
-  private async getAliasMap() {
+  private async getAliasMap(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}) {
-    // delay after which alias map will be fetched again
+    const aliasResponse = await retryCatch({
-    const RETRY_INTERVAL = 5000;
+      maxRetries: 10,
-    // maximum number of retries
+      retryInterval: 2000,
-    const RETRY_COUNT = 3;
+      doAction: () => this.client.indices.getAlias(),
-    // create a list of old indices that are not in use
+      onFailedAttempt: (attempt, error, {maxRetries, retryInterval}) => {
-    const oldIndicesToDelete: string[] = [];
-
-    let aliases:
-      | {
-          [index: string]: {
-            /**
-             * Aliases of an index
-             */
-            aliases: {
-              [K in SCThingType]: unknown;
-            };
-          };
-        }
-      | undefined;
-
-    for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) {
-      if (typeof aliases !== 'undefined') {
-        break;
-      }
-      try {
-        const aliasResponse = await this.client.indices.getAlias({});
-        aliases = aliasResponse.body;
-      } catch (error) {
         Logger.warn('Failed getting alias map:', error);
-        Logger.warn(`Retrying in ${RETRY_INTERVAL} milliseconds. (${retry} of ${RETRY_COUNT})`);
+        Logger.warn(`Retrying in ${retryInterval} milliseconds. (${attempt} of ${maxRetries})`);
-        await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL));
+      },
-      }
+      onFail: ({maxRetries}) => {
-    }
+        throw new TypeError(`Failed to retrieve alias map after ${maxRetries} attempts!`);
+      },
+      ...retryOptions,
+    });
 
-    if (typeof aliases === 'undefined') {
+    const aliases = Object.entries(aliasResponse)
-      throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
+      .filter(([index]) => !index.startsWith('.'))
-    }
+      .map(([index, alias]) => ({
+        index,
+        alias,
+        ...parseIndexName(index),
+      }));
 
-    for (const index in aliases) {
+    for (const {type, index, source} of aliases.filter(({type, alias}) => type in alias.aliases)) {
-      if (aliases.hasOwnProperty(index)) {
+      this.aliasMap[type] = this.aliasMap[type] || {};
-        const matches = indexRegex.exec(index);
-        if (matches !== null) {
-          const type = matches[1];
-          const source = matches[2];
-
-          // check if there is an alias for the current index
-          // check that alias equals type
-          const hasAlias = type in aliases[index].aliases;
-          if (hasAlias) {
-            if (typeof this.aliasMap[type] === 'undefined') {
-              this.aliasMap[type] = {};
-            }
       this.aliasMap[type][source] = index;
-          } else {
-            oldIndicesToDelete.push(index);
-          }
-        }
-      }
     }
 
     this.ready = true;
 
-    // delete old indices that are not used in any alias
+    const unusedIndices = aliases.filter(({type, alias}) => !(type in alias.aliases)).map(({index}) => index);
-    if (oldIndicesToDelete.length > 0) {
+    if (unusedIndices.length > 0) {
       await this.client.indices.delete({
-        index: oldIndicesToDelete,
+        index: unusedIndices,
       });
       Logger.warn(`Deleted old indices: oldIndicesToDelete`);
     }
@@ -291,8 +169,8 @@ export class Elasticsearch implements Database {
    * @param uid an UID to use for the search
    * @returns an elasticsearch object containing the thing
    */
-  private async getObject(uid: SCUuid): Promise<ElasticsearchObject<SCThings> | undefined> {
+  private async getObject(uid: SCUuid): Promise<SearchHit<SCThings> | undefined> {
-    const searchResponse: ApiResponse<SearchResponse<SCThings>> = await this.client.search({
+    const searchResponse = await this.client.search<SCThings>({
       body: {
         query: {
           term: {
@@ -303,43 +181,44 @@ export class Elasticsearch implements Database {
         },
       },
       from: 0,
-      index: Elasticsearch.getListOfAllIndices(),
+      index: ALL_INDICES_QUERY,
       size: 1,
     });
 
     // return data from response
-    return searchResponse.body.hits.hits[0];
+    return searchResponse.hits.hits[0];
   }
 
-  /**
+  private async prepareBulkWrite(bulk: Bulk): Promise<{index: string; alias: string}> {
-   * Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
-   *
-   * @param bulk the bulk process that was created
-   */
-  public async bulkCreated(bulk: Bulk): Promise<void> {
-    // if our es instance is not ready yet, we cannot serve this request
     if (!this.ready) {
       throw new Error('No connection to elasticsearch established yet.');
     }
 
-    // index name for elasticsearch
+    const index = getThingIndexName(bulk.type, bulk.source, bulk);
-    const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
+    const alias = removeInvalidAliasChars(bulk.type, bulk.uid);
-
-    // there already is an index with this type and source. We will index the new one and switch the alias to it
-    // the old one is deleted
-    const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
 
     if (typeof this.aliasMap[alias] === 'undefined') {
       this.aliasMap[alias] = {};
     }
 
-    if (!indexRegex.test(index)) {
+    if (!VALID_INDEX_REGEX.test(index)) {
       throw new Error(
         `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
           Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
       );
     }
 
+    return {index, alias};
+  }
+
+  /**
+   * Should be called, when a new bulk was created. Creates a new index and applies the mapping to the index
+   *
+   * @param bulk the bulk process that was created
+   */
+  public async bulkCreated(bulk: Bulk): Promise<void> {
+    const {index} = await this.prepareBulkWrite(bulk);
+
     // re-apply the index template before each new bulk operation
     await putTemplate(this.client, bulk.type);
     await this.client.indices.create({
@@ -355,8 +234,7 @@ export class Elasticsearch implements Database {
    * @param bulk the bulk process that is expired
    */
   public async bulkExpired(bulk: Bulk): Promise<void> {
-    // index name for elasticsearch
+    const index: string = getThingIndexName(bulk.type, bulk.source, bulk);
-    const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
 
     Logger.info('Bulk expired. Deleting index', index);
 
@@ -375,31 +253,11 @@ export class Elasticsearch implements Database {
    * @param bulk the new bulk process that should replace the old one with same type and source
    */
   public async bulkUpdated(bulk: Bulk): Promise<void> {
-    // if our es instance is not ready yet, we cannot serve this request
+    const {index, alias} = await this.prepareBulkWrite(bulk);
-    if (!this.ready) {
-      throw new Error('No connection to elasticsearch established yet.');
-    }
 
-    // index name for elasticsearch
+    // create the new index if it does not exist
-    const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
-
-    // alias for the indices
-    const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
-
-    if (typeof this.aliasMap[alias] === 'undefined') {
-      this.aliasMap[alias] = {};
-    }
-
-    if (!indexRegex.test(index)) {
-      throw new Error(
-        `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
-          Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
-      );
-    }
-
-    // create the new index if it does not exists
     // eslint-disable-next-line unicorn/no-await-expression-member
-    if (!(await this.client.indices.exists({index})).body) {
+    if (!(await this.client.indices.exists({index}))) {
       // re-apply the index template before each new bulk operation
       await putTemplate(this.client, bulk.type);
       await this.client.indices.create({
@@ -412,7 +270,7 @@ export class Elasticsearch implements Database {
 
     // add our new index to the alias
     // this was type safe with @types/elasticsearch, the new package however provides no type definitions
-    const actions: IndicesUpdateAliasesParamsAction[] = [
+    const actions: IndicesUpdateAliasesAction[] = [
       {
         add: {index: index, alias: alias},
       },
@@ -427,16 +285,10 @@ export class Elasticsearch implements Database {
     }
 
     // refresh the index (fsync changes)
-    await this.client.indices.refresh({
+    await this.client.indices.refresh({index});
-      index: index,
-    });
 
     // execute our alias actions
-    await this.client.indices.updateAliases({
+    await this.client.indices.updateAliases({actions});
-      body: {
-        actions,
-      },
-    });
 
     // swap the index in our aliasMap
     this.aliasMap[alias][bulk.source] = index;
@@ -457,7 +309,7 @@ export class Elasticsearch implements Database {
   public async get(uid: SCUuid): Promise<SCThings> {
     const object = await this.getObject(uid);
 
-    if (typeof object === 'undefined') {
+    if (typeof object?._source === 'undefined') {
       throw new TypeError('Item not found.');
     }
 
@@ -467,7 +319,7 @@ export class Elasticsearch implements Database {
   /**
    * Initialize the elasticsearch database (call all needed methods)
    */
-  public async init(): Promise<void> {
+  public async init(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}): Promise<void> {
     const monitoringConfiguration = this.config.internal.monitoring;
 
     if (typeof monitoringConfiguration !== 'undefined') {
@@ -480,7 +332,7 @@ export class Elasticsearch implements Database {
       await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
     }
 
-    return this.getAliasMap();
+    return this.getAliasMap(retryOptions);
   }
 
   /**
@@ -490,7 +342,7 @@ export class Elasticsearch implements Database {
    * @param bulk the bulk process which item belongs to
    */
   public async post(object: SCThings, bulk: Bulk): Promise<void> {
-    const object_: SCThings & {creation_date: string} = {
+    const thing: SCThings & {creation_date: string} = {
       ...object,
       creation_date: moment().format(),
     };
@@ -499,7 +351,7 @@ export class Elasticsearch implements Database {
 
     // check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid)
     if (typeof item !== 'undefined') {
-      const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk);
+      const indexOfNew = getThingIndexName(thing.type, bulk.source, bulk);
       const oldIndex = item._index;
 
       // new item doesn't replace the old one
@@ -509,22 +361,23 @@ export class Elasticsearch implements Database {
       ) {
         throw new Error(
           // eslint-disable-next-line unicorn/no-null
-          `Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`,
+          `Object "${thing.uid}" already exists. Object was: ${JSON.stringify(thing, null, 2)}`,
         );
       }
     }
 
     // regular bulk update (item gets replaced when bulk is updated)
-    const searchResponse = await this.client.create({
+    const searchResponse = await this.client.create<SCThings>({
-      body: object_,
+      document: thing,
-      id: object_.uid,
+      id: thing.uid,
-      index: Elasticsearch.getIndex(object_.type, bulk.source, bulk),
+      index: getThingIndexName(thing.type, bulk.source, bulk),
       timeout: '90s',
-      type: object_.type,
     });
 
-    if (!searchResponse.body.created) {
+    if (searchResponse.result !== 'created') {
-      throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`);
+      throw new Error(
+        `Object creation Error (${searchResponse.result}: Instance was: ${JSON.stringify(thing)}`,
+      );
     }
   }
 
@@ -543,7 +396,6 @@ export class Elasticsearch implements Database {
         },
         id: object.uid,
         index: item._index,
-        type: object.type.toLowerCase(),
       });
 
       return;
@@ -562,65 +414,46 @@ export class Elasticsearch implements Database {
       throw new TypeError('Database is undefined. You have to configure the query build');
     }
 
-    // create elasticsearch configuration out of data from database configuration
     const esConfig: ElasticsearchConfig = {
       name: this.config.internal.database.name as 'elasticsearch',
       version: this.config.internal.database.version as string,
+      query: this.config.internal.database.query as
+        | ElasticsearchQueryDisMaxConfig
+        | ElasticsearchQueryQueryStringConfig
+        | undefined,
     };
 
-    if (typeof this.config.internal.database.query !== 'undefined') {
+    const query = {
-      esConfig.query = this.config.internal.database.query as
-        | ElasticsearchQueryDisMaxConfig
-        | ElasticsearchQueryQueryStringConfig;
-    }
-
-    const searchRequest: RequestParams.Search = {
-      body: {
       aggs: aggregations,
       query: buildQuery(parameters, this.config, esConfig),
-      },
       from: parameters.from,
-      index: Elasticsearch.getListOfAllIndices(),
+      index: ALL_INDICES_QUERY,
       size: parameters.size,
+      sort: typeof parameters.sort !== 'undefined' ? buildSort(parameters.sort) : undefined,
     };
-
+    const response: SearchResponse<SCThings> = await this.client.search(query);
-    if (typeof parameters.sort !== 'undefined') {
-      searchRequest.body.sort = buildSort(parameters.sort);
-    }
-
-    // perform the search against elasticsearch
-    const response: ApiResponse<SearchResponse<SCThings>> = await this.client.search(searchRequest);
-
-    // gather pagination information
-    const pagination = {
-      count: response.body.hits.hits.length,
-      offset: typeof parameters.from === 'number' ? parameters.from : 0,
-      total: response.body.hits.total,
-    };
-
-    // gather statistics about this search
-    const stats = {
-      time: response.body.took,
-    };
-
-    // we only directly return the _source documents
-    // elasticsearch provides much more information, the user shouldn't see
-    const data = response.body.hits.hits.map(hit => {
-      return hit._source; // SCThing
-    });
-
-    let facets: SCFacet[] = [];
-
-    // read the aggregations from elasticsearch and parse them to facets by our configuration
-    if (typeof response.body.aggregations !== 'undefined') {
-      facets = parseAggregations(response.body.aggregations as AggregationResponse);
-    }
 
     return {
-      data,
+      data: response.hits.hits
-      facets,
+        .map(hit => {
-      pagination,
+          // we only directly return the _source documents
-      stats,
+          // elasticsearch provides much more information, the user shouldn't see
+          return hit._source;
+        })
+        .filter(noUndefined),
+      facets:
+        typeof response.aggregations !== 'undefined'
+          ? parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>)
+          : [],
+      pagination: {
+        count: response.hits.hits.length,
+        offset: typeof parameters.from === 'number' ? parameters.from : 0,
+        total:
+          typeof response.hits.total === 'number' ? response.hits.total : response.hits.total?.value ?? 0,
+      },
+      stats: {
+        time: response.took,
+      },
     };
   }
 }

src/storage/elasticsearch/monitoring.ts

@@ -13,7 +13,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch';
+import {Client} from '@elastic/elasticsearch';
+import {SearchRequest} from '@elastic/elasticsearch/lib/api/types';
 import {
   SCMonitoringConfiguration,
   SCMonitoringLogAction,
@@ -23,9 +24,6 @@ import {
   SCThings,
 } from '@openstapps/core';
 import {Logger} from '@openstapps/logger';
-// we only have the @types package because some things type definitions are still missing from the official
-// @elastic/elasticsearch package
-import {SearchResponse} from 'elasticsearch';
 import cron from 'node-cron';
 import {MailQueue} from '../../notification/mail-queue';
 
@@ -131,12 +129,11 @@ export async function setUp(
 
       cron.schedule(trigger.executionTime, async () => {
         // execute watch (search->condition->action)
-        const result: ApiResponse<SearchResponse<SCThings>> = await esClient.search(
+        const result = await esClient.search<SCThings>(watcher.query as SearchRequest);
-          watcher.query as RequestParams.Search,
-        );
 
         // check conditions
-        const total = result.body.hits.total;
+        const total =
+          typeof result.hits.total === 'number' ? result.hits.total : result.hits.total?.value ?? -1;
 
         for (const condition of watcher.conditions) {
           if (conditionFails(condition, total)) {

src/storage/elasticsearch/query.ts

@@ -1,501 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-import {
-  SCBackendConfigurationSearchBoostingContext,
-  SCBackendConfigurationSearchBoostingType,
-  SCConfigFile,
-  SCSearchBooleanFilter,
-  SCSearchContext,
-  SCSearchFilter,
-  SCSearchQuery,
-  SCSearchSort,
-  SCSportCoursePriceGroup,
-  SCThingsField,
-} from '@openstapps/core';
-import {
-  ElasticsearchConfig,
-  ESBooleanFilter,
-  ESBooleanFilterArguments,
-  ESDateRange,
-  ESDateRangeFilter,
-  ESFunctionScoreQuery,
-  ESFunctionScoreQueryFunction,
-  ESGenericRange,
-  ESGenericSort,
-  ESGeoBoundingBoxFilter,
-  ESGeoDistanceFilter,
-  ESGeoDistanceFilterArguments,
-  ESGeoDistanceSort,
-  ESGeoDistanceSortArguments,
-  ESGeoShapeFilter,
-  ESNumericRangeFilter,
-  ESRangeFilter,
-  ESTermFilter,
-  ESTypeFilter,
-  ScriptSort,
-} from './types/elasticsearch';
-
-/**
- * Escapes any reserved character that would otherwise not be accepted by Elasticsearch
- *
- * Elasticsearch as the following reserved characters:
- * + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
- * It is possible to use all, with the exception of < and >, of them by escaping them with a \
- * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
- *
- * @param string_ the string to escape the characters from
- */
-function escapeESReservedCharacters(string_: string): string {
-  return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
-}
-
-/**
- * Builds a boolean filter. Returns an elasticsearch boolean filter
- *
- * @param booleanFilter a search boolean filter for the retrieval of the data
- * @returns elasticsearch boolean arguments object
- */
-export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<unknown> {
-  const result: ESBooleanFilterArguments<unknown> = {
-    minimum_should_match: 0,
-    must: [],
-    must_not: [],
-    should: [],
-  };
-
-  if (booleanFilter.arguments.operation === 'and') {
-    result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
-  }
-
-  if (booleanFilter.arguments.operation === 'or') {
-    result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
-    result.minimum_should_match = 1;
-  }
-
-  if (booleanFilter.arguments.operation === 'not') {
-    result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
-  }
-
-  return result;
-}
-
-/**
- * Converts Array of Filters to elasticsearch query-syntax
- *
- * @param filter A search filter for the retrieval of the data
- */
-export function buildFilter(
-  filter: SCSearchFilter,
-):
-  | ESTermFilter
-  | ESGeoDistanceFilter
-  | ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter>
-  | ESGeoShapeFilter
-  | ESBooleanFilter<unknown>
-  | ESRangeFilter {
-  switch (filter.type) {
-    case 'value':
-      return Array.isArray(filter.arguments.value)
-        ? {
-            terms: {
-              [`${filter.arguments.field}.raw`]: filter.arguments.value,
-            },
-          }
-        : {
-            term: {
-              [`${filter.arguments.field}.raw`]: filter.arguments.value,
-            },
-          };
-    case 'availability':
-      const scope = filter.arguments.scope?.charAt(0) ?? 's';
-      const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
-
-      return {
-        range: {
-          [filter.arguments.field]: {
-            gte: `${time}/${scope}`,
-            lt: `${time}+1${scope}/${scope}`,
-            relation: 'intersects',
-          },
-        },
-      };
-    case 'distance':
-      const geoObject: ESGeoDistanceFilterArguments = {
-        distance: `${filter.arguments.distance}m`,
-        [`${filter.arguments.field}.point.coordinates`]: {
-          lat: filter.arguments.position[1],
-          lon: filter.arguments.position[0],
-        },
-      };
-
-      return {
-        geo_distance: geoObject,
-      };
-    case 'boolean':
-      return {
-        bool: buildBooleanFilter(filter),
-      };
-    case 'numeric range':
-      const numericRangeObject: ESGenericRange<number> = {
-        relation: filter.arguments.relation,
-      };
-      if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
-        numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
-      } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
-        numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
-      }
-      if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
-        numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
-      } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
-        numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
-      }
-
-      const numericRangeFilter: ESNumericRangeFilter = {range: {}};
-      numericRangeFilter.range[filter.arguments.field] = numericRangeObject;
-
-      return numericRangeFilter;
-    case 'date range':
-      const dateRangeObject: ESDateRange = {
-        format: filter.arguments.format,
-        time_zone: filter.arguments.timeZone,
-        relation: filter.arguments.relation,
-      };
-      if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
-        dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
-      } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
-        dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
-      }
-      if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
-        dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
-      } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
-        dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
-      }
-
-      const dateRangeFilter: ESDateRangeFilter = {range: {}};
-      dateRangeFilter.range[filter.arguments.field] = dateRangeObject;
-
-      return dateRangeFilter;
-    case 'geo':
-      // TODO: on ES upgrade, use just geo_shape filters
-      const geoShapeFilter: ESGeoShapeFilter = {
-        geo_shape: {
-          /**
-           * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
-           */
-          // @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type.
-          ignore_unmapped: true,
-          [`${filter.arguments.field}.polygon`]: {
-            shape: filter.arguments.shape,
-            relation: filter.arguments.spatialRelation,
-          },
-        },
-      };
-
-      if (
-        (typeof filter.arguments.spatialRelation === 'undefined' ||
-          filter.arguments.spatialRelation === 'intersects') &&
-        filter.arguments.shape.type === 'envelope'
-      ) {
-        return {
-          bool: {
-            minimum_should_match: 1,
-            should: [
-              geoShapeFilter,
-              {
-                geo_bounding_box: {
-                  /**
-                   * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
-                   */
-                  ignore_unmapped: true,
-                  [`${filter.arguments.field}.point.coordinates`]: {
-                    top_left: filter.arguments.shape.coordinates[0],
-                    bottom_right: filter.arguments.shape.coordinates[1],
-                  },
-                },
-              },
-            ],
-          },
-        };
-      }
-
-      return geoShapeFilter;
-  }
-}
-
-/**
- * Builds scoring functions from boosting config
- *
- * @param boostings Backend boosting configuration for contexts and types
- * @param context The context of the app from where the search was initiated
- */
-function buildFunctions(
-  boostings: SCBackendConfigurationSearchBoostingContext,
-  context: SCSearchContext | undefined,
-): ESFunctionScoreQueryFunction[] {
-  // default context
-  let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes(
-    boostings['default' as SCSearchContext],
-  );
-
-  if (typeof context !== 'undefined' && context !== 'default') {
-    // specific context provided, extend default context with additional boosts
-    functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
-  }
-
-  return functions;
-}
-
-/**
- * Creates boost functions for all type boost configurations
- *
- * @param boostingTypes Array of type boosting configurations
- */
-function buildFunctionsForBoostingTypes(
-  boostingTypes: SCBackendConfigurationSearchBoostingType[],
-): ESFunctionScoreQueryFunction[] {
-  const functions: ESFunctionScoreQueryFunction[] = [];
-
-  for (const boostingForOneSCType of boostingTypes) {
-    const typeFilter: ESTypeFilter = {
-      type: {
-        value: boostingForOneSCType.type,
-      },
-    };
-
-    functions.push({
-      filter: typeFilter,
-      weight: boostingForOneSCType.factor,
-    });
-
-    if (typeof boostingForOneSCType.fields !== 'undefined') {
-      const fields = boostingForOneSCType.fields;
-
-      for (const fieldName in boostingForOneSCType.fields) {
-        if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
-          const boostingForOneField = fields[fieldName];
-
-          for (const value in boostingForOneField) {
-            if (boostingForOneField.hasOwnProperty(value)) {
-              const factor = boostingForOneField[value];
-
-              // build term filter
-              const termFilter: ESTermFilter = {
-                term: {},
-              };
-              termFilter.term[`${fieldName}.raw`] = value;
-
-              functions.push({
-                filter: {
-                  bool: {
-                    must: [typeFilter, termFilter],
-                    should: [],
-                  },
-                },
-                weight: factor,
-              });
-            }
-          }
-        }
-      }
-    }
-  }
-
-  return functions;
-}
-
-/**
- * Builds body for Elasticsearch requests
- *
- * @param parameters Parameters for querying the backend
- * @param defaultConfig Default configuration of the backend
- * @param elasticsearchConfig Elasticsearch configuration
- * @returns ElasticsearchQuery (body of a search-request)
- */
-export function buildQuery(
-  parameters: SCSearchQuery,
-  defaultConfig: SCConfigFile,
-  elasticsearchConfig: ElasticsearchConfig,
-): ESFunctionScoreQuery {
-  // if config provides an minMatch parameter we use query_string instead of match query
-  let query;
-  if (typeof elasticsearchConfig.query === 'undefined') {
-    query = {
-      query_string: {
-        analyzer: 'search_german',
-        default_field: 'name',
-        minimum_should_match: '90%',
-        query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
-      },
-    };
-  } else if (elasticsearchConfig.query.queryType === 'query_string') {
-    query = {
-      query_string: {
-        analyzer: 'search_german',
-        default_field: 'name',
-        minimum_should_match: elasticsearchConfig.query.minMatch,
-        query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
-      },
-    };
-  } else if (elasticsearchConfig.query.queryType === 'dis_max') {
-    if (parameters.query !== '*') {
-      query = {
-        dis_max: {
-          boost: 1.2,
-          queries: [
-            {
-              match: {
-                name: {
-                  boost: elasticsearchConfig.query.matchBoosting,
-                  cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
-                  fuzziness: elasticsearchConfig.query.fuzziness,
-                  query: typeof parameters.query !== 'string' ? '*' : parameters.query,
-                },
-              },
-            },
-            {
-              query_string: {
-                analyzer: 'search_german',
-                default_field: 'name',
-                minimum_should_match: elasticsearchConfig.query.minMatch,
-                query:
-                  typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
-              },
-            },
-          ],
-          tie_breaker: elasticsearchConfig.query.tieBreaker,
-        },
-      };
-    }
-  } else {
-    throw new Error(
-      'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
-    );
-  }
-
-  const functionScoreQuery: ESFunctionScoreQuery = {
-    function_score: {
-      functions: buildFunctions(defaultConfig.internal.boostings, parameters.context),
-      query: {
-        bool: {
-          minimum_should_match: 0, // if we have no should, nothing can match
-          must: [],
-          should: [],
-        },
-      },
-      score_mode: 'multiply',
-    },
-  };
-
-  const mustMatch = functionScoreQuery.function_score.query.bool.must;
-
-  if (Array.isArray(mustMatch)) {
-    if (typeof query !== 'undefined') {
-      mustMatch.push(query);
-    }
-
-    if (typeof parameters.filter !== 'undefined') {
-      mustMatch.push(buildFilter(parameters.filter));
-    }
-  }
-
-  return functionScoreQuery;
-}
-
-/**
- * converts query to
- *
- * @param sorts Sorting rules to apply to the data that is being queried
- * @returns an array of sort queries
- */
-export function buildSort(sorts: SCSearchSort[]): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
-  return sorts.map(sort => {
-    switch (sort.type) {
-      case 'generic':
-        const esGenericSort: ESGenericSort = {};
-        esGenericSort[sort.arguments.field] = sort.order;
-
-        return esGenericSort;
-      case 'ducet':
-        const esDucetSort: ESGenericSort = {};
-        esDucetSort[`${sort.arguments.field}.sort`] = sort.order;
-
-        return esDucetSort;
-      case 'distance':
-        const arguments_: ESGeoDistanceSortArguments = {
-          mode: 'avg',
-          order: sort.order,
-          unit: 'm',
-        };
-
-        arguments_[`${sort.arguments.field}.point.coordinates`] = {
-          lat: sort.arguments.position[1],
-          lon: sort.arguments.position[0],
-        };
-
-        return {
-          _geo_distance: arguments_,
-        };
-      case 'price':
-        return {
-          _script: {
-            order: sort.order,
-            script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
-            type: 'number' as const,
-          },
-        };
-    }
-  });
-}
-
-/**
- * Provides a script for sorting search results by prices
- *
- * @param universityRole User group which consumes university services
- * @param field Field in which wanted offers with prices are located
- */
-export function buildPriceSortScript(
-  universityRole: keyof SCSportCoursePriceGroup,
-  field: SCThingsField,
-): string {
-  return `
-  // initialize the sort value with the maximum
-  double price = Double.MAX_VALUE;
-
-  // if we have any offers
-  if (params._source.containsKey('${field}')) {
-    // iterate through all offers
-    for (offer in params._source.${field}) {
-      // if this offer contains a role specific price
-      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
-        // if the role specific price is smaller than the cheapest we found
-        if (offer.prices.${universityRole} < price) {
-          // set the role specific price as cheapest for now
-          price = offer.prices.${universityRole};
-        }
-      } else { // we have no role specific price for our role in this offer
-        // if the default price of this offer is lower than the cheapest we found
-        if (offer.price < price) {
-          // set this price as the cheapest
-          price = offer.price;
-        }
-      }
-    }
-  }
-
-  // return cheapest price for our role
-  return price;
-  `;
-}

src/storage/elasticsearch/query/boost/boost-functions.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
+import {SCBackendConfigurationSearchBoostingType} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Creates boost functions for all type boost configurations
+ *
+ * @param boostingTypes Array of type boosting configurations
+ */
+export function buildFunctionsForBoostingTypes(
+  boostingTypes: SCBackendConfigurationSearchBoostingType[],
+): QueryDslFunctionScoreContainer[] {
+  const functions: QueryDslFunctionScoreContainer[] = [];
+
+  for (const boostingForOneSCType of boostingTypes) {
+    const typeFilter: QueryDslSpecificQueryContainer<'term'> = {
+      term: {
+        type: boostingForOneSCType.type,
+      },
+    };
+
+    functions.push({
+      filter: typeFilter,
+      weight: boostingForOneSCType.factor,
+    });
+
+    if (typeof boostingForOneSCType.fields !== 'undefined') {
+      const fields = boostingForOneSCType.fields;
+
+      for (const fieldName in boostingForOneSCType.fields) {
+        if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
+          const boostingForOneField = fields[fieldName];
+
+          for (const value in boostingForOneField) {
+            if (boostingForOneField.hasOwnProperty(value)) {
+              const factor = boostingForOneField[value];
+
+              // build term filter
+              const termFilter: QueryDslSpecificQueryContainer<'term'> = {
+                term: {},
+              };
+              termFilter.term[`${fieldName}.raw`] = value;
+
+              functions.push({
+                filter: {
+                  bool: {
+                    must: [typeFilter, termFilter],
+                    should: [],
+                  },
+                },
+                weight: factor,
+              });
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return functions;
+}

src/storage/elasticsearch/query/boost/scoring-functions.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
+import {SCBackendConfigurationSearchBoostingContext, SCSearchContext} from '@openstapps/core';
+import {buildFunctionsForBoostingTypes} from './boost-functions';
+
+/**
+ * Builds scoring functions from boosting config
+ *
+ * @param boostings Backend boosting configuration for contexts and types
+ * @param context The context of the app from where the search was initiated
+ */
+export function buildScoringFunctions(
+  boostings: SCBackendConfigurationSearchBoostingContext,
+  context: SCSearchContext | undefined,
+): QueryDslFunctionScoreContainer[] {
+  // default context
+  let functions = buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]);
+
+  if (typeof context !== 'undefined' && context !== 'default') {
+    // specific context provided, extend default context with additional boosts
+    functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
+  }
+
+  return functions;
+}

src/storage/elasticsearch/query/filter.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
+import {SCSearchFilter} from '@openstapps/core';
+import {buildBooleanFilter} from './filters/boolean';
+import {buildAvailabilityFilter} from './filters/availability';
+import {buildDateRangeFilter} from './filters/date-range';
+import {buildDistanceFilter} from './filters/distance';
+import {buildGeoFilter} from './filters/geo';
+import {buildNumericRangeFilter} from './filters/numeric-range';
+import {buildValueFilter} from './filters/value';
+
+/**
+ * Converts Array of Filters to elasticsearch query-syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildFilter(filter: SCSearchFilter): QueryDslQueryContainer {
+  switch (filter.type) {
+    case 'value':
+      return buildValueFilter(filter);
+    case 'availability':
+      return buildAvailabilityFilter(filter);
+    case 'distance':
+      return buildDistanceFilter(filter);
+    case 'boolean':
+      return buildBooleanFilter(filter);
+    case 'numeric range':
+      return buildNumericRangeFilter(filter);
+    case 'date range':
+      return buildDateRangeFilter(filter);
+    case 'geo':
+      return buildGeoFilter(filter);
+  }
+}

src/storage/elasticsearch/query/filters/availability.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCSearchAvailabilityFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Converts an availability filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildAvailabilityFilter(
+  filter: SCSearchAvailabilityFilter,
+): QueryDslSpecificQueryContainer<'range'> {
+  const scope = filter.arguments.scope?.charAt(0) ?? 's';
+  const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
+
+  return {
+    range: {
+      [filter.arguments.field]: {
+        gte: `${time}/${scope}`,
+        lt: `${time}+1${scope}/${scope}`,
+        relation: 'intersects',
+      },
+    },
+  };
+}

src/storage/elasticsearch/query/filters/boolean.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types';
+import {SCSearchBooleanFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+import {buildFilter} from '../filter';
+
+/**
+ * Converts a boolean filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildBooleanFilter(filter: SCSearchBooleanFilter): QueryDslSpecificQueryContainer<'bool'> {
+  const result: QueryDslBoolQuery = {
+    minimum_should_match: 0,
+    must: [],
+    must_not: [],
+    should: [],
+  };
+
+  if (filter.arguments.operation === 'and') {
+    result.must = filter.arguments.filters.map(it => buildFilter(it));
+  }
+
+  if (filter.arguments.operation === 'or') {
+    result.should = filter.arguments.filters.map(it => buildFilter(it));
+    result.minimum_should_match = 1;
+  }
+
+  if (filter.arguments.operation === 'not') {
+    result.must_not = filter.arguments.filters.map(it => buildFilter(it));
+  }
+
+  return {
+    bool: result,
+  };
+}

src/storage/elasticsearch/query/filters/date-range.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types';
+import {SCSearchDateRangeFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Converts a date range filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildDateRangeFilter(
+  filter: SCSearchDateRangeFilter,
+): QueryDslSpecificQueryContainer<'range'> {
+  const dateRangeObject: QueryDslDateRangeQuery = {
+    format: filter.arguments.format,
+    time_zone: filter.arguments.timeZone,
+    relation: filter.arguments.relation,
+  };
+  if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
+    dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
+  } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
+    dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
+  }
+  if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
+    dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
+  } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
+    dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
+  }
+
+  return {
+    range: {
+      [filter.arguments.field]: dateRangeObject,
+    },
+  };
+}

src/storage/elasticsearch/query/filters/distance.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types';
+import {SCSearchDistanceFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Converts a distance filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildDistanceFilter(
+  filter: SCSearchDistanceFilter,
+): QueryDslSpecificQueryContainer<'geo_distance'> {
+  const geoObject: QueryDslGeoDistanceQuery = {
+    distance: `${filter.arguments.distance}m`,
+    [`${filter.arguments.field}.point.coordinates`]: {
+      lat: filter.arguments.position[1],
+      lon: filter.arguments.position[0],
+    },
+  };
+
+  return {
+    geo_distance: geoObject,
+  };
+}

src/storage/elasticsearch/query/filters/geo.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCGeoFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Converts a geo filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> {
+  return {
+    geo_shape: {
+      ignore_unmapped: true,
+      [`${filter.arguments.field}.polygon`]: {
+        shape: filter.arguments.shape,
+        relation: filter.arguments.spatialRelation,
+      },
+    },
+  };
+}

src/storage/elasticsearch/query/filters/numeric-range.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types';
+import {SCSearchNumericRangeFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Converts a numeric range filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildNumericRangeFilter(
+  filter: SCSearchNumericRangeFilter,
+): QueryDslSpecificQueryContainer<'range'> {
+  const numericRangeObject: QueryDslNumberRangeQuery = {
+    relation: filter.arguments.relation,
+  };
+  if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
+    numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
+  } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
+    numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
+  }
+  if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
+    numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
+  } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
+    numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
+  }
+
+  return {
+    range: {
+      [filter.arguments.field]: numericRangeObject,
+    },
+  };
+}

src/storage/elasticsearch/query/filters/value.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCSearchValueFilter} from '@openstapps/core';
+import {QueryDslSpecificQueryContainer} from '../../types/util';
+
+/**
+ * Converts a value filter to elasticsearch syntax
+ *
+ * @param filter A search filter for the retrieval of the data
+ */
+export function buildValueFilter(
+  filter: SCSearchValueFilter,
+): QueryDslSpecificQueryContainer<'term'> | QueryDslSpecificQueryContainer<'terms'> {
+  return Array.isArray(filter.arguments.value)
+    ? {
+        terms: {
+          [`${filter.arguments.field}.raw`]: filter.arguments.value,
+        },
+      }
+    : {
+        term: {
+          [`${filter.arguments.field}.raw`]: filter.arguments.value,
+        },
+      };
+}

src/storage/elasticsearch/query/query.ts

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
+import {SCConfigFile, SCSearchQuery} from '@openstapps/core';
+import {ElasticsearchConfig} from '../types/elasticsearch-config';
+import {buildFilter} from './filter';
+import {buildScoringFunctions} from './boost/scoring-functions';
+
+/**
+ * Builds body for Elasticsearch requests
+ *
+ * @param parameters Parameters for querying the backend
+ * @param defaultConfig Default configuration of the backend
+ * @param elasticsearchConfig Elasticsearch configuration
+ * @returns ElasticsearchQuery (body of a search-request)
+ */
+export function buildQuery(
+  parameters: SCSearchQuery,
+  defaultConfig: SCConfigFile,
+  elasticsearchConfig: ElasticsearchConfig,
+): QueryDslQueryContainer {
+  // if config provides an minMatch parameter we use query_string instead of match query
+  let query;
+  if (typeof elasticsearchConfig.query === 'undefined') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: '90%',
+        query: typeof parameters.query !== 'string' ? '*' : parameters.query,
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'query_string') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: elasticsearchConfig.query.minMatch,
+        query: typeof parameters.query !== 'string' ? '*' : parameters.query,
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'dis_max') {
+    if (typeof parameters.query === 'string' && parameters.query !== '*') {
+      query = {
+        dis_max: {
+          boost: 1.2,
+          queries: [
+            {
+              match: {
+                name: {
+                  boost: elasticsearchConfig.query.matchBoosting,
+                  fuzziness: elasticsearchConfig.query.fuzziness,
+                  query: parameters.query,
+                },
+              },
+            },
+            {
+              query_string: {
+                default_field: 'name',
+                minimum_should_match: elasticsearchConfig.query.minMatch,
+                query: parameters.query,
+              },
+            },
+          ],
+          tie_breaker: elasticsearchConfig.query.tieBreaker,
+        },
+      };
+    }
+  } else {
+    throw new Error(
+      'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
+    );
+  }
+
+  const functionScoreQuery: QueryDslQueryContainer = {
+    function_score: {
+      functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
+      query: {
+        bool: {
+          minimum_should_match: 0, // if we have no should, nothing can match
+          must: [],
+          should: [],
+        },
+      },
+      score_mode: 'multiply',
+    },
+  };
+
+  const mustMatch = functionScoreQuery.function_score?.query?.bool?.must;
+
+  if (Array.isArray(mustMatch)) {
+    if (typeof query !== 'undefined') {
+      mustMatch.push(query);
+    }
+
+    if (typeof parameters.filter !== 'undefined') {
+      mustMatch.push(buildFilter(parameters.filter));
+    }
+  }
+
+  return functionScoreQuery;
+}

src/storage/elasticsearch/query/sort.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Sort} from '@elastic/elasticsearch/lib/api/types';
+import {SCSearchSort} from '@openstapps/core';
+import {buildDistanceSort} from './sort/distance';
+import {buildDucetSort} from './sort/ducet';
+import {buildGenericSort} from './sort/generic';
+import {buildPriceSort} from './sort/price';
+
+/**
+ * converts query to
+ *
+ * @param sorts Sorting rules to apply to the data that is being queried
+ * @returns an array of sort queries
+ */
+export function buildSort(sorts: SCSearchSort[]): Sort {
+  return sorts.map(sort => {
+    switch (sort.type) {
+      case 'generic':
+        return buildGenericSort(sort);
+      case 'ducet':
+        return buildDucetSort(sort);
+      case 'distance':
+        return buildDistanceSort(sort);
+      case 'price':
+        return buildPriceSort(sort);
+    }
+  });
+}

src/storage/elasticsearch/query/sort/distance.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
+import {SCDistanceSort} from '@openstapps/core';
+
+/**
+ * Converts a distance sort to elasticsearch syntax
+ *
+ * @param sort A sorting definition
+ */
+export function buildDistanceSort(sort: SCDistanceSort): SortOptions {
+  return {
+    _geo_distance: {
+      mode: 'avg',
+      order: sort.order,
+      unit: 'm',
+      [`${sort.arguments.field}.point.coordinates`]: {
+        lat: sort.arguments.position[1],
+        lon: sort.arguments.position[0],
+      },
+    },
+  };
+}

src/storage/elasticsearch/query/sort/ducet.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
+import {SCDucetSort} from '@openstapps/core';
+
+/**
+ * Converts a ducet sort to elasticsearch syntax
+ *
+ * @param sort A sorting definition
+ */
+export function buildDucetSort(sort: SCDucetSort): SortOptions {
+  return {
+    [`${sort.arguments.field}.sort`]: sort.order,
+  };
+}

src/storage/elasticsearch/query/sort/generic.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
+import {SCGenericSort} from '@openstapps/core';
+
+/**
+ * Converts a generic sort to elasticsearch syntax
+ *
+ * @param sort A sorting definition
+ */
+export function buildGenericSort(sort: SCGenericSort): SortOptions {
+  return {
+    [sort.arguments.field]: sort.order,
+  };
+}

src/storage/elasticsearch/query/sort/price.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
+import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
+
+/**
+ * Converts a price sort to elasticsearch syntax
+ *
+ * @param sort A sorting definition
+ */
+export function buildPriceSort(sort: SCPriceSort): SortOptions {
+  return {
+    _script: {
+      order: sort.order,
+      script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+      type: 'number' as const,
+    },
+  };
+}
+
+/**
+ * Provides a script for sorting search results by prices
+ *
+ * @param universityRole User group which consumes university services
+ * @param field Field in which wanted offers with prices are located
+ */
+export function buildPriceSortScript(
+  universityRole: keyof SCSportCoursePriceGroup,
+  field: SCThingsField,
+): string {
+  return `
+  // initialize the sort value with the maximum
+  double price = Double.MAX_VALUE;
+
+  // if we have any offers
+  if (params._source.containsKey('${field}')) {
+    // iterate through all offers
+    for (offer in params._source.${field}) {
+      // if this offer contains a role specific price
+      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
+        // if the role specific price is smaller than the cheapest we found
+        if (offer.prices.${universityRole} < price) {
+          // set the role specific price as cheapest for now
+          price = offer.prices.${universityRole};
+        }
+      } else { // we have no role specific price for our role in this offer
+        // if the default price of this offer is lower than the cheapest we found
+        if (offer.price < price) {
+          // set this price as the cheapest
+          price = offer.price;
+        }
+      }
+    }
+  }
+
+  // return cheapest price for our role
+  return price;
+  `;
+}

src/storage/elasticsearch/templating.ts

@@ -29,17 +29,6 @@ export const aggregations = JSON.parse(
   readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
 ) as AggregationSchema;
 
-/**
- * Re-applies all interfaces for every type
- *
- * @param client An elasticsearch client to use
- */
-export async function refreshAllTemplates(client: Client) {
-  for (const type of Object.values(SCThingType)) {
-    await putTemplate(client, type as SCThingType);
-  }
-}
-
 /**
  * Prepares all indices
  *

src/storage/elasticsearch/types/elasticsearch-config.ts

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * A configuration for using the Dis Max Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryDisMaxConfig {
+  /**
+   * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
+   */
+  cutoffFrequency: number;
+
+  /**
+   * The maximum allowed Levenshtein Edit Distance (or number of edits)
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
+   */
+  fuzziness: number | string;
+
+  /**
+   * Increase the importance (relevance score) of a field
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
+   */
+  matchBoosting: number;
+
+  /**
+   * Minimal number (or percentage) of words that should match in a query
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
+   */
+  minMatch: string;
+
+  /**
+   * Type of the query - in this case 'dis_max' which is a union of its subqueries
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
+   */
+  queryType: 'dis_max';
+
+  /**
+   * Changes behavior of default calculation of the score when multiple results match
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
+   */
+  tieBreaker: number;
+}
+
+/**
+ * A configuration for using Query String Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryQueryStringConfig {
+  /**
+   * Minimal number (or percentage) of words that should match in a query
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
+   */
+  minMatch: string;
+
+  /**
+   * Type of the query - in this case 'query_string' which uses a query parser in order to parse content
+   *
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
+   */
+  queryType: 'query_string';
+}
+
+/**
+ * An config file for the elasticsearch database interface
+ *
+ * The config file extends the SCConfig file by further defining how the database property
+ */
+export interface ElasticsearchConfigFile {
+  /**
+   * Configuration that is not visible to clients
+   */
+  internal: {
+    /**
+     * Database configuration
+     */
+    database: ElasticsearchConfig;
+  };
+}
+
+/**
+ * An elasticsearch configuration
+ */
+export interface ElasticsearchConfig {
+  /**
+   * Name of the database
+   */
+  name: 'elasticsearch';
+
+  /**
+   * Configuration for using queries
+   */
+  query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
+
+  /**
+   * Version of the used elasticsearch
+   */
+  version: string;
+}

src/storage/elasticsearch/types/elasticsearch.ts

@@ -1,605 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-import {SCThing, SCThingType} from '@openstapps/core';
-// we only have the @types package because some things type definitions are still missing from the official
-import {NameList} from 'elasticsearch';
-import {Polygon, Position} from 'geojson';
-
-/**
- * An elasticsearch aggregation bucket
- */
-interface Bucket {
-  /**
-   * Number of documents in the aggregation bucket
-   */
-  doc_count: number;
-
-  /**
-   * Text representing the documents in the bucket
-   */
-  key: string;
-}
-
-/**
- * An elasticsearch aggregation response
- */
-export interface AggregationResponse {
-  /**
-   * The individual aggregations
-   */
-  [field: string]: BucketAggregation | NestedAggregation;
-}
-
-/**
- * An elasticsearch bucket aggregation
- */
-export interface BucketAggregation {
-  /**
-   * Buckets in an aggregation
-   */
-  buckets: Bucket[];
-
-  /**
-   * Number of documents in an aggregation
-   */
-  doc_count?: number;
-}
-
-/**
- * An aggregation that contains more aggregations nested inside
- */
-export interface NestedAggregation {
-  /**
-   * Number of documents in an aggregation
-   */
-  doc_count: number;
-
-  /**
-   * Any nested responses
-   */
-  [name: string]: BucketAggregation | number;
-}
-
-/**
- * A configuration for using the Dis Max Query
- *
- * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
- * explanation of what the parameters mean
- */
-export interface ElasticsearchQueryDisMaxConfig {
-  /**
-   * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
-   */
-  cutoffFrequency: number;
-
-  /**
-   * The maximum allowed Levenshtein Edit Distance (or number of edits)
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
-   */
-  fuzziness: number | string;
-
-  /**
-   * Increase the importance (relevance score) of a field
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
-   */
-  matchBoosting: number;
-
-  /**
-   * Minimal number (or percentage) of words that should match in a query
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
-   */
-  minMatch: string;
-
-  /**
-   * Type of the query - in this case 'dis_max' which is a union of its subqueries
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
-   */
-  queryType: 'dis_max';
-
-  /**
-   * Changes behavior of default calculation of the score when multiple results match
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
-   */
-  tieBreaker: number;
-}
-
-/**
- * A configuration for using Query String Query
- *
- * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
- * explanation of what the parameters mean
- */
-export interface ElasticsearchQueryQueryStringConfig {
-  /**
-   * Minimal number (or percentage) of words that should match in a query
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
-   */
-  minMatch: string;
-
-  /**
-   * Type of the query - in this case 'query_string' which uses a query parser in order to parse content
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
-   */
-  queryType: 'query_string';
-}
-
-/**
- * A hit in an elasticsearch search result
- *
- * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
- */
-export interface ElasticsearchObject<T extends SCThing> {
-  /**
-   * Unique identifier of a document (object)
-   */
-  _id: string;
-
-  /**
-   * The index to which the document belongs
-   */
-  _index: string;
-
-  /**
-   * Relevancy of the document to a query
-   */
-  _score: number;
-
-  /**
-   * The original JSON representing the body of the document
-   */
-  _source: T;
-
-  /**
-   * The document's mapping type
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
-   */
-  _type: string;
-
-  /**
-   * Version of the document
-   */
-  _version?: number;
-
-  /**
-   * Used to index the same field in different ways for different purposes
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
-   */
-  fields?: NameList;
-
-  /**
-   * Used to highlight search results on one or more fields
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
-   */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  highlight?: any;
-
-  /**
-   * Used in when nested/children documents match the query
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
-   */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  inner_hits?: any;
-
-  /**
-   * Queries that matched for documents in results
-   */
-  matched_queries?: string[];
-
-  /**
-   * Sorting definition
-   */
-  sort?: string[];
-}
-
-/**
- * An config file for the elasticsearch database interface
- *
- * The config file extends the SCConfig file by further defining how the database property
- */
-export interface ElasticsearchConfigFile {
-  /**
-   * Configuration that is not visible to clients
-   */
-  internal: {
-    /**
-     * Database configuration
-     */
-    database: ElasticsearchConfig;
-  };
-}
-
-/**
- * An elasticsearch configuration
- */
-export interface ElasticsearchConfig {
-  /**
-   * Name of the database
-   */
-  name: 'elasticsearch';
-
-  /**
-   * Configuration for using queries
-   */
-  query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
-
-  /**
-   * Version of the used elasticsearch
-   */
-  version: string;
-}
-
-/**
- * An elasticsearch term filter
- */
-export type ESTermFilter =
-  | {
-      /**
-       * Definition of a term to match
-       */
-      term: {
-        [fieldName: string]: string;
-      };
-    }
-  | {
-      /**
-       * Definition of terms to match (or)
-       */
-      terms: {
-        [fieldName: string]: string[];
-      };
-    };
-
-export interface ESGenericRange<T> {
-  /**
-   * Greater than field
-   */
-  gt?: T;
-
-  /**
-   * Greater or equal than field
-   */
-  gte?: T;
-
-  /**
-   * Less than field
-   */
-  lt?: T;
-
-  /**
-   * Less or equal than field
-   */
-  lte?: T;
-
-  /**
-   * Relation of the range to a range field
-   *
-   * Intersects: Both ranges intersect
-   * Contains: Search range contains field range
-   * Within: Field range contains search range
-   */
-  relation?: 'intersects' | 'within' | 'contains';
-}
-
-interface ESGenericRangeFilter<G, T extends ESGenericRange<G>> {
-  /**
-   * Range filter definition
-   */
-  range: {
-    [fieldName: string]: T;
-  };
-}
-
-export interface ESDateRange extends ESGenericRange<string> {
-  /**
-   * Optional date format override
-   */
-  format?: string;
-
-  /**
-   * Optional timezone specifier
-   */
-  time_zone?: string;
-}
-
-export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<number>>;
-export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
-export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
-
-/**
- * An elasticsearch type filter
- */
-export interface ESTypeFilter {
-  /**
-   * Type filter definition
-   */
-  type: {
-    /**
-     * Type name (SCThingType) to filter with
-     */
-    value: SCThingType;
-  };
-}
-
-/**
- * Filter arguments for an elasticsearch geo distance filter
- */
-export interface ESGeoDistanceFilterArguments {
-  /**
-   * The radius of the circle centred on the specified location
-   */
-  distance: string;
-
-  [fieldName: string]:
-    | {
-        /**
-         * Latitude
-         */
-        lat: number;
-
-        /**
-         * Longitude
-         */
-        lon: number;
-      }
-    | string;
-}
-
-/**
- * An elasticsearch geo distance filter
- */
-export interface ESGeoDistanceFilter {
-  /**
-   * @see ESGeoDistanceFilterArguments
-   */
-  geo_distance: ESGeoDistanceFilterArguments;
-}
-
-/**
- * A rectangular geo shape, representing the top-left and bottom-right corners
- *
- * This is an extension of the Geojson type
- * http://geojson.org/geojson-spec.html
- */
-export interface ESEnvelope {
-  /**
-   * The top-left and bottom-right corners of the bounding box
-   */
-  coordinates: [Position, Position];
-
-  /**
-   * The type of the geometry
-   */
-  type: 'envelope';
-}
-
-/**
- * An Elasticsearch geo bounding box filter
- *
- * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
- */
-export interface ESGeoBoundingBoxFilter {
-  /**
-   * An Elasticsearch geo bounding box filter
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
-   */
-  geo_bounding_box: {
-    [fieldName: string]: {
-      /**
-       * Geo Shape
-       */
-      bottom_right: Position;
-
-      /**
-       * Geo Shape
-       */
-      top_left: Position;
-    };
-  };
-}
-
-/**
- * An Elasticsearch geo shape filter
- *
- * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html
- */
-export interface ESGeoShapeFilter {
-  geo_shape: {
-    [fieldName: string]: {
-      /**
-       * Relation of the two shapes
-       *
-       * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_spatial_relations
-       */
-      relation?: 'intersects' | 'disjoint' | 'within' | 'contains';
-
-      /**
-       * Geo Shape
-       */
-      shape: Polygon | ESEnvelope;
-    };
-  };
-}
-
-/**
- * Filter arguments for an elasticsearch boolean filter
- *
- * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
- */
-export interface ESBooleanFilterArguments<T> {
-  /**
-   * Minimal number (or percentage) of words that should match in a query
-   *
-   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
-   */
-  minimum_should_match?: number;
-
-  /**
-   * The clause (query) must appear in matching documents and will contribute to the score.
-   */
-  must?: T[];
-
-  /**
-   * The clause (query) must not appear in the matching documents.
-   */
-  must_not?: T[];
-
-  /**
-   * The clause (query) should appear in the matching document.
-   */
-  should?: T[];
-}
-
-/**
- * An elasticsearch boolean filter
- */
-export interface ESBooleanFilter<T> {
-  /**
-   * @see ESBooleanFilterArguments
-   */
-  bool: ESBooleanFilterArguments<T>;
-}
-
-/**
- * An elasticsearch function score query
- *
- * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
- */
-export interface ESFunctionScoreQuery {
-  /**
-   * Function score definition
-   */
-  function_score: {
-    /**
-     * Functions that compute score for query results (documents)
-     *
-     * @see ESFunctionScoreQueryFunction
-     */
-    functions: ESFunctionScoreQueryFunction[];
-
-    /**
-     * @see ESBooleanFilter
-     */
-    query: ESBooleanFilter<unknown>;
-
-    /**
-     * Specifies how the computed scores are combined
-     */
-    score_mode: 'multiply';
-  };
-}
-
-/**
- * An function for an elasticsearch functions score query
- */
-export interface ESFunctionScoreQueryFunction {
-  /**
-   * Function is applied only if a document matches the given filtering query
-   */
-  filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;
-
-  /**
-   * Weight (importance) of the filter
-   */
-  weight: number;
-}
-
-/**
- * An elasticsearch generic sort
- */
-export interface ESGenericSort {
-  [field: string]: string;
-}
-
-/**
- * Sort arguments for an elasticsearch geo distance sort
- */
-export interface ESGeoDistanceSortArguments {
-  /**
-   * What value to pick for sorting
-   */
-  mode: 'avg' | 'max' | 'median' | 'min';
-
-  /**
-   * Order
-   */
-  order: 'asc' | 'desc';
-
-  /**
-   * Value unit
-   */
-  unit: 'm';
-
-  [field: string]:
-    | {
-        /**
-         * Latitude
-         */
-        lat: number;
-
-        /**
-         * Longitude
-         */
-        lon: number;
-      }
-    | string;
-}
-
-/**
- * An elasticsearch geo distance sort
- */
-export interface ESGeoDistanceSort {
-  /**
-   * @see ESGeoDistanceFilterArguments
-   */
-  _geo_distance: ESGeoDistanceSortArguments;
-}
-
-/**
- * An elasticsearch script sort
- */
-export interface ScriptSort {
-  /**
-   * A script
-   */
-  _script: {
-    /**
-     * Order
-     */
-    order: 'asc' | 'desc';
-
-    /**
-     * The custom script used for sorting
-     */
-    script: string;
-
-    /**
-     * What type is being sorted
-     */
-    type: 'number' | 'string';
-  };
-}

src/storage/elasticsearch/types/guards.ts

@@ -1,69 +0,0 @@
-/*
- * Copyright (C) 2019-2021 StApps
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-import {
-  ESAggMatchAllFilter,
-  ESAggTypeFilter,
-  ESNestedAggregation,
-  ESTermsFilter,
-} from '@openstapps/es-mapping-generator/src/types/aggregation';
-import {BucketAggregation, NestedAggregation} from './elasticsearch';
-
-/**
- * Checks if the type is a BucketAggregation
- *
- * @param agg the type to check
- */
-export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
-  return typeof agg !== 'number';
-}
-
-/**
- * Checks if the type is a NestedAggregation
- *
- * @param agg the type to check
- */
-export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
-  return typeof (agg as BucketAggregation).buckets === 'undefined';
-}
-
-/**
- * Checks if the parameter is of type ESTermsFilter
- *
- * @param agg the value to check
- */
-export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
-  return typeof (agg as ESTermsFilter).terms !== 'undefined';
-}
-
-/**
- * Checks if the parameter is of type ESTermsFilter
- *
- * @param agg the value to check
- */
-export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
-  return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
-}
-
-/**
- * Checks if the parameter is of type
- *
- * @param filter the filter to narrow the type of
- */
-export function isESAggMatchAllFilter(
-  filter: ESAggTypeFilter | ESAggMatchAllFilter,
-): filter is ESAggMatchAllFilter {
-  return filter.hasOwnProperty('match_all');
-}

src/storage/elasticsearch/types/util.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
+
+export type QueryDslSpecificQueryContainer<T extends keyof QueryDslQueryContainer> = Required<
+  Pick<QueryDslQueryContainer, T>
+>;

src/storage/elasticsearch/util/alias.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Logger} from '@openstapps/logger';
+
+/**
+ * Checks for invalid character in alias names and removes them
+ *
+ * @param alias The alias name
+ * @param uid   The UID of the current bulk (for debugging purposes)
+ */
+export function removeInvalidAliasChars(alias: string, uid: string | undefined): string {
+  let formattedAlias = alias;
+
+  // spaces are included in some types, replace them with underscores
+  if (formattedAlias.includes(' ')) {
+    formattedAlias = formattedAlias.trim();
+    formattedAlias = formattedAlias.split(' ').join('_');
+  }
+  // List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
+  for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
+    if (formattedAlias.includes(value)) {
+      formattedAlias = formattedAlias.replace(value, '');
+      Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
+        having the same alias despite having different types, as invalid characters are removed automatically.
+        New alias name is "${formattedAlias}."`);
+    }
+  }
+  for (const value of ['-', '_', '+']) {
+    if (formattedAlias.charAt(0) === value) {
+      formattedAlias = formattedAlias.slice(1);
+      Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
+        alias despite having different types, as invalid characters are removed automatically.
+        New alias name is "${formattedAlias}."`);
+    }
+  }
+  if (formattedAlias === '.' || formattedAlias === '..') {
+    Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
+      another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
+
+    return 'alias_placeholder';
+  }
+  if (formattedAlias.includes(':')) {
+    Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
+      Elasticsearch versions!`);
+  }
+
+  return formattedAlias;
+}

src/storage/elasticsearch/util/index.ts

@@ -0,0 +1,63 @@
+import {SCBulkResponse, SCThingType, SCUuid} from '@openstapps/core';
+
+/**
+ * Length of the index UID used for generation of its name
+ */
+export const INDEX_UID_LENGTH = 8;
+
+/**
+ * A string which matches all indices
+ */
+export const ALL_INDICES_QUERY = 'stapps_*_*_*';
+
+/**
+ * Matches index names such as stapps_<type>_<source>_<random suffix>
+ */
+export const VALID_INDEX_REGEX = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
+
+export interface ParsedIndexName {
+  type: SCThingType;
+  source: string;
+  randomSuffix: string;
+}
+
+/**
+ *
+ */
+export function parseIndexName(index: string): ParsedIndexName {
+  const match = VALID_INDEX_REGEX.exec(index);
+  if (!match) {
+    throw new SyntaxError(`Invalid index name ${index}!`);
+  }
+
+  return {
+    type: match[1] as SCThingType,
+    source: match[2],
+    randomSuffix: match[3],
+  };
+}
+
+/**
+ * Gets the index name in elasticsearch for one SCThingType
+ *
+ * @param type SCThingType of data in the index
+ * @param source source of data in the index
+ * @param bulk bulk process which created this index
+ */
+export function getThingIndexName(type: SCThingType, source: string, bulk: SCBulkResponse) {
+  let out = type.toLowerCase();
+  while (out.includes(' ')) {
+    out = out.replace(' ', '_');
+  }
+
+  return `stapps_${out}_${source}_${getIndexUID(bulk.uid)}`;
+}
+
+/**
+ * Provides the index UID (for its name) from the bulk UID
+ *
+ * @param uid Bulk UID
+ */
+export function getIndexUID(uid: SCUuid) {
+  return uid.slice(0, Math.max(0, INDEX_UID_LENGTH));
+}

src/storage/elasticsearch/util/no-undefined.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Type guard for filter functions
+ */
+export function noUndefined<T>(item: T | undefined): item is T {
+  return typeof item !== 'undefined';
+}

src/storage/elasticsearch/util/retry.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2022 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+export interface RetryOptions<T> {
+  maxRetries: number;
+  retryInterval: number;
+  doAction: () => Promise<T>;
+  onFailedAttempt: (attempt: number, error: unknown, options: RetryOptions<T>) => void;
+  onFail: (options: RetryOptions<T>) => never;
+}
+
+/**
+ * Retries a throwing function at a set interval, until a maximum amount of attempts
+ */
+export async function retryCatch<T>(options: RetryOptions<T>): Promise<T> {
+  for (let attempt = 0; attempt < options.maxRetries; attempt++) {
+    try {
+      return await options.doAction();
+    } catch (error) {
+      options.onFailedAttempt(attempt, error, options);
+      await new Promise(resolve => setTimeout(resolve, options.retryInterval));
+    }
+  }
+
+  options.onFail(options);
+}

test/common.ts

@@ -16,6 +16,7 @@
 import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCThingType, SCUuid} from '@openstapps/core';
 import {Express} from 'express';
 import moment from 'moment';
+import {getIndexUID} from '../src/storage/elasticsearch/util';
 import {configureApp} from '../src/app';
 import express from 'express';
 import http from 'http';
@@ -24,7 +25,6 @@ import {MailQueue} from '../src/notification/mail-queue';
 import {Bulk, BulkStorage} from '../src/storage/bulk-storage';
 import getPort from 'get-port';
 import {Database} from '../src/storage/database';
-import {Elasticsearch} from '../src/storage/elasticsearch/elasticsearch';
 import {v4} from 'uuid';
 
 /**
@@ -147,5 +147,4 @@ export const getTransport = (verified: boolean) => {
   };
 };
 
-export const getIndex = (uid?: string) =>
+export const getIndex = (uid?: string) => `stapps_footype_foosource_${uid ?? getIndexUID(v4())}`;
-  `stapps_footype_foosource_${uid ?? Elasticsearch.getIndexUID(v4())}`;

test/storage/elasticsearch/aggregations.spec.ts

@@ -13,13 +13,13 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+import {AggregateName, AggregationsMultiTermsBucket} from '@elastic/elasticsearch/lib/api/types';
 import {SCFacet, SCThingType} from '@openstapps/core';
 import {expect} from 'chai';
 import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
-import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch';
 
 describe('Aggregations', function () {
-  const aggregations: AggregationResponse = {
+  const aggregations: Record<AggregateName, Partial<AggregationsMultiTermsBucket>> = {
     'catalog': {
       'doc_count': 4,
       'superCatalogs.categories': {
@@ -76,14 +76,6 @@ describe('Aggregations', function () {
         buckets: [],
       },
     },
-    'fooType': {
-      buckets: [
-        {
-          doc_count: 321,
-          key: 'foo',
-        },
-      ],
-    },
     '@all': {
       doc_count: 17,
       type: {
@@ -102,33 +94,6 @@ describe('Aggregations', function () {
   };
 
   const expectedFacets: SCFacet[] = [
-    {
-      buckets: [
-        {
-          count: 13,
-          key: 'person',
-        },
-        {
-          count: 4,
-          key: 'catalog',
-        },
-      ],
-      field: 'type',
-    },
-    {
-      buckets: [
-        {
-          count: 8,
-          key: 'foobar',
-        },
-        {
-          count: 2,
-          key: 'bar',
-        },
-      ],
-      field: 'categories',
-      onlyOnType: SCThingType.AcademicEvent,
-    },
     {
       buckets: [
         {
@@ -153,7 +118,33 @@ describe('Aggregations', function () {
       field: 'categories',
       onlyOnType: SCThingType.Catalog,
     },
-    // no fooType as it doesn't appear in the aggregation schema
+    {
+      buckets: [
+        {
+          count: 8,
+          key: 'foobar',
+        },
+        {
+          count: 2,
+          key: 'bar',
+        },
+      ],
+      field: 'categories',
+      onlyOnType: SCThingType.AcademicEvent,
+    },
+    {
+      buckets: [
+        {
+          count: 13,
+          key: 'person',
+        },
+        {
+          count: 4,
+          key: 'catalog',
+        },
+      ],
+      field: 'type',
+    },
   ];
 
   it('should parse the aggregations providing the appropriate facets', function () {

test/storage/elasticsearch/common.spec.ts

@@ -1,90 +0,0 @@
-/*
- * Copyright (C) 2020 StApps
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-import {
-  ESAggMatchAllFilter,
-  ESAggTypeFilter,
-  ESNestedAggregation,
-  ESTermsFilter,
-} from '@openstapps/es-mapping-generator/src/types/aggregation';
-import {expect} from 'chai';
-import {
-  isNestedAggregation,
-  isBucketAggregation,
-  isESTermsFilter,
-  isESAggMatchAllFilter,
-  isESNestedAggregation,
-} from '../../../lib/storage/elasticsearch/types/guards';
-import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';
-
-describe('Common', function () {
-  const bucketAggregation: BucketAggregation = {buckets: []};
-  const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
-  const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
-
-  describe('isBucketAggregation', function () {
-    it('should be false for a number', function () {
-      expect(isBucketAggregation(123)).to.be.false;
-    });
-
-    it('should be true for a bucket aggregation', function () {
-      expect(isBucketAggregation(bucketAggregation)).to.be.true;
-    });
-  });
-
-  describe('isNestedAggregation', function () {
-    it('should be false for a bucket aggregation', function () {
-      expect(isNestedAggregation(bucketAggregation)).to.be.false;
-    });
-
-    it('should be true for a nested aggregation', function () {
-      const nestedAggregation: NestedAggregation = {doc_count: 123};
-
-      expect(isNestedAggregation(nestedAggregation)).to.be.true;
-    });
-  });
-
-  describe('isESTermsFilter', function () {
-    it('should be false for an elasticsearch nested aggregation', function () {
-      expect(isESTermsFilter(esNestedAggregation)).to.be.false;
-    });
-
-    it('should be true for an elasticsearch terms filter', function () {
-      expect(isESTermsFilter(esTermsFilter)).to.be.true;
-    });
-  });
-
-  describe('isESNestedAggregation', function () {
-    it('should be false for an elasticsearch terms filter', function () {
-      expect(isESNestedAggregation(esTermsFilter)).to.be.false;
-    });
-
-    it('should be true for an elasticsearch nested aggregation', function () {
-      expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
-    });
-  });
-
-  describe('isESAggMatchAllFilter', function () {
-    it('should be false for an elasticsearch aggregation type filter', function () {
-      const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
-      expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
-    });
-
-    it('should be true for an elasticsearch aggregation match all filter', function () {
-      const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
-      expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
-    });
-  });
-});

test/storage/elasticsearch/elasticsearch.spec.ts

@@ -14,7 +14,14 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {Client, Diagnostic} from '@elastic/elasticsearch';
+import Indices from '@elastic/elasticsearch/lib/api/api/indices';
+import {
+  CreateResponse,
+  SearchHit,
+  SearchResponse,
+  SortCombinations,
+} from '@elastic/elasticsearch/lib/api/types';
 import {
   SCBook,
   SCBulkResponse,
@@ -30,22 +37,32 @@ import {Logger} from '@openstapps/logger';
 import {SMTP} from '@openstapps/logger/lib/smtp';
 import {expect, use} from 'chai';
 import chaiAsPromised from 'chai-as-promised';
-import {SearchResponse} from 'elasticsearch';
 import mockedEnv from 'mocked-env';
-import sinon from 'sinon';
+import {ALL_INDICES_QUERY, parseIndexName} from '../../../src/storage/elasticsearch/util';
+import * as queryModule from '../../../src/storage/elasticsearch/query/query';
+import * as sortModule from '../../../src/storage/elasticsearch/query/sort';
+import sinon, {SinonStub} from 'sinon';
+import {getIndexUID, getThingIndexName, INDEX_UID_LENGTH} from '../../../src/storage/elasticsearch/util';
+import * as utilModule from '../../../src/storage/elasticsearch/util';
+import {removeInvalidAliasChars} from '../../../src/storage/elasticsearch/util/alias';
 import {configFile} from '../../../src/common';
 import {MailQueue} from '../../../src/notification/mail-queue';
 import {aggregations} from '../../../src/storage/elasticsearch/templating';
-import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch';
 import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
 import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
-import * as query from '../../../src/storage/elasticsearch/query';
 import * as templating from '../../../src/storage/elasticsearch/templating';
 import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common';
 import fs from 'fs';
 
 use(chaiAsPromised);
 
+/**
+ *
+ */
+function searchResponse<T>(...hits: SearchHit<T>[]): SearchResponse<T> {
+  return {hits: {hits}, took: 0, timed_out: false, _shards: {total: 1, failed: 0, successful: 1}};
+}
+
 describe('Elasticsearch', function () {
   // increase timeout for the suite
   this.timeout(DEFAULT_TEST_TIMEOUT);
@@ -83,6 +100,14 @@ describe('Elasticsearch', function () {
     });
   });
 
+  describe('getAliasMap', function () {
+    it('should fail after retries', async function () {
+      const es = new Elasticsearch(configFile);
+      sandbox.stub(es.client.indices, 'getAlias').throws();
+      await expect(es.init({maxRetries: 1, retryInterval: 10})).to.be.rejected;
+    });
+  });
+
   describe('getIndex (including getIndexUID)', function () {
     const type = 'foo bar type';
     const source = 'foo_source';
@@ -95,59 +120,63 @@ describe('Elasticsearch', function () {
     };
 
     it('should provide index UID from the provided UID', function () {
-      const indexUID = Elasticsearch.getIndexUID(bulk.uid);
+      const indexUID = getIndexUID(bulk.uid);
 
-      expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
+      expect(indexUID.length).to.be.equal(INDEX_UID_LENGTH);
       // test starting and ending character
       expect(indexUID[0]).to.be.equal(bulk.uid[0]);
-      expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
+      expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[INDEX_UID_LENGTH - 1]);
     });
 
     it('should provide index name from the provided data', function () {
-      expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
+      expect(getThingIndexName(type as SCThingType, source, bulk)).to.be.equal(
-        `stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
+        `stapps_${type.split(' ').join('_')}_${source}_${getIndexUID(bulk.uid)}`,
       );
     });
+
+    it('should reject invalid index names', function () {
+      expect(() => parseIndexName(':)')).to.throw(SyntaxError);
+    });
   });
 
   describe('removeAliasChars', function () {
     it('should remove spaces from both ends', function () {
-      expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(removeInvalidAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
     });
 
     it('should replace inner spaces with underscores', function () {
-      expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
+      expect(removeInvalidAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
     });
 
     it('should remove invalid characters', function () {
-      expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(removeInvalidAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
     });
 
     it('should remove invalid starting characters', function () {
-      expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(removeInvalidAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
-      expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(removeInvalidAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
-      expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(removeInvalidAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
     });
 
     it('should replace with a placeholder in case of invalid alias', function () {
-      expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
+      expect(removeInvalidAliasChars('.', 'bulk-uid')).to.contain('placeholder');
-      expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
+      expect(removeInvalidAliasChars('..', 'bulk-uid')).to.contain('placeholder');
     });
 
     it('should work with common cases', function () {
       expect(
-        Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
+        removeInvalidAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
       ).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
-      expect(
+      expect(removeInvalidAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid')).to.be.equal(
-        Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
+        'THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG',
-      ).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
+      );
     });
 
     it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
       const sandbox = sinon.createSandbox();
       const loggerWarnStub = sandbox.stub(Logger, 'warn');
 
-      expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
+      expect(removeInvalidAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
       expect(loggerWarnStub.called).to.be.true;
     });
   });
@@ -182,7 +211,7 @@ describe('Elasticsearch', function () {
     it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
       const error = new Error('Foo Error');
       const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
-      sandbox.stub(Client.prototype, 'on').yields(error);
+      sandbox.stub(Diagnostic.prototype, 'on').yields(error);
 
       new Elasticsearch(configFile);
 
@@ -192,7 +221,7 @@ describe('Elasticsearch', function () {
     it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
       const fakeResponse = {foo: 'bar'};
       const loggerLogStub = sandbox.stub(Logger, 'log');
-      sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
+      sandbox.stub(Diagnostic.prototype, 'on').yields(null, fakeResponse);
 
       new Elasticsearch(configFile);
       expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
@@ -254,26 +283,24 @@ describe('Elasticsearch', function () {
     describe('Operations with bundle/index', async function () {
       const sandbox = sinon.createSandbox();
       let es: Elasticsearch;
+      let createStub: SinonStub;
+      let deleteStub: SinonStub;
+      let refreshStub: SinonStub;
+      let updateAliasesStub: SinonStub;
+      let existsStub: SinonStub;
       const oldIndex = 'stapps_footype_foosource_oldindex';
 
       beforeEach(function () {
+        sandbox
+          .stub(Indices.prototype, 'getAlias')
+          .resolves({[oldIndex]: {aliases: {[SCThingType.Book]: {}}}} as any);
+        sandbox.stub(Indices.prototype, 'putTemplate').resolves({} as any);
+        createStub = sandbox.stub(Indices.prototype, 'create').resolves({} as any);
+        deleteStub = sandbox.stub(Indices.prototype, 'delete').resolves({} as any);
+        existsStub = sandbox.stub(Indices.prototype, 'exists').resolves({} as any);
+        refreshStub = sandbox.stub(Indices.prototype, 'refresh').resolves({} as any);
+        updateAliasesStub = sandbox.stub(Indices.prototype, 'updateAliases').resolves({} as any);
         es = new Elasticsearch(configFile);
-        es.client.indices = {
-          // @ts-expect-error not assignable
-          getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
-          // @ts-expect-error not assignable
-          putTemplate: () => Promise.resolve({}),
-          // @ts-expect-error not assignable
-          create: () => Promise.resolve({}),
-          // @ts-expect-error not assignable
-          delete: () => Promise.resolve({}),
-          // @ts-expect-error not assignable
-          exists: () => Promise.resolve({}),
-          // @ts-expect-error not assignable
-          refresh: () => Promise.resolve({}),
-          // @ts-expect-error not assignable
-          updateAliases: () => Promise.resolve({}),
-        };
       });
 
       afterEach(function () {
@@ -286,8 +313,8 @@ describe('Elasticsearch', function () {
         });
 
         it('should reject (throw an error) if the index name is not valid', async function () {
-          sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`);
           sandbox.createStubInstance(Client, {});
+          sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex}`);
           await es.init();
 
           return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
@@ -295,9 +322,8 @@ describe('Elasticsearch', function () {
 
         it('should create a new index', async function () {
           const index = getIndex();
-          sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+          sandbox.stub(utilModule, 'getThingIndexName').returns(index);
           const putTemplateStub = sandbox.stub(templating, 'putTemplate');
-          const createStub = sandbox.stub(es.client.indices, 'create');
           await es.init();
 
           await es.bulkCreated(bulk);
@@ -313,21 +339,19 @@ describe('Elasticsearch', function () {
           sandbox.restore();
         });
         it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
-          sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+          sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
-          const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
 
           await es.bulkExpired({...bulk, state: 'in progress'});
 
-          expect(clientDeleteStub.called).to.be.true;
+          expect(deleteStub.called).to.be.true;
         });
 
         it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
-          sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+          sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
-          const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
 
           await es.bulkExpired({...bulk, state: 'done'});
 
-          expect(clientDeleteStub.called).to.be.false;
+          expect(deleteStub.called).to.be.false;
         });
       });
 
@@ -337,13 +361,23 @@ describe('Elasticsearch', function () {
         });
 
         it('should reject if the index name is not valid', async function () {
-          sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`);
+          sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex()}`);
           sandbox.createStubInstance(Client, {});
           await es.init();
 
           return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
         });
 
+        it("should create templates if index doesn't exist", async function () {
+          await es.init();
+          existsStub.resolves(false);
+          const putTemplateSpy = sandbox.spy(templating, 'putTemplate');
+          await es.bulkUpdated(bulk);
+
+          expect(createStub.called).to.be.true;
+          expect(putTemplateSpy.called).to.be.true;
+        });
+
         it('should create a new index', async function () {
           const index = getIndex();
           const expectedRefreshActions = [
@@ -354,15 +388,12 @@ describe('Elasticsearch', function () {
               remove: {index: oldIndex, alias: SCThingType.Book},
             },
           ];
-          sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+          sandbox.stub(utilModule, 'getThingIndexName').returns(index);
           sandbox.stub(es, 'aliasMap').value({
             [SCThingType.Book]: {
               [bulk.source]: oldIndex,
             },
           });
-          const refreshStub = sandbox.stub(es.client.indices, 'refresh');
-          const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
-          const deleteStub = sandbox.stub(es.client.indices, 'delete');
           sandbox.stub(templating, 'putTemplate');
           await es.init();
 
@@ -371,9 +402,7 @@ describe('Elasticsearch', function () {
           expect(refreshStub.calledWith({index})).to.be.true;
           expect(
             updateAliasesStub.calledWith({
-              body: {
               actions: expectedRefreshActions,
-              },
             }),
           ).to.be.true;
           expect(deleteStub.called).to.be.true;
@@ -394,20 +423,19 @@ describe('Elasticsearch', function () {
       });
 
       it('should reject if object is not found', async function () {
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse());
 
         return expect(es.get('123')).to.rejectedWith('found');
       });
 
       it('should provide the thing if object is found', async function () {
-        const foundObject: ElasticsearchObject<SCMessage> = {
+        const foundObject: SearchHit<SCMessage> = {
           _id: '',
           _index: '',
           _score: 0,
-          _type: '',
           _source: message as SCMessage,
         };
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse(foundObject));
 
         return expect(await es.get('123')).to.be.eql(message);
       });
@@ -428,56 +456,54 @@ describe('Elasticsearch', function () {
       it('should not post if the object already exists in an index which will not be rolled over', async function () {
         const index = getIndex();
         const oldIndex = index.replace('foosource', 'barsource');
-        const object: ElasticsearchObject<SCMessage> = {
+        const object: SearchHit<SCMessage> = {
           _id: '',
           _index: oldIndex,
           _score: 0,
-          _type: '',
           _source: message as SCMessage,
         };
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse(object));
-        sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+        sandbox.stub(utilModule, 'getThingIndexName').returns(index);
 
-        return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
+        return expect(es.post(object._source!, bulk)).to.rejectedWith('exist');
       });
 
       it('should not reject if the object already exists but in an index which will be rolled over', async function () {
-        const object: ElasticsearchObject<SCMessage> = {
+        const object: SearchHit<SCMessage> = {
           _id: '',
           _index: getIndex(),
           _score: 0,
-          _type: '',
           _source: message as SCMessage,
         };
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse(object));
         // return index name with different generated UID (see getIndex method)
-        sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+        sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
 
-        return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist');
+        return expect(es.post(object._source!, bulk)).to.not.rejectedWith('exist');
       });
 
       it('should reject if there is an object creation error on the elasticsearch side', async function () {
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse());
-        sandbox.stub(es.client, 'create').resolves({body: {created: false}});
+        sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse);
 
         return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
       });
 
       it('should create a new object', async function () {
         let caughtParameter: any;
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse());
         // @ts-expect-error call
         const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
           caughtParameter = parameter;
-          return Promise.resolve({body: {created: true}});
+          return Promise.resolve({result: 'created'});
         });
 
         await es.post(message as SCMessage, bulk);
 
         expect(createStub.called).to.be.true;
-        expect(caughtParameter.body).to.be.eql({
+        expect(caughtParameter.document).to.be.eql({
           ...message,
-          creation_date: caughtParameter.body.creation_date,
+          creation_date: caughtParameter.document.creation_date,
         });
       });
     });
@@ -493,29 +519,27 @@ describe('Elasticsearch', function () {
         sandbox.restore();
       });
       it('should reject to put if the object does not already exist', async function () {
-        const object: ElasticsearchObject<SCMessage> = {
+        const object: SearchHit<SCMessage> = {
           _id: '',
           _index: getIndex(),
           _score: 0,
-          _type: '',
           _source: message as SCMessage,
         };
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse());
 
-        return expect(es.put(object._source)).to.rejectedWith('exist');
+        return expect(es.put(object._source!)).to.rejectedWith('exist');
       });
 
       // noinspection JSUnusedLocalSymbols
       it('should update the object if it already exists', async function () {
         let caughtParameter: any;
-        const object: ElasticsearchObject<SCMessage> = {
+        const object: SearchHit<SCMessage> = {
           _id: '',
           _index: getIndex(),
           _score: 0,
-          _type: '',
           _source: message as SCMessage,
         };
-        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        sandbox.stub(es.client, 'search').resolves(searchResponse(object));
         // @ts-expect-error unused
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
@@ -523,7 +547,7 @@ describe('Elasticsearch', function () {
           return Promise.resolve({body: {created: true}});
         });
 
-        await es.put(object._source);
+        await es.put(object._source!);
 
         expect(caughtParameter.body.doc).to.be.eql(object._source);
       });
@@ -532,18 +556,16 @@ describe('Elasticsearch', function () {
     describe('search', async function () {
       let es: Elasticsearch;
       const sandbox = sinon.createSandbox();
-      const objectMessage: ElasticsearchObject<SCMessage> = {
+      const objectMessage: SearchHit<SCMessage> = {
         _id: '123',
         _index: getIndex(),
         _score: 0,
-        _type: '',
         _source: message as SCMessage,
       };
-      const objectBook: ElasticsearchObject<SCBook> = {
+      const objectBook: SearchHit<SCBook> = {
         _id: '321',
         _index: getIndex(),
         _score: 0,
-        _type: '',
         _source: book as SCBook,
       };
       const fakeEsAggregations = {
@@ -565,26 +587,16 @@ describe('Elasticsearch', function () {
           },
         },
       };
-      const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
+      const fakeSearchResponse: SearchResponse<SCThings> = {
-        body: {
         took: 12,
         timed_out: false,
         // @ts-expect-error not assignable
         _shards: {},
-          // @ts-expect-error not assignable
         hits: {
           hits: [objectMessage, objectBook],
           total: 123,
         },
         aggregations: fakeEsAggregations,
-        },
-        headers: {},
-        // @ts-expect-error not assignable
-        meta: {},
-        // @ts-expect-error not assignable
-        statusCode: {},
-        // @ts-expect-error not assignable
-        warnings: {},
       };
       let searchStub: sinon.SinonStub;
       before(function () {
@@ -625,9 +637,9 @@ describe('Elasticsearch', function () {
         const {pagination} = await es.search({from});
 
         expect(pagination).to.be.eql({
-          count: fakeSearchResponse.body!.hits.hits.length,
+          count: fakeSearchResponse.hits.hits.length,
           offset: from,
-          total: fakeSearchResponse.body!.hits.total,
+          total: fakeSearchResponse.hits.total,
         });
       });
 
@@ -659,22 +671,20 @@ describe('Elasticsearch', function () {
             },
           },
         };
-        const fakeResponse = {foo: 'bar'};
+        const fakeResponse = {foo: 'bar'} as SortCombinations;
         const fakeBuildSortResponse = [fakeResponse];
         // @ts-expect-error not assignable
-        sandbox.stub(query, 'buildQuery').returns(fakeResponse);
+        sandbox.stub(queryModule, 'buildQuery').returns(fakeResponse);
-        sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
+        sandbox.stub(sortModule, 'buildSort').returns(fakeBuildSortResponse);
 
         await es.search(parameters);
 
         sandbox.assert.calledWithMatch(searchStub, {
-          body: {
           aggs: aggregations,
           query: fakeResponse,
           sort: fakeBuildSortResponse,
-          },
           from: parameters.from,
-          index: Elasticsearch.getListOfAllIndices(),
+          index: ALL_INDICES_QUERY,
           size: parameters.size,
         });
       });

test/storage/elasticsearch/monitoring.spec.ts

@@ -14,7 +14,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {Client} from '@elastic/elasticsearch';
+import {SearchResponse} from '@elastic/elasticsearch/lib/api/types';
 import {
   SCMonitoringConfiguration,
   SCMonitoringLogAction,
@@ -23,7 +24,6 @@ import {
   SCThings,
 } from '@openstapps/core';
 import {Logger} from '@openstapps/logger';
-import {SearchResponse} from 'elasticsearch';
 import {MailQueue} from '../../../src/notification/mail-queue';
 import {setUp} from '../../../src/storage/elasticsearch/monitoring';
 
@@ -111,8 +111,7 @@ describe('Monitoring', async function () {
   });
 
   it('should log errors where conditions failed', async function () {
-    const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
+    const fakeSearchResponse: SearchResponse<SCThings> = {
-      body: {
       took: 12,
       timed_out: false,
       // @ts-expect-error not assignable
@@ -121,7 +120,6 @@ describe('Monitoring', async function () {
       hits: {
         total: 123,
       },
-      },
     };
     const fakeClient = new Client({node: 'http://foohost:9200'});
     const loggerErrorStub = sandbox.stub(Logger, 'error');

test/storage/elasticsearch/query.spec.ts

@@ -25,25 +25,14 @@ import {
   SCThingType,
 } from '@openstapps/core';
 import {expect} from 'chai';
-import {
+import {buildFilter} from '../../../src/storage/elasticsearch/query/filter';
-  ESDateRangeFilter,
+import {buildBooleanFilter} from '../../../src/storage/elasticsearch/query/filters/boolean';
-  ESRangeFilter,
+import {buildQuery} from '../../../src/storage/elasticsearch/query/query';
-  ESNumericRangeFilter,
+import {buildSort} from '../../../src/storage/elasticsearch/query/sort';
-  ElasticsearchConfig,
+import {ElasticsearchConfig} from '../../../src/storage/elasticsearch/types/elasticsearch-config';
-  ESBooleanFilter,
+import {QueryDslSpecificQueryContainer} from '../../../src/storage/elasticsearch/types/util';
-  ESGenericSort,
-  ESGeoDistanceFilter,
-  ESGeoDistanceSort,
-  ESTermFilter,
-  ScriptSort,
-} from '../../../src/storage/elasticsearch/types/elasticsearch';
 import {configFile} from '../../../src/common';
-import {
+import {SortCombinations} from '@elastic/elasticsearch/lib/api/types';
-  buildBooleanFilter,
-  buildFilter,
-  buildQuery,
-  buildSort,
-} from '../../../src/storage/elasticsearch/query';
 
 describe('Query', function () {
   describe('buildBooleanFilter', function () {
@@ -74,7 +63,7 @@ describe('Query', function () {
       or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
       not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
     };
-    const expectedEsFilters: Array<ESTermFilter> = [
+    const expectedEsFilters: Array<QueryDslSpecificQueryContainer<'term'>> = [
       {
         term: {
           'type.raw': 'catalog',
@@ -88,20 +77,20 @@ describe('Query', function () {
     ];
 
     it('should create appropriate elasticsearch "and" filter argument', function () {
-      const {must} = buildBooleanFilter(booleanFilters.and);
+      const {must} = buildBooleanFilter(booleanFilters.and).bool;
 
       expect(must).to.be.eql(expectedEsFilters);
     });
 
     it('should create appropriate elasticsearch "or" filter argument', function () {
-      const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
+      const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or).bool;
 
       expect(should).to.be.eql(expectedEsFilters);
       expect(minimum_should_match).to.be.equal(1);
     });
 
     it('should create appropriate elasticsearch "not" filter argument', function () {
-      const {must_not} = buildBooleanFilter(booleanFilters.not);
+      const {must_not} = buildBooleanFilter(booleanFilters.not).bool;
 
       expect(must_not).to.be.eql(expectedEsFilters);
     });
@@ -196,6 +185,10 @@ describe('Query', function () {
 
       expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
     });
+
+    it('should accept other search contexts', function () {
+      expect(buildQuery({context: 'place', ...parameters}, config, esConfig)).to.be.an('object');
+    });
   });
 
   describe('buildFilter', function () {
@@ -267,7 +260,7 @@ describe('Query', function () {
 
     it('should build value filter', function () {
       const filter = buildFilter(searchFilters.value);
-      const expectedFilter: ESTermFilter = {
+      const expectedFilter: QueryDslSpecificQueryContainer<'term'> = {
         term: {
           'type.raw': SCThingType.Dish,
         },
@@ -279,7 +272,7 @@ describe('Query', function () {
     it('should build numeric range filters', function () {
       for (const upperMode of ['inclusive', 'exclusive', null]) {
         for (const lowerMode of ['inclusive', 'exclusive', null]) {
-          const expectedFilter: ESNumericRangeFilter = {
+          const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
             range: {
               price: {
                 relation: undefined,
@@ -304,7 +297,7 @@ describe('Query', function () {
                 mode: bound as 'inclusive' | 'exclusive',
                 limit: out,
               };
-              expectedFilter.range.price[
+              expectedFilter.range.price![
                 `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
               ] = out;
             }
@@ -312,7 +305,7 @@ describe('Query', function () {
           setBound('upperBound', upperMode);
           setBound('lowerBound', lowerMode);
 
-          const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
+          const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'term'>;
           expect(filter).to.deep.equal(expectedFilter);
           for (const bound of ['g', 'l']) {
             // @ts-expect-error implicit any
@@ -330,7 +323,7 @@ describe('Query', function () {
     it('should build date range filters', function () {
       for (const upperMode of ['inclusive', 'exclusive', null]) {
         for (const lowerMode of ['inclusive', 'exclusive', null]) {
-          const expectedFilter: ESDateRangeFilter = {
+          const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
             range: {
               price: {
                 format: 'thisIsADummyFormat',
@@ -359,7 +352,7 @@ describe('Query', function () {
                 mode: bound as 'inclusive' | 'exclusive',
                 limit: out,
               };
-              expectedFilter.range.price[
+              expectedFilter.range.price![
                 `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
               ] = out;
             }
@@ -367,7 +360,7 @@ describe('Query', function () {
           setBound('upperBound', upperMode);
           setBound('lowerBound', lowerMode);
 
-          const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
+          const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'range'>;
           expect(filter).to.deep.equal(expectedFilter);
           for (const bound of ['g', 'l']) {
             // @ts-expect-error implicit any
@@ -394,7 +387,7 @@ describe('Query', function () {
             },
           });
 
-          const expectedFilter: ESRangeFilter = {
+          const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
             range: {
               'offers.availabilityRange': {
                 gte: `test||/${scope}`,
@@ -415,7 +408,7 @@ describe('Query', function () {
           },
         });
 
-        const expectedFilter: ESRangeFilter = {
+        const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
           range: {
             'offers.availabilityRange': {
               gte: 'test||/s',
@@ -436,7 +429,7 @@ describe('Query', function () {
           },
         });
 
-        const expectedFilter: ESRangeFilter = {
+        const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
           range: {
             'offers.availabilityRange': {
               gte: `test||/d`,
@@ -456,7 +449,7 @@ describe('Query', function () {
           },
         });
 
-        const expectedFilter: ESRangeFilter = {
+        const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
           range: {
             'offers.availabilityRange': {
               gte: `now/d`,
@@ -470,7 +463,7 @@ describe('Query', function () {
 
     it('should build distance filter', function () {
       const filter = buildFilter(searchFilters.distance);
-      const expectedFilter: ESGeoDistanceFilter = {
+      const expectedFilter: QueryDslSpecificQueryContainer<'geo_distance'> = {
         geo_distance: {
           'distance': '1000m',
           'geo.point.coordinates': {
@@ -486,10 +479,6 @@ describe('Query', function () {
     it('should build geo filter for shapes and points', function () {
       const filter = buildFilter(searchFilters.geoPoint);
       const expectedFilter = {
-        bool: {
-          minimum_should_match: 1,
-          should: [
-            {
         geo_shape: {
           'geo.polygon': {
             relation: undefined,
@@ -503,18 +492,6 @@ describe('Query', function () {
           },
           'ignore_unmapped': true,
         },
-            },
-            {
-              geo_bounding_box: {
-                'geo.point.coordinates': {
-                  bottom_right: [50.123, 8.123],
-                  top_left: [50.123, 8.123],
-                },
-                'ignore_unmapped': true,
-              },
-            },
-          ],
-        },
       };
 
       expect(filter).to.be.eql(expectedFilter);
@@ -543,7 +520,7 @@ describe('Query', function () {
 
     it('should build boolean filter', function () {
       const filter = buildFilter(searchFilters.boolean);
-      const expectedFilter: ESBooleanFilter<any> = {
+      const expectedFilter: QueryDslSpecificQueryContainer<'bool'> = {
         bool: {
           minimum_should_match: 0,
           must: [
@@ -604,8 +581,8 @@ describe('Query', function () {
         },
       },
     ];
-    let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
+    let sorts: SortCombinations[] = [];
-    const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
+    const expectedSorts: {[key: string]: SortCombinations} = {
       ducet: {
         'name.sort': 'desc',
       },
@@ -632,7 +609,7 @@ describe('Query', function () {
       },
     };
     before(function () {
-      sorts = buildSort(searchSCSearchSort);
+      sorts = buildSort(searchSCSearchSort) as SortCombinations[];
     });
 
     it('should build ducet sort', function () {
@@ -649,10 +626,10 @@ describe('Query', function () {
 
     it('should build price sort', function () {
       const priceSortNoScript = {
-        ...sorts[3],
+        ...(sorts[3] as any),
         _script: {
-          ...(sorts[3] as ScriptSort)._script,
+          ...(sorts[3] as any)._script,
-          script: (expectedSorts.price as ScriptSort)._script.script,
+          script: (expectedSorts.price as any)._script.script,
         },
       };
       expect(priceSortNoScript).to.be.eql(expectedSorts.price);

tsconfig.json

@@ -2,7 +2,9 @@
   "extends": "./node_modules/@openstapps/configuration/tsconfig.json",
   "compilerOptions": {
     "resolveJsonModule": true,
-    "useUnknownInCatchVariables": false
+    "skipLibCheck": true,
+    "useUnknownInCatchVariables": false,
+    "lib": ["ES2020"]
   },
   "exclude": [
     "./config/",