feat: add backend

src/storage/BulkStorage.ts

@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBulkRequest, SCThingTypes } from '@openstapps/core';
+import * as moment from 'moment';
+import * as NodeCache from 'node-cache';
+import { promisify } from 'util';
+import { v4 } from 'uuid';
+import { logger } from '../common';
+import { Database } from './Database';
+
+export type BulkOperation = 'create' | 'expired' | 'update';
+
+/**
+ * Describes an indexing process
+ */
+export class Bulk implements SCBulkRequest {
+
+  /**
+   * Expiration of the bulk
+   *
+   * If the bulk is not finished before the expiration date is hit, the bulk
+   * and all data associated with it will be deleted
+   */
+  expiration: string;
+
+  /**
+   * The data source of the bulk
+   *
+   * Bulks with same type and source will be replaced and the data will be
+   * updated when the bulk is marked as done
+   */
+  source: string;
+
+  /**
+   * State of the bulk
+   *
+   * Data can be indexed for this bulk as long as the state is `in progress`
+   * and the bulk is not expired
+   *
+   * When the bulk is marked as `done` it replaces the previous bulk with
+   * the same source and type. The data will be availabe to the user when
+   * the bulk switches to done
+   */
+  state: 'in progress' | 'done';
+
+  /**
+   * Type of data in the bulk
+   */
+  type: SCThingTypes;
+
+  /**
+   * Unique identifier of the bulk
+   */
+  uid: string;
+
+  /**
+   * Creates a new bulk process
+   * @param request
+   */
+  constructor(request: SCBulkRequest) {
+    this.uid = v4();
+    this.state = 'in progress';
+
+    if (typeof request.expiration === 'string') {
+      this.expiration = request.expiration;
+    } else {
+      this.expiration = moment().add(1, 'hour').toISOString();
+    }
+    // when should this process be finished
+    // where does the process come from
+    this.source = request.source;
+    // which type of data is this process about to index
+    this.type = request.type;
+  }
+}
+
+/**
+ * Cache for bulk-processes
+ */
+export class BulkStorage {
+
+  private cache: NodeCache;
+
+  /**
+   * Creates a new BulkStorage
+   * @param database the database that is controlled by this bulk storage
+   */
+  constructor(public database: Database) {
+
+    // a bulk lives 60 minutes if no expiration is given
+    // the cache is checked every 60 seconds
+    this.cache = new NodeCache({ stdTTL: 3600, checkperiod: 60 });
+
+    this.cache.on('expired', (_key, bulk: Bulk) => {
+      // if the bulk is not done
+      if (bulk.state !== 'done') {
+        // the database can delete the data associated with this bulk
+        this.database.bulkExpired(bulk);
+      }
+    });
+  }
+
+  /**
+   * Saves a bulk process and assigns to it a user-defined ttl (time-to-live)
+   * @param bulk the bulk process to save
+   * @returns the bulk process that was saved
+   */
+  private async save(bulk: Bulk): Promise<Bulk> {
+    const expirationInSeconds = moment(bulk.expiration).diff(moment.now()) / 1000;
+    logger.info('Bulk expires in ', expirationInSeconds, 'seconds');
+
+    // save the item in the cache with it's expected expiration
+    await promisify<string, Bulk, number>(this.cache.set)(bulk.uid, bulk, expirationInSeconds);
+    return bulk;
+  }
+
+  /**
+   * Create and save a new bulk process
+   * @param bulkRequest a request for a new bulk process
+   * @returns a promise that contains the new bulk process
+   */
+  public async create(bulkRequest: SCBulkRequest): Promise<Bulk> {
+    const bulk = new Bulk(bulkRequest);
+    bulk.source = bulkRequest.source;
+    bulk.type = bulkRequest.type;
+
+    await this.save(bulk);
+
+    // tell the database that the bulk was created
+    await this.database.bulkCreated(bulk);
+    return bulk;
+  }
+
+  /**
+   * Delete a bulk process
+   * @param uid uid of the bulk process
+   * @returns a promise that contains the deleted bulk process
+   */
+  public async delete(uid: string): Promise<Bulk> {
+    const bulk = await this.read(uid);
+
+    if (typeof bulk === 'undefined') {
+      throw new Error(`Bulk that should be deleted was not found. UID was "${uid}"`);
+    }
+
+    // delete the bulk process from the cache
+    await promisify<string>(this.cache.del)(uid);
+
+    // tell the database to handle the expiration of the bulk
+    await this.database.bulkExpired(bulk);
+
+    return bulk;
+  }
+
+  /**
+   * Update an old bulk process (replace it with the new one)
+   * @param bulk new bulk process
+   * @returns an empty promise
+   */
+  public async markAsDone(bulk: Bulk): Promise<void> {
+    bulk.state = 'done';
+    await this.save(bulk);
+
+    // tell the database that this is the new bulk
+    this.database.bulkUpdated(bulk);
+    return;
+  }
+
+  /**
+   * Read an existing bulk process
+   * @param uid uid of the bulk process
+   * @returns a promise that contains a bulk
+   */
+  public async read(uid: string): Promise<Bulk | undefined> {
+    return await promisify<string, any>(this.cache.get)(uid);
+  }
+
+}

src/storage/Database.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCSearchQuery, SCSearchResponse, SCThings, SCUuid } from '@openstapps/core';
+import {Bulk} from './BulkStorage';
+
+export interface DatabaseConstructor {
+  new (...args: any): Database;
+}
+
+export interface Database {
+
+  /**
+   * Gets called if a bulk was created
+   *
+   * The database should
+   * @param bulk
+   */
+  bulkCreated(bulk: Bulk): Promise<void>;
+
+  /**
+   * Gets called if a bulk expires
+   *
+   * The database should delete all data that is associtated with this bulk
+   * @param bulk
+   */
+  bulkExpired(bulk: Bulk): Promise<void>;
+
+  /**
+   * Gets called if a bulk was updated
+   *
+   * If the database holds a bulk with the same type and source as the given
+   * bulk it should be replaced by the given one
+   *
+   * @param bulk
+   */
+  bulkUpdated(bulk: Bulk): Promise<void>;
+
+  /**
+   * Get a single document
+   * @param uid
+   */
+  get(uid: SCUuid): Promise<SCThings>;
+
+  /**
+   * Add a thing to an existing bulk
+   * @param object
+   * @param bulkId
+   */
+  post(thing: SCThings, bulk: Bulk): Promise<void>;
+
+  /**
+   * Replace an existing thing in any Bulk
+   *
+   * Currently it is not possible to put an non-existing object
+   *
+   * @param thing
+   */
+  put(thing: SCThings): Promise<void>;
+
+  /**
+   * Search for things
+   * @param params
+   */
+   search(params: SCSearchQuery): Promise<SCSearchResponse>;
+}

src/storage/elasticsearch/Elasticsearch.ts

@@ -0,0 +1,508 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCBulkResponse,
+  SCConfigFile,
+  SCFacet,
+  SCSearchQuery,
+  SCSearchResponse,
+  SCThings,
+  SCThingTypes,
+  SCUuid,
+} from '@openstapps/core';
+import * as ES from 'elasticsearch';
+import * as moment from 'moment';
+import { logger } from '../../common';
+import { MailQueue } from '../../notification/MailQueue';
+import { Bulk } from '../BulkStorage';
+import { Database } from '../Database';
+import { buildAggregations, parseAggregations } from './aggregations';
+import { AggregationSchema, ElasticsearchConfig, ElasticsearchObject } from './common';
+import * as Monitoring from './monitoring';
+import { buildQuery, buildSort } from './query';
+import { putTemplate } from './templating';
+
+// this will match index names such as stapps_<type>_<source>_<random suffix>
+const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
+
+/**
+ * A database interface for elasticsearch
+ */
+export class Elasticsearch implements Database {
+
+  aggregationsSchema: AggregationSchema;
+
+  /**
+   * Holds a map of all elasticsearch indices that are available to search
+   */
+  aliasMap: {
+    // each scType has a alias which can contain multiple sources
+    [scType: string]: {
+      // each source is assigned a index name in elasticsearch
+      [source: string]: string;
+    },
+  };
+  client: ES.Client;
+  ready: boolean;
+
+  /**
+   * Create a new interface for elasticsearch
+   * @param config an assembled config file
+   * @param mailQueue a mailqueue for monitoring
+   */
+  constructor(private config: SCConfigFile, mailQueue?: MailQueue) {
+
+    if (!config.internal.database || typeof config.internal.database.version === 'undefined') {
+      throw new Error('Database version is undefined. Check you config file');
+    }
+
+    const options = {
+      apiVersion: config.internal.database.version,
+      host: this.getElasticsearchUrl(),
+      log: 'error',
+    };
+
+    // enable verbose logging for all request to elasticsearch
+    if (process.env.ES_DEBUG === 'true') {
+      options.log = 'trace';
+    }
+
+    this.client = new ES.Client(options);
+    this.aliasMap = {};
+    this.ready = false;
+
+    this.aggregationsSchema = buildAggregations(this.config.internal.aggregations);
+
+    this.getAliasMap();
+
+    const monitoringConfiguration = this.config.internal.monitoring;
+
+    if (typeof monitoringConfiguration !== 'undefined') {
+      if (typeof mailQueue === 'undefined') {
+        throw new Error('Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.');
+      }
+      // read all watches and schedule searches on the client
+      Monitoring.setUp(monitoringConfiguration, this.client, mailQueue);
+    }
+
+  }
+
+  /**
+   * Tests if an object already exists
+   *
+   * Returns Elasticsearch Object if it exists
+   */
+  private async doesItemExist(object: SCThings): Promise<{exists: boolean; object?: ElasticsearchObject<SCThings>}> {
+    const searchResponse = await this.client.search<SCThings>({
+      body: {
+        query: {
+          term: {
+            'uid.raw': {
+              value: object.uid,
+            },
+          },
+        },
+      },
+      from: 0,
+      index: this.getListOfAllIndices(),
+      size: 1,
+    });
+
+    if (searchResponse.hits.total > 1) {
+      return {
+        exists: true,
+        object: searchResponse.hits.hits[0],
+      };
+    }
+
+    return {
+      exists: false,
+    };
+  }
+
+  /**
+   * Gets a map which contains each alias and all indices that are associated with each alias
+   */
+  private async getAliasMap() {
+
+    // create a list of old indices that are not in use
+    const oldIndicesToDelete: string[] = [];
+
+    let aliases: {
+      [index: string]: {
+        aliases: {
+          [K in SCThingTypes]: any
+        },
+      },
+    };
+
+    try {
+      aliases = await this.client.indices.getAlias({});
+    } catch (error) {
+      logger.error('Failed getting alias map:', error);
+      setTimeout(() => {
+        this.getAliasMap();
+      }, 5000); // retry in 5 seconds
+      return;
+    }
+
+    for (const index in aliases) {
+      if (aliases.hasOwnProperty(index)) {
+
+        const matches = indexRegex.exec(index);
+        if (matches !== null) {
+          const type = matches[1];
+          const source = matches[2];
+
+          // check if there is an alias for the current index
+          // check that alias equals type
+          const hasAlias = type in aliases[index].aliases;
+          if (hasAlias) {
+            if (typeof this.aliasMap[type] === 'undefined') {
+              this.aliasMap[type] = {};
+            }
+            this.aliasMap[type][source] = index;
+          } else {
+            oldIndicesToDelete.push(index);
+          }
+        }
+      }
+    }
+
+    this.ready = true;
+
+    // delete old indices that are not used in any alias
+    if (oldIndicesToDelete.length > 0) {
+      await this.client.indices.delete({
+        index: oldIndicesToDelete,
+      });
+      logger.warn('Deleted old indices: ' + oldIndicesToDelete);
+    }
+
+    logger.ok('Read alias map from elasticsearch: ' + JSON.stringify(this.aliasMap, null, 2));
+  }
+
+  /**
+   * Get the url of elasticsearch
+   */
+  private getElasticsearchUrl(): string {
+    // check if we have a docker link
+    if (process.env.ES_PORT_9200_TCP_ADDR !== undefined && process.env.ES_PORT_9200_TCP_PORT !== undefined) {
+      return process.env.ES_PORT_9200_TCP_ADDR + ':' + process.env.ES_PORT_9200_TCP_PORT;
+    }
+
+    // default
+    return 'localhost:9200';
+  }
+
+  /**
+   * Gets the index name in elasticsearch for one SCThingType
+   * @param type SCThingType of data in the index
+   * @param source source of data in the index
+   * @param bulk bulk process which created this index
+   */
+  private getIndex(type: SCThingTypes, source: string, bulk: SCBulkResponse) {
+    return `stapps_${type.toLowerCase().replace(' ', '_')}_${source}_${bulk.uid.substring(0, 8)}`;
+  }
+
+  /**
+   * Generates a string which matches all indices
+   */
+  private getListOfAllIndices(): string {
+    // map each SC type in upper camel case
+    return 'stapps_*_*_*';
+  }
+
+  /**
+   * Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
+   * @param bulk the bulk process that was created
+   */
+  public async bulkCreated(bulk: Bulk): Promise<void> {
+    // if our es instance is not ready yet, we cannot serve this request
+    if (!this.ready) {
+      throw new Error('No connection to elasticsearch established yet.');
+    }
+
+    // index name for elasticsearch
+    const index: string = this.getIndex(bulk.type, bulk.source, bulk);
+
+    // there already is an index with this type and source. We will index the new one and switch the alias to it
+    // the old one is deleted
+    const alias = bulk.type;
+
+    if (typeof this.aliasMap[alias] === 'undefined') {
+      this.aliasMap[alias] = {};
+    }
+
+    if (!indexRegex.test(index)) {
+      throw new Error(
+          'Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.\n' +
+          'Make sure to set the bulk "source" and "type" to names consisting of the characters above.',
+      );
+    }
+
+    // re-apply the index template before each new bulk operation
+    await putTemplate(this.client);
+    await this.client.indices.create({
+      index,
+    });
+
+    logger.info('Created index', index);
+  }
+
+  /**
+   * Should be called when a bulk process is expired. The index that was created with this bulk gets deleted
+   * @param bulk the bulk process that is expired
+   */
+  public async bulkExpired(bulk: Bulk): Promise<void> {
+    // index name for elasticsearch
+    const index: string = this.getIndex(bulk.type, bulk.source, bulk);
+
+    logger.info('Bulk expired. Deleting index', index);
+
+    // don't delete indices that are in use already
+    if (bulk.state !== 'done') {
+      logger.info('deleting obsolete index', index);
+      return await this.client.indices.delete({ index });
+    }
+  }
+
+  /**
+   * Should be called when a bulk process is updated (replaced by a newer bulk). This will replace the old
+   * index and publish all data, that was index in the new instead
+   * @param bulk the new bulk process that should replace the old one with same type and source
+   */
+  public async bulkUpdated(bulk: Bulk): Promise<void> {
+    // if our es instance is not ready yet, we cannot serve this request
+    if (!this.ready) {
+      throw new Error('Elasticsearch not ready');
+    }
+
+    // index name for elasticsearch
+    const index: string = this.getIndex(bulk.type, bulk.source, bulk);
+
+    // alias for the indices
+    const alias = bulk.type;
+
+    if (typeof this.aliasMap[alias] === 'undefined') {
+      this.aliasMap[alias] = {};
+    }
+
+    if (!indexRegex.test(index)) {
+      throw new Error(
+          'Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.\n' +
+          'Make sure to set the bulk "source" and "type" to names consisting of the characters above.',
+      );
+    }
+
+    // create the new index if it does not exists
+    if (!(await this.client.indices.exists({ index }))) {
+      // re-apply the index template before each new bulk operation
+      await putTemplate(this.client);
+      await this.client.indices.create({
+        index,
+      });
+    }
+
+    // get the old index from our aliasMap
+    const oldIndex: string = this.aliasMap[alias][bulk.source];
+
+    // add our new index to the alias
+    const actions: ES.IndicesUpdateAliasesParamsAction[] = [
+      {
+        add: { index: index, alias: alias },
+      },
+    ];
+
+    // remove our old index if it exists
+    if (typeof oldIndex === 'string') {
+      actions.push({
+        remove: { index: oldIndex, alias: alias },
+      });
+    }
+
+    // refresh the index (fsync changes)
+    await this.client.indices.refresh({
+      index,
+    });
+
+    // execute our alias actions
+    await this.client.indices.updateAliases({
+      body: {
+        actions,
+      },
+    });
+
+    // swap the index in our aliasMap
+    this.aliasMap[alias][bulk.source] = index;
+    if (typeof oldIndex === 'string') {
+      // delete the old index
+      await this.client.indices.delete({ index: oldIndex });
+      logger.info('deleted old index', oldIndex);
+    }
+    logger.info('swapped alias index alias', oldIndex, '=>', index);
+  }
+
+  /**
+   * Gets an SCThing from all indexed data
+   * @param uid uid of an SCThing
+   */
+  public async get(uid: SCUuid): Promise<SCThings> {
+    const searchResponse = await this.client.search({
+      body: {
+        query: {
+          term: {
+            uid,
+          },
+        },
+      },
+      index: this.getListOfAllIndices(),
+    });
+
+    // get data from response
+    const hits = searchResponse.hits.hits;
+
+    if (hits.length !== 1) {
+      throw new Error('No unique item found.');
+    } else {
+      return hits[0]._source as SCThings;
+    }
+  }
+
+  /**
+   * Add an item to an index
+   * @param object the SCThing to add to the index
+   * @param bulk the bulk process which item belongs to
+   */
+  public async post(object: SCThings, bulk: Bulk): Promise<void> {
+
+    const obj: SCThings & {creation_date: string} = {
+      ...object,
+      creation_date: moment().format(),
+    };
+
+    const itemMeta = await this.doesItemExist(obj);
+
+    // we have to check that the item will get replaced if the index is rolled over
+    if (itemMeta.exists && typeof itemMeta.object !== 'undefined') {
+      const indexOfNew = this.getIndex(obj.type, bulk.source, bulk);
+      const oldIndex = itemMeta.object._index;
+
+      // new item doesn't replace the old one
+      if (oldIndex.substring(0, oldIndex.length - 9) !== indexOfNew.substring(0, indexOfNew.length - 9)) {
+        throw new Error(
+          'Object \"' + obj.uid + '\" already exists. Object was: ' +
+          JSON.stringify(obj, null, 2),
+        );
+      }
+    }
+
+    // regular bulk update (item gets replaced when bulk is updated)
+    const searchResponse = await this.client.create({
+      body: obj,
+      id: obj.uid,
+      index: this.getIndex(obj.type, bulk.source, bulk),
+      timeout: '90s',
+      type: obj.type,
+    });
+
+    if (!searchResponse.created) {
+      throw new Error('Object creation Error: Instance was: ' + JSON.stringify(obj));
+    }
+  }
+
+  /**
+   * Put (update) an existing item
+   * @param object SCThing to put
+   */
+  public async put(object: SCThings) {
+
+    const itemMeta = await this.doesItemExist(object);
+
+    if (itemMeta.exists && typeof itemMeta.object !== 'undefined') {
+      return await this.client.update({
+        body: {
+          doc: object,
+        },
+        id: object.uid,
+        index: itemMeta.object._index,
+        type: object.type.toLowerCase(),
+      });
+    }
+
+    throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.');
+  }
+
+  /**
+   * Search all indexed data
+   * @param params search query
+   */
+  public async search(params: SCSearchQuery): Promise<SCSearchResponse> {
+
+    if (typeof this.config.internal.database === 'undefined') {
+      throw new Error('Database is undefined. You have to configure the query build');
+    }
+
+    const searchRequest: ES.SearchParams = {
+      body: {
+        aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes)
+        query: buildQuery(params, this.config, this.config.internal.database as ElasticsearchConfig),
+      },
+      from: params.from,
+      index: this.getListOfAllIndices(),
+      size: params.size,
+    };
+
+    if (typeof params.sort !== 'undefined') {
+      searchRequest.body.sort = buildSort(params.sort);
+    }
+
+    // perform the search against elasticsearch
+    const response = await this.client.search<SCThings>(searchRequest);
+
+    // gather pagination information
+    const pagination = {
+      count: response.hits.hits.length,
+      offset: (typeof params.from === 'number') ? params.from : 0,
+      total: response.hits.total,
+    };
+
+    // gather statistics about this search
+    const stats = {
+      time: response.took,
+    };
+
+    // we only directly return the _source documents
+    // elasticsearch provides much more information, the user shouldn't see
+    const data = response.hits.hits.map((hit) => {
+      return hit._source; // SCThing
+    });
+
+    let facets: SCFacet[] = [];
+
+    // read the aggregations from elasticsearch and parse them to facets by our configuration
+    if (typeof response.aggregations !== 'undefined') {
+      facets = parseAggregations(this.aggregationsSchema, response.aggregations);
+    }
+
+    return {
+      data,
+      facets,
+      pagination,
+      stats,
+    };
+  }
+}

src/storage/elasticsearch/aggregations.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBackendAggregationConfiguration, SCFacet, SCThingTypes } from '@openstapps/core';
+import { AggregationSchema } from './common';
+
+export type aggregationType = SCThingTypes | '@all';
+
+/**
+ * Builds the aggregation
+ * @returns a schema to tell elasticsearch which aggregations to collect
+ */
+export function buildAggregations(aggsConfig: SCBackendAggregationConfiguration[]): AggregationSchema {
+
+  const result: AggregationSchema = {};
+
+  aggsConfig.forEach((aggregation) => {
+
+    result[aggregation.fieldName] = {
+      terms: {
+        field: aggregation.fieldName + '.raw',
+        size: 1000,
+      },
+    };
+  });
+
+  return result;
+}
+
+/**
+ * An elasticsearch aggregation bucket
+ */
+interface Bucket {
+  doc_count: number;
+  key: string;
+}
+
+/**
+ * An elasticsearch aggregation response
+ */
+interface AggregationResponse {
+  [field: string]: {
+    buckets: Bucket[];
+    doc_count?: number;
+  };
+}
+
+/**
+ * Parses elasticsearch aggregations (response from es) to facets for the app
+ * @param aggregationSchema - aggregation-schema for elasticsearch
+ * @param aggregations - aggregations response from elasticsearch
+ */
+export function parseAggregations(
+  aggregationSchema: AggregationSchema,
+  aggregations: AggregationResponse): SCFacet[] {
+
+  const facets: SCFacet[] = [];
+
+  const aggregationNames = Object.keys(aggregations);
+
+  aggregationNames.forEach((aggregationName) => {
+    const buckets = aggregations[aggregationName].buckets;
+
+    const facet: SCFacet = {
+      buckets: buckets.map((bucket) => {
+        const facetBucket: { [value: string]: number } = {};
+        facetBucket[bucket.key] = bucket.doc_count;
+        return facetBucket;
+      }),
+      field: aggregationSchema[aggregationName].terms.field + '.raw',
+    };
+
+    facets.push(facet);
+  });
+  return facets;
+}

src/storage/elasticsearch/common.ts

@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCThingTypes } from '@openstapps/core';
+import { SCThing } from '@openstapps/core';
+
+/**
+ * An elasticsearch bucket aggregation
+ * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.5/search-aggregations-bucket.html
+ */
+export interface AggregationSchema {
+  [aggregationName: string]: ESTermsFilter;
+}
+
+/**
+ * A configuration for using the Dis Max Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryDisMaxConfig {
+  cutoffFrequency: number;
+  fuzziness: number;
+  matchBoosting: number;
+  minMatch: string;
+  queryType: 'dis_max';
+  tieBreaker: number;
+}
+
+/**
+ * A configuration for using Query String Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryQueryStringConfig {
+  minMatch: string;
+  queryType: 'query_string';
+}
+
+/**
+ * A hit in an elastiscsearch search result
+ */
+export interface ElasticsearchObject<T extends SCThing> {
+  _id: string;
+  _index: string;
+  _score: number;
+  _source: T;
+  _type: string;
+  _version?: number;
+  fields?: any;
+  highlight?: any;
+  inner_hits?: any;
+  matched_queries?: string[];
+  sort?: string[];
+}
+
+/**
+ * An config file for the elasticsearch database interface
+ *
+ * The config file extends the SCConfig file by further defining how the database property
+ */
+export interface ElasticsearchConfigFile {
+  internal: {
+    database: ElasticsearchConfig;
+  };
+}
+
+/**
+ * An elasticsearch configuration
+ */
+export interface ElasticsearchConfig {
+  name: 'elasticsearch';
+  query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
+  version: string;
+}
+
+/**
+ * An elasticsearch term filter
+ */
+export interface ESTermFilter {
+  term: {
+    [fieldName: string]: string;
+  };
+}
+
+/**
+ * An elasticsearch terms filter
+ */
+export interface ESTermsFilter {
+  terms: {
+    field: string;
+    size?: number;
+  };
+}
+
+/**
+ * An elasticsearch type filter
+ */
+export interface ESTypeFilter {
+  type: {
+    value: SCThingTypes;
+  };
+}
+
+/**
+ * Filter arguments for an elasticsearch geo distance filter
+ */
+export interface ESGeoDistanceFilterArguments {
+  distance: string;
+  [fieldName: string]: {
+    lat: number;
+    lon: number;
+  } | string;
+}
+
+/**
+ * An elasticsearch geo distance filter
+ */
+export interface ESGeoDistanceFilter {
+  geo_distance: ESGeoDistanceFilterArguments;
+}
+
+/**
+ * Filter arguments for an elasticsearch boolean filter
+ */
+export interface ESBooleanFilterArguments<T> {
+  minimum_should_match?: number;
+  must?: T[];
+  must_not?: T[];
+  should?: T[];
+}
+
+/**
+ * An elasticsearch boolean filter
+ */
+export interface ESBooleanFilter<T> {
+  bool: ESBooleanFilterArguments<T>;
+}
+
+/**
+ * An elasticsearch function score query
+ */
+export interface ESFunctionScoreQuery {
+  function_score: {
+    functions: ESFunctionScoreQueryFunction[];
+    query: ESBooleanFilter<any>;
+    score_mode: 'multiply';
+  };
+}
+
+/**
+ * An function for an elasticsearch functions score query
+ */
+export interface ESFunctionScoreQueryFunction {
+  filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;
+  weight: number;
+}
+
+/**
+ * An elasticsearch ducet sort
+ */
+export interface ESDucetSort {
+  [field: string]: string;
+}
+
+/**
+ * Sort arguments for an elasticsearch geo distance sort
+ */
+export interface ESGeoDistanceSortArguments {
+  mode: 'avg' | 'max' | 'median' | 'min';
+  order: 'asc' | 'desc';
+  unit: 'm';
+  [field: string]: {
+    lat: number;
+    lon: number;
+  } | string;
+}
+
+/**
+ * An elasticsearch geo distance sort
+ */
+export interface ESGeoDistanceSort {
+  _geo_distance: ESGeoDistanceSortArguments;
+}
+
+/**
+ * An elasticsearch script sort
+ */
+export interface ScriptSort {
+  _script: {
+    order: 'asc' | 'desc';
+    script: string;
+    type: 'number' | 'string';
+  };
+}

src/storage/elasticsearch/monitoring.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCMonitoringConfiguration,
+  SCMonitoringLogAction,
+  SCMonitoringMailAction,
+  SCMonitoringMaximumLengthCondition,
+  SCMonitoringMinimumLengthCondition,
+} from '@openstapps/core';
+import * as ES from 'elasticsearch';
+import * as cron from 'node-cron';
+import { logger } from '../../common';
+import { MailQueue } from '../../notification/MailQueue';
+
+/**
+ * Check if the given condition fails on the given number of results and the condition
+ * @param condition condition
+ * @param total number of results
+ */
+function conditionFails(
+  condition: SCMonitoringMaximumLengthCondition | SCMonitoringMinimumLengthCondition,
+  total: number,
+) {
+  if (condition.type === 'MaximumLength') {
+    return maxConditionFails(condition.length, total);
+  }
+  return minConditionFails(condition.length, total);
+}
+
+/**
+ * Check if the min condition fails
+ * @param minimumLength
+ * @param total
+ */
+function minConditionFails(minimumLength: number, total: number) {
+  return typeof minimumLength === 'number' && minimumLength > total;
+}
+
+/**
+ * Check if the max condition fails
+ * @param maximumLength
+ * @param total
+ */
+function maxConditionFails(maximumLength: number, total: number) {
+  return typeof maximumLength === 'number' && maximumLength < total;
+}
+
+/**
+ * Run all the given actions
+ * @param actions actions to perform
+ * @param watcherName name of watcher that wants to perform them
+ * @param triggerName name of trigger that triggered the watcher
+ * @param total total number of results of the query
+ * @param mailQueue mailQueue to execute mail actions
+ */
+export function runActions(
+  actions: Array<SCMonitoringLogAction | SCMonitoringMailAction>,
+  watcherName: string,
+  triggerName: string,
+  total: number,
+  mailQueue: MailQueue,
+) {
+
+  actions.forEach((action) => {
+    if (action.type === 'log') {
+      logger.error(
+        action.prefix,
+        `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`, `Found ${total} hits instead`,
+        action.message,
+      );
+    } else {
+      mailQueue.push({
+        subject: action.subject,
+        text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'\n` +
+          action.message +
+          `Found ${total} hits instead`,
+        to: action.recipients,
+      });
+    }
+  });
+}
+
+/**
+ * Set up the triggers for the configured watchers
+ * @param monitoringConfig configuration of the monitoring
+ * @param esClient elasticsearch client
+ * @param mailQueue mailQueue for mail actions
+ */
+export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES.Client, mailQueue: MailQueue) {
+
+  // set up Watches
+  monitoringConfig.watchers.forEach((watcher) => {
+
+    // make a schedule for each trigger
+    watcher.triggers.forEach((trigger) => {
+      switch (trigger.executionTime) {
+        case 'hourly':
+          trigger.executionTime = '5 * * * *';
+          break;
+        case 'daily':
+          trigger.executionTime = '5 0 * * *';
+          break;
+        case 'weekly':
+          trigger.executionTime = '5 0 * * 0';
+          break;
+        case 'monthly':
+          trigger.executionTime = '5 0 * 0 * *';
+      }
+
+      cron.schedule(trigger.executionTime, async () => {
+        // execute watch (search->condition->action)
+        const result = await esClient.search(watcher.query);
+
+        // check conditions
+        const total = result.hits.total;
+
+        watcher.conditions.forEach((condition) => {
+          if (conditionFails(condition, total)) {
+            runActions(watcher.actions, watcher.name, trigger.name, total, mailQueue);
+          }
+        });
+      });
+    });
+
+  });
+
+  logger.log('Scheduled ' + monitoringConfig.watchers.length + ' watches');
+}

src/storage/elasticsearch/query.ts

@@ -0,0 +1,408 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCBackendConfigurationSearchBoosting,
+  SCConfigFile,
+  SCSearchBooleanFilter,
+  SCSearchFilter,
+  SCSearchQuery,
+  SCSearchSort,
+  SCSportCoursePriceGroup,
+  SCThingsField,
+  SCThingTypes,
+} from '@openstapps/core';
+import { ElasticsearchConfig, ScriptSort } from './common';
+import {
+  ESBooleanFilter,
+  ESBooleanFilterArguments,
+  ESDucetSort,
+  ESFunctionScoreQuery,
+  ESFunctionScoreQueryFunction,
+  ESGeoDistanceFilter,
+  ESGeoDistanceFilterArguments,
+  ESGeoDistanceSort,
+  ESGeoDistanceSortArguments,
+  ESTermFilter,
+  ESTypeFilter,
+} from './common';
+
+/**
+ * Builds a boolean filter. Returns an elasticsearch boolean filter
+ */
+export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<any> {
+
+  const result: ESBooleanFilterArguments<any> = {
+    minimum_should_match: 0,
+    must: [],
+    must_not: [],
+    should: [],
+  };
+
+  if (booleanFilter.arguments.operation === 'and') {
+    result.must = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
+  }
+
+  if (booleanFilter.arguments.operation === 'or') {
+    result.should = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
+    result.minimum_should_match = 1;
+  }
+
+  if (booleanFilter.arguments.operation === 'not') {
+    result.must_not = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
+  }
+
+  return result;
+}
+
+/**
+ * Converts Array of Filters to elasticsearch query-syntax
+ * @param filter
+ */
+export function buildFilter(filter: SCSearchFilter): ESTermFilter | ESGeoDistanceFilter | ESBooleanFilter<any> {
+
+  switch (filter.type) {
+    case 'value':
+      const filterObj: { [field: string]: string } = {};
+      filterObj[filter.arguments.field + '.raw'] = filter.arguments.value;
+      return {
+        term: filterObj,
+      };
+    case 'availability':
+      const startRangeFilter: { [field: string]: { lte: string } } = {};
+      startRangeFilter[filter.arguments.fromField] = {
+        lte: 'now',
+      };
+
+      const endRangeFilter: { [field: string]: { gte: string } } = {};
+      endRangeFilter[filter.arguments.toField] = {
+        gte: 'now',
+      };
+
+      return {
+        bool: {
+          should: [
+            {
+              bool: {
+                must: [
+                  {
+                    range: startRangeFilter,
+                  },
+                  {
+                    range: endRangeFilter,
+                  },
+                ],
+              },
+            },
+            {
+              bool: {
+                must_not: [
+                  {
+                    exists: {
+                      field: filter.arguments.fromField,
+                    },
+                  },
+                  {
+                    exists: {
+                      field: filter.arguments.toField,
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      };
+    case 'distance':
+      const geoObject: ESGeoDistanceFilterArguments = {
+        distance: filter.arguments.distanceInM + 'm',
+      };
+      geoObject[filter.arguments.field] = {
+        lat: filter.arguments.lat,
+        lon: filter.arguments.lon,
+      };
+      return {
+        geo_distance: geoObject,
+      };
+    case 'boolean':
+      return {
+        bool: buildBooleanFilter(filter),
+      };
+    default:
+      throw new Error('Unknown Filter type');
+  }
+}
+
+/**
+ * Builds scorings functions from boosting config
+ * @param boosting
+ * @returns
+ */
+export function buildFunctions(boosting: SCBackendConfigurationSearchBoosting[]): ESFunctionScoreQueryFunction[] {
+
+  const functions: ESFunctionScoreQueryFunction[] = [];
+
+  // add a good scoring subset from config file
+  boosting.forEach((boostingForOneSCType) => {
+    const typeFilter: ESTypeFilter = {
+      type: {
+        value: boostingForOneSCType.type,
+      },
+    };
+
+    functions.push({
+      filter: typeFilter,
+      weight: boostingForOneSCType.factor,
+    });
+
+    if (typeof boostingForOneSCType.fields !== 'undefined') {
+
+      const fields = boostingForOneSCType.fields;
+
+      Object.keys(boostingForOneSCType.fields).forEach((fieldName) => {
+
+        const boostingForOneField = fields[fieldName];
+
+        Object.keys(boostingForOneField).forEach((value) => {
+          const factor = boostingForOneField[value];
+
+          // build term filter
+          const termFilter: ESTermFilter = {
+            term: {},
+          };
+          termFilter.term[fieldName + '.raw'] = value;
+
+          functions.push({
+            filter: {
+              bool: {
+                must: [
+                  typeFilter,
+                  termFilter,
+                ],
+                should: [],
+              },
+            },
+            weight: factor,
+          });
+        });
+      });
+    }
+  });
+
+  return functions;
+}
+/**
+ * Builds body for Elasticsearch requests
+ * @param params
+ * @param defaultConfig
+ * @returns ElasticsearchQuery (body of a search-request)
+ */
+export function buildQuery(
+  params: SCSearchQuery,
+  defaultConfig: SCConfigFile,
+  elasticsearchConfig: ElasticsearchConfig,
+): ESFunctionScoreQuery {
+
+  // if a sort is used it, we may have to narrow down the types so the sort is executable
+  let typeFiltersToAppend: ESTypeFilter[] = [];
+
+  if (typeof params.sort !== 'undefined') {
+    params.sort.forEach((sort) => {
+      // types that the sort is supported on
+      const types: SCThingTypes[] = [];
+
+      defaultConfig.backend.sortableFields
+        .filter((sortableField) => {
+          return sortableField.fieldName === sort.arguments.field && sortableField.sortTypes.indexOf(sort.type) > -1;
+        })
+        .forEach((sortableField) => {
+          if (typeof sortableField.onlyOnTypes !== 'undefined') {
+            sortableField.onlyOnTypes.forEach((scType) => {
+              if (types.indexOf(scType) === -1) {
+                types.push(scType);
+              }
+            });
+          }
+        });
+
+      if (types.length > 0) {
+        typeFiltersToAppend = types.map((type) => {
+          return {
+            type: {
+              value: type,
+            },
+          };
+        });
+      }
+    });
+  }
+
+  // if config provides an minMatch parameter we use query_string instead of match query
+  let query;
+  if (typeof elasticsearchConfig.query === 'undefined') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: '90%',
+        query: (typeof params.query !== 'string') ? '*' : params.query,
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'query_string') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: elasticsearchConfig.query.minMatch,
+        query: (typeof params.query !== 'string') ? '*' : params.query,
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'dis_max') {
+    if (params.query !== '*') {
+      query = {
+        dis_max: {
+          boost: 1.2,
+          queries: [
+            {
+              match: {
+                name: {
+                  boost: elasticsearchConfig.query.matchBoosting,
+                  cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
+                  fuzziness: elasticsearchConfig.query.fuzziness,
+                  query: (typeof params.query !== 'string') ? '*' : params.query,
+                },
+              },
+            },
+            {
+              query_string: {
+                analyzer: 'search_german',
+                default_field: 'name',
+                minimum_should_match: elasticsearchConfig.query.fuzziness,
+                query: (typeof params.query !== 'string') ? '*' : params.query,
+              },
+            },
+          ],
+          tie_breaker: elasticsearchConfig.query.tieBreaker,
+        },
+
+      };
+    } else {
+      throw new Error('Query Type is not supported. Check your config file and reconfigure your elasticsearch query');
+    }
+  }
+
+  const functionScoreQuery: ESFunctionScoreQuery = {
+    function_score: {
+      functions: buildFunctions(defaultConfig.internal.boostings),
+      query: {
+        bool: {
+          minimum_should_match: 0, // if we have no should, nothing can match
+          must: [],
+          should: [],
+        },
+      },
+      score_mode: 'multiply',
+    },
+  };
+
+  const mustMatch = functionScoreQuery.function_score.query.bool.must;
+
+  if (Array.isArray(mustMatch)) {
+    if (typeof query !== 'undefined') {
+      mustMatch.push(query);
+    }
+
+    if (typeof params.filter !== 'undefined') {
+      mustMatch.push(buildFilter(params.filter));
+    }
+
+    // add type filters for sorts
+    mustMatch.push.apply(mustMatch, typeFiltersToAppend);
+  }
+  return functionScoreQuery;
+}
+
+/**
+ * converts query to
+ * @param params
+ * @param sortableFields
+ * @returns an array of sort queries
+ */
+export function buildSort(
+  sorts: SCSearchSort[],
+): Array<ESDucetSort | ESGeoDistanceSort | ScriptSort> {
+  return sorts.map((sort) => {
+    switch (sort.type) {
+      case 'ducet':
+        const ducetSort: ESDucetSort = {};
+        ducetSort[sort.arguments.field + '.sort'] = sort.order;
+        return ducetSort;
+      case 'distance':
+        const args: ESGeoDistanceSortArguments = {
+          mode: 'avg',
+          order: sort.order,
+          unit: 'm',
+        };
+
+        args[sort.arguments.field] = {
+          lat: sort.arguments.lat,
+          lon: sort.arguments.lon,
+        };
+
+        return {
+          _geo_distance: args,
+        };
+      case 'price':
+        return {
+          _script: {
+            order: sort.order,
+            script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+            type: 'number' as 'number',
+          },
+        };
+    }
+  });
+}
+
+export function buildPriceSortScript(universityRole: keyof SCSportCoursePriceGroup, field: SCThingsField): string {
+  return `
+  // initialize the sort value with the maximum
+  double price = Double.MAX_VALUE;
+
+  // if we have any offers
+  if (params._source.containsKey('${field}')) {
+    // iterate through all offers
+    for (offer in params._source.${field}) {
+      // if this offer contains a role specific price
+      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
+        // if the role specific price is smaller than the cheapest we found
+        if (offer.prices.${universityRole} < price) {
+          // set the role specific price as cheapest for now
+          price = offer.prices.${universityRole};
+        }
+      } else { // we have no role specific price for our role in this offer
+        // if the default price of this offer is lower than the cheapest we found
+        if (offer.price < price) {
+          // set this price as the cheapest
+          price = offer.price;
+        }
+      }
+    }
+  }
+
+  // return cheapest price for our role
+  return price;
+  `;
+}

src/storage/elasticsearch/templates/address.field.template.json

@@ -0,0 +1,19 @@
+{
+  "properties": {
+    "addressCountry": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "addressLocality": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "addressRegion": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "streetAddress": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "postalCode": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/article.sc-type.template.json

@@ -0,0 +1,25 @@
+{
+  "properties": {
+    "authors": {
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "publishers": {
+      "_typeRef": [
+        "person.sc-type.template.json",
+        "organization.sc-type.template.json"
+      ]
+    },
+    "datePublished": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "inLanguages": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "keywords": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "articleBody": {
+      "_fieldRef": "text.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/base.template.json

@@ -0,0 +1,91 @@
+{
+  "template": "stapps_*",
+  "settings": {
+    "max_result_window": 30000,
+    "mapping.total_fields.limit": 2000,
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "analysis": {
+      "filter": {
+        "german_stemmer": {
+          "type":       "stemmer",
+          "language":   "german"
+        },
+        "german_stop": {
+          "type": "stop",
+          "stopwords": "_german_"
+        },
+        "german_phonebook": {
+          "type": "icu_collation",
+          "language": "de",
+          "country": "DE",
+          "variant": "@collation=phonebook"
+        }
+      },
+      "tokenizer": {
+        "stapps_ngram": {
+          "type": "ngram",
+          "min_gram": 4,
+          "max_gram": 7
+        }
+      },
+      "analyzer": {
+        "search_german": {
+          "tokenizer": "stapps_ngram",
+          "filter": [
+            "lowercase",
+            "german_stop",
+            "german_stemmer"
+          ]
+        },
+        "ducet_sort": {
+          "tokenizer": "keyword",
+          "filter": [
+            "german_phonebook"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "_default_": {
+      "properties": {
+        "creation_date": {
+          "type": "date",
+          "store": true
+        },
+        "uid": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "type": {
+          "_fieldRef": "sortableKeyword.field.template.json"
+        },
+        "name": {
+          "_fieldRef": "text.field.template.json"
+        },
+        "alternateNames": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "categories": {
+          "_fieldRef": "sortableKeyword.field.template.json"
+        },
+        "url": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "description": {
+          "_fieldRef": "text.field.template.json"
+        },
+        "image": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        }
+      },
+      "_source": {
+        "excludes": [
+          "creation_date"
+        ]
+      },
+      "date_detection": false,
+      "dynamic_templates": []
+    }
+  }
+}

src/storage/elasticsearch/templates/book.sc-type.template.json

@@ -0,0 +1,31 @@
+{
+  "properties": {
+    "authors": {
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "bookEdition": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "isbn": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "numberOfPages": {
+      "type": "integer"
+    },
+    "publishers": {
+      "_typeRef": "organization.sc-type.template.json"
+    },
+    "datePublished": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "inLanguages": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "keywords": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "articleBody": {
+      "_fieldRef": "text.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/catalog.sc-type.template.json

@@ -0,0 +1,22 @@
+{
+  "properties": {
+    "level": {
+      "type": "integer",
+      "fields": {
+        "raw": {
+          "type": "integer"
+        }
+      }
+    },
+    "semester": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "superCatalog": {
+      "_typeRef": "catalog.sc-type.template.json"
+    },
+    "superCatalogs": {
+      "type": "nested",
+      "_typeRef": "catalog.sc-type.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/date.sc-type.template.json

@@ -0,0 +1,47 @@
+{
+  "properties": {
+    "startDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "endDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "startTime": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "endTime": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "dayOfWeek": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "performers": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "duration": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "frequency": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "frequencyMultiplier": {
+      "type": "float"
+    },
+    "repeatFrequency": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "superEvent": {
+      "_typeRef": "event.sc-type.template.json"
+    },
+    "exceptions": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "dates": {
+      "_fieldRef": "filterableDate.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/diff.sc-type.template.json

@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "dateCreated": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "action": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/dish.sc-type.template.json

@@ -0,0 +1,22 @@
+{
+  "properties": {
+    "availabilityStarts": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "availabilityEnds": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "offers": {
+      "_typeRef": "offers.sc-type.template.json"
+    },
+    "characteristics": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "additives": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/event.sc-type.template.json

@@ -0,0 +1,47 @@
+{
+  "properties": {
+    "subType": {
+      "_fieldRef": "sortableKeyword.field.template.json"
+    },
+    "categories": {
+      "_fieldRef": "sortableKeyword.field.template.json"
+    },
+    "previousStartDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "organizers": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "performers": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "attendees": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "catalogs": {
+      "type": "nested",
+      "_typeRef": "catalog.sc-type.template.json"
+    },
+    "maximumParticipants": {
+      "type": "integer"
+    },
+    "superEvent": {
+      "_typeRef": "event.sc-type.template.json"
+    },
+    "subProperties": {
+      "_fieldRef": "eventSubProperties.field.template.json"
+    },
+    "startDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "endDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/eventSubProperties.field.template.json

@@ -0,0 +1,16 @@
+{
+  "properties": {
+    "id": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "semester": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "majors": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "originalCategory": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/filterableDate.field.template.json

@@ -0,0 +1,8 @@
+{
+  "type": "date",
+  "fields": {
+    "raw": {
+      "type": "keyword"
+    }
+  }
+}

src/storage/elasticsearch/templates/filterableKeyword.field.template.json

@@ -0,0 +1,8 @@
+{
+  "type": "keyword",
+  "fields": {
+    "raw": {
+      "type": "keyword"
+    }
+  }
+}

src/storage/elasticsearch/templates/floorplan.sc-type.template.json

@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "floor": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/jobs.field.template.json

@@ -0,0 +1,26 @@
+{
+  "properties": {
+    "jobTitle": {
+      "type": "text"
+    },
+    "worksFor": {
+      "_typeRef": "organization.sc-type.template.json"
+    },
+    "workLocation": {
+      "properties": {
+        "email": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "faxNumber": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "telephone": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "openingHours": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        }
+      }
+    }
+  }
+}

src/storage/elasticsearch/templates/offers.sc-type.template.json

@@ -0,0 +1,24 @@
+{
+  "properties": {
+    "price": {
+      "type": "double"
+    },
+    "prices": {
+      "type": "nested",
+      "properties": {
+        "alumni": {
+          "type": "double"
+        },
+        "student": {
+          "type": "double"
+        },
+        "employee": {
+          "type": "double"
+        },
+        "guest": {
+          "type": "double"
+        }
+      }
+    }
+  }
+}

src/storage/elasticsearch/templates/organization.sc-type.template.json

@@ -0,0 +1,7 @@
+{
+  "properties": {
+    "address": {
+      "_fieldRef": "address.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/person.sc-type.template.json

@@ -0,0 +1,47 @@
+{
+  "properties": {
+    "honorificPrefix": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "honorificSuffix": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "givenName": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "additionalName": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "familyName": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "birthDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "gender": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "email": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "faxNumber": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "telephone": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "adress": {
+      "type": "text"
+    },
+    "affiliations": {
+      "type": "nested",
+      "_typeRef": "organization.sc-type.template.json"
+    },
+    "homeLocation": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "nationality": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/place.sc-type.template.json

@@ -0,0 +1,33 @@
+{
+  "properties": {
+    "subType": {
+      "_fieldRef": "sortableKeyword.field.template.json"
+    },
+    "address": {
+      "_fieldRef": "address.field.template.json"
+    },
+    "openingHours": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "geo": {
+      "properties": {
+        "point": {
+          "properties": {
+            "coordinates": {
+              "type": "geo_point"
+            }
+          }
+        },
+        "polygon": {
+          "type": "geo_shape"
+        }
+      }
+    },
+    "superPlace": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "subProperties": {
+      "_fieldRef": "placeSubProperties.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/placeSubProperties.field.template.json

@@ -0,0 +1,38 @@
+{
+  "properties": {
+    "floors": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "floor": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "paymentAccepted": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "roomCharacterization": {
+      "type": "nested",
+      "properties": {
+        "inventory": {
+          "properties": {
+            "key": {
+              "type": "keyword",
+              "fields": {
+                "raw": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "value": {
+              "type": "integer",
+              "fields": {
+                "raw": {
+                  "type": "integer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/storage/elasticsearch/templates/prices.field.template.json

@@ -0,0 +1,14 @@
+{
+  "type": "nested",
+  "properties": {
+    "student": {
+      "type": "float"
+    },
+    "employee": {
+      "type": "float"
+    },
+    "guest": {
+      "type": "float"
+    }
+  }
+}

src/storage/elasticsearch/templates/sortableKeyword.field.template.json

@@ -0,0 +1,15 @@
+{
+  "type": "text",
+  "analyzer": "search_german",
+  "fields": {
+    "raw": {
+      "type": "keyword",
+      "ignore_above": 10000
+    },
+    "sort": {
+      "fielddata": true,
+      "type": "text",
+      "analyzer": "ducet_sort"
+    }
+  }
+}

src/storage/elasticsearch/templates/text.field.template.json

@@ -0,0 +1,10 @@
+{
+  "type": "text",
+  "fielddata": true,
+  "analyzer": "search_german",
+  "fields": {
+    "raw": {
+      "type": "keyword"
+    }
+  }
+}

src/storage/elasticsearch/templating.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Client} from 'elasticsearch';
+import {readdir, readFile} from 'fs-extra';
+import { resolve } from 'path';
+import { logger } from '../../common';
+
+/**
+ * Assembles an elasticsearch template with all resolved subType-references
+ * @param templateType
+ * @param templates
+ * @param inline
+ * @returns
+ */
+function assembleElasticsearchTemplate(templateType: string, templates: {[key: string]: any}, inline: number): any {
+
+  const templateBase = JSON.parse(JSON.stringify(templates[templateType]));
+
+  if (inline) {
+    delete templateBase.dynamic_templates;
+  }
+
+  // these have no properties to replace
+  const excludeBaseFields = [
+    'filterableKeyword.field.template.json',
+    'sortableKeyword.field.template.json',
+    'text.field.template.json',
+    'filterableDate.field.template.json',
+  ];
+
+  if (excludeBaseFields.indexOf(templateType) === -1) {
+
+    try {
+      // extend the template by the properties of the basetemplate
+      templateBase.properties = Object.assign(
+        templateBase.properties,
+        templates['base.template.json'].mappings._default_.properties,
+      );
+    } catch (e) {
+      logger.error('Failed to merge properties on: ' + templateType);
+      throw e;
+    }
+    const fieldKeys = Object.keys(templateBase.properties);
+
+    fieldKeys.forEach((fieldKey) => {
+
+      const field = templateBase.properties[fieldKey];
+      const keys = Object.keys(field);
+
+      // we have subtype-references to replace
+      if (keys.indexOf('_typeRef') > -1) {
+        // if we are already inline of a superObject, we don't resolve types
+        if (inline > 1) {
+          delete templateBase.properties[fieldKey];
+        } else {
+          // we have more than one reference
+          if (Array.isArray(field._typeRef)) {
+            let obj = {};
+            field._typeRef.forEach((subType: string) => {
+              obj = Object.assign(obj, assembleElasticsearchTemplate(subType, templates, inline + 1));
+            });
+            templateBase.properties[fieldKey] = obj;
+          } else {
+            templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._typeRef, templates, inline + 1);
+          }
+        }
+      } else if (keys.indexOf('_fieldRef') > -1) {
+        templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._fieldRef, templates, inline + 1);
+      }
+    });
+  }
+  return templateBase;
+}
+
+/**
+ * Reads all template files and returns the assembled template
+ */
+export async function getElasticsearchTemplate(): Promise<any> {
+
+  // readIM all templates
+  const elasticsearchFolder = resolve('.', 'src', 'storage', 'elasticsearch', 'templates');
+  const templates: {[key: string]: any} = {};
+
+  const fileNames = await readdir(elasticsearchFolder);
+
+  const availableTypes = fileNames.filter((fileName) => {
+    return Array.isArray(fileName.match(/\w*\.sc-type\.template\.json/i));
+  }).map((fileName) => {
+    return fileName.substring(0, fileName.indexOf('.sc-type.template.json'));
+  });
+
+  const promises = fileNames.map(async (fileName) => {
+    const file = await readFile(resolve(elasticsearchFolder, fileName), 'utf8');
+
+    try {
+      templates[fileName] = JSON.parse(file.toString());
+    } catch (jsonParsingError) {
+      logger.error('Failed parsing file: ' + fileName);
+      throw jsonParsingError;
+    }
+  });
+
+  await Promise.all(promises);
+
+  const template = templates['base.template.json'];
+
+  availableTypes.forEach((configType) => {
+    template.mappings[configType.toLowerCase()] =
+      assembleElasticsearchTemplate(configType + '.sc-type.template.json', templates, 0);
+  });
+
+  // this is like the base type (StappsCoreThing)
+  const baseProperties = template.mappings._default_.properties;
+  Object.keys(baseProperties).forEach((basePropertyName) => {
+    let field = baseProperties[basePropertyName];
+    field = templates[field._fieldRef];
+    template.mappings._default_.properties[basePropertyName] = field;
+  });
+
+  return template;
+}
+
+/**
+ * Puts a new global template
+ * @param client
+ */
+export async function putTemplate(client: Client): Promise<void> {
+  return client.indices.putTemplate({
+    body: await getElasticsearchTemplate(),
+    name: 'global',
+  });
+}