feat: add backend

src/app.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCNotFoundErrorResponse, SCUnsupportedMediaTypeErrorResponse } from '@openstapps/core';
+import { SCValidator } from '@openstapps/core-validator';
+import * as bodyParser from 'body-parser';
+import * as config from 'config';
+import * as cors from 'cors';
+import * as express from 'express';
+import * as morgan from 'morgan';
+import { logger, mailer } from './common';
+import { MailQueue } from './notification/MailQueue';
+import { bulkAddRouter } from './routes/BulkAddRoute';
+import { bulkDoneRouter } from './routes/BulkDoneRoute';
+import { bulkRouter } from './routes/BulkRoute';
+import { indexRouter } from './routes/IndexRoute';
+import { multiSearchRouter } from './routes/MultiSearchRoute';
+import { searchRouter } from './routes/SearchRoute';
+import { thingUpdateRouter } from './routes/ThingUpdateRoute';
+import { BulkStorage } from './storage/BulkStorage';
+import { DatabaseConstructor } from './storage/Database';
+import { Elasticsearch } from './storage/elasticsearch/Elasticsearch';
+
+export const app = express();
+
+// only accept json as content type for all requests
+app.use(bodyParser.json({
+  limit: '500kb',
+  type: (req) => {
+    const contentType = typeof req.headers['Content-Type'] === 'string' ?
+      req.headers['Content-Type'] : req.headers['content-type'];
+    if (typeof contentType === 'string' && contentType.match(/^application\/json/)) {
+      return true;
+    } else {
+      throw new SCUnsupportedMediaTypeErrorResponse(process.env.NODE_ENV !== 'production');
+    }
+  },
+})); // 500kb should be reasonably large
+
+// use morgan as a request logger
+// request loggers have to be the first middleware to be set in express
+app.use(morgan('dev'));
+
+const databases: {[name: string]: DatabaseConstructor} = {
+  elasticsearch: Elasticsearch,
+};
+
+// validate config file
+export const scValidator = new SCValidator('./node_modules/@openstapps/core/lib/schema/');
+scValidator.feedValidator();
+
+// validate the config file
+const configValidation = scValidator.validate(config.util.toObject(), 'ConfigFile');
+
+// validation failed
+if (configValidation.errors.length > 0) {
+  throw new Error(
+    'Validation of config file failed. Errors were: ' +
+    JSON.stringify(configValidation.errors),
+  );
+}
+
+// check if a database name was given
+if (!config.has('internal.database.name')) {
+  throw new Error('You have to configure a database');
+}
+
+if (typeof mailer !== 'undefined') {
+  // set a mailQueue to use the backend mailer
+  if (config.has('internal.monitoring')) {
+    app.set('mailQueue', new MailQueue(mailer));
+  }
+}
+
+const database =
+  new databases[config.get<string>('internal.database.name')](
+    config.util.toObject(),
+    app.get('mailQueue'),
+  );
+
+if (typeof database === 'undefined') {
+  throw new Error('No implementation for configured database found. Please check your configuration.');
+}
+
+logger.ok('Validated config file sucessfully');
+
+// make the validator available on the app
+app.set('validator', scValidator);
+
+// treats /foo and /foo/ as two different routes
+// see http://expressjs.com/en/api.html#app.set
+app.set('strict routing', true);
+
+// make the bulk storage available to all http middlewares/routes
+app.set(
+  'bulk',
+  new BulkStorage(database),
+);
+
+const corsOptions = {
+  allowedHeaders: [
+    'DNT',
+    'Keep-Alive',
+    'User-Agent',
+    'X-Requested-With',
+    'If-Modified-Since',
+    'Cache-Control',
+    'Content-Type',
+    'X-StApps-Version',
+  ],
+  credentials: true,
+  maxAge: 1728000,
+  methods: ['GET', 'POST', 'PUT', 'OPTIONS'],
+  optionsSuccessStatus: 204,
+};
+
+// allow all origins on all routes
+app.use(cors(corsOptions));
+// TODO: See if it can handle options request with no content-type
+
+// allow cors preflight requests on every route
+app.options('*', cors(corsOptions));
+
+app.set('isProductiveEnvironment', process.env.NODE_ENV !== 'production');
+
+// load routes before plugins
+// they now can be used or overwritten by any plugin
+app.use(
+  bulkAddRouter,
+  bulkDoneRouter,
+  bulkRouter,
+  indexRouter,
+  multiSearchRouter,
+  searchRouter,
+  thingUpdateRouter,
+);
+
+// add a route for a missing resource (404)
+app.use((_req, res) => {
+  const errorResponse = new SCNotFoundErrorResponse(process.env.NODE_ENV !== 'production');
+  res.status(errorResponse.statusCode);
+  res.json(errorResponse);
+});
+
+// TODO: implement a route to register plugins

src/cli.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import * as http from 'http';
+import { app } from './app';
+import { logger } from './common';
+
+/**
+ * Get port from environment and store in Express.
+ */
+const port = normalizePort(process.env.PORT || '3000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+const server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+function normalizePort(value: string) {
+  const portNumber = parseInt(value, 10);
+
+  if (isNaN(portNumber)) {
+    // named pipe
+    return value;
+  }
+
+  if (portNumber >= 0) {
+    // port number
+    return portNumber;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+function onError(error: Error | any) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  const bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      logger.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      logger.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+function onListening() {
+  const addr = server.address();
+  const bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  logger.ok('Listening on ' + bind);
+}

src/common.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { Logger } from '@openstapps/logger';
+import { BackendTransport } from './notification/BackendTransport';
+
+export const mailer = BackendTransport.getTransportInstance();
+
+export const logger = new Logger(mailer);

src/notification/BackendTransport.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SMTP} from '@openstapps/logger/lib/SMTP';
+import {Transport, TransportWithVerification} from '@openstapps/logger/lib/Transport';
+
+export function isTransportWithVerification(instance: Transport): instance is TransportWithVerification {
+  return typeof (instance as TransportWithVerification).verify === 'function';
+}
+
+/**
+ * Singleton for getting only one transport service
+ *
+ * In the future this may support more than loading SMTP as a transport.
+ */
+export class BackendTransport {
+
+  private static _instance: BackendTransport;
+  private waitingForVerification: boolean;
+  protected transport: SMTP | undefined;
+
+  public static getTransportInstance(): SMTP | undefined {
+    if (this._instance) {
+      return this._instance.transport;
+    }
+
+    this._instance = new this();
+    return this._instance.transport;
+  }
+
+  private constructor() {
+    // get SMTP instance for the time
+    // in the future we may implement some other transport services which can be selected
+    // via the configuration files
+    try {
+      this.transport = SMTP.getInstance();
+    } catch (error) {
+      if (process.env.ALLOW_NO_TRANSPORT === 'true') {
+        /* tslint:disable-next-line:no-console */
+        console.warn('SMTP error was ignored.');
+      } else {
+        throw error;
+      }
+    }
+
+    if (typeof this.transport !== 'undefined' && isTransportWithVerification(this.transport)) {
+      this.waitingForVerification = true;
+
+      this.transport.verify().then((message) => {
+        if (typeof message === 'string') {
+          // tslint:disable-next-line:no-console
+          console.log(message);
+        }
+      }).catch((err) => {
+        throw err;
+      });
+    } else {
+      this.waitingForVerification = false;
+    }
+  }
+
+  public isWaitingForVerification(): boolean {
+    return this.waitingForVerification;
+  }
+}

src/notification/MailQueue.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.nse along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SMTP } from '@openstapps/logger/lib/SMTP';
+import { MailOptions } from 'nodemailer/lib/sendmail-transport';
+import * as Queue from 'promise-queue';
+import { logger } from '../common';
+/**
+ * A queue that can send mails in serial
+ */
+export class MailQueue {
+
+  /**
+   * A queue that saves mails, before the transport is ready. When
+   * the transport gets ready this mails are getting pushed in to
+   * the normal queue.
+   */
+  dryQueue: MailOptions[];
+
+  /**
+   * A queue that saves mails, that are being sent in series
+   */
+  queue: Queue;
+
+  /**
+   * A counter for the number of verifications that failed
+   */
+  verificationCounter: number;
+
+  /**
+   * Creates a mail queue
+   * @param transport
+   */
+  constructor(private transport: SMTP) {
+
+    this.queue = new Queue(1);
+
+    // this queue saves all request when the transport is not ready yet
+    this.dryQueue = [];
+
+    this.verificationCounter = 0;
+
+    // if the transport can be verified it should check if it was done...
+    this.checkForVerification();
+  }
+
+  /**
+   * Verify the given transport
+   */
+  private checkForVerification() {
+
+    if (this.verificationCounter > 5) {
+      throw new Error('Failed to initialize the SMTP transport for the mail queue');
+    }
+
+    if (!this.transport.isVerified()) {
+      this.verificationCounter++;
+      setTimeout(() => {
+        logger.warn('Transport not verified yet. Trying to send mails here...');
+        this.checkForVerification();
+      }, 5000);
+    } else {
+      logger.ok('Transport for mail queue was verified. We can send mails now');
+      // if the transport finally was verified send all our mails from the dry queue
+      this.dryQueue.forEach((mail) => {
+        this.queue.add<string>(() => (this.transport as SMTP).sendMail(mail));
+      });
+    }
+  }
+
+  /**
+   * Push a mail into the queue so it gets send when the queue is ready
+   * @param mail
+   */
+  public push(mail: MailOptions) {
+    if (!this.transport.isVerified()) { // the transport has verification, but is not verified yet
+      // push to a dry queue which gets pushed to the real queue when the transport is verified
+      this.dryQueue.push(mail);
+    } else {
+      this.queue.add<string>(() => (this.transport as SMTP).sendMail(mail));
+    }
+  }
+}

src/routes/BulkAddRoute.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBulkAddRequest, SCBulkAddResponse, SCBulkAddRoute, SCNotFoundErrorResponse } from '@openstapps/core';
+import { logger } from '../common';
+import { BulkStorage } from '../storage/BulkStorage';
+import { createRoute } from './Route';
+
+const bulkRouteModel = new SCBulkAddRoute();
+
+/**
+ * Implementation of the bulk add route (SCBulkAddRoute)
+ */
+export const bulkAddRouter = createRoute<SCBulkAddResponse>(
+  bulkRouteModel,
+  async (request: SCBulkAddRequest, app, params) => {
+
+    if (!params || typeof params.UID !== 'string') {
+      throw new Error('UID of Bulk was not given, but route with obligatory parameter was called');
+    }
+
+    const bulkMemory: BulkStorage = app.get('bulk');
+    const bulk = await bulkMemory.read(params.UID);
+
+    if (typeof bulk === 'undefined') {
+      logger.warn(`Bulk with ${params.UID} not found.`);
+      throw new SCNotFoundErrorResponse(app.get('isProductiveEnvironment'));
+    }
+
+    await bulkMemory.database.post(request, bulk);
+
+    return {};
+  },
+);

src/routes/BulkDoneRoute.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBulkDoneRequest, SCBulkDoneResponse, SCBulkDoneRoute, SCNotFoundErrorResponse } from '@openstapps/core';
+import { logger } from '../common';
+import { BulkStorage } from '../storage/BulkStorage';
+import { createRoute } from './Route';
+
+const bulkDoneRouteModel = new SCBulkDoneRoute();
+
+/**
+ * Implementation of the bulk done request route (SCBulkDoneRoute)
+ */
+export const bulkDoneRouter = createRoute<SCBulkDoneResponse>(
+  bulkDoneRouteModel,
+  async (_request: SCBulkDoneRequest, app, params) => {
+
+    if (!params || typeof params.UID !== 'string') {
+      throw new Error('UID of Bulk was not given, but route with obligatory parameter was called');
+    }
+
+    const bulkMemory: BulkStorage = app.get('bulk');
+    const bulk = await bulkMemory.read(params.UID);
+
+    if (typeof bulk === 'undefined') {
+      logger.warn(`Bulk with ${params.UID} not found.`);
+      throw new SCNotFoundErrorResponse(app.get('isProductiveEnvironment'));
+    }
+
+    bulk.state = 'done';
+    await bulkMemory.markAsDone(bulk);
+    return {};
+  },
+);

src/routes/BulkRoute.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBulkRequest, SCBulkResponse, SCBulkRoute } from '@openstapps/core';
+import { BulkStorage } from '../storage/BulkStorage';
+import { createRoute } from './Route';
+
+const bulkRouteModel = new SCBulkRoute();
+
+/**
+ * Implementation of the bulk request route (SCBulkRoute)
+ */
+export const bulkRouter = createRoute<SCBulkResponse>(
+  bulkRouteModel,
+  async (request: SCBulkRequest, app) => {
+    const bulkMemory: BulkStorage = app.get('bulk');
+    return await bulkMemory.create(request);
+  },
+);

src/routes/HTTPTypes.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+export type HTTPVerb = 'all' |
+  'get' |
+  'post' |
+  'put' |
+  'delete' |
+  'patch' |
+  'options' |
+  'head' |
+  'checkout' |
+  'copy' |
+  'lock' |
+  'merge' |
+  'mkactivity' |
+  'mkcol' |
+  'move' |
+  'm-search' |
+  'notify' |
+  'purge' |
+  'report' |
+  'search' |
+  'subscribe' |
+  'trace' |
+  'unlock' |
+  'unsubscribe';
+
+export function isHttpMethod(method: string): method is HTTPVerb {
+  return ['get', 'post', 'put'].indexOf(method) > -1;
+}

src/routes/IndexRoute.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCConfigFile, SCIndexResponse, SCIndexRoute } from '@openstapps/core';
+import * as config from 'config';
+import { createRoute } from './Route';
+
+const indexRouteModel = new SCIndexRoute();
+
+/**
+ * Implementation of the index route (SCIndexRoute)
+ */
+export const indexRouter = createRoute<SCIndexResponse>(
+  indexRouteModel,
+  async (_request: SCIndexRoute, _app) => {
+    const configObject: SCConfigFile = config.util.toObject();
+    delete configObject.internal;
+    return configObject;
+  },
+);

src/routes/MultiSearchRoute.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCMultiSearchRequest,
+  SCMultiSearchResponse,
+  SCMultiSearchRoute,
+  SCSearchResponse,
+  SCTooManyRequestsErrorResponse,
+} from '@openstapps/core';
+import { BulkStorage } from '../storage/BulkStorage';
+import { createRoute } from './Route';
+
+const multiSearchRouteModel = new SCMultiSearchRoute();
+
+/**
+ * Implementation of the multi search route (SCMultiSearchRoute)
+ */
+export const multiSearchRouter = createRoute<SCMultiSearchResponse | SCTooManyRequestsErrorResponse>(
+  multiSearchRouteModel,
+  async (request: SCMultiSearchRequest, app) => {
+
+    const bulkMemory: BulkStorage = app.get('bulk');
+    const queryNames = Object.keys(request);
+
+    if (queryNames.length > 5) {
+      return new SCTooManyRequestsErrorResponse(app.get('isProductiveEnvironment'));
+    }
+
+    // get a map of promises for each query
+    const searchRequests = queryNames.map((queryName) => {
+      return bulkMemory.database.search(request[queryName]);
+    });
+
+    const listOfSearchResponses = await Promise.all(searchRequests);
+
+    const response: { [queryName: string]: SCSearchResponse } = {};
+    queryNames.forEach((queryName, index) => {
+      response[queryName] = listOfSearchResponses[index];
+    });
+
+    return response;
+  },
+);

src/routes/Route.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCInternalServerErrorResponse,
+  SCMethodNotAllowedErrorResponse,
+  SCRoute,
+  SCValidationErrorResponse,
+} from '@openstapps/core';
+import { SCValidator } from '@openstapps/core-validator';
+import { Application, Router } from 'express';
+import PromiseRouter from 'express-promise-router';
+import { logger } from '../common';
+import { isHttpMethod } from './HTTPTypes';
+
+/**
+ * Creates a router from a route class (model of a route) and a handler function which implements the logic
+ *
+ * The given router performs a request and respone validation, sets status codes and checks if the given handler
+ * only returns errors that are allowed for the client to see
+ *
+ * @param routeClass
+ * @param handler
+ */
+export function createRoute<RETURNTYPE>(
+  routeClass: SCRoute,
+  handler: (validatedBody: any, app: Application, params?: { [parameterName: string]: string }) => Promise<RETURNTYPE>,
+): Router {
+  // create router
+  const router = PromiseRouter({ mergeParams: true });
+
+  // create route
+  // the given type has no index signature so we have to cast to get the IRouteHandler when a HTTP method is given
+  const route = router.route(routeClass.urlFragment);
+
+  // get route parameters (path parameters)
+  if (Array.isArray(routeClass.obligatoryParameters) && routeClass.obligatoryParameters.length > 0) {
+    routeClass.obligatoryParameters.forEach((parameterName) => {
+      router.param(parameterName, async (_req, _res, next, _parameterValue: string) => {
+
+        // if (typeof req.params === 'undefined') {
+        //   req.params = {};
+        // }
+
+        // set parameter values on request object
+        // req.params[parameterName] = parameterValue;
+        // hand over the request to the next handler (our method route handler)
+        next();
+      });
+    });
+  }
+
+  const verb = routeClass.method.toLowerCase();
+
+  // check if route has a valid http verb
+  if (isHttpMethod(verb)) {
+    // create a route handler for the given HTTP method
+    route[verb](async (req, res) => {
+
+      try {
+        // get the core validator from the app
+        const validator: SCValidator = req.app.get('validator');
+
+        // validate request
+        const requestValidation = validator.validate(req.body, routeClass.requestBodyName.substring(2));
+
+        if (requestValidation.errors.length > 0) {
+          const error = new SCValidationErrorResponse(
+            requestValidation.errors,
+            req.app.get('isProductiveEnvironment'),
+          );
+          res.status(error.statusCode);
+          res.json(error);
+          logger.warn(error);
+          return;
+        }
+
+        const params: { [parameterName: string]: string } = {};
+
+        if (Array.isArray(routeClass.obligatoryParameters) && routeClass.obligatoryParameters) {
+          // copy over parameter values from request object
+          // the parameter values were set in the parameter handler of the route
+          routeClass.obligatoryParameters.forEach((parameterName) => {
+            params[parameterName] = req.params[parameterName];
+          });
+        }
+
+        // hand over request to handler with path parameters
+        const response = await handler(req.body, req.app, params);
+
+        // validate response generated by handler
+        const responseValidation = validator.validate(response, routeClass.responseBodyName.substring(2));
+
+        if (responseValidation.errors.length > 0) {
+          const validationError = new SCValidationErrorResponse(
+            responseValidation.errors,
+            req.app.get('isProductiveEnvironment'),
+          );
+          const internalServerError = new SCInternalServerErrorResponse(
+            validationError,
+            req.app.get('isProductiveEnvironment'),
+          );
+          res.status(internalServerError.statusCode);
+          res.json(internalServerError);
+          logger.warn(internalServerError);
+          return;
+        }
+
+        // set status code
+        res.status(routeClass.statusCodeSuccess);
+
+        // respond
+        res.json(response);
+      } catch (error) {
+        // if the error response is allowed on the route
+        if (routeClass.errorNames.indexOf(error.constructor.name) > -1) {
+          // respond with the error from the handler
+          res.status(error.statusCode);
+          res.json(error);
+          logger.warn(error);
+        } else {
+          // the error is not allowed so something went wrong
+          const internalServerError = new SCInternalServerErrorResponse(
+            error,
+            req.app.get('isProductiveEnvironment'),
+          );
+          res.status(internalServerError.statusCode);
+          res.json(internalServerError);
+          logger.error(error);
+        }
+      }
+    });
+  } else {
+    throw new Error('Invalid HTTP verb in route definition. Please check route definitions in `@openstapps/core`');
+  }
+
+  // return a SCMethodNotAllowedErrorResponse on all other HTTP methods
+  route.all((req, res) => {
+    const error = new SCMethodNotAllowedErrorResponse(req.app.get('isProductiveEnvironment'));
+    res.status(error.statusCode);
+    res.json(error);
+    logger.warn(error);
+  });
+
+  // return router
+  return router;
+}

src/routes/SearchRoute.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCSearchRequest, SCSearchResponse, SCSearchRoute } from '@openstapps/core';
+import { BulkStorage } from '../storage/BulkStorage';
+import { createRoute } from './Route';
+
+const searchRouteModel = new SCSearchRoute();
+
+/**
+ * Implementation of the search route (SCSearchRoute)
+ */
+export const searchRouter = createRoute<SCSearchResponse>(searchRouteModel, async ( request: SCSearchRequest, app) => {
+  const bulkMemory: BulkStorage = app.get('bulk');
+  return await bulkMemory.database.search(request);
+});

src/routes/ThingUpdateRoute.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCThingUpdateRequest, SCThingUpdateResponse, SCThingUpdateRoute } from '@openstapps/core';
+import { BulkStorage } from '../storage/BulkStorage';
+import { createRoute } from './Route';
+
+const thingUpdateRouteModel = new SCThingUpdateRoute();
+
+/**
+ * Implementation of the thing update route (SCThingUpdateRoute)
+ */
+export const thingUpdateRouter = createRoute<SCThingUpdateResponse>(
+  thingUpdateRouteModel,
+  async (request: SCThingUpdateRequest, app) => {
+    const bulkMemory: BulkStorage = app.get('bulk');
+    await bulkMemory.database.put(request);
+    return {};
+  },
+);

src/storage/BulkStorage.ts

@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBulkRequest, SCThingTypes } from '@openstapps/core';
+import * as moment from 'moment';
+import * as NodeCache from 'node-cache';
+import { promisify } from 'util';
+import { v4 } from 'uuid';
+import { logger } from '../common';
+import { Database } from './Database';
+
+export type BulkOperation = 'create' | 'expired' | 'update';
+
+/**
+ * Describes an indexing process
+ */
+export class Bulk implements SCBulkRequest {
+
+  /**
+   * Expiration of the bulk
+   *
+   * If the bulk is not finished before the expiration date is hit, the bulk
+   * and all data associated with it will be deleted
+   */
+  expiration: string;
+
+  /**
+   * The data source of the bulk
+   *
+   * Bulks with same type and source will be replaced and the data will be
+   * updated when the bulk is marked as done
+   */
+  source: string;
+
+  /**
+   * State of the bulk
+   *
+   * Data can be indexed for this bulk as long as the state is `in progress`
+   * and the bulk is not expired
+   *
+   * When the bulk is marked as `done` it replaces the previous bulk with
+   * the same source and type. The data will be availabe to the user when
+   * the bulk switches to done
+   */
+  state: 'in progress' | 'done';
+
+  /**
+   * Type of data in the bulk
+   */
+  type: SCThingTypes;
+
+  /**
+   * Unique identifier of the bulk
+   */
+  uid: string;
+
+  /**
+   * Creates a new bulk process
+   * @param request
+   */
+  constructor(request: SCBulkRequest) {
+    this.uid = v4();
+    this.state = 'in progress';
+
+    if (typeof request.expiration === 'string') {
+      this.expiration = request.expiration;
+    } else {
+      this.expiration = moment().add(1, 'hour').toISOString();
+    }
+    // when should this process be finished
+    // where does the process come from
+    this.source = request.source;
+    // which type of data is this process about to index
+    this.type = request.type;
+  }
+}
+
+/**
+ * Cache for bulk-processes
+ */
+export class BulkStorage {
+
+  private cache: NodeCache;
+
+  /**
+   * Creates a new BulkStorage
+   * @param database the database that is controlled by this bulk storage
+   */
+  constructor(public database: Database) {
+
+    // a bulk lives 60 minutes if no expiration is given
+    // the cache is checked every 60 seconds
+    this.cache = new NodeCache({ stdTTL: 3600, checkperiod: 60 });
+
+    this.cache.on('expired', (_key, bulk: Bulk) => {
+      // if the bulk is not done
+      if (bulk.state !== 'done') {
+        // the database can delete the data associated with this bulk
+        this.database.bulkExpired(bulk);
+      }
+    });
+  }
+
+  /**
+   * Saves a bulk process and assigns to it a user-defined ttl (time-to-live)
+   * @param bulk the bulk process to save
+   * @returns the bulk process that was saved
+   */
+  private async save(bulk: Bulk): Promise<Bulk> {
+    const expirationInSeconds = moment(bulk.expiration).diff(moment.now()) / 1000;
+    logger.info('Bulk expires in ', expirationInSeconds, 'seconds');
+
+    // save the item in the cache with it's expected expiration
+    await promisify<string, Bulk, number>(this.cache.set)(bulk.uid, bulk, expirationInSeconds);
+    return bulk;
+  }
+
+  /**
+   * Create and save a new bulk process
+   * @param bulkRequest a request for a new bulk process
+   * @returns a promise that contains the new bulk process
+   */
+  public async create(bulkRequest: SCBulkRequest): Promise<Bulk> {
+    const bulk = new Bulk(bulkRequest);
+    bulk.source = bulkRequest.source;
+    bulk.type = bulkRequest.type;
+
+    await this.save(bulk);
+
+    // tell the database that the bulk was created
+    await this.database.bulkCreated(bulk);
+    return bulk;
+  }
+
+  /**
+   * Delete a bulk process
+   * @param uid uid of the bulk process
+   * @returns a promise that contains the deleted bulk process
+   */
+  public async delete(uid: string): Promise<Bulk> {
+    const bulk = await this.read(uid);
+
+    if (typeof bulk === 'undefined') {
+      throw new Error(`Bulk that should be deleted was not found. UID was "${uid}"`);
+    }
+
+    // delete the bulk process from the cache
+    await promisify<string>(this.cache.del)(uid);
+
+    // tell the database to handle the expiration of the bulk
+    await this.database.bulkExpired(bulk);
+
+    return bulk;
+  }
+
+  /**
+   * Update an old bulk process (replace it with the new one)
+   * @param bulk new bulk process
+   * @returns an empty promise
+   */
+  public async markAsDone(bulk: Bulk): Promise<void> {
+    bulk.state = 'done';
+    await this.save(bulk);
+
+    // tell the database that this is the new bulk
+    this.database.bulkUpdated(bulk);
+    return;
+  }
+
+  /**
+   * Read an existing bulk process
+   * @param uid uid of the bulk process
+   * @returns a promise that contains a bulk
+   */
+  public async read(uid: string): Promise<Bulk | undefined> {
+    return await promisify<string, any>(this.cache.get)(uid);
+  }
+
+}

src/storage/Database.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCSearchQuery, SCSearchResponse, SCThings, SCUuid } from '@openstapps/core';
+import {Bulk} from './BulkStorage';
+
+export interface DatabaseConstructor {
+  new (...args: any): Database;
+}
+
+export interface Database {
+
+  /**
+   * Gets called if a bulk was created
+   *
+   * The database should
+   * @param bulk
+   */
+  bulkCreated(bulk: Bulk): Promise<void>;
+
+  /**
+   * Gets called if a bulk expires
+   *
+   * The database should delete all data that is associtated with this bulk
+   * @param bulk
+   */
+  bulkExpired(bulk: Bulk): Promise<void>;
+
+  /**
+   * Gets called if a bulk was updated
+   *
+   * If the database holds a bulk with the same type and source as the given
+   * bulk it should be replaced by the given one
+   *
+   * @param bulk
+   */
+  bulkUpdated(bulk: Bulk): Promise<void>;
+
+  /**
+   * Get a single document
+   * @param uid
+   */
+  get(uid: SCUuid): Promise<SCThings>;
+
+  /**
+   * Add a thing to an existing bulk
+   * @param object
+   * @param bulkId
+   */
+  post(thing: SCThings, bulk: Bulk): Promise<void>;
+
+  /**
+   * Replace an existing thing in any Bulk
+   *
+   * Currently it is not possible to put an non-existing object
+   *
+   * @param thing
+   */
+  put(thing: SCThings): Promise<void>;
+
+  /**
+   * Search for things
+   * @param params
+   */
+   search(params: SCSearchQuery): Promise<SCSearchResponse>;
+}

src/storage/elasticsearch/Elasticsearch.ts

@@ -0,0 +1,508 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCBulkResponse,
+  SCConfigFile,
+  SCFacet,
+  SCSearchQuery,
+  SCSearchResponse,
+  SCThings,
+  SCThingTypes,
+  SCUuid,
+} from '@openstapps/core';
+import * as ES from 'elasticsearch';
+import * as moment from 'moment';
+import { logger } from '../../common';
+import { MailQueue } from '../../notification/MailQueue';
+import { Bulk } from '../BulkStorage';
+import { Database } from '../Database';
+import { buildAggregations, parseAggregations } from './aggregations';
+import { AggregationSchema, ElasticsearchConfig, ElasticsearchObject } from './common';
+import * as Monitoring from './monitoring';
+import { buildQuery, buildSort } from './query';
+import { putTemplate } from './templating';
+
+// this will match index names such as stapps_<type>_<source>_<random suffix>
+const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
+
+/**
+ * A database interface for elasticsearch
+ */
+export class Elasticsearch implements Database {
+
+  aggregationsSchema: AggregationSchema;
+
+  /**
+   * Holds a map of all elasticsearch indices that are available to search
+   */
+  aliasMap: {
+    // each scType has a alias which can contain multiple sources
+    [scType: string]: {
+      // each source is assigned a index name in elasticsearch
+      [source: string]: string;
+    },
+  };
+  client: ES.Client;
+  ready: boolean;
+
+  /**
+   * Create a new interface for elasticsearch
+   * @param config an assembled config file
+   * @param mailQueue a mailqueue for monitoring
+   */
+  constructor(private config: SCConfigFile, mailQueue?: MailQueue) {
+
+    if (!config.internal.database || typeof config.internal.database.version === 'undefined') {
+      throw new Error('Database version is undefined. Check you config file');
+    }
+
+    const options = {
+      apiVersion: config.internal.database.version,
+      host: this.getElasticsearchUrl(),
+      log: 'error',
+    };
+
+    // enable verbose logging for all request to elasticsearch
+    if (process.env.ES_DEBUG === 'true') {
+      options.log = 'trace';
+    }
+
+    this.client = new ES.Client(options);
+    this.aliasMap = {};
+    this.ready = false;
+
+    this.aggregationsSchema = buildAggregations(this.config.internal.aggregations);
+
+    this.getAliasMap();
+
+    const monitoringConfiguration = this.config.internal.monitoring;
+
+    if (typeof monitoringConfiguration !== 'undefined') {
+      if (typeof mailQueue === 'undefined') {
+        throw new Error('Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.');
+      }
+      // read all watches and schedule searches on the client
+      Monitoring.setUp(monitoringConfiguration, this.client, mailQueue);
+    }
+
+  }
+
+  /**
+   * Tests if an object already exists
+   *
+   * Returns Elasticsearch Object if it exists
+   */
+  private async doesItemExist(object: SCThings): Promise<{exists: boolean; object?: ElasticsearchObject<SCThings>}> {
+    const searchResponse = await this.client.search<SCThings>({
+      body: {
+        query: {
+          term: {
+            'uid.raw': {
+              value: object.uid,
+            },
+          },
+        },
+      },
+      from: 0,
+      index: this.getListOfAllIndices(),
+      size: 1,
+    });
+
+    if (searchResponse.hits.total > 1) {
+      return {
+        exists: true,
+        object: searchResponse.hits.hits[0],
+      };
+    }
+
+    return {
+      exists: false,
+    };
+  }
+
+  /**
+   * Gets a map which contains each alias and all indices that are associated with each alias
+   */
+  private async getAliasMap() {
+
+    // create a list of old indices that are not in use
+    const oldIndicesToDelete: string[] = [];
+
+    let aliases: {
+      [index: string]: {
+        aliases: {
+          [K in SCThingTypes]: any
+        },
+      },
+    };
+
+    try {
+      aliases = await this.client.indices.getAlias({});
+    } catch (error) {
+      logger.error('Failed getting alias map:', error);
+      setTimeout(() => {
+        this.getAliasMap();
+      }, 5000); // retry in 5 seconds
+      return;
+    }
+
+    for (const index in aliases) {
+      if (aliases.hasOwnProperty(index)) {
+
+        const matches = indexRegex.exec(index);
+        if (matches !== null) {
+          const type = matches[1];
+          const source = matches[2];
+
+          // check if there is an alias for the current index
+          // check that alias equals type
+          const hasAlias = type in aliases[index].aliases;
+          if (hasAlias) {
+            if (typeof this.aliasMap[type] === 'undefined') {
+              this.aliasMap[type] = {};
+            }
+            this.aliasMap[type][source] = index;
+          } else {
+            oldIndicesToDelete.push(index);
+          }
+        }
+      }
+    }
+
+    this.ready = true;
+
+    // delete old indices that are not used in any alias
+    if (oldIndicesToDelete.length > 0) {
+      await this.client.indices.delete({
+        index: oldIndicesToDelete,
+      });
+      logger.warn('Deleted old indices: ' + oldIndicesToDelete);
+    }
+
+    logger.ok('Read alias map from elasticsearch: ' + JSON.stringify(this.aliasMap, null, 2));
+  }
+
+  /**
+   * Get the url of elasticsearch
+   */
+  private getElasticsearchUrl(): string {
+    // check if we have a docker link
+    if (process.env.ES_PORT_9200_TCP_ADDR !== undefined && process.env.ES_PORT_9200_TCP_PORT !== undefined) {
+      return process.env.ES_PORT_9200_TCP_ADDR + ':' + process.env.ES_PORT_9200_TCP_PORT;
+    }
+
+    // default
+    return 'localhost:9200';
+  }
+
+  /**
+   * Gets the index name in elasticsearch for one SCThingType
+   * @param type SCThingType of data in the index
+   * @param source source of data in the index
+   * @param bulk bulk process which created this index
+   */
+  private getIndex(type: SCThingTypes, source: string, bulk: SCBulkResponse) {
+    return `stapps_${type.toLowerCase().replace(' ', '_')}_${source}_${bulk.uid.substring(0, 8)}`;
+  }
+
+  /**
+   * Generates a string which matches all indices
+   */
+  private getListOfAllIndices(): string {
+    // map each SC type in upper camel case
+    return 'stapps_*_*_*';
+  }
+
+  /**
+   * Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
+   * @param bulk the bulk process that was created
+   */
+  public async bulkCreated(bulk: Bulk): Promise<void> {
+    // if our es instance is not ready yet, we cannot serve this request
+    if (!this.ready) {
+      throw new Error('No connection to elasticsearch established yet.');
+    }
+
+    // index name for elasticsearch
+    const index: string = this.getIndex(bulk.type, bulk.source, bulk);
+
+    // there already is an index with this type and source. We will index the new one and switch the alias to it
+    // the old one is deleted
+    const alias = bulk.type;
+
+    if (typeof this.aliasMap[alias] === 'undefined') {
+      this.aliasMap[alias] = {};
+    }
+
+    if (!indexRegex.test(index)) {
+      throw new Error(
+          'Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.\n' +
+          'Make sure to set the bulk "source" and "type" to names consisting of the characters above.',
+      );
+    }
+
+    // re-apply the index template before each new bulk operation
+    await putTemplate(this.client);
+    await this.client.indices.create({
+      index,
+    });
+
+    logger.info('Created index', index);
+  }
+
+  /**
+   * Should be called when a bulk process is expired. The index that was created with this bulk gets deleted
+   * @param bulk the bulk process that is expired
+   */
+  public async bulkExpired(bulk: Bulk): Promise<void> {
+    // index name for elasticsearch
+    const index: string = this.getIndex(bulk.type, bulk.source, bulk);
+
+    logger.info('Bulk expired. Deleting index', index);
+
+    // don't delete indices that are in use already
+    if (bulk.state !== 'done') {
+      logger.info('deleting obsolete index', index);
+      return await this.client.indices.delete({ index });
+    }
+  }
+
+  /**
+   * Should be called when a bulk process is updated (replaced by a newer bulk). This will replace the old
+   * index and publish all data, that was index in the new instead
+   * @param bulk the new bulk process that should replace the old one with same type and source
+   */
+  public async bulkUpdated(bulk: Bulk): Promise<void> {
+    // if our es instance is not ready yet, we cannot serve this request
+    if (!this.ready) {
+      throw new Error('Elasticsearch not ready');
+    }
+
+    // index name for elasticsearch
+    const index: string = this.getIndex(bulk.type, bulk.source, bulk);
+
+    // alias for the indices
+    const alias = bulk.type;
+
+    if (typeof this.aliasMap[alias] === 'undefined') {
+      this.aliasMap[alias] = {};
+    }
+
+    if (!indexRegex.test(index)) {
+      throw new Error(
+          'Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.\n' +
+          'Make sure to set the bulk "source" and "type" to names consisting of the characters above.',
+      );
+    }
+
+    // create the new index if it does not exists
+    if (!(await this.client.indices.exists({ index }))) {
+      // re-apply the index template before each new bulk operation
+      await putTemplate(this.client);
+      await this.client.indices.create({
+        index,
+      });
+    }
+
+    // get the old index from our aliasMap
+    const oldIndex: string = this.aliasMap[alias][bulk.source];
+
+    // add our new index to the alias
+    const actions: ES.IndicesUpdateAliasesParamsAction[] = [
+      {
+        add: { index: index, alias: alias },
+      },
+    ];
+
+    // remove our old index if it exists
+    if (typeof oldIndex === 'string') {
+      actions.push({
+        remove: { index: oldIndex, alias: alias },
+      });
+    }
+
+    // refresh the index (fsync changes)
+    await this.client.indices.refresh({
+      index,
+    });
+
+    // execute our alias actions
+    await this.client.indices.updateAliases({
+      body: {
+        actions,
+      },
+    });
+
+    // swap the index in our aliasMap
+    this.aliasMap[alias][bulk.source] = index;
+    if (typeof oldIndex === 'string') {
+      // delete the old index
+      await this.client.indices.delete({ index: oldIndex });
+      logger.info('deleted old index', oldIndex);
+    }
+    logger.info('swapped alias index alias', oldIndex, '=>', index);
+  }
+
+  /**
+   * Gets an SCThing from all indexed data
+   * @param uid uid of an SCThing
+   */
+  public async get(uid: SCUuid): Promise<SCThings> {
+    const searchResponse = await this.client.search({
+      body: {
+        query: {
+          term: {
+            uid,
+          },
+        },
+      },
+      index: this.getListOfAllIndices(),
+    });
+
+    // get data from response
+    const hits = searchResponse.hits.hits;
+
+    if (hits.length !== 1) {
+      throw new Error('No unique item found.');
+    } else {
+      return hits[0]._source as SCThings;
+    }
+  }
+
+  /**
+   * Add an item to an index
+   * @param object the SCThing to add to the index
+   * @param bulk the bulk process which item belongs to
+   */
+  public async post(object: SCThings, bulk: Bulk): Promise<void> {
+
+    const obj: SCThings & {creation_date: string} = {
+      ...object,
+      creation_date: moment().format(),
+    };
+
+    const itemMeta = await this.doesItemExist(obj);
+
+    // we have to check that the item will get replaced if the index is rolled over
+    if (itemMeta.exists && typeof itemMeta.object !== 'undefined') {
+      const indexOfNew = this.getIndex(obj.type, bulk.source, bulk);
+      const oldIndex = itemMeta.object._index;
+
+      // new item doesn't replace the old one
+      if (oldIndex.substring(0, oldIndex.length - 9) !== indexOfNew.substring(0, indexOfNew.length - 9)) {
+        throw new Error(
+          'Object \"' + obj.uid + '\" already exists. Object was: ' +
+          JSON.stringify(obj, null, 2),
+        );
+      }
+    }
+
+    // regular bulk update (item gets replaced when bulk is updated)
+    const searchResponse = await this.client.create({
+      body: obj,
+      id: obj.uid,
+      index: this.getIndex(obj.type, bulk.source, bulk),
+      timeout: '90s',
+      type: obj.type,
+    });
+
+    if (!searchResponse.created) {
+      throw new Error('Object creation Error: Instance was: ' + JSON.stringify(obj));
+    }
+  }
+
+  /**
+   * Put (update) an existing item
+   * @param object SCThing to put
+   */
+  public async put(object: SCThings) {
+
+    const itemMeta = await this.doesItemExist(object);
+
+    if (itemMeta.exists && typeof itemMeta.object !== 'undefined') {
+      return await this.client.update({
+        body: {
+          doc: object,
+        },
+        id: object.uid,
+        index: itemMeta.object._index,
+        type: object.type.toLowerCase(),
+      });
+    }
+
+    throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.');
+  }
+
+  /**
+   * Search all indexed data
+   * @param params search query
+   */
+  public async search(params: SCSearchQuery): Promise<SCSearchResponse> {
+
+    if (typeof this.config.internal.database === 'undefined') {
+      throw new Error('Database is undefined. You have to configure the query build');
+    }
+
+    const searchRequest: ES.SearchParams = {
+      body: {
+        aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes)
+        query: buildQuery(params, this.config, this.config.internal.database as ElasticsearchConfig),
+      },
+      from: params.from,
+      index: this.getListOfAllIndices(),
+      size: params.size,
+    };
+
+    if (typeof params.sort !== 'undefined') {
+      searchRequest.body.sort = buildSort(params.sort);
+    }
+
+    // perform the search against elasticsearch
+    const response = await this.client.search<SCThings>(searchRequest);
+
+    // gather pagination information
+    const pagination = {
+      count: response.hits.hits.length,
+      offset: (typeof params.from === 'number') ? params.from : 0,
+      total: response.hits.total,
+    };
+
+    // gather statistics about this search
+    const stats = {
+      time: response.took,
+    };
+
+    // we only directly return the _source documents
+    // elasticsearch provides much more information, the user shouldn't see
+    const data = response.hits.hits.map((hit) => {
+      return hit._source; // SCThing
+    });
+
+    let facets: SCFacet[] = [];
+
+    // read the aggregations from elasticsearch and parse them to facets by our configuration
+    if (typeof response.aggregations !== 'undefined') {
+      facets = parseAggregations(this.aggregationsSchema, response.aggregations);
+    }
+
+    return {
+      data,
+      facets,
+      pagination,
+      stats,
+    };
+  }
+}

src/storage/elasticsearch/aggregations.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCBackendAggregationConfiguration, SCFacet, SCThingTypes } from '@openstapps/core';
+import { AggregationSchema } from './common';
+
+export type aggregationType = SCThingTypes | '@all';
+
+/**
+ * Builds the aggregation
+ * @returns a schema to tell elasticsearch which aggregations to collect
+ */
+export function buildAggregations(aggsConfig: SCBackendAggregationConfiguration[]): AggregationSchema {
+
+  const result: AggregationSchema = {};
+
+  aggsConfig.forEach((aggregation) => {
+
+    result[aggregation.fieldName] = {
+      terms: {
+        field: aggregation.fieldName + '.raw',
+        size: 1000,
+      },
+    };
+  });
+
+  return result;
+}
+
+/**
+ * An elasticsearch aggregation bucket
+ */
+interface Bucket {
+  doc_count: number;
+  key: string;
+}
+
+/**
+ * An elasticsearch aggregation response
+ */
+interface AggregationResponse {
+  [field: string]: {
+    buckets: Bucket[];
+    doc_count?: number;
+  };
+}
+
+/**
+ * Parses elasticsearch aggregations (response from es) to facets for the app
+ * @param aggregationSchema - aggregation-schema for elasticsearch
+ * @param aggregations - aggregations response from elasticsearch
+ */
+export function parseAggregations(
+  aggregationSchema: AggregationSchema,
+  aggregations: AggregationResponse): SCFacet[] {
+
+  const facets: SCFacet[] = [];
+
+  const aggregationNames = Object.keys(aggregations);
+
+  aggregationNames.forEach((aggregationName) => {
+    const buckets = aggregations[aggregationName].buckets;
+
+    const facet: SCFacet = {
+      buckets: buckets.map((bucket) => {
+        const facetBucket: { [value: string]: number } = {};
+        facetBucket[bucket.key] = bucket.doc_count;
+        return facetBucket;
+      }),
+      field: aggregationSchema[aggregationName].terms.field + '.raw',
+    };
+
+    facets.push(facet);
+  });
+  return facets;
+}

src/storage/elasticsearch/common.ts

@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import { SCThingTypes } from '@openstapps/core';
+import { SCThing } from '@openstapps/core';
+
+/**
+ * An elasticsearch bucket aggregation
+ * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.5/search-aggregations-bucket.html
+ */
+export interface AggregationSchema {
+  [aggregationName: string]: ESTermsFilter;
+}
+
+/**
+ * A configuration for using the Dis Max Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryDisMaxConfig {
+  cutoffFrequency: number;
+  fuzziness: number;
+  matchBoosting: number;
+  minMatch: string;
+  queryType: 'dis_max';
+  tieBreaker: number;
+}
+
+/**
+ * A configuration for using Query String Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryQueryStringConfig {
+  minMatch: string;
+  queryType: 'query_string';
+}
+
+/**
+ * A hit in an elastiscsearch search result
+ */
+export interface ElasticsearchObject<T extends SCThing> {
+  _id: string;
+  _index: string;
+  _score: number;
+  _source: T;
+  _type: string;
+  _version?: number;
+  fields?: any;
+  highlight?: any;
+  inner_hits?: any;
+  matched_queries?: string[];
+  sort?: string[];
+}
+
+/**
+ * An config file for the elasticsearch database interface
+ *
+ * The config file extends the SCConfig file by further defining how the database property
+ */
+export interface ElasticsearchConfigFile {
+  internal: {
+    database: ElasticsearchConfig;
+  };
+}
+
+/**
+ * An elasticsearch configuration
+ */
+export interface ElasticsearchConfig {
+  name: 'elasticsearch';
+  query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
+  version: string;
+}
+
+/**
+ * An elasticsearch term filter
+ */
+export interface ESTermFilter {
+  term: {
+    [fieldName: string]: string;
+  };
+}
+
+/**
+ * An elasticsearch terms filter
+ */
+export interface ESTermsFilter {
+  terms: {
+    field: string;
+    size?: number;
+  };
+}
+
+/**
+ * An elasticsearch type filter
+ */
+export interface ESTypeFilter {
+  type: {
+    value: SCThingTypes;
+  };
+}
+
+/**
+ * Filter arguments for an elasticsearch geo distance filter
+ */
+export interface ESGeoDistanceFilterArguments {
+  distance: string;
+  [fieldName: string]: {
+    lat: number;
+    lon: number;
+  } | string;
+}
+
+/**
+ * An elasticsearch geo distance filter
+ */
+export interface ESGeoDistanceFilter {
+  geo_distance: ESGeoDistanceFilterArguments;
+}
+
+/**
+ * Filter arguments for an elasticsearch boolean filter
+ */
+export interface ESBooleanFilterArguments<T> {
+  minimum_should_match?: number;
+  must?: T[];
+  must_not?: T[];
+  should?: T[];
+}
+
+/**
+ * An elasticsearch boolean filter
+ */
+export interface ESBooleanFilter<T> {
+  bool: ESBooleanFilterArguments<T>;
+}
+
+/**
+ * An elasticsearch function score query
+ */
+export interface ESFunctionScoreQuery {
+  function_score: {
+    functions: ESFunctionScoreQueryFunction[];
+    query: ESBooleanFilter<any>;
+    score_mode: 'multiply';
+  };
+}
+
+/**
+ * An function for an elasticsearch functions score query
+ */
+export interface ESFunctionScoreQueryFunction {
+  filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;
+  weight: number;
+}
+
+/**
+ * An elasticsearch ducet sort
+ */
+export interface ESDucetSort {
+  [field: string]: string;
+}
+
+/**
+ * Sort arguments for an elasticsearch geo distance sort
+ */
+export interface ESGeoDistanceSortArguments {
+  mode: 'avg' | 'max' | 'median' | 'min';
+  order: 'asc' | 'desc';
+  unit: 'm';
+  [field: string]: {
+    lat: number;
+    lon: number;
+  } | string;
+}
+
+/**
+ * An elasticsearch geo distance sort
+ */
+export interface ESGeoDistanceSort {
+  _geo_distance: ESGeoDistanceSortArguments;
+}
+
+/**
+ * An elasticsearch script sort
+ */
+export interface ScriptSort {
+  _script: {
+    order: 'asc' | 'desc';
+    script: string;
+    type: 'number' | 'string';
+  };
+}

src/storage/elasticsearch/monitoring.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCMonitoringConfiguration,
+  SCMonitoringLogAction,
+  SCMonitoringMailAction,
+  SCMonitoringMaximumLengthCondition,
+  SCMonitoringMinimumLengthCondition,
+} from '@openstapps/core';
+import * as ES from 'elasticsearch';
+import * as cron from 'node-cron';
+import { logger } from '../../common';
+import { MailQueue } from '../../notification/MailQueue';
+
+/**
+ * Check if the given condition fails on the given number of results and the condition
+ * @param condition condition
+ * @param total number of results
+ */
+function conditionFails(
+  condition: SCMonitoringMaximumLengthCondition | SCMonitoringMinimumLengthCondition,
+  total: number,
+) {
+  if (condition.type === 'MaximumLength') {
+    return maxConditionFails(condition.length, total);
+  }
+  return minConditionFails(condition.length, total);
+}
+
+/**
+ * Check if the min condition fails
+ * @param minimumLength
+ * @param total
+ */
+function minConditionFails(minimumLength: number, total: number) {
+  return typeof minimumLength === 'number' && minimumLength > total;
+}
+
+/**
+ * Check if the max condition fails
+ * @param maximumLength
+ * @param total
+ */
+function maxConditionFails(maximumLength: number, total: number) {
+  return typeof maximumLength === 'number' && maximumLength < total;
+}
+
+/**
+ * Run all the given actions
+ * @param actions actions to perform
+ * @param watcherName name of watcher that wants to perform them
+ * @param triggerName name of trigger that triggered the watcher
+ * @param total total number of results of the query
+ * @param mailQueue mailQueue to execute mail actions
+ */
+export function runActions(
+  actions: Array<SCMonitoringLogAction | SCMonitoringMailAction>,
+  watcherName: string,
+  triggerName: string,
+  total: number,
+  mailQueue: MailQueue,
+) {
+
+  actions.forEach((action) => {
+    if (action.type === 'log') {
+      logger.error(
+        action.prefix,
+        `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`, `Found ${total} hits instead`,
+        action.message,
+      );
+    } else {
+      mailQueue.push({
+        subject: action.subject,
+        text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'\n` +
+          action.message +
+          `Found ${total} hits instead`,
+        to: action.recipients,
+      });
+    }
+  });
+}
+
+/**
+ * Set up the triggers for the configured watchers
+ * @param monitoringConfig configuration of the monitoring
+ * @param esClient elasticsearch client
+ * @param mailQueue mailQueue for mail actions
+ */
+export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES.Client, mailQueue: MailQueue) {
+
+  // set up Watches
+  monitoringConfig.watchers.forEach((watcher) => {
+
+    // make a schedule for each trigger
+    watcher.triggers.forEach((trigger) => {
+      switch (trigger.executionTime) {
+        case 'hourly':
+          trigger.executionTime = '5 * * * *';
+          break;
+        case 'daily':
+          trigger.executionTime = '5 0 * * *';
+          break;
+        case 'weekly':
+          trigger.executionTime = '5 0 * * 0';
+          break;
+        case 'monthly':
+          trigger.executionTime = '5 0 * 0 * *';
+      }
+
+      cron.schedule(trigger.executionTime, async () => {
+        // execute watch (search->condition->action)
+        const result = await esClient.search(watcher.query);
+
+        // check conditions
+        const total = result.hits.total;
+
+        watcher.conditions.forEach((condition) => {
+          if (conditionFails(condition, total)) {
+            runActions(watcher.actions, watcher.name, trigger.name, total, mailQueue);
+          }
+        });
+      });
+    });
+
+  });
+
+  logger.log('Scheduled ' + monitoringConfig.watchers.length + ' watches');
+}

src/storage/elasticsearch/query.ts

@@ -0,0 +1,408 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCBackendConfigurationSearchBoosting,
+  SCConfigFile,
+  SCSearchBooleanFilter,
+  SCSearchFilter,
+  SCSearchQuery,
+  SCSearchSort,
+  SCSportCoursePriceGroup,
+  SCThingsField,
+  SCThingTypes,
+} from '@openstapps/core';
+import { ElasticsearchConfig, ScriptSort } from './common';
+import {
+  ESBooleanFilter,
+  ESBooleanFilterArguments,
+  ESDucetSort,
+  ESFunctionScoreQuery,
+  ESFunctionScoreQueryFunction,
+  ESGeoDistanceFilter,
+  ESGeoDistanceFilterArguments,
+  ESGeoDistanceSort,
+  ESGeoDistanceSortArguments,
+  ESTermFilter,
+  ESTypeFilter,
+} from './common';
+
+/**
+ * Builds a boolean filter. Returns an elasticsearch boolean filter
+ */
+export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<any> {
+
+  const result: ESBooleanFilterArguments<any> = {
+    minimum_should_match: 0,
+    must: [],
+    must_not: [],
+    should: [],
+  };
+
+  if (booleanFilter.arguments.operation === 'and') {
+    result.must = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
+  }
+
+  if (booleanFilter.arguments.operation === 'or') {
+    result.should = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
+    result.minimum_should_match = 1;
+  }
+
+  if (booleanFilter.arguments.operation === 'not') {
+    result.must_not = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
+  }
+
+  return result;
+}
+
+/**
+ * Converts Array of Filters to elasticsearch query-syntax
+ * @param filter
+ */
+export function buildFilter(filter: SCSearchFilter): ESTermFilter | ESGeoDistanceFilter | ESBooleanFilter<any> {
+
+  switch (filter.type) {
+    case 'value':
+      const filterObj: { [field: string]: string } = {};
+      filterObj[filter.arguments.field + '.raw'] = filter.arguments.value;
+      return {
+        term: filterObj,
+      };
+    case 'availability':
+      const startRangeFilter: { [field: string]: { lte: string } } = {};
+      startRangeFilter[filter.arguments.fromField] = {
+        lte: 'now',
+      };
+
+      const endRangeFilter: { [field: string]: { gte: string } } = {};
+      endRangeFilter[filter.arguments.toField] = {
+        gte: 'now',
+      };
+
+      return {
+        bool: {
+          should: [
+            {
+              bool: {
+                must: [
+                  {
+                    range: startRangeFilter,
+                  },
+                  {
+                    range: endRangeFilter,
+                  },
+                ],
+              },
+            },
+            {
+              bool: {
+                must_not: [
+                  {
+                    exists: {
+                      field: filter.arguments.fromField,
+                    },
+                  },
+                  {
+                    exists: {
+                      field: filter.arguments.toField,
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      };
+    case 'distance':
+      const geoObject: ESGeoDistanceFilterArguments = {
+        distance: filter.arguments.distanceInM + 'm',
+      };
+      geoObject[filter.arguments.field] = {
+        lat: filter.arguments.lat,
+        lon: filter.arguments.lon,
+      };
+      return {
+        geo_distance: geoObject,
+      };
+    case 'boolean':
+      return {
+        bool: buildBooleanFilter(filter),
+      };
+    default:
+      throw new Error('Unknown Filter type');
+  }
+}
+
+/**
+ * Builds scorings functions from boosting config
+ * @param boosting
+ * @returns
+ */
+export function buildFunctions(boosting: SCBackendConfigurationSearchBoosting[]): ESFunctionScoreQueryFunction[] {
+
+  const functions: ESFunctionScoreQueryFunction[] = [];
+
+  // add a good scoring subset from config file
+  boosting.forEach((boostingForOneSCType) => {
+    const typeFilter: ESTypeFilter = {
+      type: {
+        value: boostingForOneSCType.type,
+      },
+    };
+
+    functions.push({
+      filter: typeFilter,
+      weight: boostingForOneSCType.factor,
+    });
+
+    if (typeof boostingForOneSCType.fields !== 'undefined') {
+
+      const fields = boostingForOneSCType.fields;
+
+      Object.keys(boostingForOneSCType.fields).forEach((fieldName) => {
+
+        const boostingForOneField = fields[fieldName];
+
+        Object.keys(boostingForOneField).forEach((value) => {
+          const factor = boostingForOneField[value];
+
+          // build term filter
+          const termFilter: ESTermFilter = {
+            term: {},
+          };
+          termFilter.term[fieldName + '.raw'] = value;
+
+          functions.push({
+            filter: {
+              bool: {
+                must: [
+                  typeFilter,
+                  termFilter,
+                ],
+                should: [],
+              },
+            },
+            weight: factor,
+          });
+        });
+      });
+    }
+  });
+
+  return functions;
+}
+/**
+ * Builds body for Elasticsearch requests
+ * @param params
+ * @param defaultConfig
+ * @returns ElasticsearchQuery (body of a search-request)
+ */
+export function buildQuery(
+  params: SCSearchQuery,
+  defaultConfig: SCConfigFile,
+  elasticsearchConfig: ElasticsearchConfig,
+): ESFunctionScoreQuery {
+
+  // if a sort is used it, we may have to narrow down the types so the sort is executable
+  let typeFiltersToAppend: ESTypeFilter[] = [];
+
+  if (typeof params.sort !== 'undefined') {
+    params.sort.forEach((sort) => {
+      // types that the sort is supported on
+      const types: SCThingTypes[] = [];
+
+      defaultConfig.backend.sortableFields
+        .filter((sortableField) => {
+          return sortableField.fieldName === sort.arguments.field && sortableField.sortTypes.indexOf(sort.type) > -1;
+        })
+        .forEach((sortableField) => {
+          if (typeof sortableField.onlyOnTypes !== 'undefined') {
+            sortableField.onlyOnTypes.forEach((scType) => {
+              if (types.indexOf(scType) === -1) {
+                types.push(scType);
+              }
+            });
+          }
+        });
+
+      if (types.length > 0) {
+        typeFiltersToAppend = types.map((type) => {
+          return {
+            type: {
+              value: type,
+            },
+          };
+        });
+      }
+    });
+  }
+
+  // if config provides an minMatch parameter we use query_string instead of match query
+  let query;
+  if (typeof elasticsearchConfig.query === 'undefined') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: '90%',
+        query: (typeof params.query !== 'string') ? '*' : params.query,
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'query_string') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: elasticsearchConfig.query.minMatch,
+        query: (typeof params.query !== 'string') ? '*' : params.query,
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'dis_max') {
+    if (params.query !== '*') {
+      query = {
+        dis_max: {
+          boost: 1.2,
+          queries: [
+            {
+              match: {
+                name: {
+                  boost: elasticsearchConfig.query.matchBoosting,
+                  cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
+                  fuzziness: elasticsearchConfig.query.fuzziness,
+                  query: (typeof params.query !== 'string') ? '*' : params.query,
+                },
+              },
+            },
+            {
+              query_string: {
+                analyzer: 'search_german',
+                default_field: 'name',
+                minimum_should_match: elasticsearchConfig.query.fuzziness,
+                query: (typeof params.query !== 'string') ? '*' : params.query,
+              },
+            },
+          ],
+          tie_breaker: elasticsearchConfig.query.tieBreaker,
+        },
+
+      };
+    } else {
+      throw new Error('Query Type is not supported. Check your config file and reconfigure your elasticsearch query');
+    }
+  }
+
+  const functionScoreQuery: ESFunctionScoreQuery = {
+    function_score: {
+      functions: buildFunctions(defaultConfig.internal.boostings),
+      query: {
+        bool: {
+          minimum_should_match: 0, // if we have no should, nothing can match
+          must: [],
+          should: [],
+        },
+      },
+      score_mode: 'multiply',
+    },
+  };
+
+  const mustMatch = functionScoreQuery.function_score.query.bool.must;
+
+  if (Array.isArray(mustMatch)) {
+    if (typeof query !== 'undefined') {
+      mustMatch.push(query);
+    }
+
+    if (typeof params.filter !== 'undefined') {
+      mustMatch.push(buildFilter(params.filter));
+    }
+
+    // add type filters for sorts
+    mustMatch.push.apply(mustMatch, typeFiltersToAppend);
+  }
+  return functionScoreQuery;
+}
+
+/**
+ * converts query to
+ * @param params
+ * @param sortableFields
+ * @returns an array of sort queries
+ */
+export function buildSort(
+  sorts: SCSearchSort[],
+): Array<ESDucetSort | ESGeoDistanceSort | ScriptSort> {
+  return sorts.map((sort) => {
+    switch (sort.type) {
+      case 'ducet':
+        const ducetSort: ESDucetSort = {};
+        ducetSort[sort.arguments.field + '.sort'] = sort.order;
+        return ducetSort;
+      case 'distance':
+        const args: ESGeoDistanceSortArguments = {
+          mode: 'avg',
+          order: sort.order,
+          unit: 'm',
+        };
+
+        args[sort.arguments.field] = {
+          lat: sort.arguments.lat,
+          lon: sort.arguments.lon,
+        };
+
+        return {
+          _geo_distance: args,
+        };
+      case 'price':
+        return {
+          _script: {
+            order: sort.order,
+            script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+            type: 'number' as 'number',
+          },
+        };
+    }
+  });
+}
+
+export function buildPriceSortScript(universityRole: keyof SCSportCoursePriceGroup, field: SCThingsField): string {
+  return `
+  // initialize the sort value with the maximum
+  double price = Double.MAX_VALUE;
+
+  // if we have any offers
+  if (params._source.containsKey('${field}')) {
+    // iterate through all offers
+    for (offer in params._source.${field}) {
+      // if this offer contains a role specific price
+      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
+        // if the role specific price is smaller than the cheapest we found
+        if (offer.prices.${universityRole} < price) {
+          // set the role specific price as cheapest for now
+          price = offer.prices.${universityRole};
+        }
+      } else { // we have no role specific price for our role in this offer
+        // if the default price of this offer is lower than the cheapest we found
+        if (offer.price < price) {
+          // set this price as the cheapest
+          price = offer.price;
+        }
+      }
+    }
+  }
+
+  // return cheapest price for our role
+  return price;
+  `;
+}

src/storage/elasticsearch/templates/address.field.template.json

@@ -0,0 +1,19 @@
+{
+  "properties": {
+    "addressCountry": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "addressLocality": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "addressRegion": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "streetAddress": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "postalCode": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/article.sc-type.template.json

@@ -0,0 +1,25 @@
+{
+  "properties": {
+    "authors": {
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "publishers": {
+      "_typeRef": [
+        "person.sc-type.template.json",
+        "organization.sc-type.template.json"
+      ]
+    },
+    "datePublished": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "inLanguages": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "keywords": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "articleBody": {
+      "_fieldRef": "text.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/base.template.json

@@ -0,0 +1,91 @@
+{
+  "template": "stapps_*",
+  "settings": {
+    "max_result_window": 30000,
+    "mapping.total_fields.limit": 2000,
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "analysis": {
+      "filter": {
+        "german_stemmer": {
+          "type":       "stemmer",
+          "language":   "german"
+        },
+        "german_stop": {
+          "type": "stop",
+          "stopwords": "_german_"
+        },
+        "german_phonebook": {
+          "type": "icu_collation",
+          "language": "de",
+          "country": "DE",
+          "variant": "@collation=phonebook"
+        }
+      },
+      "tokenizer": {
+        "stapps_ngram": {
+          "type": "ngram",
+          "min_gram": 4,
+          "max_gram": 7
+        }
+      },
+      "analyzer": {
+        "search_german": {
+          "tokenizer": "stapps_ngram",
+          "filter": [
+            "lowercase",
+            "german_stop",
+            "german_stemmer"
+          ]
+        },
+        "ducet_sort": {
+          "tokenizer": "keyword",
+          "filter": [
+            "german_phonebook"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "_default_": {
+      "properties": {
+        "creation_date": {
+          "type": "date",
+          "store": true
+        },
+        "uid": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "type": {
+          "_fieldRef": "sortableKeyword.field.template.json"
+        },
+        "name": {
+          "_fieldRef": "text.field.template.json"
+        },
+        "alternateNames": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "categories": {
+          "_fieldRef": "sortableKeyword.field.template.json"
+        },
+        "url": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "description": {
+          "_fieldRef": "text.field.template.json"
+        },
+        "image": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        }
+      },
+      "_source": {
+        "excludes": [
+          "creation_date"
+        ]
+      },
+      "date_detection": false,
+      "dynamic_templates": []
+    }
+  }
+}

src/storage/elasticsearch/templates/book.sc-type.template.json

@@ -0,0 +1,31 @@
+{
+  "properties": {
+    "authors": {
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "bookEdition": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "isbn": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "numberOfPages": {
+      "type": "integer"
+    },
+    "publishers": {
+      "_typeRef": "organization.sc-type.template.json"
+    },
+    "datePublished": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "inLanguages": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "keywords": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "articleBody": {
+      "_fieldRef": "text.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/catalog.sc-type.template.json

@@ -0,0 +1,22 @@
+{
+  "properties": {
+    "level": {
+      "type": "integer",
+      "fields": {
+        "raw": {
+          "type": "integer"
+        }
+      }
+    },
+    "semester": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "superCatalog": {
+      "_typeRef": "catalog.sc-type.template.json"
+    },
+    "superCatalogs": {
+      "type": "nested",
+      "_typeRef": "catalog.sc-type.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/date.sc-type.template.json

@@ -0,0 +1,47 @@
+{
+  "properties": {
+    "startDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "endDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "startTime": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "endTime": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "dayOfWeek": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "performers": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "duration": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "frequency": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "frequencyMultiplier": {
+      "type": "float"
+    },
+    "repeatFrequency": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "superEvent": {
+      "_typeRef": "event.sc-type.template.json"
+    },
+    "exceptions": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "dates": {
+      "_fieldRef": "filterableDate.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/diff.sc-type.template.json

@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "dateCreated": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "action": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/dish.sc-type.template.json

@@ -0,0 +1,22 @@
+{
+  "properties": {
+    "availabilityStarts": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "availabilityEnds": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "offers": {
+      "_typeRef": "offers.sc-type.template.json"
+    },
+    "characteristics": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "additives": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/event.sc-type.template.json

@@ -0,0 +1,47 @@
+{
+  "properties": {
+    "subType": {
+      "_fieldRef": "sortableKeyword.field.template.json"
+    },
+    "categories": {
+      "_fieldRef": "sortableKeyword.field.template.json"
+    },
+    "previousStartDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "organizers": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "performers": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "attendees": {
+      "type": "nested",
+      "_typeRef": "person.sc-type.template.json"
+    },
+    "catalogs": {
+      "type": "nested",
+      "_typeRef": "catalog.sc-type.template.json"
+    },
+    "maximumParticipants": {
+      "type": "integer"
+    },
+    "superEvent": {
+      "_typeRef": "event.sc-type.template.json"
+    },
+    "subProperties": {
+      "_fieldRef": "eventSubProperties.field.template.json"
+    },
+    "startDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "endDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/eventSubProperties.field.template.json

@@ -0,0 +1,16 @@
+{
+  "properties": {
+    "id": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "semester": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "majors": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "originalCategory": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/filterableDate.field.template.json

@@ -0,0 +1,8 @@
+{
+  "type": "date",
+  "fields": {
+    "raw": {
+      "type": "keyword"
+    }
+  }
+}

src/storage/elasticsearch/templates/filterableKeyword.field.template.json

@@ -0,0 +1,8 @@
+{
+  "type": "keyword",
+  "fields": {
+    "raw": {
+      "type": "keyword"
+    }
+  }
+}

src/storage/elasticsearch/templates/floorplan.sc-type.template.json

@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "place": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "floor": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/jobs.field.template.json

@@ -0,0 +1,26 @@
+{
+  "properties": {
+    "jobTitle": {
+      "type": "text"
+    },
+    "worksFor": {
+      "_typeRef": "organization.sc-type.template.json"
+    },
+    "workLocation": {
+      "properties": {
+        "email": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "faxNumber": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "telephone": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        },
+        "openingHours": {
+          "_fieldRef": "filterableKeyword.field.template.json"
+        }
+      }
+    }
+  }
+}

src/storage/elasticsearch/templates/offers.sc-type.template.json

@@ -0,0 +1,24 @@
+{
+  "properties": {
+    "price": {
+      "type": "double"
+    },
+    "prices": {
+      "type": "nested",
+      "properties": {
+        "alumni": {
+          "type": "double"
+        },
+        "student": {
+          "type": "double"
+        },
+        "employee": {
+          "type": "double"
+        },
+        "guest": {
+          "type": "double"
+        }
+      }
+    }
+  }
+}

src/storage/elasticsearch/templates/organization.sc-type.template.json

@@ -0,0 +1,7 @@
+{
+  "properties": {
+    "address": {
+      "_fieldRef": "address.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/person.sc-type.template.json

@@ -0,0 +1,47 @@
+{
+  "properties": {
+    "honorificPrefix": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "honorificSuffix": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "givenName": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "additionalName": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "familyName": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "birthDate": {
+      "_fieldRef": "filterableDate.field.template.json"
+    },
+    "gender": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "email": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "faxNumber": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "telephone": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "adress": {
+      "type": "text"
+    },
+    "affiliations": {
+      "type": "nested",
+      "_typeRef": "organization.sc-type.template.json"
+    },
+    "homeLocation": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "nationality": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/place.sc-type.template.json

@@ -0,0 +1,33 @@
+{
+  "properties": {
+    "subType": {
+      "_fieldRef": "sortableKeyword.field.template.json"
+    },
+    "address": {
+      "_fieldRef": "address.field.template.json"
+    },
+    "openingHours": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "geo": {
+      "properties": {
+        "point": {
+          "properties": {
+            "coordinates": {
+              "type": "geo_point"
+            }
+          }
+        },
+        "polygon": {
+          "type": "geo_shape"
+        }
+      }
+    },
+    "superPlace": {
+      "_typeRef": "place.sc-type.template.json"
+    },
+    "subProperties": {
+      "_fieldRef": "placeSubProperties.field.template.json"
+    }
+  }
+}

src/storage/elasticsearch/templates/placeSubProperties.field.template.json

@@ -0,0 +1,38 @@
+{
+  "properties": {
+    "floors": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "floor": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "paymentAccepted": {
+      "_fieldRef": "filterableKeyword.field.template.json"
+    },
+    "roomCharacterization": {
+      "type": "nested",
+      "properties": {
+        "inventory": {
+          "properties": {
+            "key": {
+              "type": "keyword",
+              "fields": {
+                "raw": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "value": {
+              "type": "integer",
+              "fields": {
+                "raw": {
+                  "type": "integer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/storage/elasticsearch/templates/prices.field.template.json

@@ -0,0 +1,14 @@
+{
+  "type": "nested",
+  "properties": {
+    "student": {
+      "type": "float"
+    },
+    "employee": {
+      "type": "float"
+    },
+    "guest": {
+      "type": "float"
+    }
+  }
+}

src/storage/elasticsearch/templates/sortableKeyword.field.template.json

@@ -0,0 +1,15 @@
+{
+  "type": "text",
+  "analyzer": "search_german",
+  "fields": {
+    "raw": {
+      "type": "keyword",
+      "ignore_above": 10000
+    },
+    "sort": {
+      "fielddata": true,
+      "type": "text",
+      "analyzer": "ducet_sort"
+    }
+  }
+}

src/storage/elasticsearch/templates/text.field.template.json

@@ -0,0 +1,10 @@
+{
+  "type": "text",
+  "fielddata": true,
+  "analyzer": "search_german",
+  "fields": {
+    "raw": {
+      "type": "keyword"
+    }
+  }
+}

src/storage/elasticsearch/templating.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Client} from 'elasticsearch';
+import {readdir, readFile} from 'fs-extra';
+import { resolve } from 'path';
+import { logger } from '../../common';
+
+/**
+ * Assembles an elasticsearch template with all resolved subType-references
+ * @param templateType
+ * @param templates
+ * @param inline
+ * @returns
+ */
+function assembleElasticsearchTemplate(templateType: string, templates: {[key: string]: any}, inline: number): any {
+
+  const templateBase = JSON.parse(JSON.stringify(templates[templateType]));
+
+  if (inline) {
+    delete templateBase.dynamic_templates;
+  }
+
+  // these have no properties to replace
+  const excludeBaseFields = [
+    'filterableKeyword.field.template.json',
+    'sortableKeyword.field.template.json',
+    'text.field.template.json',
+    'filterableDate.field.template.json',
+  ];
+
+  if (excludeBaseFields.indexOf(templateType) === -1) {
+
+    try {
+      // extend the template by the properties of the basetemplate
+      templateBase.properties = Object.assign(
+        templateBase.properties,
+        templates['base.template.json'].mappings._default_.properties,
+      );
+    } catch (e) {
+      logger.error('Failed to merge properties on: ' + templateType);
+      throw e;
+    }
+    const fieldKeys = Object.keys(templateBase.properties);
+
+    fieldKeys.forEach((fieldKey) => {
+
+      const field = templateBase.properties[fieldKey];
+      const keys = Object.keys(field);
+
+      // we have subtype-references to replace
+      if (keys.indexOf('_typeRef') > -1) {
+        // if we are already inline of a superObject, we don't resolve types
+        if (inline > 1) {
+          delete templateBase.properties[fieldKey];
+        } else {
+          // we have more than one reference
+          if (Array.isArray(field._typeRef)) {
+            let obj = {};
+            field._typeRef.forEach((subType: string) => {
+              obj = Object.assign(obj, assembleElasticsearchTemplate(subType, templates, inline + 1));
+            });
+            templateBase.properties[fieldKey] = obj;
+          } else {
+            templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._typeRef, templates, inline + 1);
+          }
+        }
+      } else if (keys.indexOf('_fieldRef') > -1) {
+        templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._fieldRef, templates, inline + 1);
+      }
+    });
+  }
+  return templateBase;
+}
+
+/**
+ * Reads all template files and returns the assembled template
+ */
+export async function getElasticsearchTemplate(): Promise<any> {
+
+  // readIM all templates
+  const elasticsearchFolder = resolve('.', 'src', 'storage', 'elasticsearch', 'templates');
+  const templates: {[key: string]: any} = {};
+
+  const fileNames = await readdir(elasticsearchFolder);
+
+  const availableTypes = fileNames.filter((fileName) => {
+    return Array.isArray(fileName.match(/\w*\.sc-type\.template\.json/i));
+  }).map((fileName) => {
+    return fileName.substring(0, fileName.indexOf('.sc-type.template.json'));
+  });
+
+  const promises = fileNames.map(async (fileName) => {
+    const file = await readFile(resolve(elasticsearchFolder, fileName), 'utf8');
+
+    try {
+      templates[fileName] = JSON.parse(file.toString());
+    } catch (jsonParsingError) {
+      logger.error('Failed parsing file: ' + fileName);
+      throw jsonParsingError;
+    }
+  });
+
+  await Promise.all(promises);
+
+  const template = templates['base.template.json'];
+
+  availableTypes.forEach((configType) => {
+    template.mappings[configType.toLowerCase()] =
+      assembleElasticsearchTemplate(configType + '.sc-type.template.json', templates, 0);
+  });
+
+  // this is like the base type (StappsCoreThing)
+  const baseProperties = template.mappings._default_.properties;
+  Object.keys(baseProperties).forEach((basePropertyName) => {
+    let field = baseProperties[basePropertyName];
+    field = templates[field._fieldRef];
+    template.mappings._default_.properties[basePropertyName] = field;
+  });
+
+  return template;
+}
+
+/**
+ * Puts a new global template
+ * @param client
+ */
+export async function putTemplate(client: Client): Promise<void> {
+  return client.indices.putTemplate({
+    body: await getElasticsearchTemplate(),
+    name: 'global',
+  });
+}