refactor: move backend to monorepo

backend/test/app.spec.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2019, 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCNotFoundErrorResponse,
+  SCRequestBodyTooLargeErrorResponse,
+  SCSyntaxErrorResponse,
+  SCUnsupportedMediaTypeErrorResponse,
+} from '@openstapps/core';
+import {expect} from 'chai';
+import {configFile, DEFAULT_TIMEOUT} from '../src/common';
+import {DEFAULT_TEST_TIMEOUT} from './common';
+import {testApp} from './tests-setup';
+import sinon from 'sinon';
+import mockedEnv from 'mocked-env';
+
+describe('App', async function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+  const sandbox = sinon.createSandbox();
+
+  afterEach(function () {
+    sandbox.restore();
+  });
+
+  it('should exit process if there is 20 seconds of pause after a request during the integration test', async function () {
+    const clock = sandbox.useFakeTimers();
+    const processExitStub = sandbox.stub(process, 'exit');
+    // fake NODE_ENV as integration test
+    const restore = mockedEnv({
+      NODE_ENV: 'integration-test',
+    });
+    await testApp.post('/');
+    // fake timeout of default timeout
+    clock.tick(DEFAULT_TIMEOUT);
+
+    expect(processExitStub.called).to.be.true;
+
+    // restore env variables
+    restore();
+    // terminate faking of the clock (setTimeout etc.)
+    clock.restore();
+  });
+
+  it('should provide request body too large error in case of a body larger than the max size', async function () {
+    sandbox.stub(configFile.backend, 'maxRequestBodySize').value('3');
+
+    const {status} = await testApp
+      .post('/')
+      .set('Content-Type', 'application/json')
+      .send({some: 'data larger than 1 byte'});
+
+    expect(status).to.equal(new SCRequestBodyTooLargeErrorResponse().statusCode);
+  });
+
+  it('should implement CORS', async function () {
+    // @ts-expect error
+    const {headers} = await testApp.options('/');
+
+    expect(headers['access-control-allow-origin']).to.be.equal('*');
+  });
+
+  it('should provide unsupported media type error in case of a non-json body', async function () {
+    const responseNoType = await testApp.post('/non-existing-route').send();
+
+    const responseOtherType = await testApp
+      .post('/non-existing-route')
+      .set('Content-Type', 'application/foo')
+      .send();
+
+    expect(responseNoType.status).to.equal(new SCUnsupportedMediaTypeErrorResponse().statusCode);
+    expect(responseOtherType.status).to.equal(new SCUnsupportedMediaTypeErrorResponse().statusCode);
+  });
+
+  it('should provide syntax error if not able to parse the expected JSON body', async function () {
+    const {status} = await testApp
+      .post('/non-existing-route')
+      .set('Content-Type', 'application/json')
+      .send('this is not a JSON');
+
+    expect(status).to.equal(new SCSyntaxErrorResponse('Any message').statusCode);
+  });
+
+  it('should provide route not found error for non-registered routes', async function () {
+    const {status} = await testApp
+      .post('/non-existing-route')
+      .set('Content-Type', 'application/json')
+      .send({});
+
+    expect(status).to.equal(new SCNotFoundErrorResponse().statusCode);
+  });
+});

backend/test/common.spec.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {yearSlice} from '../config/default';
+import {expect} from 'chai';
+
+describe('Common', function () {
+  describe('yearSlice', function () {
+    it('should provide correct ascending month number ranges', function () {
+      expect(yearSlice(1, 12)).to.eql([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]);
+    });
+
+    it('should provide correct month number ranges for year rollovers', function () {
+      expect(yearSlice(12, 1)).to.eql([12, 1]);
+    });
+
+    it('should provide correct month number ranges for a whole year', function () {
+      expect(yearSlice(12, 12)).to.eql([12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]);
+    });
+  });
+});

backend/test/common.ts

@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2019, 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCThingType, SCUuid} from '@openstapps/core';
+import {Express} from 'express';
+import moment from 'moment';
+import {configureApp} from '../src/app';
+import express from 'express';
+import http from 'http';
+import {configFile} from '../src/common';
+import {MailQueue} from '../src/notification/mail-queue';
+import {Bulk, BulkStorage} from '../src/storage/bulk-storage';
+import getPort from 'get-port';
+import {Database} from '../src/storage/database';
+import {Elasticsearch} from '../src/storage/elasticsearch/elasticsearch';
+import {v4} from 'uuid';
+
+/**
+ * Adds routers and configures an (express) app
+ *
+ */
+export async function startApp(): Promise<Express> {
+  const app = express();
+
+  await configureApp(app, {elasticsearch: ElasticsearchMock});
+
+  const server = http.createServer(app);
+
+  // get a random free port
+  const port = await getPort();
+  server.listen(port);
+
+  server.on('error', error => {
+    throw error;
+  });
+
+  return new Promise(resolve =>
+    server.on('listening', () => {
+      app.set('bulk', bulkStorageMock);
+      resolve(app);
+    }),
+  );
+}
+
+/**
+ * An elasticsearch mock
+ */
+export class ElasticsearchMock implements Database {
+  // @ts-expect-error never read
+  private bulk: Bulk | undefined;
+
+  private storageMock = new Map<string, SCThings>();
+
+  constructor(_configFile: SCConfigFile, _mailQueue?: MailQueue) {
+    // Nothing to do here
+  }
+
+  bulkCreated(bulk: Bulk): Promise<void> {
+    this.bulk = bulk;
+    return Promise.resolve(undefined);
+  }
+
+  bulkExpired(_bulk: Bulk): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  bulkUpdated(_bulk: Bulk): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  get(uid: SCUuid): Promise<SCThings> {
+    // @ts-expect-error incompatible types
+    return Promise.resolve(this.storageMock.get(uid));
+  }
+
+  init(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  post(_thing: SCThings, _bulk: Bulk): Promise<void> {
+    return Promise.resolve();
+  }
+
+  put(thing: SCThings): Promise<void> {
+    this.storageMock.set(thing.uid, thing);
+    return Promise.resolve();
+  }
+
+  search(_parameters: SCSearchQuery): Promise<SCSearchResponse> {
+    return Promise.resolve({
+      data: [],
+      facets: [],
+      pagination: {count: 0, offset: 0, total: 0},
+      stats: {time: 0},
+    });
+  }
+}
+
+export const bulkStorageMock = new BulkStorage(new ElasticsearchMock(configFile));
+
+export const bulk: Bulk = {
+  expiration: moment().add(3600, 'seconds').format(),
+  source: 'some_source',
+  state: 'in progress',
+  type: SCThingType.Book,
+  uid: '',
+};
+
+export class FooError extends Error {}
+
+export const DEFAULT_TEST_TIMEOUT = 10_000;
+
+export const TRANSPORT_SEND_RESPONSE = 'Send Response';
+
+export const getTransport = (verified: boolean) => {
+  return {
+    cc: undefined,
+    from: undefined,
+    recipients: undefined,
+    transportAgent: undefined,
+    verified: undefined,
+    isVerified(): boolean {
+      return verified;
+    },
+    send(_subject: string, _message: string): Promise<string> {
+      return Promise.resolve('');
+    },
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    sendMail(_mail: any): Promise<string> {
+      return Promise.resolve(TRANSPORT_SEND_RESPONSE);
+    },
+    verify(): Promise<boolean> {
+      return Promise.resolve(false);
+    },
+  };
+};
+
+export const getIndex = (uid?: string) =>
+  `stapps_footype_foosource_${uid ?? Elasticsearch.getIndexUID(v4())}`;

backend/test/notification/backend-transport.spec.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SMTP} from '@openstapps/logger/lib/smtp';
+import {Transport} from '@openstapps/logger/lib/transport';
+import {expect} from 'chai';
+import mockedEnv from 'mocked-env';
+import {BackendTransport, isTransportWithVerification} from '../../src/notification/backend-transport';
+import sinon from 'sinon';
+
+describe('Backend transport', function () {
+  describe('isTransportWithVerification', function () {
+    it('should return false if transport is not verifiable', function () {
+      expect(isTransportWithVerification({} as Transport)).to.be.false;
+      expect(isTransportWithVerification({verify: 'foo'} as unknown as Transport)).to.be.false;
+    });
+
+    it('should return true if transport is verifiable', function () {
+      // a transport which has verify function should be verifiable
+      // eslint-disable-next-line @typescript-eslint/no-empty-function
+      expect(isTransportWithVerification({verify: () => {}} as unknown as Transport)).to.be.true;
+    });
+  });
+
+  describe('BackendTransport', async function () {
+    const sandbox = sinon.createSandbox();
+
+    afterEach(function () {
+      BackendTransport.destroy();
+      sandbox.restore();
+    });
+
+    it('should provide only one instance of the transport', function () {
+      // @ts-expect-error not assignable
+      sandbox.stub(SMTP, 'getInstance').callsFake(() => {
+        return {};
+      });
+      const transport1 = BackendTransport.getTransportInstance();
+      const transport2 = BackendTransport.getTransportInstance();
+      expect(transport1).to.be.equal(transport2);
+    });
+
+    it('should not throw in case of error getting SMTP instance when transport not allowed', function () {
+      sandbox.stub(SMTP, 'getInstance').throws('Foo Error');
+      const restore = mockedEnv({
+        ALLOW_NO_TRANSPORT: 'true',
+      });
+
+      expect(() => BackendTransport.getTransportInstance()).to.not.throw();
+
+      // restore env variables
+      restore();
+    });
+
+    it('should throw in case of error getting SMTP instance when transport is allowed', function () {
+      sandbox.stub(SMTP, 'getInstance').throws('Foo Error');
+      const restore = mockedEnv({
+        ALLOW_NO_TRANSPORT: undefined,
+      });
+
+      expect(() => BackendTransport.getTransportInstance()).to.throw();
+
+      // restore env variables
+      restore();
+    });
+
+    it('should provide information that the transport if waiting for its verification', function () {
+      // @ts-expect-error not assignable
+      sandbox.stub(SMTP, 'getInstance').callsFake(() => {
+        return {verify: () => Promise.resolve(true)};
+      });
+
+      expect(BackendTransport.getInstance().isWaitingForVerification()).to.be.true;
+    });
+
+    it('should provide information that the transport if not waiting for its verification after the verification is over', function () {
+      sinon.stub(SMTP.prototype, 'verify').resolves(true);
+
+      expect(BackendTransport.getInstance().isWaitingForVerification()).to.be.false;
+    });
+  });
+});

backend/test/notification/mail-queue.spec.ts

@@ -0,0 +1,107 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Logger} from '@openstapps/logger';
+import sinon from 'sinon';
+import {MailQueue} from '../../src/notification/mail-queue';
+import {expect} from 'chai';
+import Queue from 'promise-queue';
+import {MailOptions} from 'nodemailer/lib/sendmail-transport';
+import {getTransport, TRANSPORT_SEND_RESPONSE} from '../common';
+
+describe('MailQueue', async function () {
+  const sandbox = sinon.createSandbox();
+  let clock: sinon.SinonFakeTimers;
+
+  beforeEach(() => {
+    clock = sandbox.useFakeTimers();
+  });
+
+  afterEach(function () {
+    clock.restore();
+    sandbox.restore();
+  });
+
+  it('should fail after maximal number of verification checks', function () {
+    const loggerStub = sandbox.spy(Logger, 'warn');
+    const test = () => {
+      // @ts-expect-error not assignable
+      new MailQueue(getTransport(false));
+      // fake that VERIFICATION_TIMEOUT was reached more times (one more) than MAX_VERIFICATION_ATTEMPTS
+      clock.tick(MailQueue.VERIFICATION_TIMEOUT * (MailQueue.MAX_VERIFICATION_ATTEMPTS + 1));
+    };
+
+    expect(() => test()).to.throw();
+    expect(loggerStub.callCount).to.be.equal(MailQueue.MAX_VERIFICATION_ATTEMPTS);
+  });
+
+  it('should add queued mails to the queue for sending when transport is verified', async function () {
+    const queueAddStub = sandbox.stub(Queue.prototype, 'add');
+    const numberOfMails = 3;
+    const transport = getTransport(false);
+    // @ts-expect-error not assignable
+    const mailQueue = new MailQueue(transport);
+    const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'};
+    for (let i = 0; i < numberOfMails; i++) {
+      await mailQueue.push(mail);
+    }
+
+    // fake that transport is verified
+    transport.isVerified = () => true;
+    clock.tick(MailQueue.VERIFICATION_TIMEOUT);
+
+    expect(queueAddStub.callCount).to.be.equal(numberOfMails);
+  });
+
+  it('should not add SMTP sending tasks to queue when transport is not verified', function () {
+    const queueAddStub = sandbox.stub(Queue.prototype, 'add');
+    // @ts-expect-error not assignable
+    const mailQueue = new MailQueue(getTransport(false));
+    const mail: MailOptions = {};
+    mailQueue.push(mail);
+
+    expect(queueAddStub.called).to.be.false;
+  });
+
+  it('should add SMTP sending tasks to queue when transport is verified', function () {
+    const queueAddStub = sandbox.stub(Queue.prototype, 'add');
+    const transport = getTransport(false);
+    // @ts-expect-error not assignable
+    const mailQueue = new MailQueue(transport);
+    const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'};
+    // fake that transport is verified
+    transport.isVerified = () => true;
+    mailQueue.push(mail);
+
+    expect(queueAddStub.called).to.be.true;
+  });
+
+  it('should send SMTP mails when transport is verified', async function () {
+    let caught: any;
+    sandbox.stub(Queue.prototype, 'add').callsFake(async promiseGenerator => {
+      caught = await promiseGenerator();
+    });
+    const transport = getTransport(false);
+    // @ts-expect-error not assignable
+    const mailQueue = new MailQueue(transport);
+    const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'};
+    // fake that transport is verified
+    transport.isVerified = () => true;
+    await mailQueue.push(mail);
+
+    expect(caught).to.be.equal(TRANSPORT_SEND_RESPONSE);
+  });
+});

backend/test/routes/bulk-route.spec.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright (C) 2019, 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCBulkAddRoute,
+  SCBulkDoneRoute,
+  SCBulkRequest,
+  SCBulkRoute,
+  SCNotFoundErrorResponse,
+} from '@openstapps/core';
+import {expect} from 'chai';
+import {instance as book} from '@openstapps/core/test/resources/indexable/Book.1.json';
+import {bulk, DEFAULT_TEST_TIMEOUT} from '../common';
+import {testApp} from '../tests-setup';
+
+describe('Bulk routes', async function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+
+  const request: SCBulkRequest = {
+    expiration: bulk.expiration,
+    source: bulk.source,
+    type: bulk.type,
+  };
+  const bulkRoute = new SCBulkRoute();
+  const bulkAddRoute = new SCBulkAddRoute();
+  const bulkDoneRoute = new SCBulkDoneRoute();
+
+  // afterEach(async function() {
+  // TODO: Delete saved bulks
+  // });
+
+  it('should create bulk', async function () {
+    const {status, body, error} = await testApp
+      .post(bulkRoute.urlPath)
+      .set('Content-Type', 'application/json')
+      .send(request);
+
+    expect(status).to.be.equal(bulkRoute.statusCodeSuccess);
+    expect(error).to.be.equal(false);
+    expect(body.uid).to.be.a('string');
+    expect(body).to.deep.equal({...bulk, uid: body.uid});
+  });
+
+  it('should return (throw) error if a bulk with the provided UID cannot be found when adding to a bulk', async function () {
+    await testApp.post(bulkRoute.urlPath).set('Content-Type', 'application/json').send(request);
+    const bulkAddRouteUrlPath = bulkAddRoute.urlPath.toLocaleLowerCase().replace(':uid', 'a-wrong-uid');
+
+    const {status} = await testApp
+      .post(bulkAddRouteUrlPath)
+      .set('Content-Type', 'application/json')
+      .send(book);
+
+    expect(status).to.be.equal(new SCNotFoundErrorResponse().statusCode);
+  });
+
+  it('should add to a created bulk', async function () {
+    const response = await testApp
+      .post(bulkRoute.urlPath)
+      .set('Content-Type', 'application/json')
+      .send(request);
+    const bulkAddRouteurlPath = bulkAddRoute.urlPath.toLocaleLowerCase().replace(':uid', response.body.uid);
+
+    const {status, body} = await testApp
+      .post(bulkAddRouteurlPath)
+      .set('Content-Type', 'application/json')
+      .send(book);
+
+    expect(status).to.be.equal(bulkAddRoute.statusCodeSuccess);
+    expect(body).to.be.deep.equal({});
+  });
+
+  it('should return (throw) error if a bulk with the provided UID cannot be found when closing a bulk (done)', async function () {
+    await testApp.post(bulkRoute.urlPath).set('Content-Type', 'application/json').send(request);
+    const bulkDoneRouteurlPath = bulkDoneRoute.urlPath.toLocaleLowerCase().replace(':uid', 'a-wrong-uid');
+
+    const {status} = await testApp
+      .post(bulkDoneRouteurlPath)
+      .set('Content-Type', 'application/json')
+      .send({});
+
+    expect(status).to.be.equal(new SCNotFoundErrorResponse().statusCode);
+  });
+
+  it('should close the bulk (done)', async function () {
+    const response = await testApp
+      .post(bulkRoute.urlPath)
+      .set('Content-Type', 'application/json')
+      .send(request);
+    const bulkDoneRouteurlPath = bulkDoneRoute.urlPath.toLocaleLowerCase().replace(':uid', response.body.uid);
+
+    const response2 = await testApp
+      .post(bulkDoneRouteurlPath)
+      .set('Content-Type', 'application/json')
+      .send({});
+
+    expect(response2.status).to.be.equal(bulkDoneRoute.statusCodeSuccess);
+  });
+});

backend/test/routes/http-types.spec.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCRouteHttpVerbs} from '@openstapps/core';
+import {expect} from 'chai';
+import {isHttpMethod} from '../../src/routes/http-types';
+
+describe('Is HTTP method', async function () {
+  it('should allow valid (predefined) http methods', async function () {
+    // take methods names from SCRouteHttpVerbs
+    const validMethods = Object.values(SCRouteHttpVerbs).map(v => v.toLowerCase());
+
+    for (const method of validMethods) {
+      expect(isHttpMethod(method)).to.be.true;
+    }
+  });
+
+  it('should disallow http methods that are not valid (predefined)', async function () {
+    const invalidMethods = ['foo', 'bar'];
+
+    for (const method of invalidMethods) {
+      expect(isHttpMethod(method)).to.be.false;
+    }
+  });
+});

backend/test/routes/index-route.spec.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2019, 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {DEFAULT_TEST_TIMEOUT} from '../common';
+import {testApp} from '../tests-setup';
+import {SCIndexRequest, SCIndexRoute} from '@openstapps/core';
+import {expect} from 'chai';
+
+describe('Index route', async function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+  const indexRoute = new SCIndexRoute();
+
+  it('should respond with both app and backend configuration', async function () {
+    const request: SCIndexRequest = {};
+
+    const response = await testApp
+      .post(indexRoute.urlPath)
+      .set('Content-Type', 'application/json')
+      .send(request);
+
+    expect(response.type).to.equal('application/json');
+    expect(response.status).to.equal(indexRoute.statusCodeSuccess);
+    expect(response.body).to.haveOwnProperty('app');
+    expect(response.body).to.haveOwnProperty('backend');
+    expect(response.body).to.haveOwnProperty('auth');
+  });
+});

backend/test/routes/plugin-register-route.spec.ts

@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCNotFoundErrorResponse,
+  SCPluginAdd,
+  SCPluginAlreadyRegisteredErrorResponse,
+  SCPluginRegisterResponse,
+  SCPluginRegisterRoute,
+  SCPluginRemove,
+  SCValidationErrorResponse,
+} from '@openstapps/core';
+import nock from 'nock';
+import {configFile, plugins} from '../../src/common';
+import {pluginRegisterHandler} from '../../src/routes/plugin-register-route';
+import {expect, use} from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import {instance as registerRequest} from '@openstapps/core/test/resources/PluginRegisterRequest.1.json';
+import {DEFAULT_TEST_TIMEOUT} from '../common';
+import {testApp} from '../tests-setup';
+
+// for using promises in expectations (to.eventually.be...)
+use(chaiAsPromised);
+
+// cast it because of "TS2322: Type 'string' is not assignable to type '"add"'"
+export const registerAddRequest: SCPluginAdd = registerRequest as SCPluginAdd;
+
+export const registerRemoveRequest: SCPluginRemove = {
+  action: 'remove',
+  route: registerAddRequest.plugin.route,
+};
+
+describe('Plugin registration', async function () {
+  const bodySuccess: SCPluginRegisterResponse = {success: true};
+  describe('Middleware', async function () {
+    after(function () {
+      // remove plugins
+      plugins.clear();
+      configFile.app.features = {};
+    });
+
+    it('should register a plugin', async function () {
+      // register one plugin
+      const response = await pluginRegisterHandler(registerAddRequest, {});
+
+      expect(response).to.deep.equal(bodySuccess) &&
+        expect(plugins.size).to.equal(1) &&
+        expect(configFile.app.features.plugins!['Foo Plugin']).to.not.be.empty;
+    });
+
+    it('should allow re-registering the same plugin', async function () {
+      // register one plugin
+      await pluginRegisterHandler(registerAddRequest, {});
+
+      // register the same plugin again
+      const response = await pluginRegisterHandler(registerAddRequest, {});
+
+      return (
+        expect(response).to.deep.equal(bodySuccess) &&
+        expect(plugins.size).to.equal(1) &&
+        expect(configFile.app.features.plugins!['Foo Plugin']).to.not.be.empty
+      );
+    });
+
+    it('should show an error if a plugin has already been registered', async function () {
+      // register one plugin
+      await pluginRegisterHandler(registerAddRequest, {});
+
+      // create new request for adding a plugin with only name that changed
+      const registerAddRequestAltered: SCPluginAdd = {
+        ...registerAddRequest,
+        plugin: {...registerAddRequest.plugin, name: registerAddRequest.plugin.name + 'foo'},
+      };
+
+      // register the same plugin again
+      expect(pluginRegisterHandler(registerAddRequestAltered, {}))
+        .to.eventually.be.rejectedWith('Plugin already registered')
+        .and.be.an.instanceOf(SCPluginAlreadyRegisteredErrorResponse)
+        // check that the right plugin information (of the previously registered plugin) is provided with the error
+        .and.have.property('additionalData', registerAddRequest.plugin);
+    });
+
+    it('should remove plugin if it exists', async function () {
+      // register one plugin
+      await pluginRegisterHandler(registerAddRequest, {});
+
+      expect(plugins.size).to.equal(1);
+
+      const response = await pluginRegisterHandler(registerRemoveRequest, {});
+
+      expect(response).to.deep.equal(bodySuccess) &&
+        expect(plugins.size).to.equal(0) &&
+        expect(configFile.app.features.plugins).to.be.empty;
+    });
+
+    it('should throw a "not found" error when removing a plugin whose registered route does not exist', async function () {
+      // register one plugin
+      await pluginRegisterHandler(registerAddRequest, {});
+
+      expect(pluginRegisterHandler({...registerRemoveRequest, route: '/not-foo'}, {}))
+        .to.eventually.be.rejectedWith('Resource not found')
+        .and.be.an.instanceOf(SCNotFoundErrorResponse);
+    });
+  });
+
+  describe('Routes', async function () {
+    // increase timeout for the suite
+    this.timeout(DEFAULT_TEST_TIMEOUT);
+    const pluginRegisterRoute = new SCPluginRegisterRoute();
+    const validationError = new SCValidationErrorResponse([]);
+    const notFoundError = new SCNotFoundErrorResponse();
+    // @ts-expect-error not assignable
+    const alreadyRegisteredError = new SCPluginAlreadyRegisteredErrorResponse('Foo Message', {});
+
+    afterEach(async function () {
+      // remove routes
+      plugins.clear();
+      // clean up request mocks (fixes issue with receiving response from mock from previous test case)
+      nock.cleanAll();
+    });
+
+    it('should provide "not found" (404) if plugin/route is not registered', async function () {
+      // lets simulate that the plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+      // mock response of the plugin address
+      nock('http://foo.com:1234')
+        .post('/foo')
+        .reply(200, {result: [{foo: 'bar'}, {bar: 'foo'}]});
+
+      const {status} = await testApp
+        .post('/not-foo')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({query: 'foo'});
+
+      expect(status).to.be.equal(notFoundError.statusCode);
+    });
+
+    it('should respond with bad request if when providing invalid request', async function () {
+      const {status} = await testApp
+        .post(pluginRegisterRoute.urlPath)
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({foo: 'bar'});
+
+      expect(status).to.be.equal(validationError.statusCode);
+    });
+
+    it('should respond with an error if something went wrong with a valid request', async function () {
+      // lets simulate that a plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+      // registering a different plugin for the same route causes the expected error
+      const registerAddRequestAltered = {
+        ...registerAddRequest,
+        plugin: {...registerAddRequest.plugin, name: 'FooBar Plugin'},
+      };
+
+      const {status, body} = await testApp
+        .post(pluginRegisterRoute.urlPath)
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send(registerAddRequestAltered);
+
+      expect(status).to.be.equal(alreadyRegisteredError.statusCode);
+      expect(body).to.haveOwnProperty('name', 'SCPluginAlreadyRegisteredError');
+    });
+
+    it('should respond with success when de-registering already registered plugin', async function () {
+      // lets simulate that a plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+
+      const {status, body} = await testApp
+        .post('/plugin/register')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send(registerRemoveRequest);
+
+      expect(status).to.be.equal(new SCPluginRegisterRoute().statusCodeSuccess);
+      expect(body).to.be.deep.equal(bodySuccess);
+    });
+
+    it('should response with 404 when deleting a plugin which was not registered', async function () {
+      // lets simulate that the plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+
+      const {status} = await testApp
+        .post('/plugin/register')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        // using a different route for the remove request
+        .send({action: 'remove', route: '/not-foo'} as SCPluginRemove);
+
+      expect(status).to.be.equal(notFoundError.statusCode);
+    });
+  });
+});

backend/test/routes/route.spec.ts

@@ -0,0 +1,223 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCInternalServerErrorResponse,
+  SCMethodNotAllowedErrorResponse,
+  SCRoute,
+  SCRouteHttpVerbs,
+  SCValidationErrorResponse,
+} from '@openstapps/core';
+import * as bodyParser from 'body-parser';
+import sinon from 'sinon';
+import {expect} from 'chai';
+import {Application} from 'express';
+import {validator} from '../../src/common';
+import {createRoute} from '../../src/routes/route';
+import express, {Express} from 'express';
+import supertest from 'supertest';
+import {Logger} from '@openstapps/logger';
+import {DEFAULT_TEST_TIMEOUT} from '../common';
+
+interface ReturnType {
+  foo: boolean;
+}
+
+describe('Create route', async function () {
+  let routeClass: SCRoute;
+  let handler: (
+    validatedBody: any,
+    app: Application,
+    parameters?: {[parameterName: string]: string},
+  ) => Promise<ReturnType>;
+  let app: Express;
+  const statusCodeSuccess = 222;
+  const bodySuccess = {foo: true};
+  const sandbox = sinon.createSandbox();
+  const validationError = new SCValidationErrorResponse([]);
+  const internalServerError = new SCInternalServerErrorResponse();
+
+  beforeEach(function () {
+    app = express();
+    app.use(bodyParser.json());
+    routeClass = {
+      errorNames: [],
+      method: SCRouteHttpVerbs.POST,
+      requestBodyName: 'fooBodyName',
+      responseBodyName: 'barBodyName',
+      statusCodeSuccess: statusCodeSuccess,
+      urlPath: '/foo',
+    };
+    handler = (_request, _app) => {
+      return Promise.resolve(bodySuccess);
+    };
+  });
+
+  afterEach(function () {
+    sandbox.restore();
+  });
+
+  it('should complain (throw an error) if provided method is not a valid HTTP verb', async function () {
+    // put a "method" which is not a valid HTTP verb and pretend that it is defined in SCRouteHttpVerbs
+    routeClass.method = 'update' as SCRouteHttpVerbs;
+    expect(() => createRoute<any, any>(routeClass, handler)).to.throw(Error);
+  });
+
+  it('should complain (throw an error) if used method is other than defined in the route creation', async function () {
+    const methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
+    // @ts-expect-error not assignable
+    sandbox.stub(validator, 'validate').returns({errors: []});
+    let error: any = {};
+    sandbox.stub(Logger, 'warn').callsFake(error_ => {
+      error = error_;
+    });
+    const router = createRoute<any, any>(routeClass, handler);
+    await app.use(router);
+
+    const response = await supertest(app)
+      // use method other than defined ("get" is not the method of the route)
+      .get(routeClass.urlPath)
+      .send();
+
+    expect(response.status).to.be.equal(methodNotAllowedError.statusCode);
+    expect(error).to.be.instanceOf(SCMethodNotAllowedErrorResponse);
+  });
+
+  it('should provide a route which returns handler response and success code', async function () {
+    // @ts-expect-error not assignable
+    sandbox.stub(validator, 'validate').returns({errors: []});
+    const router = createRoute<any, any>(routeClass, handler);
+    app.use(router);
+
+    const response = await supertest(app).post(routeClass.urlPath).send();
+
+    expect(response.status).to.be.equal(statusCodeSuccess);
+    expect(response.body).to.be.deep.equal(bodySuccess);
+  });
+
+  it('should complain (throw an error) if provided request is not valid', async function () {
+    this.timeout(DEFAULT_TEST_TIMEOUT);
+    const body = {invalid: 'request'};
+    const router = createRoute<any, any>(routeClass, handler);
+    app.use(router);
+    const startApp = supertest(app);
+    const validatorStub = sandbox.stub(validator, 'validate');
+    // @ts-expect-error not assignable
+    validatorStub.withArgs(body, routeClass.requestBodyName).returns({errors: [new Error('Foo Error')]});
+
+    const response = await startApp
+      .post(routeClass.urlPath)
+      .set('Content-Type', 'application/json')
+      .set('Accept', 'application/json')
+      .send(body);
+
+    expect(response.status).to.be.equal(validationError.statusCode);
+  });
+
+  it('should complain (throw an error) if response got through the handler is not valid', async function () {
+    const router = createRoute<any, any>(routeClass, handler);
+    await app.use(router);
+    const startApp = supertest(app);
+    // @ts-expect-error not assignable
+    const validatorStub = sandbox.stub(validator, 'validate').returns({errors: []});
+    validatorStub
+      .withArgs(bodySuccess, routeClass.responseBodyName)
+      // @ts-expect-error not assignable
+      .returns({errors: [new Error('Foo Error')]});
+
+    const response = await startApp.post(routeClass.urlPath).send();
+
+    expect(response.status).to.be.equal(internalServerError.statusCode);
+  });
+
+  it('should return internal server error if error response not allowed', async function () {
+    class FooErrorResponse {
+      statusCode: number;
+
+      name: string;
+
+      message: string;
+
+      constructor(statusCode: number, name: string, message: string) {
+        this.statusCode = statusCode;
+        this.name = name;
+        this.message = message;
+      }
+    }
+    class BarErrorResponse {
+      statusCode: number;
+
+      constructor(statusCode: number) {
+        this.statusCode = statusCode;
+      }
+    }
+    const routeClassWithErrorNames: SCRoute = {
+      ...routeClass,
+      errorNames: [FooErrorResponse],
+    };
+    const barErrorResponse = new BarErrorResponse(599);
+
+    const handlerThatThrows = () => {
+      throw barErrorResponse;
+    };
+    const router = createRoute<any, any>(routeClassWithErrorNames, handlerThatThrows);
+    await app.use(router);
+    const startApp = supertest(app);
+
+    // @ts-expect-error not assignable
+    sandbox.stub(validator, 'validate').returns({errors: []});
+
+    const response = await startApp.post(routeClass.urlPath).send();
+
+    expect(response.status).to.be.equal(internalServerError.statusCode);
+  });
+
+  it('should return the exact error if error response is allowed', async function () {
+    class FooErrorResponse {
+      statusCode: number;
+
+      name: string;
+
+      message: string;
+
+      constructor(statusCode: number, name: string, message: string) {
+        this.statusCode = statusCode;
+        this.name = name;
+        this.message = message;
+      }
+    }
+    const routeClassWithErrorNames: SCRoute = {
+      ...routeClass,
+      errorNames: [FooErrorResponse],
+    };
+
+    const fooErrorResponse = new FooErrorResponse(598, 'Foo Error', 'Foo Error occurred');
+
+    const handlerThatThrows = () => {
+      throw fooErrorResponse;
+    };
+    const router = createRoute<any, any>(routeClassWithErrorNames, handlerThatThrows);
+    await app.use(router);
+    const startApp = supertest(app);
+
+    // @ts-expect-error not assignable
+    sandbox.stub(validator, 'validate').returns({errors: []});
+
+    const response = await startApp.post(routeClass.urlPath).send();
+
+    expect(response.status).to.be.equal(fooErrorResponse.statusCode);
+  });
+});

backend/test/routes/search-route.spec.ts

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2019, 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCMethodNotAllowedErrorResponse,
+  SCMultiSearchRoute,
+  SCSearchRoute,
+  SCSyntaxErrorResponse,
+  SCTooManyRequestsErrorResponse,
+} from '@openstapps/core';
+import {expect} from 'chai';
+import {configFile} from '../../src/common';
+import {DEFAULT_TEST_TIMEOUT} from '../common';
+import {testApp} from '../tests-setup';
+import sinon from 'sinon';
+
+describe('Search route', async function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+  const searchRoute = new SCSearchRoute();
+  const multiSearchRoute = new SCMultiSearchRoute();
+  const syntaxError = new SCSyntaxErrorResponse('Foo Message');
+  const methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
+  const tooManyRequestsError = new SCTooManyRequestsErrorResponse();
+
+  it('should reject GET, PUT with a valid search query', async function () {
+    // const expectedParams = JSON.parse(JSON.stringify(defaultParams));
+    const {status} = await testApp.get('/search').set('Accept', 'application/json').send({
+      query: 'Some search terms',
+    });
+
+    expect(status).to.equal(methodNotAllowedError.statusCode);
+  });
+
+  describe('Basic POST /search', async function () {
+    it('should accept empty JSON object', async function () {
+      const {status} = await testApp.post(searchRoute.urlPath).set('Accept', 'application/json').send({});
+
+      expect(status).to.equal(searchRoute.statusCodeSuccess);
+    });
+
+    it('should accept valid search request', async function () {
+      const {status} = await testApp.post(searchRoute.urlPath).set('Accept', 'application/json').send({
+        query: 'Some search terms',
+      });
+
+      expect(status).to.equal(searchRoute.statusCodeSuccess);
+    });
+
+    it('should respond with bad request on invalid search request (body)', async function () {
+      const {status} = await testApp
+        .post(searchRoute.urlPath)
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({
+          some: {invalid: 'search'},
+        });
+
+      expect(status).to.equal(syntaxError.statusCode);
+    });
+  });
+
+  describe('Basic POST /multi/search', async function () {
+    it('should respond with bad request on invalid search request (empty JSON object as body)', async function () {
+      const {status} = await testApp
+        .post(multiSearchRoute.urlPath)
+        .set('Accept', 'application/json')
+        .send({});
+
+      expect(status).to.equal(multiSearchRoute.statusCodeSuccess);
+    });
+
+    it('should accept valid search request', async function () {
+      const {status} = await testApp
+        .post(multiSearchRoute.urlPath)
+        .set('Accept', 'application/json')
+        .send({
+          one: {query: 'Some search terms for one search'},
+          two: {query: 'Some search terms for another search'},
+        });
+
+      expect(status).to.equal(multiSearchRoute.statusCodeSuccess);
+    });
+
+    it('should respond with too many requests error if the number of sub-queries exceed their max number', async function () {
+      const sandbox = sinon.createSandbox();
+      sandbox.stub(configFile.backend, 'maxMultiSearchRouteQueries').value(2);
+
+      const {status} = await testApp
+        .post(multiSearchRoute.urlPath)
+        .set('Content-Type', 'application/json')
+        .send({
+          one: {},
+          two: {},
+          three: {},
+        });
+
+      expect(status).to.equal(tooManyRequestsError.statusCode);
+      sandbox.restore();
+    });
+  });
+});

backend/test/routes/thing-update-route.spec.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCThingUpdateRoute} from '@openstapps/core';
+import chaiAsPromised from 'chai-as-promised';
+import {bulkStorageMock, DEFAULT_TEST_TIMEOUT} from '../common';
+import {expect, use} from 'chai';
+import {instance as book} from '@openstapps/core/test/resources/indexable/Book.1.json';
+import {testApp} from '../tests-setup';
+
+use(chaiAsPromised);
+
+describe('Thing update route', async function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+  const thingUpdateRoute = new SCThingUpdateRoute();
+
+  it('should update a thing', async function () {
+    const thingUpdateRouteurlPath = thingUpdateRoute.urlPath
+      .toLocaleLowerCase()
+      .replace(':type', book.type)
+      .replace(':uid', book.uid);
+
+    const {status} = await testApp
+      .put(thingUpdateRouteurlPath)
+      .set('Content-Type', 'application/json')
+      .send(book);
+
+    expect(status).to.equal(thingUpdateRoute.statusCodeSuccess);
+    expect(bulkStorageMock.database.get(book.uid)).to.eventually.be.deep.equal(book);
+  });
+});

backend/test/routes/virtual-plugin-route.spec.ts

@@ -0,0 +1,241 @@
+/* eslint-disable unicorn/consistent-function-scoping,@typescript-eslint/no-explicit-any */
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCInternalServerErrorResponse, SCPluginMetaData, SCValidationErrorResponse} from '@openstapps/core';
+import {expect, use} from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import {Request} from 'express';
+import got, {Options} from 'got';
+import nock from 'nock';
+import sinon from 'sinon';
+import {mockReq} from 'sinon-express-mock';
+import {plugins, validator} from '../../src/common';
+import {virtualPluginRoute} from '../../src/routes/virtual-plugin-route';
+import {DEFAULT_TEST_TIMEOUT, FooError} from '../common';
+import {registerAddRequest} from './plugin-register-route.spec';
+import {testApp} from '../tests-setup';
+
+use(chaiAsPromised);
+
+const plugin = registerAddRequest.plugin;
+
+describe('Virtual plugin routes', async function () {
+  describe('Middleware', async function () {
+    const sandbox = sinon.createSandbox();
+    /**
+     * Internal method which provides information about the specific error inside of an internal server error
+     *
+     * @param request Express request
+     * @param plugin Plugin information (metadata)
+     * @param specificError Class of a specific error
+     */
+    async function testRejection(request: Request, plugin: SCPluginMetaData, specificError: object) {
+      // eslint-disable-next-line unicorn/error-message
+      let thrownError: Error = new Error();
+      try {
+        await virtualPluginRoute(request, plugin);
+      } catch (error) {
+        thrownError = error;
+      }
+      // return virtualPluginRoute(req, plugin).should.be.rejectedWith(SCInternalServerErrorResponse); was not working for some reason
+      expect(thrownError).to.be.instanceOf(SCInternalServerErrorResponse);
+      expect((thrownError as SCInternalServerErrorResponse).additionalData).to.be.instanceOf(specificError);
+    }
+
+    afterEach(function () {
+      // clean up request mocks (fixes issue with receiving response from mock from previous test case)
+      nock.cleanAll();
+      // restore everything to default methods (remove spies and stubs)
+      sandbox.restore();
+    });
+
+    it('should forward body of the request to address and route of the plugin', async function () {
+      const request = {
+        body: {
+          query: 'bar',
+        },
+      };
+      // spy the post method of got
+      // @ts-expect-error not assignable
+      const gotStub = sandbox.stub(got, 'post').returns({body: {}});
+      // @ts-expect-error not assignable
+      sandbox.stub(validator, 'validate').returns({errors: []});
+      const request_ = mockReq(request);
+
+      await virtualPluginRoute(request_, plugin);
+
+      expect(gotStub.args[0][0]).to.equal(plugin.route.slice(1));
+      expect(((gotStub.args[0] as any)[1] as Options).prefixUrl).to.equal(plugin.address);
+      expect(((gotStub.args[0] as any)[1] as Options).json).to.equal(request_.body);
+    });
+
+    it('should provide data from the plugin when its route is called', async function () {
+      const request = {
+        body: {
+          query: 'bar',
+        },
+      };
+      const response = {
+        result: [{foo: 'bar'}, {bar: 'foo'}],
+      };
+      // mock response of the plugin's address
+      nock('http://foo.com:1234').post('/foo').reply(200, response);
+      const request_ = mockReq(request);
+
+      expect(await virtualPluginRoute(request_, plugin)).to.eql(response);
+    });
+
+    it('should throw the validation error if request is not valid', async function () {
+      const request = {
+        body: {
+          invalid_query_field: 'foo',
+        },
+      };
+      const request_ = mockReq(request);
+
+      await testRejection(request_, plugin, SCValidationErrorResponse);
+    });
+
+    it('should throw the validation error if response is not valid', async function () {
+      const request = {
+        body: {
+          query: 'foo',
+        },
+      };
+      // mock response of the plugin service
+      nock('http://foo.com:1234')
+        .post('/foo')
+        .reply(200, {invalid_result: ['foo bar']});
+      const request_ = mockReq(request);
+
+      await testRejection(request_, plugin, SCValidationErrorResponse);
+    });
+
+    it('should throw error if there is a problem with reaching the address of a plugin', async function () {
+      const request = {
+        body: {
+          query: 'foo',
+        },
+      };
+
+      // fake that post method of got throws an error
+      sandbox.stub(got, 'post').callsFake(() => {
+        throw new FooError();
+      });
+      const request_ = mockReq(request);
+
+      await testRejection(request_, plugin, FooError);
+    });
+  });
+
+  describe('Routes', async function () {
+    // increase timeout for the suite
+    this.timeout(DEFAULT_TEST_TIMEOUT);
+    const sandbox = sinon.createSandbox();
+    // http status code
+    const OK = 200;
+    const internalServerError = new SCInternalServerErrorResponse();
+
+    afterEach(async function () {
+      // remove routes
+      plugins.clear();
+      // // restore everything to default methods (remove stubs)
+      sandbox.restore();
+      // clean up request mocks (fixes issue with receiving response from mock from previous test case)
+      nock.cleanAll();
+    });
+
+    it('should properly provide the response of a plugin', async function () {
+      // lets simulate that the plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+      // mock responses of the plugin, depending on the body sent
+      nock('http://foo.com:1234')
+        .post('/foo', {query: 'foo'})
+        .reply(200, {result: [{foo: 'foo'}, {bar: 'foo'}]});
+      nock('http://foo.com:1234')
+        .post('/foo', {query: 'bar'})
+        .reply(200, {result: [{foo: 'bar'}, {bar: 'bar'}]});
+
+      const fooResponse = await testApp
+        .post('/foo')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({query: 'foo'});
+      const barResponse = await testApp
+        .post('/foo')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({query: 'bar'});
+
+      expect(fooResponse.status).to.be.equal(OK);
+      expect(fooResponse.body).to.be.deep.equal({result: [{foo: 'foo'}, {bar: 'foo'}]});
+      expect(barResponse.status).to.be.equal(OK);
+      expect(barResponse.body).to.be.deep.equal({result: [{foo: 'bar'}, {bar: 'bar'}]});
+    });
+
+    it('should return error response if plugin address is not responding', async function () {
+      // lets simulate that the plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+
+      class FooError extends Error {}
+      // fake that got's post method throws an error
+      sandbox.stub(got, 'post').callsFake(() => {
+        throw new FooError();
+      });
+
+      const {status} = await testApp
+        .post('/foo')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({query: 'foo'});
+
+      expect(status).to.be.equal(internalServerError.statusCode);
+    });
+
+    it('should return the validation error response if plugin request is not valid', async function () {
+      // lets simulate that the plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+
+      const {status, body} = await testApp
+        .post('/foo')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        // using number for query instead of (in request schema) required text
+        .send({query: 123});
+
+      expect(status).to.be.equal(502);
+      expect(body.additionalData).to.haveOwnProperty('name', 'ValidationError');
+    });
+
+    it('should return the validation error response if plugin response is not valid', async function () {
+      // lets simulate that the plugin is already registered
+      plugins.set(registerAddRequest.plugin.route, registerAddRequest.plugin);
+      // mock response of the plugin address
+      nock('http://foo.com:1234')
+        .post('/foo')
+        .reply(OK, {not_valid_field: ['foo bar']});
+
+      const {status, body} = await testApp
+        .post('/foo')
+        .set('Content-Type', 'application/json')
+        .set('Accept', 'application/json')
+        .send({query: 'foo'});
+
+      expect(status).to.be.equal(internalServerError.statusCode);
+      expect(body.additionalData).to.haveOwnProperty('name', 'ValidationError');
+    });
+  });
+});

backend/test/storage/bulk-storage.spec.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCBulkRequest, SCThingType} from '@openstapps/core';
+import moment from 'moment';
+// eslint-disable-next-line unicorn/import-style
+import util from 'util';
+import {configFile} from '../../src/common';
+import {Bulk, BulkStorage} from '../../src/storage/bulk-storage';
+import {expect} from 'chai';
+import {ElasticsearchMock} from '../common';
+import sinon from 'sinon';
+import NodeCache from 'node-cache';
+
+describe('Bulk Storage', function () {
+  describe('Bulk', function () {
+    let bulkRequest: SCBulkRequest;
+    beforeEach(function () {
+      bulkRequest = {source: 'some_source', type: SCThingType.Book};
+    });
+
+    it('should create a bulk with the given expiration', function () {
+      const expiration = moment().add(3600, 'seconds').toISOString();
+      bulkRequest.expiration = expiration;
+
+      const bulk = new Bulk(bulkRequest);
+
+      expect(bulk.expiration).to.be.equal(expiration);
+      expect(bulk.state).to.be.equal('in progress');
+      expect(bulk.uid).to.not.be.undefined;
+    });
+
+    it('should fallback and set expiration when it is not provided', function () {
+      const bulk = new Bulk(bulkRequest);
+
+      expect(bulk.expiration).to.not.be.undefined;
+    });
+  });
+
+  describe('BulkStorage', async function () {
+    const sandbox = sinon.createSandbox();
+    const bulkRequest = {source: 'some_source', type: SCThingType.Book};
+    const bulk = new Bulk(bulkRequest);
+    bulk.uid = '123';
+    let esMock: sinon.SinonStub;
+    let database: ElasticsearchMock;
+
+    beforeEach(function () {
+      database = new ElasticsearchMock(configFile);
+      esMock = sandbox.stub(database, 'bulkExpired');
+    });
+
+    afterEach(function () {
+      sandbox.restore();
+    });
+
+    it('should call appropriate database clean-up method on expire', async function () {
+      sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
+      new BulkStorage(database);
+
+      expect(esMock.calledWith(bulk)).to.be.true;
+    });
+
+    it("should not call appropriate database clean-up method on expire if bulk's state is done", async function () {
+      bulk.state = 'done';
+      sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
+      new BulkStorage(database);
+
+      expect(esMock.called).to.be.false;
+    });
+
+    it('should throw an error if the bulk for deletion cannot be read', async function () {
+      sandbox.stub(BulkStorage.prototype, 'read').callsFake(() => undefined);
+      const bulkStorage = new BulkStorage(database);
+
+      return expect(bulkStorage.delete('123')).to.be.rejected;
+    });
+
+    it('should delete a bulk', async function () {
+      const readStub = sandbox.stub(BulkStorage.prototype, 'read').callsFake(() => bulk);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      let caught: any;
+      sandbox.stub(NodeCache.prototype, 'del').callsFake(() => (caught = 123));
+      // force call
+      sandbox
+        .stub(util, 'promisify')
+        // eslint-disable-next-line @typescript-eslint/no-empty-function,unicorn/consistent-function-scoping
+        .callsFake(() => () => {})
+        // eslint-disable-next-line unicorn/no-null
+        .yields(null);
+      const bulkStorage = new BulkStorage(database);
+
+      await bulkStorage.delete(bulk.uid);
+
+      expect(readStub.called).to.be.true;
+      expect(caught).to.be.equal(123);
+      expect(esMock.called).to.be.true;
+    });
+
+    it('should read an existing bulk', async function () {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      let caught: any;
+      sandbox.stub(NodeCache.prototype, 'get').callsFake(() => (caught = 123));
+      // force call
+      sandbox
+        .stub(util, 'promisify')
+        // eslint-disable-next-line unicorn/consistent-function-scoping,@typescript-eslint/no-empty-function
+        .callsFake(() => () => {})
+        // eslint-disable-next-line unicorn/no-null
+        .yields(null);
+      const bulkStorage = new BulkStorage(database);
+
+      await bulkStorage.read(bulk.uid);
+
+      expect(caught).to.be.equal(123);
+    });
+    ``;
+  });
+});

backend/test/storage/elasticsearch/aggregations.spec.ts

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCFacet, SCThingType} from '@openstapps/core';
+import {expect} from 'chai';
+import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
+import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch';
+
+describe('Aggregations', function () {
+  const aggregations: AggregationResponse = {
+    'catalog': {
+      'doc_count': 4,
+      'superCatalogs.categories': {
+        buckets: [],
+      },
+      'academicTerm.acronym': {
+        buckets: [
+          {
+            key: 'SoSe 2020',
+            doc_count: 2,
+          },
+        ],
+      },
+      'superCatalog.categories': {
+        buckets: [],
+      },
+      'categories': {
+        buckets: [
+          {
+            key: 'foo',
+            doc_count: 1,
+          },
+          {
+            key: 'bar',
+            doc_count: 3,
+          },
+        ],
+      },
+    },
+    'person': {
+      'doc_count': 13,
+      'homeLocations.categories': {
+        buckets: [],
+      },
+    },
+    'academic event': {
+      'doc_count': 0,
+      'academicTerms.acronym': {
+        buckets: [],
+      },
+      'categories': {
+        buckets: [
+          {
+            key: 'foobar',
+            doc_count: 8,
+          },
+          {
+            key: 'bar',
+            doc_count: 2,
+          },
+        ],
+      },
+      'creativeWorks.keywords': {
+        buckets: [],
+      },
+    },
+    'fooType': {
+      buckets: [
+        {
+          doc_count: 321,
+          key: 'foo',
+        },
+      ],
+    },
+    '@all': {
+      doc_count: 17,
+      type: {
+        buckets: [
+          {
+            key: 'person',
+            doc_count: 13,
+          },
+          {
+            key: 'catalog',
+            doc_count: 4,
+          },
+        ],
+      },
+    },
+  };
+
+  const expectedFacets: SCFacet[] = [
+    {
+      buckets: [
+        {
+          count: 13,
+          key: 'person',
+        },
+        {
+          count: 4,
+          key: 'catalog',
+        },
+      ],
+      field: 'type',
+    },
+    {
+      buckets: [
+        {
+          count: 8,
+          key: 'foobar',
+        },
+        {
+          count: 2,
+          key: 'bar',
+        },
+      ],
+      field: 'categories',
+      onlyOnType: SCThingType.AcademicEvent,
+    },
+    {
+      buckets: [
+        {
+          count: 2,
+          key: 'SoSe 2020',
+        },
+      ],
+      field: 'academicTerm.acronym',
+      onlyOnType: SCThingType.Catalog,
+    },
+    {
+      buckets: [
+        {
+          count: 1,
+          key: 'foo',
+        },
+        {
+          count: 3,
+          key: 'bar',
+        },
+      ],
+      field: 'categories',
+      onlyOnType: SCThingType.Catalog,
+    },
+    // no fooType as it doesn't appear in the aggregation schema
+  ];
+
+  it('should parse the aggregations providing the appropriate facets', function () {
+    const facets = parseAggregations(aggregations);
+
+    expect(facets).to.be.eql(expectedFacets);
+  });
+});

backend/test/storage/elasticsearch/common.spec.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  ESAggMatchAllFilter,
+  ESAggTypeFilter,
+  ESNestedAggregation,
+  ESTermsFilter,
+} from '@openstapps/es-mapping-generator/src/types/aggregation';
+import {expect} from 'chai';
+import {
+  isNestedAggregation,
+  isBucketAggregation,
+  isESTermsFilter,
+  isESAggMatchAllFilter,
+  isESNestedAggregation,
+} from '../../../lib/storage/elasticsearch/types/guards';
+import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';
+
+describe('Common', function () {
+  const bucketAggregation: BucketAggregation = {buckets: []};
+  const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
+  const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
+
+  describe('isBucketAggregation', function () {
+    it('should be false for a number', function () {
+      expect(isBucketAggregation(123)).to.be.false;
+    });
+
+    it('should be true for a bucket aggregation', function () {
+      expect(isBucketAggregation(bucketAggregation)).to.be.true;
+    });
+  });
+
+  describe('isNestedAggregation', function () {
+    it('should be false for a bucket aggregation', function () {
+      expect(isNestedAggregation(bucketAggregation)).to.be.false;
+    });
+
+    it('should be true for a nested aggregation', function () {
+      const nestedAggregation: NestedAggregation = {doc_count: 123};
+
+      expect(isNestedAggregation(nestedAggregation)).to.be.true;
+    });
+  });
+
+  describe('isESTermsFilter', function () {
+    it('should be false for an elasticsearch nested aggregation', function () {
+      expect(isESTermsFilter(esNestedAggregation)).to.be.false;
+    });
+
+    it('should be true for an elasticsearch terms filter', function () {
+      expect(isESTermsFilter(esTermsFilter)).to.be.true;
+    });
+  });
+
+  describe('isESNestedAggregation', function () {
+    it('should be false for an elasticsearch terms filter', function () {
+      expect(isESNestedAggregation(esTermsFilter)).to.be.false;
+    });
+
+    it('should be true for an elasticsearch nested aggregation', function () {
+      expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
+    });
+  });
+
+  describe('isESAggMatchAllFilter', function () {
+    it('should be false for an elasticsearch aggregation type filter', function () {
+      const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
+      expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
+    });
+
+    it('should be true for an elasticsearch aggregation match all filter', function () {
+      const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
+      expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
+    });
+  });
+});

backend/test/storage/elasticsearch/elasticsearch.spec.ts

@@ -0,0 +1,683 @@
+/* eslint-disable @typescript-eslint/no-explicit-any,unicorn/no-null */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {
+  SCBook,
+  SCBulkResponse,
+  SCConfigFile,
+  SCMessage,
+  SCSearchQuery,
+  SCThings,
+  SCThingType,
+} from '@openstapps/core';
+import {instance as book} from '@openstapps/core/test/resources/indexable/Book.1.json';
+import {instance as message} from '@openstapps/core/test/resources/indexable/Message.1.json';
+import {Logger} from '@openstapps/logger';
+import {SMTP} from '@openstapps/logger/lib/smtp';
+import {expect, use} from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import {SearchResponse} from 'elasticsearch';
+import mockedEnv from 'mocked-env';
+import sinon from 'sinon';
+import {configFile} from '../../../src/common';
+import {MailQueue} from '../../../src/notification/mail-queue';
+import {aggregations} from '../../../src/storage/elasticsearch/templating';
+import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch';
+import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
+import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
+import * as query from '../../../src/storage/elasticsearch/query';
+import * as templating from '../../../src/storage/elasticsearch/templating';
+import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common';
+import fs from 'fs';
+
+use(chaiAsPromised);
+
+describe('Elasticsearch', function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+  const sandbox = sinon.createSandbox();
+
+  before(function () {
+    // eslint-disable-next-line no-console
+    console.log('before');
+    sandbox.stub(fs, 'readFileSync').returns('{}');
+  });
+  after(function () {
+    sandbox.restore();
+  });
+
+  describe('getElasticsearchUrl', function () {
+    it('should provide custom elasticsearch URL if defined', function () {
+      const customAddress = 'http://foo-address:9200';
+      const restore = mockedEnv({
+        ES_ADDR: customAddress,
+      });
+
+      expect(Elasticsearch.getElasticsearchUrl()).to.be.equal(customAddress);
+      // restore env variables
+      restore();
+    });
+
+    it('should provide local URL as fallback', function () {
+      const restore = mockedEnv({
+        ES_ADDR: undefined,
+      });
+
+      expect(Elasticsearch.getElasticsearchUrl()).to.match(/(https?:\/\/)?localhost(:\d+)?/);
+      // restore env variables
+      restore();
+    });
+  });
+
+  describe('getIndex (including getIndexUID)', function () {
+    const type = 'foo bar type';
+    const source = 'foo_source';
+    const bulk: SCBulkResponse = {
+      expiration: '',
+      source: '',
+      state: 'in progress',
+      type: SCThingType.Semester,
+      uid: 'bulk-uid-123-123-123',
+    };
+
+    it('should provide index UID from the provided UID', function () {
+      const indexUID = Elasticsearch.getIndexUID(bulk.uid);
+
+      expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
+      // test starting and ending character
+      expect(indexUID[0]).to.be.equal(bulk.uid[0]);
+      expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
+    });
+
+    it('should provide index name from the provided data', function () {
+      expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
+        `stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
+      );
+    });
+  });
+
+  describe('removeAliasChars', function () {
+    it('should remove spaces from both ends', function () {
+      expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
+    });
+
+    it('should replace inner spaces with underscores', function () {
+      expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
+    });
+
+    it('should remove invalid characters', function () {
+      expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
+    });
+
+    it('should remove invalid starting characters', function () {
+      expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+    });
+
+    it('should replace with a placeholder in case of invalid alias', function () {
+      expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
+      expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
+    });
+
+    it('should work with common cases', function () {
+      expect(
+        Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
+      ).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
+      expect(
+        Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
+      ).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
+    });
+
+    it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
+      const sandbox = sinon.createSandbox();
+      const loggerWarnStub = sandbox.stub(Logger, 'warn');
+
+      expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
+      expect(loggerWarnStub.called).to.be.true;
+    });
+  });
+
+  describe('constructor', async function () {
+    const sandbox = sinon.createSandbox();
+    afterEach(function () {
+      sandbox.restore();
+    });
+
+    it('should complain (throw an error) if database in config is undefined', function () {
+      const config: SCConfigFile = {...configFile, internal: {...configFile.internal, database: undefined}};
+
+      expect(() => new Elasticsearch(config)).to.throw(Error);
+    });
+
+    it('should complain (throw an error) if database version is not a string', function () {
+      const config: SCConfigFile = {
+        ...configFile,
+        internal: {
+          ...configFile.internal,
+          database: {
+            name: 'foo',
+            version: 123,
+          },
+        },
+      };
+
+      expect(() => new Elasticsearch(config)).to.throw(Error);
+    });
+
+    it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
+      const error = new Error('Foo Error');
+      const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
+      sandbox.stub(Client.prototype, 'on').yields(error);
+
+      new Elasticsearch(configFile);
+
+      expect(loggerErrorStub.calledWith(error)).to.be.true;
+    });
+
+    it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
+      const fakeResponse = {foo: 'bar'};
+      const loggerLogStub = sandbox.stub(Logger, 'log');
+      sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
+
+      new Elasticsearch(configFile);
+      expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
+
+      const restore = mockedEnv({
+        ES_DEBUG: 'true',
+      });
+      new Elasticsearch(configFile);
+
+      expect(loggerLogStub.calledWith(fakeResponse)).to.be.true;
+      // restore env variables
+      restore();
+    });
+
+    describe('init', async function () {
+      const sandbox = sinon.createSandbox();
+      after(function () {
+        sandbox.restore();
+      });
+
+      it('should complain (throw an error) if monitoring is set but mail queue is undefined', async function () {
+        const config: SCConfigFile = {
+          ...configFile,
+          internal: {
+            ...configFile.internal,
+            monitoring: {
+              actions: [],
+              watchers: [],
+            },
+          },
+        };
+
+        const es = new Elasticsearch(config);
+
+        return expect(es.init()).to.be.rejected;
+      });
+
+      it('should setup the monitoring if there is monitoring is set and mail queue is defined', function () {
+        const config: SCConfigFile = {
+          ...configFile,
+          internal: {
+            ...configFile.internal,
+            monitoring: {
+              actions: [],
+              watchers: [],
+            },
+          },
+        };
+        const monitoringSetUpStub = sandbox.stub(Monitoring, 'setUp');
+
+        const es = new Elasticsearch(config, new MailQueue(getTransport(false) as unknown as SMTP));
+
+        es.init();
+
+        expect(monitoringSetUpStub.called).to.be.true;
+      });
+    });
+
+    describe('Operations with bundle/index', async function () {
+      const sandbox = sinon.createSandbox();
+      let es: Elasticsearch;
+      const oldIndex = 'stapps_footype_foosource_oldindex';
+
+      beforeEach(function () {
+        es = new Elasticsearch(configFile);
+        es.client.indices = {
+          // @ts-expect-error not assignable
+          getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
+          // @ts-expect-error not assignable
+          putTemplate: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          create: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          delete: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          exists: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          refresh: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          updateAliases: () => Promise.resolve({}),
+        };
+      });
+
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      describe('bulkCreated', async function () {
+        it('should reject (throw an error) if the connection to elasticsearch is not established', async function () {
+          return expect(es.bulkCreated(bulk)).to.be.rejectedWith('elasticsearch');
+        });
+
+        it('should reject (throw an error) if the index name is not valid', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`);
+          sandbox.createStubInstance(Client, {});
+          await es.init();
+
+          return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
+        });
+
+        it('should create a new index', async function () {
+          const index = getIndex();
+          sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+          const putTemplateStub = sandbox.stub(templating, 'putTemplate');
+          const createStub = sandbox.stub(es.client.indices, 'create');
+          await es.init();
+
+          await es.bulkCreated(bulk);
+
+          expect(putTemplateStub.called).to.be.true;
+          expect(createStub.calledWith({index})).to.be.true;
+        });
+      });
+
+      describe('bulkExpired', async function () {
+        const sandbox = sinon.createSandbox();
+        afterEach(function () {
+          sandbox.restore();
+        });
+        it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+          const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
+
+          await es.bulkExpired({...bulk, state: 'in progress'});
+
+          expect(clientDeleteStub.called).to.be.true;
+        });
+
+        it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+          const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
+
+          await es.bulkExpired({...bulk, state: 'done'});
+
+          expect(clientDeleteStub.called).to.be.false;
+        });
+      });
+
+      describe('bulkUpdated', function () {
+        it('should reject if the connection to elasticsearch is not established', async function () {
+          return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('elasticsearch');
+        });
+
+        it('should reject if the index name is not valid', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`);
+          sandbox.createStubInstance(Client, {});
+          await es.init();
+
+          return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
+        });
+
+        it('should create a new index', async function () {
+          const index = getIndex();
+          const expectedRefreshActions = [
+            {
+              add: {index: index, alias: SCThingType.Book},
+            },
+            {
+              remove: {index: oldIndex, alias: SCThingType.Book},
+            },
+          ];
+          sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+          sandbox.stub(es, 'aliasMap').value({
+            [SCThingType.Book]: {
+              [bulk.source]: oldIndex,
+            },
+          });
+          const refreshStub = sandbox.stub(es.client.indices, 'refresh');
+          const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
+          const deleteStub = sandbox.stub(es.client.indices, 'delete');
+          sandbox.stub(templating, 'putTemplate');
+          await es.init();
+
+          await es.bulkUpdated(bulk);
+
+          expect(refreshStub.calledWith({index})).to.be.true;
+          expect(
+            updateAliasesStub.calledWith({
+              body: {
+                actions: expectedRefreshActions,
+              },
+            }),
+          ).to.be.true;
+          expect(deleteStub.called).to.be.true;
+        });
+      });
+    });
+
+    describe('get', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      it('should reject if object is not found', async function () {
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+
+        return expect(es.get('123')).to.rejectedWith('found');
+      });
+
+      it('should provide the thing if object is found', async function () {
+        const foundObject: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: '',
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
+
+        return expect(await es.get('123')).to.be.eql(message);
+      });
+    });
+
+    describe('post', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      it('should not post if the object already exists in an index which will not be rolled over', async function () {
+        const index = getIndex();
+        const oldIndex = index.replace('foosource', 'barsource');
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: oldIndex,
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+
+        return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
+      });
+
+      it('should not reject if the object already exists but in an index which will be rolled over', async function () {
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: getIndex(),
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        // return index name with different generated UID (see getIndex method)
+        sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+
+        return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist');
+      });
+
+      it('should reject if there is an object creation error on the elasticsearch side', async function () {
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        sandbox.stub(es.client, 'create').resolves({body: {created: false}});
+
+        return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
+      });
+
+      it('should create a new object', async function () {
+        let caughtParameter: any;
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        // @ts-expect-error call
+        const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
+          caughtParameter = parameter;
+          return Promise.resolve({body: {created: true}});
+        });
+
+        await es.post(message as SCMessage, bulk);
+
+        expect(createStub.called).to.be.true;
+        expect(caughtParameter.body).to.be.eql({
+          ...message,
+          creation_date: caughtParameter.body.creation_date,
+        });
+      });
+    });
+
+    describe('put', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+      afterEach(function () {
+        sandbox.restore();
+      });
+      it('should reject to put if the object does not already exist', async function () {
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: getIndex(),
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+
+        return expect(es.put(object._source)).to.rejectedWith('exist');
+      });
+
+      // noinspection JSUnusedLocalSymbols
+      it('should update the object if it already exists', async function () {
+        let caughtParameter: any;
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: getIndex(),
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        // @ts-expect-error unused
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
+          caughtParameter = parameters;
+          return Promise.resolve({body: {created: true}});
+        });
+
+        await es.put(object._source);
+
+        expect(caughtParameter.body.doc).to.be.eql(object._source);
+      });
+    });
+
+    describe('search', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+      const objectMessage: ElasticsearchObject<SCMessage> = {
+        _id: '123',
+        _index: getIndex(),
+        _score: 0,
+        _type: '',
+        _source: message as SCMessage,
+      };
+      const objectBook: ElasticsearchObject<SCBook> = {
+        _id: '321',
+        _index: getIndex(),
+        _score: 0,
+        _type: '',
+        _source: book as SCBook,
+      };
+      const fakeEsAggregations = {
+        '@all': {
+          doc_count: 17,
+          type: {
+            doc_count_error_upper_bound: 0,
+            sum_other_doc_count: 0,
+            buckets: [
+              {
+                key: 'person',
+                doc_count: 13,
+              },
+              {
+                key: 'catalog',
+                doc_count: 4,
+              },
+            ],
+          },
+        },
+      };
+      const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
+        body: {
+          took: 12,
+          timed_out: false,
+          // @ts-expect-error not assignable
+          _shards: {},
+          // @ts-expect-error not assignable
+          hits: {
+            hits: [objectMessage, objectBook],
+            total: 123,
+          },
+          aggregations: fakeEsAggregations,
+        },
+        headers: {},
+        // @ts-expect-error not assignable
+        meta: {},
+        // @ts-expect-error not assignable
+        statusCode: {},
+        // @ts-expect-error not assignable
+        warnings: {},
+      };
+      let searchStub: sinon.SinonStub;
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+      beforeEach(function () {
+        searchStub = sandbox.stub(es.client, 'search').resolves(fakeSearchResponse);
+      });
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      it('should provide appropriate data and facets', async function () {
+        const fakeFacets = [
+          {
+            buckets: [
+              {
+                count: 13,
+                key: 'person',
+              },
+              {
+                count: 4,
+                key: 'catalog',
+              },
+            ],
+            field: 'type',
+          },
+        ];
+
+        const {data, facets} = await es.search({});
+
+        expect(data).to.be.eql([objectMessage._source, objectBook._source]);
+        expect(facets).to.be.eql(fakeFacets);
+      });
+
+      it('should provide pagination from params', async function () {
+        const from = 30;
+        const {pagination} = await es.search({from});
+
+        expect(pagination).to.be.eql({
+          count: fakeSearchResponse.body!.hits.hits.length,
+          offset: from,
+          total: fakeSearchResponse.body!.hits.total,
+        });
+      });
+
+      it('should have fallback to zero if from is not given through params', async function () {
+        const {pagination} = await es.search({});
+
+        expect(pagination.offset).to.be.equal(0);
+      });
+
+      it('should build the search request properly', async function () {
+        const parameters: SCSearchQuery = {
+          query: 'mathematics',
+          from: 30,
+          size: 5,
+          sort: [
+            {
+              type: 'ducet',
+              order: 'desc',
+              arguments: {
+                field: 'name',
+              },
+            },
+          ],
+          filter: {
+            type: 'value',
+            arguments: {
+              field: 'type',
+              value: SCThingType.AcademicEvent,
+            },
+          },
+        };
+        const fakeResponse = {foo: 'bar'};
+        const fakeBuildSortResponse = [fakeResponse];
+        // @ts-expect-error not assignable
+        sandbox.stub(query, 'buildQuery').returns(fakeResponse);
+        sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
+
+        await es.search(parameters);
+
+        sandbox.assert.calledWithMatch(searchStub, {
+          body: {
+            aggs: aggregations,
+            query: fakeResponse,
+            sort: fakeBuildSortResponse,
+          },
+          from: parameters.from,
+          index: Elasticsearch.getListOfAllIndices(),
+          size: parameters.size,
+        });
+      });
+    });
+  });
+});

backend/test/storage/elasticsearch/monitoring.spec.ts

@@ -0,0 +1,136 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {
+  SCMonitoringConfiguration,
+  SCMonitoringLogAction,
+  SCMonitoringMailAction,
+  SCMonitoringWatcher,
+  SCThings,
+} from '@openstapps/core';
+import {Logger} from '@openstapps/logger';
+import {SearchResponse} from 'elasticsearch';
+import {MailQueue} from '../../../src/notification/mail-queue';
+import {setUp} from '../../../src/storage/elasticsearch/monitoring';
+
+import {getTransport} from '../../common';
+import {expect} from 'chai';
+import sinon from 'sinon';
+import cron from 'node-cron';
+
+describe('Monitoring', async function () {
+  const sandbox = sinon.createSandbox();
+  const logAction: SCMonitoringLogAction = {
+    message: 'Foo monitoring message',
+    prefix: 'Backend Monitoring',
+    type: 'log',
+  };
+  const mailAction: SCMonitoringMailAction = {
+    message: 'Bar monitoring message',
+    recipients: ['xyz@xyz.com'],
+    subject: 'Backend Monitoring',
+    type: 'mail',
+  };
+  let transport: any;
+  let mailQueue: any;
+  beforeEach(async function () {
+    transport = getTransport(true);
+    mailQueue = new MailQueue(transport);
+    cronScheduleStub = sandbox.stub(cron, 'schedule');
+  });
+  afterEach(async function () {
+    sandbox.restore();
+  });
+  // const sandbox = sinon.createSandbox();
+  let cronScheduleStub: sinon.SinonStub;
+  const minLengthWatcher: SCMonitoringWatcher = {
+    actions: [logAction, mailAction],
+    conditions: [
+      {
+        length: 10,
+        type: 'MinimumLength',
+      },
+    ],
+    name: 'foo watcher',
+    query: {foo: 'bar'},
+    triggers: [
+      {
+        executionTime: 'monthly',
+        name: 'beginning of month',
+      },
+      {
+        executionTime: 'daily',
+        name: 'every night',
+      },
+    ],
+  };
+  const maxLengthWatcher: SCMonitoringWatcher = {
+    actions: [logAction, mailAction],
+    conditions: [
+      {
+        length: 30,
+        type: 'MaximumLength',
+      },
+    ],
+    name: 'foo watcher',
+    query: {bar: 'foo'},
+    triggers: [
+      {
+        executionTime: 'hourly',
+        name: 'every hour',
+      },
+      {
+        executionTime: 'weekly',
+        name: 'every week',
+      },
+    ],
+  };
+  const monitoringConfig: SCMonitoringConfiguration = {
+    actions: [logAction, mailAction],
+    watchers: [minLengthWatcher, maxLengthWatcher],
+  };
+
+  it('should create a schedule for each trigger', async function () {
+    await setUp(monitoringConfig, new Client({node: 'http://foohost:9200'}), mailQueue);
+
+    expect(cronScheduleStub.callCount).to.be.equal(4);
+  });
+
+  it('should log errors where conditions failed', async function () {
+    const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
+      body: {
+        took: 12,
+        timed_out: false,
+        // @ts-expect-error not assignable
+        _shards: {},
+        // @ts-expect-error not assignable
+        hits: {
+          total: 123,
+        },
+      },
+    };
+    const fakeClient = new Client({node: 'http://foohost:9200'});
+    const loggerErrorStub = sandbox.stub(Logger, 'error');
+    const mailQueueSpy = sinon.spy(mailQueue, 'push');
+    cronScheduleStub.yields();
+    sandbox.stub(fakeClient, 'search').resolves(fakeSearchResponse);
+    await setUp(monitoringConfig, fakeClient, mailQueue);
+
+    expect(loggerErrorStub.callCount).to.be.equal(2);
+    expect(mailQueueSpy.callCount).to.be.equal(2);
+  });
+});

backend/test/storage/elasticsearch/query.spec.ts

@@ -0,0 +1,661 @@
+/* eslint-disable @typescript-eslint/no-explicit-any,unicorn/no-null */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCConfigFile,
+  SCSearchBooleanFilter,
+  SCSearchDateRangeFilter,
+  SCSearchFilter,
+  SCSearchNumericRangeFilter,
+  SCSearchQuery,
+  SCSearchSort,
+  SCThingType,
+} from '@openstapps/core';
+import {expect} from 'chai';
+import {
+  ESDateRangeFilter,
+  ESRangeFilter,
+  ESNumericRangeFilter,
+  ElasticsearchConfig,
+  ESBooleanFilter,
+  ESGenericSort,
+  ESGeoDistanceFilter,
+  ESGeoDistanceSort,
+  ESTermFilter,
+  ScriptSort,
+} from '../../../src/storage/elasticsearch/types/elasticsearch';
+import {configFile} from '../../../src/common';
+import {
+  buildBooleanFilter,
+  buildFilter,
+  buildQuery,
+  buildSort,
+} from '../../../src/storage/elasticsearch/query';
+
+describe('Query', function () {
+  describe('buildBooleanFilter', function () {
+    const booleanFilter: SCSearchBooleanFilter = {
+      arguments: {
+        operation: 'and',
+        filters: [
+          {
+            type: 'value',
+            arguments: {
+              field: 'type',
+              value: SCThingType.Catalog,
+            },
+          },
+          {
+            type: 'value',
+            arguments: {
+              field: 'type',
+              value: SCThingType.Building,
+            },
+          },
+        ],
+      },
+      type: 'boolean',
+    };
+    const booleanFilters: {[key: string]: SCSearchBooleanFilter} = {
+      and: booleanFilter,
+      or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
+      not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
+    };
+    const expectedEsFilters: Array<ESTermFilter> = [
+      {
+        term: {
+          'type.raw': 'catalog',
+        },
+      },
+      {
+        term: {
+          'type.raw': 'building',
+        },
+      },
+    ];
+
+    it('should create appropriate elasticsearch "and" filter argument', function () {
+      const {must} = buildBooleanFilter(booleanFilters.and);
+
+      expect(must).to.be.eql(expectedEsFilters);
+    });
+
+    it('should create appropriate elasticsearch "or" filter argument', function () {
+      const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
+
+      expect(should).to.be.eql(expectedEsFilters);
+      expect(minimum_should_match).to.be.equal(1);
+    });
+
+    it('should create appropriate elasticsearch "not" filter argument', function () {
+      const {must_not} = buildBooleanFilter(booleanFilters.not);
+
+      expect(must_not).to.be.eql(expectedEsFilters);
+    });
+  });
+
+  describe('buildQuery', function () {
+    const parameters: SCSearchQuery = {
+      query: 'mathematics',
+      from: 30,
+      size: 5,
+      sort: [
+        {
+          type: 'ducet',
+          order: 'desc',
+          arguments: {
+            field: 'name',
+          },
+        },
+        {
+          type: 'ducet',
+          order: 'desc',
+          arguments: {
+            field: 'categories',
+          },
+        },
+      ],
+      filter: {
+        type: 'value',
+        arguments: {
+          field: 'type',
+          value: SCThingType.AcademicEvent,
+        },
+      },
+    };
+    let esConfig: ElasticsearchConfig = {
+      name: 'elasticsearch',
+      version: '123',
+      query: {
+        minMatch: '75%',
+        queryType: 'dis_max',
+        matchBoosting: 1.3,
+        fuzziness: 'AUTO',
+        cutoffFrequency: 0,
+        tieBreaker: 0,
+      },
+    };
+    const query = {
+      minMatch: '75%',
+      queryType: 'dis_max',
+      matchBoosting: 1.3,
+      fuzziness: 'AUTO',
+      cutoffFrequency: 0,
+      tieBreaker: 0,
+    };
+    const config: SCConfigFile = {
+      ...configFile,
+    };
+    beforeEach(function () {
+      esConfig = {
+        name: 'elasticsearch',
+        version: '123',
+      };
+    });
+
+    // TODO: check parts of received elasticsearch query for each test case
+
+    it('should build query that includes sorting when query is undefined', function () {
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should build query that includes sorting when query type is query_string', function () {
+      esConfig.query = {...query, queryType: 'query_string'};
+
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should build query that includes sorting when query type is dis_max', function () {
+      esConfig.query = {...query, queryType: 'dis_max'};
+
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should build query that includes sorting when query type is dis_max', function () {
+      esConfig.query = {...query, queryType: 'dis_max'};
+
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should reject (throw an error) if provided query type is not supported', function () {
+      // @ts-expect-error not assignable
+      esConfig.query = {...query, queryType: 'invalid_query_type'};
+
+      expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
+    });
+  });
+
+  describe('buildFilter', function () {
+    const searchFilters: {[key: string]: SCSearchFilter} = {
+      value: {
+        type: 'value',
+        arguments: {
+          field: 'type',
+          value: SCThingType.Dish,
+        },
+      },
+      distance: {
+        type: 'distance',
+        arguments: {
+          distance: 1000,
+          field: 'geo',
+          position: [50.123, 8.123],
+        },
+      },
+      geoPoint: {
+        type: 'geo',
+        arguments: {
+          field: 'geo',
+          shape: {
+            type: 'envelope',
+            coordinates: [
+              [50.123, 8.123],
+              [50.123, 8.123],
+            ],
+          },
+        },
+      },
+      geoShape: {
+        type: 'geo',
+        arguments: {
+          field: 'geo',
+          spatialRelation: 'contains',
+          shape: {
+            type: 'envelope',
+            coordinates: [
+              [50.123, 8.123],
+              [50.123, 8.123],
+            ],
+          },
+        },
+      },
+      boolean: {
+        type: 'boolean',
+        arguments: {
+          operation: 'and',
+          filters: [
+            {
+              type: 'value',
+              arguments: {
+                field: 'type',
+                value: SCThingType.Dish,
+              },
+            },
+            {
+              type: 'availability',
+              arguments: {
+                field: 'offers.availabilityRange',
+              },
+            },
+          ],
+        },
+      },
+    };
+
+    it('should build value filter', function () {
+      const filter = buildFilter(searchFilters.value);
+      const expectedFilter: ESTermFilter = {
+        term: {
+          'type.raw': SCThingType.Dish,
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build numeric range filters', function () {
+      for (const upperMode of ['inclusive', 'exclusive', null]) {
+        for (const lowerMode of ['inclusive', 'exclusive', null]) {
+          const expectedFilter: ESNumericRangeFilter = {
+            range: {
+              price: {
+                relation: undefined,
+              },
+            },
+          };
+
+          const rawFilter: SCSearchNumericRangeFilter = {
+            type: 'numeric range',
+            arguments: {
+              bounds: {},
+              field: 'price',
+            },
+          };
+
+          // eslint-disable-next-line unicorn/consistent-function-scoping
+          const setBound = (location: 'upperBound' | 'lowerBound', bound: string | null) => {
+            let out: number | null = null;
+            if (bound != undefined) {
+              out = Math.random();
+              rawFilter.arguments.bounds[location] = {
+                mode: bound as 'inclusive' | 'exclusive',
+                limit: out,
+              };
+              expectedFilter.range.price[
+                `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
+              ] = out;
+            }
+          };
+          setBound('upperBound', upperMode);
+          setBound('lowerBound', lowerMode);
+
+          const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
+          expect(filter).to.deep.equal(expectedFilter);
+          for (const bound of ['g', 'l']) {
+            // @ts-expect-error implicit any
+            const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
+            // @ts-expect-error implicit any
+            const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
+
+            // only one should exist at the same time
+            expect(inclusiveExists && exclusiveExists).to.be.false;
+          }
+        }
+      }
+    });
+
+    it('should build date range filters', function () {
+      for (const upperMode of ['inclusive', 'exclusive', null]) {
+        for (const lowerMode of ['inclusive', 'exclusive', null]) {
+          const expectedFilter: ESDateRangeFilter = {
+            range: {
+              price: {
+                format: 'thisIsADummyFormat',
+                time_zone: 'thisIsADummyTimeZone',
+                relation: 'testRelation' as any,
+              },
+            },
+          };
+
+          const rawFilter: SCSearchDateRangeFilter = {
+            type: 'date range',
+            arguments: {
+              bounds: {},
+              field: 'price',
+              relation: 'testRelation' as any,
+              format: 'thisIsADummyFormat',
+              timeZone: 'thisIsADummyTimeZone',
+            },
+          };
+
+          const setBound = (location: 'upperBound' | 'lowerBound', bound: string | null) => {
+            let out: string | null = null;
+            if (bound != undefined) {
+              out = `${location} ${bound} ${upperMode} ${lowerMode}`;
+              rawFilter.arguments.bounds[location] = {
+                mode: bound as 'inclusive' | 'exclusive',
+                limit: out,
+              };
+              expectedFilter.range.price[
+                `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
+              ] = out;
+            }
+          };
+          setBound('upperBound', upperMode);
+          setBound('lowerBound', lowerMode);
+
+          const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
+          expect(filter).to.deep.equal(expectedFilter);
+          for (const bound of ['g', 'l']) {
+            // @ts-expect-error implicit any
+            const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
+            // @ts-expect-error implicit any
+            const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
+
+            // only one should exist at the same time
+            expect(inclusiveExists && exclusiveExists).to.be.false;
+          }
+        }
+      }
+    });
+
+    it('should build availability filters', function () {
+      it('should copy scope', function () {
+        for (const scope of ['a', 'b']) {
+          const filter = buildFilter({
+            type: 'availability',
+            arguments: {
+              time: 'test',
+              scope: scope as any,
+              field: 'offers.availabilityRange',
+            },
+          });
+
+          const expectedFilter: ESRangeFilter = {
+            range: {
+              'offers.availabilityRange': {
+                gte: `test||/${scope}`,
+                lt: `test||+1${scope}/${scope}`,
+              },
+            },
+          };
+          expect(filter).to.be.eql(expectedFilter);
+        }
+      });
+
+      it('should default to second scope', function () {
+        const filter = buildFilter({
+          type: 'availability',
+          arguments: {
+            time: 'test',
+            field: 'offers.availabilityRange',
+          },
+        });
+
+        const expectedFilter: ESRangeFilter = {
+          range: {
+            'offers.availabilityRange': {
+              gte: 'test||/s',
+              lt: 'test||+1s/s',
+            },
+          },
+        };
+        expect(filter).to.be.eql(expectedFilter);
+      });
+
+      it('should add || to dates', function () {
+        const filter = buildFilter({
+          type: 'availability',
+          arguments: {
+            time: 'test',
+            scope: 'd',
+            field: 'offers.availabilityRange',
+          },
+        });
+
+        const expectedFilter: ESRangeFilter = {
+          range: {
+            'offers.availabilityRange': {
+              gte: `test||/d`,
+              lt: `test||+1d/d`,
+            },
+          },
+        };
+        expect(filter).to.be.eql(expectedFilter);
+      });
+
+      it('should default to now and not add ||', function () {
+        const filter = buildFilter({
+          type: 'availability',
+          arguments: {
+            scope: 'd',
+            field: 'offers.availabilityRange',
+          },
+        });
+
+        const expectedFilter: ESRangeFilter = {
+          range: {
+            'offers.availabilityRange': {
+              gte: `now/d`,
+              lt: `now+1d/d`,
+            },
+          },
+        };
+        expect(filter).to.be.eql(expectedFilter);
+      });
+    });
+
+    it('should build distance filter', function () {
+      const filter = buildFilter(searchFilters.distance);
+      const expectedFilter: ESGeoDistanceFilter = {
+        geo_distance: {
+          'distance': '1000m',
+          'geo.point.coordinates': {
+            lat: 8.123,
+            lon: 50.123,
+          },
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build geo filter for shapes and points', function () {
+      const filter = buildFilter(searchFilters.geoPoint);
+      const expectedFilter = {
+        bool: {
+          minimum_should_match: 1,
+          should: [
+            {
+              geo_shape: {
+                'geo.polygon': {
+                  relation: undefined,
+                  shape: {
+                    type: 'envelope',
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                  },
+                },
+                'ignore_unmapped': true,
+              },
+            },
+            {
+              geo_bounding_box: {
+                'geo.point.coordinates': {
+                  bottom_right: [50.123, 8.123],
+                  top_left: [50.123, 8.123],
+                },
+                'ignore_unmapped': true,
+              },
+            },
+          ],
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build geo filter for shapes only', function () {
+      const filter = buildFilter(searchFilters.geoShape);
+      const expectedFilter = {
+        geo_shape: {
+          'geo.polygon': {
+            relation: 'contains',
+            shape: {
+              type: 'envelope',
+              coordinates: [
+                [50.123, 8.123],
+                [50.123, 8.123],
+              ],
+            },
+          },
+          'ignore_unmapped': true,
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build boolean filter', function () {
+      const filter = buildFilter(searchFilters.boolean);
+      const expectedFilter: ESBooleanFilter<any> = {
+        bool: {
+          minimum_should_match: 0,
+          must: [
+            {
+              term: {
+                'type.raw': 'dish',
+              },
+            },
+            {
+              range: {
+                'offers.availabilityRange': {
+                  gte: 'now/s',
+                  lt: 'now+1s/s',
+                  relation: 'intersects',
+                },
+              },
+            },
+          ],
+          must_not: [],
+          should: [],
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+  });
+
+  describe('buildSort', function () {
+    const searchSCSearchSort: Array<SCSearchSort> = [
+      {
+        type: 'ducet',
+        order: 'desc',
+        arguments: {
+          field: 'name',
+        },
+      },
+      {
+        type: 'generic',
+        order: 'desc',
+        arguments: {
+          field: 'name',
+        },
+      },
+      {
+        type: 'distance',
+        order: 'desc',
+        arguments: {
+          field: 'geo',
+          position: [8.123, 50.123],
+        },
+      },
+      {
+        type: 'price',
+        order: 'asc',
+        arguments: {
+          universityRole: 'student',
+          field: 'offers.prices',
+        },
+      },
+    ];
+    let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
+    const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
+      ducet: {
+        'name.sort': 'desc',
+      },
+      generic: {
+        name: 'desc',
+      },
+      distance: {
+        _geo_distance: {
+          'mode': 'avg',
+          'order': 'desc',
+          'unit': 'm',
+          'geo.point.coordinates': {
+            lat: 50.123,
+            lon: 8.123,
+          },
+        },
+      },
+      price: {
+        _script: {
+          order: 'asc',
+          script: '\n  // foo price sort script',
+          type: 'number',
+        },
+      },
+    };
+    before(function () {
+      sorts = buildSort(searchSCSearchSort);
+    });
+
+    it('should build ducet sort', function () {
+      expect(sorts[0]).to.be.eql(expectedSorts.ducet);
+    });
+
+    it('should build generic sort', function () {
+      expect(sorts[1]).to.be.eql(expectedSorts.generic);
+    });
+
+    it('should build distance sort', function () {
+      expect(sorts[2]).to.be.eql(expectedSorts.distance);
+    });
+
+    it('should build price sort', function () {
+      const priceSortNoScript = {
+        ...sorts[3],
+        _script: {
+          ...(sorts[3] as ScriptSort)._script,
+          script: (expectedSorts.price as ScriptSort)._script.script,
+        },
+      };
+      expect(priceSortNoScript).to.be.eql(expectedSorts.price);
+    });
+  });
+});

backend/test/tests-setup.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {DEFAULT_TIMEOUT} from '../src/common';
+import {startApp} from './common';
+import supertest from 'supertest';
+
+before(async function () {
+  this.timeout(DEFAULT_TIMEOUT);
+  testApp = supertest(await startApp());
+});
+
+export let testApp: supertest.SuperTest<supertest.Test>;