refactor: move backend to monorepo

backend/test/storage/bulk-storage.spec.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCBulkRequest, SCThingType} from '@openstapps/core';
+import moment from 'moment';
+// eslint-disable-next-line unicorn/import-style
+import util from 'util';
+import {configFile} from '../../src/common';
+import {Bulk, BulkStorage} from '../../src/storage/bulk-storage';
+import {expect} from 'chai';
+import {ElasticsearchMock} from '../common';
+import sinon from 'sinon';
+import NodeCache from 'node-cache';
+
+describe('Bulk Storage', function () {
+  describe('Bulk', function () {
+    let bulkRequest: SCBulkRequest;
+    beforeEach(function () {
+      bulkRequest = {source: 'some_source', type: SCThingType.Book};
+    });
+
+    it('should create a bulk with the given expiration', function () {
+      const expiration = moment().add(3600, 'seconds').toISOString();
+      bulkRequest.expiration = expiration;
+
+      const bulk = new Bulk(bulkRequest);
+
+      expect(bulk.expiration).to.be.equal(expiration);
+      expect(bulk.state).to.be.equal('in progress');
+      expect(bulk.uid).to.not.be.undefined;
+    });
+
+    it('should fallback and set expiration when it is not provided', function () {
+      const bulk = new Bulk(bulkRequest);
+
+      expect(bulk.expiration).to.not.be.undefined;
+    });
+  });
+
+  describe('BulkStorage', async function () {
+    const sandbox = sinon.createSandbox();
+    const bulkRequest = {source: 'some_source', type: SCThingType.Book};
+    const bulk = new Bulk(bulkRequest);
+    bulk.uid = '123';
+    let esMock: sinon.SinonStub;
+    let database: ElasticsearchMock;
+
+    beforeEach(function () {
+      database = new ElasticsearchMock(configFile);
+      esMock = sandbox.stub(database, 'bulkExpired');
+    });
+
+    afterEach(function () {
+      sandbox.restore();
+    });
+
+    it('should call appropriate database clean-up method on expire', async function () {
+      sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
+      new BulkStorage(database);
+
+      expect(esMock.calledWith(bulk)).to.be.true;
+    });
+
+    it("should not call appropriate database clean-up method on expire if bulk's state is done", async function () {
+      bulk.state = 'done';
+      sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
+      new BulkStorage(database);
+
+      expect(esMock.called).to.be.false;
+    });
+
+    it('should throw an error if the bulk for deletion cannot be read', async function () {
+      sandbox.stub(BulkStorage.prototype, 'read').callsFake(() => undefined);
+      const bulkStorage = new BulkStorage(database);
+
+      return expect(bulkStorage.delete('123')).to.be.rejected;
+    });
+
+    it('should delete a bulk', async function () {
+      const readStub = sandbox.stub(BulkStorage.prototype, 'read').callsFake(() => bulk);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      let caught: any;
+      sandbox.stub(NodeCache.prototype, 'del').callsFake(() => (caught = 123));
+      // force call
+      sandbox
+        .stub(util, 'promisify')
+        // eslint-disable-next-line @typescript-eslint/no-empty-function,unicorn/consistent-function-scoping
+        .callsFake(() => () => {})
+        // eslint-disable-next-line unicorn/no-null
+        .yields(null);
+      const bulkStorage = new BulkStorage(database);
+
+      await bulkStorage.delete(bulk.uid);
+
+      expect(readStub.called).to.be.true;
+      expect(caught).to.be.equal(123);
+      expect(esMock.called).to.be.true;
+    });
+
+    it('should read an existing bulk', async function () {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      let caught: any;
+      sandbox.stub(NodeCache.prototype, 'get').callsFake(() => (caught = 123));
+      // force call
+      sandbox
+        .stub(util, 'promisify')
+        // eslint-disable-next-line unicorn/consistent-function-scoping,@typescript-eslint/no-empty-function
+        .callsFake(() => () => {})
+        // eslint-disable-next-line unicorn/no-null
+        .yields(null);
+      const bulkStorage = new BulkStorage(database);
+
+      await bulkStorage.read(bulk.uid);
+
+      expect(caught).to.be.equal(123);
+    });
+    ``;
+  });
+});

backend/test/storage/elasticsearch/aggregations.spec.ts

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCFacet, SCThingType} from '@openstapps/core';
+import {expect} from 'chai';
+import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
+import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch';
+
+describe('Aggregations', function () {
+  const aggregations: AggregationResponse = {
+    'catalog': {
+      'doc_count': 4,
+      'superCatalogs.categories': {
+        buckets: [],
+      },
+      'academicTerm.acronym': {
+        buckets: [
+          {
+            key: 'SoSe 2020',
+            doc_count: 2,
+          },
+        ],
+      },
+      'superCatalog.categories': {
+        buckets: [],
+      },
+      'categories': {
+        buckets: [
+          {
+            key: 'foo',
+            doc_count: 1,
+          },
+          {
+            key: 'bar',
+            doc_count: 3,
+          },
+        ],
+      },
+    },
+    'person': {
+      'doc_count': 13,
+      'homeLocations.categories': {
+        buckets: [],
+      },
+    },
+    'academic event': {
+      'doc_count': 0,
+      'academicTerms.acronym': {
+        buckets: [],
+      },
+      'categories': {
+        buckets: [
+          {
+            key: 'foobar',
+            doc_count: 8,
+          },
+          {
+            key: 'bar',
+            doc_count: 2,
+          },
+        ],
+      },
+      'creativeWorks.keywords': {
+        buckets: [],
+      },
+    },
+    'fooType': {
+      buckets: [
+        {
+          doc_count: 321,
+          key: 'foo',
+        },
+      ],
+    },
+    '@all': {
+      doc_count: 17,
+      type: {
+        buckets: [
+          {
+            key: 'person',
+            doc_count: 13,
+          },
+          {
+            key: 'catalog',
+            doc_count: 4,
+          },
+        ],
+      },
+    },
+  };
+
+  const expectedFacets: SCFacet[] = [
+    {
+      buckets: [
+        {
+          count: 13,
+          key: 'person',
+        },
+        {
+          count: 4,
+          key: 'catalog',
+        },
+      ],
+      field: 'type',
+    },
+    {
+      buckets: [
+        {
+          count: 8,
+          key: 'foobar',
+        },
+        {
+          count: 2,
+          key: 'bar',
+        },
+      ],
+      field: 'categories',
+      onlyOnType: SCThingType.AcademicEvent,
+    },
+    {
+      buckets: [
+        {
+          count: 2,
+          key: 'SoSe 2020',
+        },
+      ],
+      field: 'academicTerm.acronym',
+      onlyOnType: SCThingType.Catalog,
+    },
+    {
+      buckets: [
+        {
+          count: 1,
+          key: 'foo',
+        },
+        {
+          count: 3,
+          key: 'bar',
+        },
+      ],
+      field: 'categories',
+      onlyOnType: SCThingType.Catalog,
+    },
+    // no fooType as it doesn't appear in the aggregation schema
+  ];
+
+  it('should parse the aggregations providing the appropriate facets', function () {
+    const facets = parseAggregations(aggregations);
+
+    expect(facets).to.be.eql(expectedFacets);
+  });
+});

backend/test/storage/elasticsearch/common.spec.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  ESAggMatchAllFilter,
+  ESAggTypeFilter,
+  ESNestedAggregation,
+  ESTermsFilter,
+} from '@openstapps/es-mapping-generator/src/types/aggregation';
+import {expect} from 'chai';
+import {
+  isNestedAggregation,
+  isBucketAggregation,
+  isESTermsFilter,
+  isESAggMatchAllFilter,
+  isESNestedAggregation,
+} from '../../../lib/storage/elasticsearch/types/guards';
+import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';
+
+describe('Common', function () {
+  const bucketAggregation: BucketAggregation = {buckets: []};
+  const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
+  const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
+
+  describe('isBucketAggregation', function () {
+    it('should be false for a number', function () {
+      expect(isBucketAggregation(123)).to.be.false;
+    });
+
+    it('should be true for a bucket aggregation', function () {
+      expect(isBucketAggregation(bucketAggregation)).to.be.true;
+    });
+  });
+
+  describe('isNestedAggregation', function () {
+    it('should be false for a bucket aggregation', function () {
+      expect(isNestedAggregation(bucketAggregation)).to.be.false;
+    });
+
+    it('should be true for a nested aggregation', function () {
+      const nestedAggregation: NestedAggregation = {doc_count: 123};
+
+      expect(isNestedAggregation(nestedAggregation)).to.be.true;
+    });
+  });
+
+  describe('isESTermsFilter', function () {
+    it('should be false for an elasticsearch nested aggregation', function () {
+      expect(isESTermsFilter(esNestedAggregation)).to.be.false;
+    });
+
+    it('should be true for an elasticsearch terms filter', function () {
+      expect(isESTermsFilter(esTermsFilter)).to.be.true;
+    });
+  });
+
+  describe('isESNestedAggregation', function () {
+    it('should be false for an elasticsearch terms filter', function () {
+      expect(isESNestedAggregation(esTermsFilter)).to.be.false;
+    });
+
+    it('should be true for an elasticsearch nested aggregation', function () {
+      expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
+    });
+  });
+
+  describe('isESAggMatchAllFilter', function () {
+    it('should be false for an elasticsearch aggregation type filter', function () {
+      const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
+      expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
+    });
+
+    it('should be true for an elasticsearch aggregation match all filter', function () {
+      const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
+      expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
+    });
+  });
+});

backend/test/storage/elasticsearch/elasticsearch.spec.ts

@@ -0,0 +1,683 @@
+/* eslint-disable @typescript-eslint/no-explicit-any,unicorn/no-null */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {
+  SCBook,
+  SCBulkResponse,
+  SCConfigFile,
+  SCMessage,
+  SCSearchQuery,
+  SCThings,
+  SCThingType,
+} from '@openstapps/core';
+import {instance as book} from '@openstapps/core/test/resources/indexable/Book.1.json';
+import {instance as message} from '@openstapps/core/test/resources/indexable/Message.1.json';
+import {Logger} from '@openstapps/logger';
+import {SMTP} from '@openstapps/logger/lib/smtp';
+import {expect, use} from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import {SearchResponse} from 'elasticsearch';
+import mockedEnv from 'mocked-env';
+import sinon from 'sinon';
+import {configFile} from '../../../src/common';
+import {MailQueue} from '../../../src/notification/mail-queue';
+import {aggregations} from '../../../src/storage/elasticsearch/templating';
+import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch';
+import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
+import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
+import * as query from '../../../src/storage/elasticsearch/query';
+import * as templating from '../../../src/storage/elasticsearch/templating';
+import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common';
+import fs from 'fs';
+
+use(chaiAsPromised);
+
+describe('Elasticsearch', function () {
+  // increase timeout for the suite
+  this.timeout(DEFAULT_TEST_TIMEOUT);
+  const sandbox = sinon.createSandbox();
+
+  before(function () {
+    // eslint-disable-next-line no-console
+    console.log('before');
+    sandbox.stub(fs, 'readFileSync').returns('{}');
+  });
+  after(function () {
+    sandbox.restore();
+  });
+
+  describe('getElasticsearchUrl', function () {
+    it('should provide custom elasticsearch URL if defined', function () {
+      const customAddress = 'http://foo-address:9200';
+      const restore = mockedEnv({
+        ES_ADDR: customAddress,
+      });
+
+      expect(Elasticsearch.getElasticsearchUrl()).to.be.equal(customAddress);
+      // restore env variables
+      restore();
+    });
+
+    it('should provide local URL as fallback', function () {
+      const restore = mockedEnv({
+        ES_ADDR: undefined,
+      });
+
+      expect(Elasticsearch.getElasticsearchUrl()).to.match(/(https?:\/\/)?localhost(:\d+)?/);
+      // restore env variables
+      restore();
+    });
+  });
+
+  describe('getIndex (including getIndexUID)', function () {
+    const type = 'foo bar type';
+    const source = 'foo_source';
+    const bulk: SCBulkResponse = {
+      expiration: '',
+      source: '',
+      state: 'in progress',
+      type: SCThingType.Semester,
+      uid: 'bulk-uid-123-123-123',
+    };
+
+    it('should provide index UID from the provided UID', function () {
+      const indexUID = Elasticsearch.getIndexUID(bulk.uid);
+
+      expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
+      // test starting and ending character
+      expect(indexUID[0]).to.be.equal(bulk.uid[0]);
+      expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
+    });
+
+    it('should provide index name from the provided data', function () {
+      expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
+        `stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
+      );
+    });
+  });
+
+  describe('removeAliasChars', function () {
+    it('should remove spaces from both ends', function () {
+      expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
+    });
+
+    it('should replace inner spaces with underscores', function () {
+      expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
+    });
+
+    it('should remove invalid characters', function () {
+      expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
+    });
+
+    it('should remove invalid starting characters', function () {
+      expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+      expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+    });
+
+    it('should replace with a placeholder in case of invalid alias', function () {
+      expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
+      expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
+    });
+
+    it('should work with common cases', function () {
+      expect(
+        Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
+      ).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
+      expect(
+        Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
+      ).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
+    });
+
+    it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
+      const sandbox = sinon.createSandbox();
+      const loggerWarnStub = sandbox.stub(Logger, 'warn');
+
+      expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
+      expect(loggerWarnStub.called).to.be.true;
+    });
+  });
+
+  describe('constructor', async function () {
+    const sandbox = sinon.createSandbox();
+    afterEach(function () {
+      sandbox.restore();
+    });
+
+    it('should complain (throw an error) if database in config is undefined', function () {
+      const config: SCConfigFile = {...configFile, internal: {...configFile.internal, database: undefined}};
+
+      expect(() => new Elasticsearch(config)).to.throw(Error);
+    });
+
+    it('should complain (throw an error) if database version is not a string', function () {
+      const config: SCConfigFile = {
+        ...configFile,
+        internal: {
+          ...configFile.internal,
+          database: {
+            name: 'foo',
+            version: 123,
+          },
+        },
+      };
+
+      expect(() => new Elasticsearch(config)).to.throw(Error);
+    });
+
+    it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
+      const error = new Error('Foo Error');
+      const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
+      sandbox.stub(Client.prototype, 'on').yields(error);
+
+      new Elasticsearch(configFile);
+
+      expect(loggerErrorStub.calledWith(error)).to.be.true;
+    });
+
+    it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
+      const fakeResponse = {foo: 'bar'};
+      const loggerLogStub = sandbox.stub(Logger, 'log');
+      sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
+
+      new Elasticsearch(configFile);
+      expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
+
+      const restore = mockedEnv({
+        ES_DEBUG: 'true',
+      });
+      new Elasticsearch(configFile);
+
+      expect(loggerLogStub.calledWith(fakeResponse)).to.be.true;
+      // restore env variables
+      restore();
+    });
+
+    describe('init', async function () {
+      const sandbox = sinon.createSandbox();
+      after(function () {
+        sandbox.restore();
+      });
+
+      it('should complain (throw an error) if monitoring is set but mail queue is undefined', async function () {
+        const config: SCConfigFile = {
+          ...configFile,
+          internal: {
+            ...configFile.internal,
+            monitoring: {
+              actions: [],
+              watchers: [],
+            },
+          },
+        };
+
+        const es = new Elasticsearch(config);
+
+        return expect(es.init()).to.be.rejected;
+      });
+
+      it('should setup the monitoring if there is monitoring is set and mail queue is defined', function () {
+        const config: SCConfigFile = {
+          ...configFile,
+          internal: {
+            ...configFile.internal,
+            monitoring: {
+              actions: [],
+              watchers: [],
+            },
+          },
+        };
+        const monitoringSetUpStub = sandbox.stub(Monitoring, 'setUp');
+
+        const es = new Elasticsearch(config, new MailQueue(getTransport(false) as unknown as SMTP));
+
+        es.init();
+
+        expect(monitoringSetUpStub.called).to.be.true;
+      });
+    });
+
+    describe('Operations with bundle/index', async function () {
+      const sandbox = sinon.createSandbox();
+      let es: Elasticsearch;
+      const oldIndex = 'stapps_footype_foosource_oldindex';
+
+      beforeEach(function () {
+        es = new Elasticsearch(configFile);
+        es.client.indices = {
+          // @ts-expect-error not assignable
+          getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
+          // @ts-expect-error not assignable
+          putTemplate: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          create: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          delete: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          exists: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          refresh: () => Promise.resolve({}),
+          // @ts-expect-error not assignable
+          updateAliases: () => Promise.resolve({}),
+        };
+      });
+
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      describe('bulkCreated', async function () {
+        it('should reject (throw an error) if the connection to elasticsearch is not established', async function () {
+          return expect(es.bulkCreated(bulk)).to.be.rejectedWith('elasticsearch');
+        });
+
+        it('should reject (throw an error) if the index name is not valid', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`);
+          sandbox.createStubInstance(Client, {});
+          await es.init();
+
+          return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
+        });
+
+        it('should create a new index', async function () {
+          const index = getIndex();
+          sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+          const putTemplateStub = sandbox.stub(templating, 'putTemplate');
+          const createStub = sandbox.stub(es.client.indices, 'create');
+          await es.init();
+
+          await es.bulkCreated(bulk);
+
+          expect(putTemplateStub.called).to.be.true;
+          expect(createStub.calledWith({index})).to.be.true;
+        });
+      });
+
+      describe('bulkExpired', async function () {
+        const sandbox = sinon.createSandbox();
+        afterEach(function () {
+          sandbox.restore();
+        });
+        it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+          const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
+
+          await es.bulkExpired({...bulk, state: 'in progress'});
+
+          expect(clientDeleteStub.called).to.be.true;
+        });
+
+        it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+          const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
+
+          await es.bulkExpired({...bulk, state: 'done'});
+
+          expect(clientDeleteStub.called).to.be.false;
+        });
+      });
+
+      describe('bulkUpdated', function () {
+        it('should reject if the connection to elasticsearch is not established', async function () {
+          return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('elasticsearch');
+        });
+
+        it('should reject if the index name is not valid', async function () {
+          sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`);
+          sandbox.createStubInstance(Client, {});
+          await es.init();
+
+          return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
+        });
+
+        it('should create a new index', async function () {
+          const index = getIndex();
+          const expectedRefreshActions = [
+            {
+              add: {index: index, alias: SCThingType.Book},
+            },
+            {
+              remove: {index: oldIndex, alias: SCThingType.Book},
+            },
+          ];
+          sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+          sandbox.stub(es, 'aliasMap').value({
+            [SCThingType.Book]: {
+              [bulk.source]: oldIndex,
+            },
+          });
+          const refreshStub = sandbox.stub(es.client.indices, 'refresh');
+          const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
+          const deleteStub = sandbox.stub(es.client.indices, 'delete');
+          sandbox.stub(templating, 'putTemplate');
+          await es.init();
+
+          await es.bulkUpdated(bulk);
+
+          expect(refreshStub.calledWith({index})).to.be.true;
+          expect(
+            updateAliasesStub.calledWith({
+              body: {
+                actions: expectedRefreshActions,
+              },
+            }),
+          ).to.be.true;
+          expect(deleteStub.called).to.be.true;
+        });
+      });
+    });
+
+    describe('get', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      it('should reject if object is not found', async function () {
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+
+        return expect(es.get('123')).to.rejectedWith('found');
+      });
+
+      it('should provide the thing if object is found', async function () {
+        const foundObject: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: '',
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
+
+        return expect(await es.get('123')).to.be.eql(message);
+      });
+    });
+
+    describe('post', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      it('should not post if the object already exists in an index which will not be rolled over', async function () {
+        const index = getIndex();
+        const oldIndex = index.replace('foosource', 'barsource');
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: oldIndex,
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+
+        return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
+      });
+
+      it('should not reject if the object already exists but in an index which will be rolled over', async function () {
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: getIndex(),
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        // return index name with different generated UID (see getIndex method)
+        sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
+
+        return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist');
+      });
+
+      it('should reject if there is an object creation error on the elasticsearch side', async function () {
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        sandbox.stub(es.client, 'create').resolves({body: {created: false}});
+
+        return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
+      });
+
+      it('should create a new object', async function () {
+        let caughtParameter: any;
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+        // @ts-expect-error call
+        const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
+          caughtParameter = parameter;
+          return Promise.resolve({body: {created: true}});
+        });
+
+        await es.post(message as SCMessage, bulk);
+
+        expect(createStub.called).to.be.true;
+        expect(caughtParameter.body).to.be.eql({
+          ...message,
+          creation_date: caughtParameter.body.creation_date,
+        });
+      });
+    });
+
+    describe('put', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+      afterEach(function () {
+        sandbox.restore();
+      });
+      it('should reject to put if the object does not already exist', async function () {
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: getIndex(),
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
+
+        return expect(es.put(object._source)).to.rejectedWith('exist');
+      });
+
+      // noinspection JSUnusedLocalSymbols
+      it('should update the object if it already exists', async function () {
+        let caughtParameter: any;
+        const object: ElasticsearchObject<SCMessage> = {
+          _id: '',
+          _index: getIndex(),
+          _score: 0,
+          _type: '',
+          _source: message as SCMessage,
+        };
+        sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
+        // @ts-expect-error unused
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
+          caughtParameter = parameters;
+          return Promise.resolve({body: {created: true}});
+        });
+
+        await es.put(object._source);
+
+        expect(caughtParameter.body.doc).to.be.eql(object._source);
+      });
+    });
+
+    describe('search', async function () {
+      let es: Elasticsearch;
+      const sandbox = sinon.createSandbox();
+      const objectMessage: ElasticsearchObject<SCMessage> = {
+        _id: '123',
+        _index: getIndex(),
+        _score: 0,
+        _type: '',
+        _source: message as SCMessage,
+      };
+      const objectBook: ElasticsearchObject<SCBook> = {
+        _id: '321',
+        _index: getIndex(),
+        _score: 0,
+        _type: '',
+        _source: book as SCBook,
+      };
+      const fakeEsAggregations = {
+        '@all': {
+          doc_count: 17,
+          type: {
+            doc_count_error_upper_bound: 0,
+            sum_other_doc_count: 0,
+            buckets: [
+              {
+                key: 'person',
+                doc_count: 13,
+              },
+              {
+                key: 'catalog',
+                doc_count: 4,
+              },
+            ],
+          },
+        },
+      };
+      const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
+        body: {
+          took: 12,
+          timed_out: false,
+          // @ts-expect-error not assignable
+          _shards: {},
+          // @ts-expect-error not assignable
+          hits: {
+            hits: [objectMessage, objectBook],
+            total: 123,
+          },
+          aggregations: fakeEsAggregations,
+        },
+        headers: {},
+        // @ts-expect-error not assignable
+        meta: {},
+        // @ts-expect-error not assignable
+        statusCode: {},
+        // @ts-expect-error not assignable
+        warnings: {},
+      };
+      let searchStub: sinon.SinonStub;
+      before(function () {
+        es = new Elasticsearch(configFile);
+      });
+      beforeEach(function () {
+        searchStub = sandbox.stub(es.client, 'search').resolves(fakeSearchResponse);
+      });
+      afterEach(function () {
+        sandbox.restore();
+      });
+
+      it('should provide appropriate data and facets', async function () {
+        const fakeFacets = [
+          {
+            buckets: [
+              {
+                count: 13,
+                key: 'person',
+              },
+              {
+                count: 4,
+                key: 'catalog',
+              },
+            ],
+            field: 'type',
+          },
+        ];
+
+        const {data, facets} = await es.search({});
+
+        expect(data).to.be.eql([objectMessage._source, objectBook._source]);
+        expect(facets).to.be.eql(fakeFacets);
+      });
+
+      it('should provide pagination from params', async function () {
+        const from = 30;
+        const {pagination} = await es.search({from});
+
+        expect(pagination).to.be.eql({
+          count: fakeSearchResponse.body!.hits.hits.length,
+          offset: from,
+          total: fakeSearchResponse.body!.hits.total,
+        });
+      });
+
+      it('should have fallback to zero if from is not given through params', async function () {
+        const {pagination} = await es.search({});
+
+        expect(pagination.offset).to.be.equal(0);
+      });
+
+      it('should build the search request properly', async function () {
+        const parameters: SCSearchQuery = {
+          query: 'mathematics',
+          from: 30,
+          size: 5,
+          sort: [
+            {
+              type: 'ducet',
+              order: 'desc',
+              arguments: {
+                field: 'name',
+              },
+            },
+          ],
+          filter: {
+            type: 'value',
+            arguments: {
+              field: 'type',
+              value: SCThingType.AcademicEvent,
+            },
+          },
+        };
+        const fakeResponse = {foo: 'bar'};
+        const fakeBuildSortResponse = [fakeResponse];
+        // @ts-expect-error not assignable
+        sandbox.stub(query, 'buildQuery').returns(fakeResponse);
+        sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
+
+        await es.search(parameters);
+
+        sandbox.assert.calledWithMatch(searchStub, {
+          body: {
+            aggs: aggregations,
+            query: fakeResponse,
+            sort: fakeBuildSortResponse,
+          },
+          from: parameters.from,
+          index: Elasticsearch.getListOfAllIndices(),
+          size: parameters.size,
+        });
+      });
+    });
+  });
+});

backend/test/storage/elasticsearch/monitoring.spec.ts

@@ -0,0 +1,136 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {
+  SCMonitoringConfiguration,
+  SCMonitoringLogAction,
+  SCMonitoringMailAction,
+  SCMonitoringWatcher,
+  SCThings,
+} from '@openstapps/core';
+import {Logger} from '@openstapps/logger';
+import {SearchResponse} from 'elasticsearch';
+import {MailQueue} from '../../../src/notification/mail-queue';
+import {setUp} from '../../../src/storage/elasticsearch/monitoring';
+
+import {getTransport} from '../../common';
+import {expect} from 'chai';
+import sinon from 'sinon';
+import cron from 'node-cron';
+
+describe('Monitoring', async function () {
+  const sandbox = sinon.createSandbox();
+  const logAction: SCMonitoringLogAction = {
+    message: 'Foo monitoring message',
+    prefix: 'Backend Monitoring',
+    type: 'log',
+  };
+  const mailAction: SCMonitoringMailAction = {
+    message: 'Bar monitoring message',
+    recipients: ['xyz@xyz.com'],
+    subject: 'Backend Monitoring',
+    type: 'mail',
+  };
+  let transport: any;
+  let mailQueue: any;
+  beforeEach(async function () {
+    transport = getTransport(true);
+    mailQueue = new MailQueue(transport);
+    cronScheduleStub = sandbox.stub(cron, 'schedule');
+  });
+  afterEach(async function () {
+    sandbox.restore();
+  });
+  // const sandbox = sinon.createSandbox();
+  let cronScheduleStub: sinon.SinonStub;
+  const minLengthWatcher: SCMonitoringWatcher = {
+    actions: [logAction, mailAction],
+    conditions: [
+      {
+        length: 10,
+        type: 'MinimumLength',
+      },
+    ],
+    name: 'foo watcher',
+    query: {foo: 'bar'},
+    triggers: [
+      {
+        executionTime: 'monthly',
+        name: 'beginning of month',
+      },
+      {
+        executionTime: 'daily',
+        name: 'every night',
+      },
+    ],
+  };
+  const maxLengthWatcher: SCMonitoringWatcher = {
+    actions: [logAction, mailAction],
+    conditions: [
+      {
+        length: 30,
+        type: 'MaximumLength',
+      },
+    ],
+    name: 'foo watcher',
+    query: {bar: 'foo'},
+    triggers: [
+      {
+        executionTime: 'hourly',
+        name: 'every hour',
+      },
+      {
+        executionTime: 'weekly',
+        name: 'every week',
+      },
+    ],
+  };
+  const monitoringConfig: SCMonitoringConfiguration = {
+    actions: [logAction, mailAction],
+    watchers: [minLengthWatcher, maxLengthWatcher],
+  };
+
+  it('should create a schedule for each trigger', async function () {
+    await setUp(monitoringConfig, new Client({node: 'http://foohost:9200'}), mailQueue);
+
+    expect(cronScheduleStub.callCount).to.be.equal(4);
+  });
+
+  it('should log errors where conditions failed', async function () {
+    const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
+      body: {
+        took: 12,
+        timed_out: false,
+        // @ts-expect-error not assignable
+        _shards: {},
+        // @ts-expect-error not assignable
+        hits: {
+          total: 123,
+        },
+      },
+    };
+    const fakeClient = new Client({node: 'http://foohost:9200'});
+    const loggerErrorStub = sandbox.stub(Logger, 'error');
+    const mailQueueSpy = sinon.spy(mailQueue, 'push');
+    cronScheduleStub.yields();
+    sandbox.stub(fakeClient, 'search').resolves(fakeSearchResponse);
+    await setUp(monitoringConfig, fakeClient, mailQueue);
+
+    expect(loggerErrorStub.callCount).to.be.equal(2);
+    expect(mailQueueSpy.callCount).to.be.equal(2);
+  });
+});

backend/test/storage/elasticsearch/query.spec.ts

@@ -0,0 +1,661 @@
+/* eslint-disable @typescript-eslint/no-explicit-any,unicorn/no-null */
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+import {
+  SCConfigFile,
+  SCSearchBooleanFilter,
+  SCSearchDateRangeFilter,
+  SCSearchFilter,
+  SCSearchNumericRangeFilter,
+  SCSearchQuery,
+  SCSearchSort,
+  SCThingType,
+} from '@openstapps/core';
+import {expect} from 'chai';
+import {
+  ESDateRangeFilter,
+  ESRangeFilter,
+  ESNumericRangeFilter,
+  ElasticsearchConfig,
+  ESBooleanFilter,
+  ESGenericSort,
+  ESGeoDistanceFilter,
+  ESGeoDistanceSort,
+  ESTermFilter,
+  ScriptSort,
+} from '../../../src/storage/elasticsearch/types/elasticsearch';
+import {configFile} from '../../../src/common';
+import {
+  buildBooleanFilter,
+  buildFilter,
+  buildQuery,
+  buildSort,
+} from '../../../src/storage/elasticsearch/query';
+
+describe('Query', function () {
+  describe('buildBooleanFilter', function () {
+    const booleanFilter: SCSearchBooleanFilter = {
+      arguments: {
+        operation: 'and',
+        filters: [
+          {
+            type: 'value',
+            arguments: {
+              field: 'type',
+              value: SCThingType.Catalog,
+            },
+          },
+          {
+            type: 'value',
+            arguments: {
+              field: 'type',
+              value: SCThingType.Building,
+            },
+          },
+        ],
+      },
+      type: 'boolean',
+    };
+    const booleanFilters: {[key: string]: SCSearchBooleanFilter} = {
+      and: booleanFilter,
+      or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
+      not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
+    };
+    const expectedEsFilters: Array<ESTermFilter> = [
+      {
+        term: {
+          'type.raw': 'catalog',
+        },
+      },
+      {
+        term: {
+          'type.raw': 'building',
+        },
+      },
+    ];
+
+    it('should create appropriate elasticsearch "and" filter argument', function () {
+      const {must} = buildBooleanFilter(booleanFilters.and);
+
+      expect(must).to.be.eql(expectedEsFilters);
+    });
+
+    it('should create appropriate elasticsearch "or" filter argument', function () {
+      const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
+
+      expect(should).to.be.eql(expectedEsFilters);
+      expect(minimum_should_match).to.be.equal(1);
+    });
+
+    it('should create appropriate elasticsearch "not" filter argument', function () {
+      const {must_not} = buildBooleanFilter(booleanFilters.not);
+
+      expect(must_not).to.be.eql(expectedEsFilters);
+    });
+  });
+
+  describe('buildQuery', function () {
+    const parameters: SCSearchQuery = {
+      query: 'mathematics',
+      from: 30,
+      size: 5,
+      sort: [
+        {
+          type: 'ducet',
+          order: 'desc',
+          arguments: {
+            field: 'name',
+          },
+        },
+        {
+          type: 'ducet',
+          order: 'desc',
+          arguments: {
+            field: 'categories',
+          },
+        },
+      ],
+      filter: {
+        type: 'value',
+        arguments: {
+          field: 'type',
+          value: SCThingType.AcademicEvent,
+        },
+      },
+    };
+    let esConfig: ElasticsearchConfig = {
+      name: 'elasticsearch',
+      version: '123',
+      query: {
+        minMatch: '75%',
+        queryType: 'dis_max',
+        matchBoosting: 1.3,
+        fuzziness: 'AUTO',
+        cutoffFrequency: 0,
+        tieBreaker: 0,
+      },
+    };
+    const query = {
+      minMatch: '75%',
+      queryType: 'dis_max',
+      matchBoosting: 1.3,
+      fuzziness: 'AUTO',
+      cutoffFrequency: 0,
+      tieBreaker: 0,
+    };
+    const config: SCConfigFile = {
+      ...configFile,
+    };
+    beforeEach(function () {
+      esConfig = {
+        name: 'elasticsearch',
+        version: '123',
+      };
+    });
+
+    // TODO: check parts of received elasticsearch query for each test case
+
+    it('should build query that includes sorting when query is undefined', function () {
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should build query that includes sorting when query type is query_string', function () {
+      esConfig.query = {...query, queryType: 'query_string'};
+
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should build query that includes sorting when query type is dis_max', function () {
+      esConfig.query = {...query, queryType: 'dis_max'};
+
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should build query that includes sorting when query type is dis_max', function () {
+      esConfig.query = {...query, queryType: 'dis_max'};
+
+      expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
+    });
+
+    it('should reject (throw an error) if provided query type is not supported', function () {
+      // @ts-expect-error not assignable
+      esConfig.query = {...query, queryType: 'invalid_query_type'};
+
+      expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
+    });
+  });
+
+  describe('buildFilter', function () {
+    const searchFilters: {[key: string]: SCSearchFilter} = {
+      value: {
+        type: 'value',
+        arguments: {
+          field: 'type',
+          value: SCThingType.Dish,
+        },
+      },
+      distance: {
+        type: 'distance',
+        arguments: {
+          distance: 1000,
+          field: 'geo',
+          position: [50.123, 8.123],
+        },
+      },
+      geoPoint: {
+        type: 'geo',
+        arguments: {
+          field: 'geo',
+          shape: {
+            type: 'envelope',
+            coordinates: [
+              [50.123, 8.123],
+              [50.123, 8.123],
+            ],
+          },
+        },
+      },
+      geoShape: {
+        type: 'geo',
+        arguments: {
+          field: 'geo',
+          spatialRelation: 'contains',
+          shape: {
+            type: 'envelope',
+            coordinates: [
+              [50.123, 8.123],
+              [50.123, 8.123],
+            ],
+          },
+        },
+      },
+      boolean: {
+        type: 'boolean',
+        arguments: {
+          operation: 'and',
+          filters: [
+            {
+              type: 'value',
+              arguments: {
+                field: 'type',
+                value: SCThingType.Dish,
+              },
+            },
+            {
+              type: 'availability',
+              arguments: {
+                field: 'offers.availabilityRange',
+              },
+            },
+          ],
+        },
+      },
+    };
+
+    it('should build value filter', function () {
+      const filter = buildFilter(searchFilters.value);
+      const expectedFilter: ESTermFilter = {
+        term: {
+          'type.raw': SCThingType.Dish,
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build numeric range filters', function () {
+      for (const upperMode of ['inclusive', 'exclusive', null]) {
+        for (const lowerMode of ['inclusive', 'exclusive', null]) {
+          const expectedFilter: ESNumericRangeFilter = {
+            range: {
+              price: {
+                relation: undefined,
+              },
+            },
+          };
+
+          const rawFilter: SCSearchNumericRangeFilter = {
+            type: 'numeric range',
+            arguments: {
+              bounds: {},
+              field: 'price',
+            },
+          };
+
+          // eslint-disable-next-line unicorn/consistent-function-scoping
+          const setBound = (location: 'upperBound' | 'lowerBound', bound: string | null) => {
+            let out: number | null = null;
+            if (bound != undefined) {
+              out = Math.random();
+              rawFilter.arguments.bounds[location] = {
+                mode: bound as 'inclusive' | 'exclusive',
+                limit: out,
+              };
+              expectedFilter.range.price[
+                `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
+              ] = out;
+            }
+          };
+          setBound('upperBound', upperMode);
+          setBound('lowerBound', lowerMode);
+
+          const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
+          expect(filter).to.deep.equal(expectedFilter);
+          for (const bound of ['g', 'l']) {
+            // @ts-expect-error implicit any
+            const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
+            // @ts-expect-error implicit any
+            const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
+
+            // only one should exist at the same time
+            expect(inclusiveExists && exclusiveExists).to.be.false;
+          }
+        }
+      }
+    });
+
+    it('should build date range filters', function () {
+      for (const upperMode of ['inclusive', 'exclusive', null]) {
+        for (const lowerMode of ['inclusive', 'exclusive', null]) {
+          const expectedFilter: ESDateRangeFilter = {
+            range: {
+              price: {
+                format: 'thisIsADummyFormat',
+                time_zone: 'thisIsADummyTimeZone',
+                relation: 'testRelation' as any,
+              },
+            },
+          };
+
+          const rawFilter: SCSearchDateRangeFilter = {
+            type: 'date range',
+            arguments: {
+              bounds: {},
+              field: 'price',
+              relation: 'testRelation' as any,
+              format: 'thisIsADummyFormat',
+              timeZone: 'thisIsADummyTimeZone',
+            },
+          };
+
+          const setBound = (location: 'upperBound' | 'lowerBound', bound: string | null) => {
+            let out: string | null = null;
+            if (bound != undefined) {
+              out = `${location} ${bound} ${upperMode} ${lowerMode}`;
+              rawFilter.arguments.bounds[location] = {
+                mode: bound as 'inclusive' | 'exclusive',
+                limit: out,
+              };
+              expectedFilter.range.price[
+                `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
+              ] = out;
+            }
+          };
+          setBound('upperBound', upperMode);
+          setBound('lowerBound', lowerMode);
+
+          const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
+          expect(filter).to.deep.equal(expectedFilter);
+          for (const bound of ['g', 'l']) {
+            // @ts-expect-error implicit any
+            const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
+            // @ts-expect-error implicit any
+            const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
+
+            // only one should exist at the same time
+            expect(inclusiveExists && exclusiveExists).to.be.false;
+          }
+        }
+      }
+    });
+
+    it('should build availability filters', function () {
+      it('should copy scope', function () {
+        for (const scope of ['a', 'b']) {
+          const filter = buildFilter({
+            type: 'availability',
+            arguments: {
+              time: 'test',
+              scope: scope as any,
+              field: 'offers.availabilityRange',
+            },
+          });
+
+          const expectedFilter: ESRangeFilter = {
+            range: {
+              'offers.availabilityRange': {
+                gte: `test||/${scope}`,
+                lt: `test||+1${scope}/${scope}`,
+              },
+            },
+          };
+          expect(filter).to.be.eql(expectedFilter);
+        }
+      });
+
+      it('should default to second scope', function () {
+        const filter = buildFilter({
+          type: 'availability',
+          arguments: {
+            time: 'test',
+            field: 'offers.availabilityRange',
+          },
+        });
+
+        const expectedFilter: ESRangeFilter = {
+          range: {
+            'offers.availabilityRange': {
+              gte: 'test||/s',
+              lt: 'test||+1s/s',
+            },
+          },
+        };
+        expect(filter).to.be.eql(expectedFilter);
+      });
+
+      it('should add || to dates', function () {
+        const filter = buildFilter({
+          type: 'availability',
+          arguments: {
+            time: 'test',
+            scope: 'd',
+            field: 'offers.availabilityRange',
+          },
+        });
+
+        const expectedFilter: ESRangeFilter = {
+          range: {
+            'offers.availabilityRange': {
+              gte: `test||/d`,
+              lt: `test||+1d/d`,
+            },
+          },
+        };
+        expect(filter).to.be.eql(expectedFilter);
+      });
+
+      it('should default to now and not add ||', function () {
+        const filter = buildFilter({
+          type: 'availability',
+          arguments: {
+            scope: 'd',
+            field: 'offers.availabilityRange',
+          },
+        });
+
+        const expectedFilter: ESRangeFilter = {
+          range: {
+            'offers.availabilityRange': {
+              gte: `now/d`,
+              lt: `now+1d/d`,
+            },
+          },
+        };
+        expect(filter).to.be.eql(expectedFilter);
+      });
+    });
+
+    it('should build distance filter', function () {
+      const filter = buildFilter(searchFilters.distance);
+      const expectedFilter: ESGeoDistanceFilter = {
+        geo_distance: {
+          'distance': '1000m',
+          'geo.point.coordinates': {
+            lat: 8.123,
+            lon: 50.123,
+          },
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build geo filter for shapes and points', function () {
+      const filter = buildFilter(searchFilters.geoPoint);
+      const expectedFilter = {
+        bool: {
+          minimum_should_match: 1,
+          should: [
+            {
+              geo_shape: {
+                'geo.polygon': {
+                  relation: undefined,
+                  shape: {
+                    type: 'envelope',
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                  },
+                },
+                'ignore_unmapped': true,
+              },
+            },
+            {
+              geo_bounding_box: {
+                'geo.point.coordinates': {
+                  bottom_right: [50.123, 8.123],
+                  top_left: [50.123, 8.123],
+                },
+                'ignore_unmapped': true,
+              },
+            },
+          ],
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build geo filter for shapes only', function () {
+      const filter = buildFilter(searchFilters.geoShape);
+      const expectedFilter = {
+        geo_shape: {
+          'geo.polygon': {
+            relation: 'contains',
+            shape: {
+              type: 'envelope',
+              coordinates: [
+                [50.123, 8.123],
+                [50.123, 8.123],
+              ],
+            },
+          },
+          'ignore_unmapped': true,
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+
+    it('should build boolean filter', function () {
+      const filter = buildFilter(searchFilters.boolean);
+      const expectedFilter: ESBooleanFilter<any> = {
+        bool: {
+          minimum_should_match: 0,
+          must: [
+            {
+              term: {
+                'type.raw': 'dish',
+              },
+            },
+            {
+              range: {
+                'offers.availabilityRange': {
+                  gte: 'now/s',
+                  lt: 'now+1s/s',
+                  relation: 'intersects',
+                },
+              },
+            },
+          ],
+          must_not: [],
+          should: [],
+        },
+      };
+
+      expect(filter).to.be.eql(expectedFilter);
+    });
+  });
+
+  describe('buildSort', function () {
+    const searchSCSearchSort: Array<SCSearchSort> = [
+      {
+        type: 'ducet',
+        order: 'desc',
+        arguments: {
+          field: 'name',
+        },
+      },
+      {
+        type: 'generic',
+        order: 'desc',
+        arguments: {
+          field: 'name',
+        },
+      },
+      {
+        type: 'distance',
+        order: 'desc',
+        arguments: {
+          field: 'geo',
+          position: [8.123, 50.123],
+        },
+      },
+      {
+        type: 'price',
+        order: 'asc',
+        arguments: {
+          universityRole: 'student',
+          field: 'offers.prices',
+        },
+      },
+    ];
+    let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
+    const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
+      ducet: {
+        'name.sort': 'desc',
+      },
+      generic: {
+        name: 'desc',
+      },
+      distance: {
+        _geo_distance: {
+          'mode': 'avg',
+          'order': 'desc',
+          'unit': 'm',
+          'geo.point.coordinates': {
+            lat: 50.123,
+            lon: 8.123,
+          },
+        },
+      },
+      price: {
+        _script: {
+          order: 'asc',
+          script: '\n  // foo price sort script',
+          type: 'number',
+        },
+      },
+    };
+    before(function () {
+      sorts = buildSort(searchSCSearchSort);
+    });
+
+    it('should build ducet sort', function () {
+      expect(sorts[0]).to.be.eql(expectedSorts.ducet);
+    });
+
+    it('should build generic sort', function () {
+      expect(sorts[1]).to.be.eql(expectedSorts.generic);
+    });
+
+    it('should build distance sort', function () {
+      expect(sorts[2]).to.be.eql(expectedSorts.distance);
+    });
+
+    it('should build price sort', function () {
+      const priceSortNoScript = {
+        ...sorts[3],
+        _script: {
+          ...(sorts[3] as ScriptSort)._script,
+          script: (expectedSorts.price as ScriptSort)._script.script,
+        },
+      };
+      expect(priceSortNoScript).to.be.eql(expectedSorts.price);
+    });
+  });
+});