diff --git a/test/common.spec.ts b/test/common.spec.ts
index cd9eca10..bc2f31a5 100644
--- a/test/common.spec.ts
+++ b/test/common.spec.ts
@@ -22,10 +22,12 @@ describe('Common', function () {
expect(inRangeInclusive(1, [1,3])).to.be.true;
expect(inRangeInclusive(2, [1,3])).to.be.true;
expect(inRangeInclusive(1.1, [1,3])).to.be.true;
+ expect(inRangeInclusive(3, [1, 3])).to.be.true;
});
it('should provide false if the given number is not in the range', function () {
expect(inRangeInclusive(3.1, [1,3])).to.be.false;
+ expect(inRangeInclusive(0, [1, 3])).to.be.false;
});
});
});
diff --git a/test/notification/backend-transport.spec.ts b/test/notification/backend-transport.spec.ts
new file mode 100644
index 00000000..43155715
--- /dev/null
+++ b/test/notification/backend-transport.spec.ts
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {SMTP} from '@openstapps/logger/lib/smtp';
+import {Transport} from '@openstapps/logger/lib/transport';
+import {expect} from 'chai';
+import mockedEnv from 'mocked-env';
+import {BackendTransport, isTransportWithVerification} from '../../src/notification/backend-transport';
+import sinon from 'sinon';
+
+describe('Backend transport', function () {
+ describe('isTransportWithVerification', function () {
+
+ it('should return false if transport is not verifiable', function () {
+ expect(isTransportWithVerification({} as Transport)).to.be.false;
+ expect(isTransportWithVerification({verify: 'foo'} as unknown as Transport)).to.be.false;
+ });
+
+ it('should return true if transport is verifiable', function () {
+ // a transport which has verify function should be verifiable
+ expect(isTransportWithVerification({verify: () => {}} as unknown as Transport)).to.be.true;
+ });
+ });
+
+ describe('BackendTransport', async function () {
+ const sandbox = sinon.createSandbox();
+
+ afterEach(function () {
+ BackendTransport.destroy();
+ sandbox.restore();
+ });
+
+ it('should provide only one instance of the transport', function () {
+ // @ts-ignore
+ sandbox.stub(SMTP, 'getInstance').callsFake(() => {
+ return {};
+ });
+ const transport1 = BackendTransport.getTransportInstance();
+ const transport2 = BackendTransport.getTransportInstance();
+ expect(transport1).to.be.equal(transport2);
+ });
+
+ it('should not throw in case of error getting SMTP instance when transport not allowed', function () {
+ sandbox.stub(SMTP, 'getInstance').throws('Foo Error');
+ const restore = mockedEnv({
+ ALLOW_NO_TRANSPORT: 'true',
+ });
+
+ expect(() => BackendTransport.getTransportInstance()).to.not.throw();
+
+ // restore env variables
+ restore();
+ });
+
+ it('should throw in case of error getting SMTP instance when transport is allowed', function () {
+ sandbox.stub(SMTP, 'getInstance').throws('Foo Error');
+ const restore = mockedEnv({
+ ALLOW_NO_TRANSPORT: undefined,
+ });
+
+ expect(() => BackendTransport.getTransportInstance()).to.throw();
+
+ // restore env variables
+ restore();
+ });
+
+ it('should provide information that the transport if waiting for its verification', function () {
+ // @ts-ignore
+ sandbox.stub(SMTP, 'getInstance').callsFake(() => {return {verify: () => Promise.resolve(true)}});
+
+ expect(BackendTransport.getInstance().isWaitingForVerification()).to.be.true;
+ });
+
+ it('should provide information that the transport if not waiting for its verification after the verification is over', function () {
+ sinon.stub(SMTP.prototype, 'verify').resolves(true);
+
+ expect(BackendTransport.getInstance().isWaitingForVerification()).to.be.false;
+ });
+ });
+});
diff --git a/test/notification/mail-queue.spec.ts b/test/notification/mail-queue.spec.ts
new file mode 100644
index 00000000..f6a47460
--- /dev/null
+++ b/test/notification/mail-queue.spec.ts
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {Logger} from '@openstapps/logger';
+import sinon from 'sinon';
+import {MailQueue} from '../../src/notification/mail-queue';
+import {expect} from 'chai';
+import Queue from 'promise-queue';
+import {MailOptions} from 'nodemailer/lib/sendmail-transport';
+import {getTransport, TRANSPORT_SEND_RESPONSE} from '../common';
+
+describe('MailQueue', async function () {
+ const sandbox = sinon.createSandbox();
+ let clock: sinon.SinonFakeTimers;
+
+ beforeEach(() => {
+ clock = sandbox.useFakeTimers();
+ });
+
+ afterEach(function () {
+ clock.restore();
+ sandbox.restore();
+ });
+
+ it('should fail after maximal number of verification checks', function () {
+ const loggerStub = sandbox.spy(Logger, 'warn');
+ const test = () => {
+ // @ts-ignore
+ new MailQueue(getTransport(false));
+ // fake that VERIFICATION_TIMEOUT was reached more times (one more) than MAX_VERIFICATION_ATTEMPTS
+ clock.tick(MailQueue.VERIFICATION_TIMEOUT * (MailQueue.MAX_VERIFICATION_ATTEMPTS + 1));
+ };
+
+ expect(() => test()).to.throw();
+ expect(loggerStub.callCount).to.be.equal(MailQueue.MAX_VERIFICATION_ATTEMPTS);
+ });
+
+ it('should add queued mails to the queue for sending when transport is verified', async function () {
+ const queueAddStub = sandbox.stub(Queue.prototype, 'add');
+ const numberOfMails = 3;
+ let transport = getTransport(false);
+ // @ts-ignore
+ const mailQueue = new MailQueue(transport);
+ const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'};
+ for (let i = 0; i < numberOfMails; i++) {
+ await mailQueue.push(mail);
+ }
+
+ // fake that transport is verified
+ transport.isVerified = () => true;
+ clock.tick(MailQueue.VERIFICATION_TIMEOUT);
+
+ expect(queueAddStub.callCount).to.be.equal(numberOfMails);
+ });
+
+ it('should not add SMTP sending tasks to queue when transport is not verified', function () {
+ const queueAddStub = sandbox.stub(Queue.prototype, 'add');
+ // @ts-ignore
+ const mailQueue = new MailQueue(getTransport(false));
+ const mail: MailOptions = {};
+ mailQueue.push(mail);
+
+ expect(queueAddStub.called).to.be.false;
+ });
+
+ it('should add SMTP sending tasks to queue when transport is verified', function () {
+ const queueAddStub = sandbox.stub(Queue.prototype, 'add');
+ let transport = getTransport(false);
+ // @ts-ignore
+ const mailQueue = new MailQueue(transport);
+ const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'};
+ // fake that transport is verified
+ transport.isVerified = () => true;
+ mailQueue.push(mail);
+
+ expect(queueAddStub.called).to.be.true;
+ });
+
+ it('should send SMTP mails when transport is verified', async function () {
+ let caught: any;
+ sandbox.stub(Queue.prototype, 'add').callsFake(async (promiseGenerator) => {
+ caught = await promiseGenerator();
+ });
+ let transport = getTransport(false);
+ // @ts-ignore
+ const mailQueue = new MailQueue(transport);
+ const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'};
+ // fake that transport is verified
+ transport.isVerified = () => true;
+ await mailQueue.push(mail);
+
+ expect(caught).to.be.equal(TRANSPORT_SEND_RESPONSE);
+ });
+});
diff --git a/test/routes/virtual-plugin-route.spec.ts b/test/routes/virtual-plugin-route.spec.ts
index 8ee30595..2c0a11cf 100644
--- a/test/routes/virtual-plugin-route.spec.ts
+++ b/test/routes/virtual-plugin-route.spec.ts
@@ -177,16 +177,16 @@ describe('Virtual plugin routes', async function () {
.post('/foo', {query: 'bar'})
.reply(200, {result: [{foo: 'bar'}, {bar: 'bar'}]});
- const fooResponse = await testApp
- .post('/foo')
- .set('Content-Type', 'application/json')
- .set('Accept', 'application/json')
- .send({query: 'foo'});
- const barResponse = await testApp
- .post('/foo')
- .set('Content-Type', 'application/json')
- .set('Accept', 'application/json')
- .send({query: 'bar'});
+ const fooResponse = await testApp
+ .post('/foo')
+ .set('Content-Type', 'application/json')
+ .set('Accept', 'application/json')
+ .send({query: 'foo'});
+ const barResponse = await testApp
+ .post('/foo')
+ .set('Content-Type', 'application/json')
+ .set('Accept', 'application/json')
+ .send({query: 'bar'});
expect(fooResponse.status).to.be.equal(OK);
expect(fooResponse.body).to.be.deep.equal({result: [{foo: 'foo'}, {bar: 'foo'}]});
diff --git a/test/storage/bulk-storage.spec.ts b/test/storage/bulk-storage.spec.ts
new file mode 100644
index 00000000..7420f5d9
--- /dev/null
+++ b/test/storage/bulk-storage.spec.ts
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {SCBulkRequest, SCThingType} from '@openstapps/core';
+import moment from 'moment';
+import util from 'util';
+import {configFile} from '../../src/common';
+import {Bulk, BulkStorage} from '../../src/storage/bulk-storage';
+import {expect} from 'chai';
+import {ElasticsearchMock} from '../common';
+import sinon from 'sinon';
+import NodeCache from 'node-cache';
+
+describe('Bulk Storage', function () {
+ describe('Bulk', function () {
+ let bulkRequest: SCBulkRequest;
+ beforeEach(function () {
+ bulkRequest = {source: 'some_source', type: SCThingType.Book};
+ });
+
+ it('should create a bulk with the given expiration', function () {
+ const expiration = moment().add(3600, 'seconds').toISOString();
+ bulkRequest.expiration = expiration;
+
+ const bulk = new Bulk(bulkRequest);
+
+ expect(bulk.expiration).to.be.equal(expiration);
+ expect(bulk.state).to.be.equal('in progress');
+ expect(bulk.uid).to.not.be.undefined;
+ });
+
+ it('should fallback and set expiration when it is not provided', function () {
+ const bulk = new Bulk(bulkRequest);
+
+ expect(bulk.expiration).to.not.be.undefined;
+ });
+ });
+
+ describe('BulkStorage', async function () {
+ const sandbox = sinon.createSandbox();
+ const bulkRequest = {source: 'some_source', type: SCThingType.Book};
+ const bulk = new Bulk(bulkRequest);
+ bulk.uid = '123';
+ let esMock: sinon.SinonStub;
+ let database: ElasticsearchMock;
+
+ beforeEach(function () {
+ database = new ElasticsearchMock(configFile);
+ esMock = sandbox.stub(database, 'bulkExpired');
+ });
+
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ it('should call appropriate database clean-up method on expire', async function () {
+ sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
+ new BulkStorage(database);
+
+ expect(esMock.calledWith(bulk)).to.be.true;
+ });
+
+ it('should not call appropriate database clean-up method on expire if bulk\'s state is done', async function () {
+ bulk.state = 'done';
+ sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
+ new BulkStorage(database);
+
+ expect(esMock.called).to.be.false;
+ });
+
+ it('should throw an error if the bulk for deletion cannot be read', async function () {
+ sandbox.stub(BulkStorage.prototype, 'read').callsFake(async () => Promise.resolve(undefined));
+ const bulkStorage = new BulkStorage(database);
+
+ return expect(bulkStorage.delete('123')).to.be.rejected;
+ });
+
+ it('should delete a bulk', async function () {
+ const readStub = sandbox.stub(BulkStorage.prototype, 'read').callsFake(async () => Promise.resolve(bulk));
+ let caught: any;
+ sandbox.stub(NodeCache.prototype, 'del').callsFake(() => caught = 123);
+ // force call
+ sandbox.stub(util, 'promisify').callsFake(() => () => {}).yields(null);
+ const bulkStorage = new BulkStorage(database);
+
+ await bulkStorage.delete(bulk.uid);
+
+ expect(readStub.called).to.be.true;
+ expect(caught).to.be.equal(123);
+ expect(esMock.called).to.be.true;
+ });
+
+ it('should read an existing bulk', async function () {
+ let caught: any;
+ sandbox.stub(NodeCache.prototype, 'get').callsFake(() => caught = 123);
+ // force call
+ sandbox.stub(util, 'promisify').callsFake(() => () => {}).yields(null);
+ const bulkStorage = new BulkStorage(database);
+
+ await bulkStorage.read(bulk.uid);
+
+ expect(caught).to.be.equal(123);
+ });``
+ });
+});
diff --git a/test/storage/elasticsearch/aggregations.spec.ts b/test/storage/elasticsearch/aggregations.spec.ts
new file mode 100644
index 00000000..ed09c177
--- /dev/null
+++ b/test/storage/elasticsearch/aggregations.spec.ts
@@ -0,0 +1,281 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {SCFacet, SCThingType} from '@openstapps/core';
+import {expect} from 'chai';
+import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
+import {AggregationResponse, AggregationSchema} from '../../../src/storage/elasticsearch/common';
+
+describe('Aggregations', function () {
+ const schema: AggregationSchema = {
+ '@all': {
+ aggs: {
+ type: {
+ terms: {
+ field: 'type.raw',
+ size: 1000
+ }
+ }
+ },
+ filter: {
+ match_all: {}
+ }
+ },
+ 'academic event': {
+ aggs: {
+ 'academicTerms.acronym': {
+ terms: {
+ field: 'academicTerms.acronym.raw',
+ size: 1000
+ }
+ },
+ 'catalogs.categories': {
+ terms: {
+ field: 'catalogs.categories.raw',
+ size: 1000
+ }
+ },
+ categories: {
+ terms: {
+ field: 'categories.raw',
+ size: 1000
+ }
+ },
+ 'creativeWorks.keywords': {
+ terms: {
+ field: 'creativeWorks.keywords.raw',
+ size: 1000
+ }
+ },
+ majors: {
+ terms: {
+ field: 'majors.raw',
+ size: 1000
+ }
+ }
+ },
+ filter: {
+ type: {
+ value: 'academic event'
+ }
+ }
+ },
+ catalog: {
+ aggs: {
+ 'academicTerm.acronym': {
+ terms: {
+ field: 'academicTerm.acronym.raw',
+ size: 1000
+ }
+ },
+ categories: {
+ terms: {
+ field: 'categories.raw',
+ size: 1000
+ }
+ },
+ 'superCatalog.categories': {
+ terms: {
+ field: 'superCatalog.categories.raw',
+ size: 1000
+ }
+ },
+ 'superCatalogs.categories': {
+ terms: {
+ field: 'superCatalogs.categories.raw',
+ size: 1000
+ }
+ }
+ },
+ filter: {
+ type: {
+ value: 'catalog'
+ }
+ }
+ },
+ person: {
+ aggs: {
+ 'homeLocations.categories': {
+ terms: {
+ field: 'homeLocations.categories.raw',
+ size: 1000
+ }
+ }
+ },
+ filter: {
+ type: {
+ value: 'person'
+ }
+ }
+ },
+ fooType: {
+ terms: {
+ field: 'foo',
+ size: 123,
+ }
+ }
+ };
+
+ const aggregations: AggregationResponse = {
+ catalog: {
+ doc_count: 4,
+ 'superCatalogs.categories': {
+ buckets: []
+ },
+ 'academicTerm.acronym': {
+ buckets: [
+ {
+ key: 'SoSe 2020',
+ doc_count: 2
+ }
+ ]
+ },
+ 'superCatalog.categories': {
+ buckets: []
+ },
+ categories: {
+ buckets: [
+ {
+ key: 'foo',
+ doc_count: 1,
+ },
+ {
+ key: 'bar',
+ doc_count: 3,
+ },
+ ]
+ }
+ },
+ person: {
+ doc_count: 13,
+ 'homeLocations.categories': {
+ buckets: []
+ }
+ },
+ 'academic event': {
+ doc_count: 0,
+ 'academicTerms.acronym': {
+ buckets: []
+ },
+ categories: {
+ buckets: [
+ {
+ key: 'foobar',
+ doc_count: 8,
+ },
+ {
+ key: 'bar',
+ doc_count: 2,
+ },
+ ]
+ },
+ 'creativeWorks.keywords': {
+ buckets: []
+ }
+ },
+ fooType: {
+ buckets: [
+ {
+ doc_count: 321,
+ key: 'foo'
+ }
+ ],
+ },
+ '@all': {
+ doc_count: 17,
+ type: {
+ buckets: [
+ {
+ key: 'person',
+ doc_count: 13
+ },
+ {
+ key: 'catalog',
+ doc_count: 4
+ }
+ ]
+ }
+ }
+ };
+
+ const expectedFacets: SCFacet[] = [
+ {
+ buckets: [
+ {
+ count: 13,
+ 'key': 'person'
+ },
+ {
+ count: 4,
+ key: 'catalog'
+ }
+ ],
+ field: 'type',
+ },
+ {
+ buckets: [
+ {
+ count: 8,
+ key: 'foobar'
+ },
+ {
+ count: 2,
+ key: 'bar'
+ }
+ ],
+ field: 'categories',
+ onlyOnType: SCThingType.AcademicEvent,
+ },
+ {
+ buckets: [
+ {
+ count: 2,
+ key: 'SoSe 2020'
+ }
+ ],
+ field: 'academicTerm.acronym',
+ onlyOnType: SCThingType.Catalog
+ },
+ {
+ buckets: [
+ {
+ count: 1,
+ key: 'foo'
+ },
+ {
+ count: 3,
+ key: 'bar'
+ }
+ ],
+ field: 'categories',
+ onlyOnType: SCThingType.Catalog,
+ },
+ {
+ buckets: [
+ {
+ count: 321,
+ key: 'foo'
+ }
+ ],
+ field: 'fooType'
+ }
+ ];
+
+ it('should parse the aggregations providing the appropriate facets', function () {
+ const facets = parseAggregations(schema, aggregations);
+
+ expect(facets).to.be.eql(expectedFacets);
+ });
+});
diff --git a/test/storage/elasticsearch/common.spec.ts b/test/storage/elasticsearch/common.spec.ts
new file mode 100644
index 00000000..82ead6fc
--- /dev/null
+++ b/test/storage/elasticsearch/common.spec.ts
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {
+ ESAggMatchAllFilter,
+ ESAggTypeFilter,
+ ESNestedAggregation,
+ ESTermsFilter
+} from '@openstapps/core-tools/lib/mappings/aggregation-definitions';
+import { expect } from "chai";
+import {
+ BucketAggregation,
+ isBucketAggregation, isESAggMatchAllFilter, isESNestedAggregation, isESTermsFilter,
+ isNestedAggregation,
+ NestedAggregation
+} from '../../../src/storage/elasticsearch/common';
+
+describe('Common', function () {
+ const bucketAggregation: BucketAggregation = {buckets: []};
+ const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
+ const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
+
+ describe('isBucketAggregation', function () {
+ it('should be false for a number', function () {
+ expect(isBucketAggregation(123)).to.be.false;
+ });
+
+ it('should be true for a bucket aggregation', function () {
+ expect(isBucketAggregation(bucketAggregation)).to.be.true;
+ });
+ });
+
+ describe('isNestedAggregation', function () {
+ it('should be false for a bucket aggregation', function () {
+ expect(isNestedAggregation(bucketAggregation)).to.be.false;
+ });
+
+ it('should be true for a nested aggregation', function () {
+ const nestedAggregation: NestedAggregation = {doc_count: 123};
+
+ expect(isNestedAggregation(nestedAggregation)).to.be.true;
+ });
+ });
+
+ describe('isESTermsFilter', function () {
+ it('should be false for an elasticsearch nested aggregation', function () {
+ expect(isESTermsFilter(esNestedAggregation)).to.be.false;
+ });
+
+ it('should be true for an elasticsearch terms filter', function () {
+ expect(isESTermsFilter(esTermsFilter)).to.be.true;
+ });
+ });
+
+ describe('isESNestedAggregation', function () {
+ it('should be false for an elasticsearch terms filter', function () {
+ expect(isESNestedAggregation(esTermsFilter)).to.be.false;
+ });
+
+ it('should be true for an elasticsearch nested aggregation', function () {
+ expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
+ });
+ });
+
+ describe('isESAggMatchAllFilter', function () {
+ it('should be false for an elasticsearch aggregation type filter', function () {
+ const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
+ expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
+ });
+
+ it('should be true for an elasticsearch aggregation match all filter', function () {
+ const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
+ expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
+ });
+ });
+});
diff --git a/test/storage/elasticsearch/elasticsearch.spec.ts b/test/storage/elasticsearch/elasticsearch.spec.ts
new file mode 100644
index 00000000..7907a5f8
--- /dev/null
+++ b/test/storage/elasticsearch/elasticsearch.spec.ts
@@ -0,0 +1,641 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {SCBook, SCBulkResponse, SCConfigFile, SCMessage, SCSearchQuery, SCThings, SCThingType} from '@openstapps/core';
+import {instance as book} from '@openstapps/core/test/resources/Book.1.json';
+import {instance as message} from '@openstapps/core/test/resources/Message.1.json';
+import {Logger} from '@openstapps/logger';
+import {SMTP} from '@openstapps/logger/lib/smtp';
+import {expect, use} from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import {SearchResponse} from 'elasticsearch';
+import mockedEnv from 'mocked-env';
+import sinon from 'sinon';
+import {configFile} from '../../../src/common';
+import {MailQueue} from '../../../src/notification/mail-queue';
+import * as aggregations from '../../../src/storage/elasticsearch/aggregations';
+import {ElasticsearchObject} from '../../../src/storage/elasticsearch/common';
+import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
+import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
+import * as query from '../../../src/storage/elasticsearch/query';
+import * as templating from '../../../src/storage/elasticsearch/templating';
+import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, index} from '../../common';
+import fs from 'fs';
+
+use(chaiAsPromised);
+
+describe('Elasticsearch', function () {
+ // increase timeout for the suite
+ this.timeout(DEFAULT_TEST_TIMEOUT);
+ const sandbox = sinon.createSandbox();
+ let checkESTemplateStub: sinon.SinonStub = sandbox.stub(templating, 'checkESTemplate');
+
+ before(function () {
+ console.log('before');
+ sandbox.stub(fs, 'readFileSync').returns('{}');
+ });
+ after(function () {
+ sandbox.restore();
+ });
+
+ describe('getElasticsearchUrl', function () {
+ it('should provide custom elasticsearch URL if defined', function () {
+ const customAddress = 'http://foo-address:9200';
+ const restore = mockedEnv({
+ ES_ADDR: customAddress
+ });
+
+ expect(Elasticsearch.getElasticsearchUrl()).to.be.equal(customAddress);
+ // restore env variables
+ restore();
+ });
+
+ it('should provide local URL as fallback', function () {
+ const restore = mockedEnv({
+ ES_ADDR: undefined
+ });
+
+ expect(Elasticsearch.getElasticsearchUrl()).to.match(/(https?:\/\/)?localhost(:\d+)?/);
+ // restore env variables
+ restore();
+ });
+ });
+
+ describe('getIndex (including getIndexUID)', function () {
+ const type = 'foo bar type';
+ const source = 'foo_source';
+ const bulk: SCBulkResponse = {
+ expiration: '',
+ source: '',
+ state: 'in progress',
+ type: SCThingType.Semester,
+ uid: 'bulk-uid-123-123-123'
+ };
+
+ it('should provide index UID from the provided UID', function () {
+ const indexUID = Elasticsearch.getIndexUID(bulk.uid);
+
+ expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
+ // test starting and ending character
+ expect(indexUID[0]).to.be.equal(bulk.uid[0]);
+ expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
+ });
+
+ it('should provide index name from the provided data', function () {
+ expect(Elasticsearch.getIndex(type as SCThingType, source, bulk))
+ .to.be.equal(`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`);
+ });
+ });
+
+ describe('removeAliasChars', function () {
+ it('should remove invalid characters', function () {
+ expect(Elasticsearch.removeAliasChars('f,o#o\\ b|ar/ * ', 'bulk-uid')).to.be.equal('foobaralias');
+ });
+
+ it('should remove invalid starting characters', function () {
+ expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+ expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+ expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
+ });
+
+ it('should replace with a placeholder in case of invalid alias', function () {
+ expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
+ expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
+ });
+
+ it('should work with common cases', function () {
+ expect(Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'))
+ .to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
+ expect(Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'))
+ .to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
+ });
+
+ it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
+ const sandbox = sinon.createSandbox();
+ const loggerWarnStub = sandbox.stub(Logger, 'warn');
+
+ expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
+ expect(loggerWarnStub.called).to.be.true;
+ });
+ });
+
+ describe('constructor', async function () {
+ const sandbox = sinon.createSandbox();
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ it('should complain (throw an error) if database in config is undefined', function () {
+ const config: SCConfigFile = {...configFile, internal: {...configFile.internal, database: undefined}};
+
+ expect(() => new Elasticsearch(config)).to.throw(Error);
+ });
+
+ it('should complain (throw an error) if database version is not a string', function () {
+ const config: SCConfigFile = {
+ ...configFile,
+ internal: {
+ ...configFile.internal,
+ database: {
+ name: 'foo',
+ version: 123
+ }
+ }
+ };
+
+ expect(() => new Elasticsearch(config)).to.throw(Error);
+ });
+
+ it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
+ const error = Error('Foo Error');
+ const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
+ sandbox.stub(Client.prototype, 'on').yields(error);
+
+ new Elasticsearch(configFile);
+
+ expect(loggerErrorStub.calledWith(error)).to.be.true;
+ });
+
+ it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
+ const fakeResponse = {foo: 'bar'};
+ const loggerLogStub = sandbox.stub(Logger, 'log');
+ sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
+
+ new Elasticsearch(configFile);
+ expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
+
+ const restore = mockedEnv({
+ 'ES_DEBUG': 'true',
+ });
+ new Elasticsearch(configFile);
+
+ expect(loggerLogStub.calledWith(fakeResponse)).to.be.true;
+ // restore env variables
+ restore();
+ });
+
+ it('should force mapping update if related process env variable is not set', async function () {
+ const restore = mockedEnv({
+ 'ES_FORCE_MAPPING_UPDATE': undefined,
+ });
+
+ new Elasticsearch(configFile);
+
+ expect(checkESTemplateStub.calledWith(false)).to.be.true;
+ // restore env variables
+ restore();
+ });
+
+ it('should force mapping update if related process env variable is set', async function () {
+ const restore = mockedEnv({
+ 'ES_FORCE_MAPPING_UPDATE': 'true',
+ });
+
+ new Elasticsearch(configFile);
+
+ expect(checkESTemplateStub.calledWith(true)).to.be.true;
+ // restore env variables
+ restore();
+ });
+ });
+
+ describe('init', async function () {
+ const sandbox = sinon.createSandbox();
+ after(function () {
+ sandbox.restore();
+ });
+
+ it('should complain (throw an error) if monitoring is set but mail queue is undefined', async function () {
+ const config: SCConfigFile = {
+ ...configFile,
+ internal: {
+ ...configFile.internal,
+ monitoring: {
+ actions: [],
+ watchers: [],
+ }
+ }
+ };
+
+ const es = new Elasticsearch(config);
+
+ return expect(es.init()).to.be.rejected;
+ });
+
+ it('should setup the monitoring if there is monitoring is set and mail queue is defined', function () {
+ const config: SCConfigFile = {
+ ...configFile,
+ internal: {
+ ...configFile.internal,
+ monitoring: {
+ actions: [],
+ watchers: [],
+ }
+ }
+ };
+ const monitoringSetUpStub = sandbox.stub(Monitoring, 'setUp');
+
+ const es = new Elasticsearch(config, new MailQueue(getTransport(false) as unknown as SMTP));
+
+ es.init();
+
+ expect(monitoringSetUpStub.called).to.be.true;
+ });
+ });
+
+ describe('Operations with bundle/index', async function () {
+ const sandbox = sinon.createSandbox();
+ let es: Elasticsearch;
+ const oldIndex = 'stapps_footype_foosource_oldindex';
+
+ beforeEach(function () {
+ es = new Elasticsearch(configFile);
+ // @ts-ignore
+ es.client.indices = {
+ // @ts-ignore
+ getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
+ // @ts-ignore
+ putTemplate: () => Promise.resolve({}),
+ // @ts-ignore
+ create: () => Promise.resolve({}),
+ // @ts-ignore
+ delete: () => Promise.resolve({}),
+ // @ts-ignore
+ exists: () => Promise.resolve({}),
+ // @ts-ignore
+ refresh: () => Promise.resolve({}),
+ // @ts-ignore
+ updateAliases: () => Promise.resolve({})
+ };
+ });
+
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ describe('bulkCreated', async function () {
+ it('should reject (throw an error) if the connection to elasticsearch is not established', async function () {
+ return expect(es.bulkCreated(bulk)).to.be.rejectedWith('elasticsearch');
+ });
+
+ it('should reject (throw an error) if the index name is not valid', async function () {
+ sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${index}`);
+ sandbox.createStubInstance(Client, {});
+ await es.init();
+
+ return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
+ });
+
+ it('should create a new index', async function () {
+ sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+ const putTemplateStub = sandbox.stub(templating, 'putTemplate');
+ const createStub = sandbox.stub(es.client.indices, 'create');
+ await es.init();
+
+ await es.bulkCreated(bulk);
+
+ expect(putTemplateStub.called).to.be.true;
+ expect(createStub.calledWith({index})).to.be.true;
+ });
+ });
+
+ describe('bulkExpired', async function () {
+ const sandbox = sinon.createSandbox();
+ afterEach(function () {
+ sandbox.restore();
+ });
+ it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
+ sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+ const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
+
+ await es.bulkExpired({...bulk, state: 'in progress'});
+
+ expect(clientDeleteStub.called).to.be.true;
+ });
+
+ it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
+ sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+ const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
+
+ await es.bulkExpired({...bulk, state: 'done'});
+
+ expect(clientDeleteStub.called).to.be.false;
+ });
+ });
+
+ describe('bulkUpdated', function () {
+ it('should reject if the connection to elasticsearch is not established', async function () {
+ return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('elasticsearch');
+ });
+
+ it('should reject if the index name is not valid', async function () {
+ sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${index}`);
+ sandbox.createStubInstance(Client, {});
+ await es.init();
+
+ return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
+ });
+
+ it('should create a new index', async function () {
+ const expectedRefreshActions = [
+ {
+ add: {index: index, alias: SCThingType.Book},
+ },
+ {
+ remove: {index: oldIndex, alias: SCThingType.Book},
+ }
+ ];
+ sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+ sandbox.stub(es, 'aliasMap').value({
+ [SCThingType.Book]: {
+ [bulk.source]: oldIndex,
+ }
+ });
+ const refreshStub = sandbox.stub(es.client.indices, 'refresh');
+ const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
+ const deleteStub = sandbox.stub(es.client.indices, 'delete');
+ sandbox.stub(templating, 'putTemplate');
+ await es.init();
+
+ await es.bulkUpdated(bulk);
+
+ expect(refreshStub.calledWith({index})).to.be.true;
+ expect(updateAliasesStub.calledWith({
+ body: {
+ actions: expectedRefreshActions
+ }
+ })
+ ).to.be.true;
+ expect(deleteStub.called).to.be.true;
+ });
+ });
+ });
+
+ describe('get', async function () {
+ let es: Elasticsearch;
+ const sandbox = sinon.createSandbox();
+
+ before(function () {
+ es = new Elasticsearch(configFile);
+ });
+
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ it('should reject if object is not found', async function () {
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}});
+
+ return expect(es.get('123')).to.rejectedWith('found');
+ });
+
+ it('should provide the thing if object is found', async function () {
+ const foundObject: ElasticsearchObject = {_id: '', _index: '', _score: 0, _type: '', _source: message as SCMessage};
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: [foundObject]}}});
+
+ return expect(await es.get('123')).to.be.eql(message);
+ });
+ });
+
+ describe('post', async function () {
+ let es: Elasticsearch;
+ const sandbox = sinon.createSandbox();
+
+ before(function () {
+ es = new Elasticsearch(configFile);
+ });
+
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ it('should not post if the object already exists in an index which will not be rolled over', async function () {
+ const oldIndex = index.replace('foosource', 'barsource');
+ const object: ElasticsearchObject = {_id: '', _index: oldIndex, _score: 0, _type: '', _source: message as SCMessage};
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: [object]}}});
+ sandbox.stub(Elasticsearch, 'getIndex').returns(index);
+
+ return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
+ });
+
+ it('should reject if there is an object creation error on the elasticsearch side', async function () {
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}});
+ sandbox.stub(es.client, 'create').resolves({body: {created: false}});
+
+ return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
+ });
+
+ it('should create a new object', async function () {
+ let caughtParam: any;
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}});
+ // @ts-ignore
+ let createStub = sandbox.stub(es.client, 'create').callsFake((param) => {
+ caughtParam = param;
+ return Promise.resolve({body: { created: true }});
+ });
+
+ await es.post(message as SCMessage, bulk);
+
+ expect(createStub.called).to.be.true;
+ expect(caughtParam.body).to.be.eql({...message, creation_date: caughtParam.body.creation_date});
+ });
+ });
+
+ describe('put', async function () {
+ let es: Elasticsearch;
+ const sandbox = sinon.createSandbox();
+
+ before(function () {
+ es = new Elasticsearch(configFile);
+ });
+ afterEach(function () {
+ sandbox.restore();
+ });
+ it('should reject to put if the object does not already exist', async function () {
+ const oldIndex = index.replace('foosource', 'barsource');
+ const object: ElasticsearchObject = {_id: '', _index: oldIndex, _score: 0, _type: '', _source: message as SCMessage};
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}});
+
+ return expect(es.put(object._source)).to.rejectedWith('exist');
+ });
+
+ it('should update the object if it already exists', async function () {
+ let caughtParam: any;
+ const oldIndex = index.replace('foosource', 'barsource');
+ const object: ElasticsearchObject = {_id: '', _index: oldIndex, _score: 0, _type: '', _source: message as SCMessage};
+ sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: [object]}}});
+ // @ts-ignore
+ const stubUpdate = sandbox.stub(es.client, 'update').callsFake((params) => {
+ caughtParam = params;
+ return Promise.resolve({body: { created: true }});
+ });
+
+ await es.put(object._source);
+
+ expect(caughtParam.body.doc).to.be.eql(object._source);
+ });
+ });
+
+ describe('search', async function () {
+ let es: Elasticsearch;
+ const sandbox = sinon.createSandbox();
+ const objectMessage: ElasticsearchObject = {_id: '123', _index: index, _score: 0, _type: '', _source: message as SCMessage};
+ const objectBook: ElasticsearchObject = {_id: '321', _index: index, _score: 0, _type: '', _source: book as SCBook};
+ const fakeEsAggregations = {
+ '@all': {
+ doc_count: 17,
+ type: {
+ doc_count_error_upper_bound: 0,
+ sum_other_doc_count: 0,
+ buckets: [
+ {
+ key: 'person',
+ doc_count: 13
+ },
+ {
+ key: 'catalog',
+ doc_count: 4
+ }
+ ]
+ }
+ }
+ };
+ const fakeSearchResponse: Partial>> = {
+ // @ts-ignore
+ body: {
+ took: 12,
+ timed_out: false,
+ // @ts-ignore
+ _shards: {},
+ // @ts-ignore
+ hits: {
+ hits: [
+ objectMessage,
+ objectBook,
+ ],
+ total: 123
+ },
+ aggregations: fakeEsAggregations
+ },
+ headers: {},
+ // @ts-ignore
+ meta: {},
+ // @ts-ignore
+ statusCode: {},
+ // @ts-ignore
+ warnings: {}
+ };
+ let searchStub: sinon.SinonStub;
+ before(function () {
+ es = new Elasticsearch(configFile);
+ });
+ beforeEach(function () {
+ searchStub = sandbox.stub(es.client, 'search').resolves(fakeSearchResponse);
+ });
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ it('should provide appropriate data and facets', async function () {
+ const fakeFacets = [
+ {
+ buckets: [
+ {
+ count: 1,
+ 'key': 'foo'
+ },
+ {
+ count: 1,
+ key: 'bar'
+ }
+ ],
+ field: 'type',
+ }
+ ];
+ const parseAggregationsStub = sandbox.stub(aggregations, 'parseAggregations').returns(fakeFacets);
+
+ const {data, facets} = await es.search({});
+
+ expect(data).to.be.eql([objectMessage._source, objectBook._source]);
+ expect(facets).to.be.eql(fakeFacets);
+ expect(parseAggregationsStub.calledWith(sinon.match.any, fakeEsAggregations)).to.be.true;
+ });
+
+ it('should provide pagination from params', async function () {
+ const from = 30;
+ const {pagination} = await es.search({from});
+
+ expect(pagination).to.be.eql({
+ count: fakeSearchResponse.body!.hits.hits.length,
+ offset: from,
+ total: fakeSearchResponse.body!.hits.total
+ });
+ });
+
+ it('should have fallback to zero if from is not given through params', async function () {
+ const {pagination} = await es.search({});
+
+ expect(pagination.offset).to.be.equal(0);
+ });
+
+ it('should build the search request properly', async function () {
+ const params: SCSearchQuery = {
+ query: 'mathematics',
+ from: 30,
+ size: 5,
+ sort: [
+ {
+ type: 'ducet',
+ order: 'desc',
+ arguments:
+ {
+ field: 'name'
+ }
+ }
+ ],
+ filter: {
+ type: 'value',
+ arguments: {
+ field: 'type',
+ value: SCThingType.AcademicEvent
+ }
+ }
+ };
+ const fakeResponse = {foo: 'bar'};
+ const fakeBuildSortResponse = [fakeResponse];
+ // @ts-ignore
+ sandbox.stub(query, 'buildQuery').returns(fakeResponse);
+ // @ts-ignore
+ sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
+
+ await es.search(params);
+
+ sandbox.assert
+ .calledWithMatch(searchStub,
+ {
+ body: {
+ aggs: es.aggregationsSchema,
+ query: fakeResponse,
+ sort: fakeBuildSortResponse
+ },
+ from: params.from,
+ index: Elasticsearch.getListOfAllIndices(),
+ size: params.size,
+ }
+ );
+ });
+ });
+});
diff --git a/test/storage/elasticsearch/monitoring.spec.ts b/test/storage/elasticsearch/monitoring.spec.ts
new file mode 100644
index 00000000..9cc3aef0
--- /dev/null
+++ b/test/storage/elasticsearch/monitoring.spec.ts
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {ApiResponse, Client} from '@elastic/elasticsearch';
+import {
+ SCMonitoringConfiguration,
+ SCMonitoringLogAction,
+ SCMonitoringMailAction,
+ SCMonitoringWatcher, SCThings
+} from '@openstapps/core';
+import {Logger} from '@openstapps/logger';
+import {SearchResponse} from 'elasticsearch';
+import {MailQueue} from '../../../src/notification/mail-queue';
+import {setUp} from '../../../src/storage/elasticsearch/monitoring';
+
+import {getTransport} from '../../common';
+import { expect } from 'chai';
+import sinon from 'sinon';
+import cron from 'node-cron';
+import * as templating from '../../../src/storage/elasticsearch/templating';
+
+describe('Monitoring', async function () {
+ const sandbox = sinon.createSandbox();
+ const logAction: SCMonitoringLogAction = {
+ message: 'Foo monitoring message',
+ prefix: 'Backend Monitoring',
+ type: 'log'
+ };
+ const mailAction: SCMonitoringMailAction = {
+ message: 'Bar monitoring message',
+ recipients: ['xyz@xyz.com'],
+ subject: 'Backend Monitoring',
+ type: 'mail'
+ };
+ let transport: any;
+ // @ts-ignore
+ let mailQueue: any;
+ beforeEach(async function () {
+ transport = getTransport(true);
+ mailQueue = new MailQueue(transport);
+ cronScheduleStub = sandbox.stub(cron, 'schedule');
+ sandbox.stub(templating, 'checkESTemplate');
+ });
+ afterEach(async function () {
+ sandbox.restore();
+ });
+ // const sandbox = sinon.createSandbox();
+ let cronScheduleStub: sinon.SinonStub
+ const minLengthWatcher: SCMonitoringWatcher = {
+ actions: [logAction, mailAction],
+ conditions: [
+ {
+ length: 10,
+ type: 'MinimumLength'
+ }
+ ],
+ name: 'foo watcher',
+ query: {foo: 'bar'},
+ triggers: [
+ {
+ executionTime: 'monthly',
+ name: 'beginning of month'
+ },
+ {
+ executionTime: 'daily',
+ name: 'every night'
+ }
+ ]
+ };
+ const maxLengthWatcher: SCMonitoringWatcher = {
+ actions: [logAction, mailAction],
+ conditions: [
+ {
+ length: 30,
+ type: 'MaximumLength'
+ }
+ ],
+ name: 'foo watcher',
+ query: {bar: 'foo'},
+ triggers: [
+ {
+ executionTime: 'hourly',
+ name: 'every hour'
+ },
+ {
+ executionTime: 'weekly',
+ name: 'every week'
+ },
+ ]
+ };
+ const monitoringConfig: SCMonitoringConfiguration = {
+ actions: [logAction, mailAction],
+ watchers: [minLengthWatcher, maxLengthWatcher]
+ };
+
+ it('should create a schedule for each trigger', async function () {
+ await setUp(monitoringConfig, new Client({node: 'http://foohost:9200'}), mailQueue);
+
+ expect(cronScheduleStub.callCount).to.be.equal(4);
+ });
+
+ it('should log errors where conditions failed', async function () {
+ const fakeSearchResponse: Partial>> = {
+ // @ts-ignore
+ body: {
+ took: 12,
+ timed_out: false,
+ // @ts-ignore
+ _shards: {},
+ // @ts-ignore
+ hits: {
+ total: 123
+ },
+ },
+ };
+ let fakeClient = new Client({node: 'http://foohost:9200'});
+ const loggerErrorStub = sandbox.stub(Logger, 'error');
+ const mailQueueSpy = sinon.spy(mailQueue, 'push');
+ cronScheduleStub.yields();
+ sandbox.stub(fakeClient, 'search').resolves(fakeSearchResponse);
+ await setUp(monitoringConfig, fakeClient, mailQueue);
+
+ expect(loggerErrorStub.callCount).to.be.equal(2);
+ expect(mailQueueSpy.callCount).to.be.equal(2);
+ });
+});
diff --git a/test/storage/elasticsearch/query.spec.ts b/test/storage/elasticsearch/query.spec.ts
new file mode 100644
index 00000000..572e649b
--- /dev/null
+++ b/test/storage/elasticsearch/query.spec.ts
@@ -0,0 +1,442 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {
+ SCConfigFile,
+ SCSearchBooleanFilter,
+ SCSearchFilter,
+ SCSearchQuery,
+ SCSearchSort,
+ SCThingType
+} from '@openstapps/core';
+import { expect } from 'chai';
+import {configFile} from '../../../src/common';
+import {
+ ElasticsearchConfig, ESBooleanFilter, ESDucetSort, ESGeoDistanceFilter,
+ ESGeoDistanceSort,
+ ESTermFilter,
+ ScriptSort
+} from '../../../src/storage/elasticsearch/common';
+import {buildBooleanFilter, buildFilter, buildQuery, buildSort} from '../../../src/storage/elasticsearch/query';
+
+describe('Query', function () {
+ describe('buildBooleanFilter', function () {
+ const booleanFilter: SCSearchBooleanFilter = {
+ arguments: {
+ operation: 'and',
+ filters: [
+ {
+ type: 'value',
+ arguments: {
+ field: 'type',
+ value: SCThingType.Catalog
+ }
+ },
+ {
+ type: 'value',
+ arguments: {
+ field: 'type',
+ value: SCThingType.Building
+ }
+ }
+ ]
+ },
+ type: 'boolean'
+ };
+ const booleanFilters: {[key: string]: SCSearchBooleanFilter} = {
+ and: booleanFilter,
+ or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
+ not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
+ };
+ const expectedEsFilters: Array = [
+ {
+ term: {
+ 'type.raw': 'catalog'
+ }
+ },
+ {
+ term: {
+ 'type.raw': 'building'
+ }
+ }
+ ];
+
+ it('should create appropriate elasticsearch "and" filter argument', function () {
+ const {must} = buildBooleanFilter(booleanFilters.and);
+
+ expect(must).to.be.eql(expectedEsFilters);
+ });
+
+ it('should create appropriate elasticsearch "or" filter argument', function () {
+ const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
+
+ expect(should).to.be.eql(expectedEsFilters);
+ expect(minimum_should_match).to.be.equal(1);
+ });
+
+ it('should create appropriate elasticsearch "not" filter argument', function () {
+ const {must_not} = buildBooleanFilter(booleanFilters.not);
+
+ expect(must_not).to.be.eql(expectedEsFilters);
+ });
+ });
+
+ describe('buildQuery', function () {
+ const params: SCSearchQuery = {
+ query: 'mathematics',
+ from: 30,
+ size: 5,
+ sort: [
+ {
+ type: 'ducet',
+ order: 'desc',
+ arguments:
+ {
+ field: 'name'
+ }
+ },
+ {
+ type: 'ducet',
+ order: 'desc',
+ arguments:
+ {
+ field: 'categories'
+ }
+ },
+ ],
+ filter: {
+ type: 'value',
+ arguments: {
+ field: 'type',
+ value: SCThingType.AcademicEvent
+ }
+ }
+ };
+ let esConfig: ElasticsearchConfig = {
+ name: 'elasticsearch',
+ version: '123',
+ query: {
+ minMatch: '75%',
+ queryType: 'dis_max',
+ matchBoosting: 1.3,
+ fuzziness: 'AUTO',
+ cutoffFrequency: 0.0,
+ tieBreaker: 0,
+ },
+ };
+ const query = {
+ minMatch: '75%',
+ queryType: 'dis_max',
+ matchBoosting: 1.3,
+ fuzziness: 'AUTO',
+ cutoffFrequency: 0.0,
+ tieBreaker: 0,
+ }
+ const config: SCConfigFile = {
+ ...configFile
+ };
+ beforeEach(function () {
+ esConfig = {
+ name: 'elasticsearch',
+ version: '123'
+ };
+ });
+
+ // TODO: check parts of received elasticsearch query for each test case
+
+ it('should build query that includes sorting when query is undefined', function () {
+ expect(buildQuery(params, config, esConfig)).to.be.an('object');
+ });
+
+ it('should build query that includes sorting when query type is query_string', function () {
+ esConfig.query = {...query, queryType: 'query_string'};
+
+ expect(buildQuery(params, config, esConfig)).to.be.an('object');
+ });
+
+ it('should build query that includes sorting when query type is dis_max', function () {
+ esConfig.query = {...query, queryType: 'dis_max'};
+
+ expect(buildQuery(params, config, esConfig)).to.be.an('object');
+ });
+
+ it('should build query that includes sorting when query type is dis_max', function () {
+ esConfig.query = {...query, queryType: 'dis_max'};
+
+ expect(buildQuery(params, config, esConfig)).to.be.an('object');
+ });
+
+ it('should reject (throw an error) if provided query type is not supported', function () {
+ // @ts-ignore
+ esConfig.query = {...query, queryType: 'invalid_query_type'};
+
+ expect(() => buildQuery(params, config, esConfig)).to.throw('query type');
+ });
+ });
+
+ describe('buildFilter', function () {
+ const searchFilters: {[key: string]: SCSearchFilter} = {
+ value: {
+ type: 'value',
+ arguments: {
+ field: 'type',
+ value: SCThingType.Dish
+ }
+ },
+ availability: {
+ type: 'availability',
+ arguments: {
+ time: '2017-01-30T12:05:00.000Z',
+ fromField: 'offers.availabilityStarts',
+ toField: 'offers.availabilityEnds'
+ }
+ },
+ distance: {
+ type: 'distance',
+ arguments: {
+ distance: 1000,
+ field: 'geo.point.coordinates',
+ position: [50.123, 8.123],
+ }
+ },
+ boolean: {
+ type: 'boolean',
+ arguments: {
+ operation: 'and',
+ filters: [
+ {
+ type: 'value',
+ arguments: {
+ field: 'type',
+ value: SCThingType.Dish,
+ }
+ },
+ {
+ type: 'availability',
+ arguments: {
+ fromField: 'offers.availabilityStarts',
+ toField: 'offers.availabilityEnds'
+ }
+ }
+ ]
+ }
+ },
+ };
+
+ it('should build value filter', function () {
+ const filter = buildFilter(searchFilters.value);
+ const expectedFilter: ESTermFilter = {
+ term: {
+ 'type.raw': SCThingType.Dish
+ }
+ };
+
+ expect(filter).to.be.eql(expectedFilter);
+ });
+
+ it('should build availability filter', function () {
+ const filter = buildFilter(searchFilters.availability);
+ const expectedFilter: ESBooleanFilter = {
+ bool: {
+ should: [
+ {
+ bool: {
+ must: [
+ {
+ range: {
+ 'offers.availabilityStarts': {
+ lte: '2017-01-30T12:05:00.000Z'
+ }
+ }
+ },
+ {
+ range: {
+ 'offers.availabilityEnds': {
+ gte: '2017-01-30T12:05:00.000Z'
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ bool: {
+ must_not: [
+ {
+ exists: {
+ field: 'offers.availabilityStarts'
+ }
+ },
+ {
+ exists: {
+ field: 'offers.availabilityEnds'
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ };
+
+ expect(filter).to.be.eql(expectedFilter);
+ });
+
+ it('should build distance filter', function () {
+ const filter = buildFilter(searchFilters.distance);
+ const expectedFilter: ESGeoDistanceFilter = {
+ geo_distance: {
+ distance: '1000m',
+ 'geo.point.coordinates': {
+ lat: 8.123,
+ lon: 50.123
+ }
+ }
+ };
+
+ expect(filter).to.be.eql(expectedFilter);
+ });
+
+ it('should build boolean filter', function () {
+ const filter = buildFilter(searchFilters.boolean);
+ const expectedFilter: ESBooleanFilter = {
+ bool: {
+ minimum_should_match: 0,
+ must: [
+ {
+ term: {
+ 'type.raw': 'dish'
+ }
+ },
+ {
+ bool: {
+ should: [
+ {
+ bool: {
+ must: [
+ {
+ range: {
+ 'offers.availabilityStarts': {
+ lte: 'now'
+ }
+ }
+ },
+ {
+ range: {
+ 'offers.availabilityEnds': {
+ gte: 'now'
+ }
+ }
+ }
+ ]
+ }
+ },
+ {
+ bool: {
+ must_not: [
+ {
+ exists: {
+ field: 'offers.availabilityStarts'
+ }
+ },
+ {
+ exists: {
+ field: 'offers.availabilityEnds'
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ],
+ must_not: [],
+ should: []
+ }
+ }
+
+ expect(filter).to.be.eql(expectedFilter);
+ });
+ });
+
+ describe('buildSort', function () {
+ const searchSCSearchSort: Array = [
+ {
+ type: 'ducet',
+ order: 'desc',
+ arguments: {
+ field: 'name'
+ },
+ },
+ {
+ type: 'distance',
+ order: 'desc',
+ arguments: {
+ field: 'geo.point',
+ position: [8.123, 50.123]
+ },
+ },
+ {
+ type: 'price',
+ order: 'asc',
+ arguments: {
+ universityRole: 'student',
+ field: 'offers.prices',
+ }
+ },
+ ];
+ let sorts: Array = [];
+ const expectedSorts: {[key: string]: ESDucetSort | ESGeoDistanceSort | ScriptSort} = {
+ ducet: {
+ 'name.sort': 'desc'
+ },
+ distance: {
+ _geo_distance: {
+ mode: 'avg',
+ order: 'desc',
+ unit: 'm',
+ 'geo.point': {
+ lat: 50.123,
+ lon: 8.123
+ }
+ }
+ },
+ price: {
+ _script: {
+ order: 'asc',
+ script: '\n // foo price sort script',
+ type: 'number'
+ }
+ }
+ };
+ before(function () {
+ sorts = buildSort(searchSCSearchSort);
+ });
+
+ it('should build ducet sort', function () {
+ expect(sorts[0]).to.be.eql(expectedSorts.ducet);
+ });
+
+ it('should build distance sort', function () {
+ expect(sorts[1]).to.be.eql(expectedSorts.distance);
+ });
+
+ it('should build price sort', function () {
+ const priceSortNoScript = {...sorts[2], _script: {...(sorts[2] as ScriptSort)._script, script: (expectedSorts.price as ScriptSort)._script.script}}
+ expect(priceSortNoScript).to.be.eql(expectedSorts.price);
+ });
+ });
+});
diff --git a/test/storage/elasticsearch/templating.spec.ts b/test/storage/elasticsearch/templating.spec.ts
new file mode 100644
index 00000000..74140098
--- /dev/null
+++ b/test/storage/elasticsearch/templating.spec.ts
@@ -0,0 +1,201 @@
+/*
+ * Copyright (C) 2020 StApps
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import {SCThingType} from '@openstapps/core';
+import * as mapping from '@openstapps/core-tools/lib/mapping';
+import {ElasticsearchTemplateCollection} from '@openstapps/core-tools/lib/mappings/mapping-definitions';
+import {Logger} from '@openstapps/logger';
+import {AggregationSchema} from '../../../src/storage/elasticsearch/common';
+import {checkESTemplate, refreshAllTemplates} from '../../../src/storage/elasticsearch/templating';
+import sinon from "sinon";
+import * as path from 'path';
+import * as common from '@openstapps/core-tools/lib/common';
+import {expect} from 'chai';
+import fs from 'fs';
+import fsExtra from 'fs-extra';
+import {Client} from '@elastic/elasticsearch';
+
+describe('templating', function () {
+ describe('checkESTemplate', function () {
+ const sandbox = sinon.createSandbox();
+ let fakeMap: { aggregations: AggregationSchema, errors: string[], mappings: ElasticsearchTemplateCollection };
+ beforeEach(function () {
+ fakeMap = {
+ aggregations: {
+ '@all': {
+ aggs: {
+ type: {
+ terms: {
+ field: 'type.raw',
+ size: 1000
+ }
+ }
+ },
+ filter: {
+ match_all: {}
+ }
+ },
+ },
+ errors: [],
+ mappings: {
+ 'template_dish': {
+ mappings: {
+ dish: {
+ // @ts-ignore just mock the mapping
+ foo: 'mapping'
+ }
+ },
+ settings: {
+ analysis: {
+ ducet_sort: {
+ filter: [
+ 'german_phonebook'
+ ],
+ tokenizer: 'keyword',
+ type: 'custom'
+ },
+ search_german: {
+ filter: [
+ 'lowercase',
+ 'german_stop',
+ 'german_stemmer'
+ ],
+ tokenizer: 'stapps_ngram',
+ type: 'custom'
+ }
+ },
+ max_result_window: 30000,
+ },
+ template: 'stapps_dish*'
+ },
+ 'template_book': {
+ mappings: {
+ book: {
+ // @ts-ignore just mock the mapping
+ foo: 'mapping'
+ }
+ },
+ settings: {
+ analysis: {
+ ducet_sort: {
+ filter: [
+ 'german_phonebook'
+ ],
+ tokenizer: 'keyword',
+ type: 'custom'
+ },
+ search_german: {
+ filter: [
+ 'lowercase',
+ 'german_stop',
+ 'german_stemmer'
+ ],
+ tokenizer: 'stapps_ngram',
+ type: 'custom'
+ }
+ },
+ max_result_window: 30000,
+ },
+ template: 'stapps_book*'
+ }
+ }
+ }
+ });
+ afterEach(function () {
+ sandbox.restore();
+ });
+
+ it('should write new templates when "force update" is true', async function () {
+ sandbox.stub(Logger, 'error').resolves();
+ sandbox.stub(fs, 'existsSync').returns(true);
+ sandbox.stub(common, 'getProjectReflection');
+ let caughtData: any = [];
+ const writeFileSyncStub = sandbox.stub(fs, 'writeFileSync');
+ sandbox.stub(path, 'resolve').returns('/foo/bar');
+ sandbox.stub(mapping, 'generateTemplate').returns(fakeMap);
+
+ checkESTemplate(true);
+
+ expect(writeFileSyncStub.callCount).to.be.gt(0);
+ for (let i = 0; i < writeFileSyncStub.callCount; i++) {
+ caughtData.push(writeFileSyncStub.getCall(i).args[1]);
+ }
+
+ expect(caughtData).to.be.eql([
+ JSON.stringify(fakeMap.mappings['template_dish'], null, 2),
+ JSON.stringify(fakeMap.mappings['template_book'], null, 2),
+ JSON.stringify(fakeMap.aggregations),
+ ]);
+ });
+
+ it('should not write new templates when "force update" is false', async function () {
+ sandbox.stub(Logger, 'error').resolves();
+ sandbox.stub(fs, 'existsSync').returns(true);
+ sandbox.stub(common, 'getProjectReflection');
+ const writeFileSyncStub = sandbox.stub(fs, 'writeFileSync');
+ sandbox.stub(path, 'resolve').returns('/foo/bar');
+ sandbox.stub(mapping, 'generateTemplate').returns(fakeMap);
+
+ checkESTemplate(false);
+
+ expect(writeFileSyncStub.called).to.be.false;
+ });
+
+ it('should terminate if there are errors in the map', async function () {
+ const processExitStub = sandbox.stub(process, 'exit');
+ const fakeMapWithErrors = {
+ ...fakeMap,
+ errors: ['Foo Error']
+ };
+ sandbox.stub(Logger, 'error').resolves();
+ sandbox.stub(fs, 'existsSync').returns(true);
+ sandbox.stub(common, 'getProjectReflection');
+ sandbox.stub(fs, 'writeFileSync');
+ sandbox.stub(path, 'resolve').returns('/foo/bar');
+ sandbox.stub(mapping, 'generateTemplate').returns(fakeMapWithErrors);
+
+ checkESTemplate(true);
+
+ expect(processExitStub.called).to.be.true;
+ });
+ });
+
+ describe('refreshAllTemplates', async function () {
+ const sandbox = sinon.createSandbox();
+ const client = {
+ indices: {
+ putTemplate: (_template: any) => {
+ }
+ }
+ }
+
+ after(function () {
+ sandbox.restore();
+ });
+
+ it('should put templates for all types', async function () {
+ const clientPutTemplateStub = sandbox.stub(client.indices, 'putTemplate');
+ sandbox.stub(fsExtra, 'readFile').resolves(Buffer.from('{"foo": "file content"}', 'utf8'));
+ await refreshAllTemplates(client as Client);
+
+ for (const type of Object.values(SCThingType)) {
+ sinon.assert.calledWith(clientPutTemplateStub, {
+ body: {foo: 'file content'},
+ name: `template_${type.split(' ').join('_')}`
+ })
+ }
+ });
+ });
+});