From 5eefa3c2e11dc0a5d0c768d5d4fb2d7a72df8d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jovan=20Kruni=C4=87?= Date: Fri, 23 Oct 2020 11:27:59 +0200 Subject: [PATCH] test: add missing storage and notification tests Closes #31 --- test/common.spec.ts | 2 + test/notification/backend-transport.spec.ts | 92 +++ test/notification/mail-queue.spec.ts | 106 +++ test/routes/virtual-plugin-route.spec.ts | 20 +- test/storage/bulk-storage.spec.ts | 117 ++++ .../elasticsearch/aggregations.spec.ts | 281 ++++++++ test/storage/elasticsearch/common.spec.ts | 88 +++ .../elasticsearch/elasticsearch.spec.ts | 641 ++++++++++++++++++ test/storage/elasticsearch/monitoring.spec.ts | 138 ++++ test/storage/elasticsearch/query.spec.ts | 442 ++++++++++++ test/storage/elasticsearch/templating.spec.ts | 201 ++++++ 11 files changed, 2118 insertions(+), 10 deletions(-) create mode 100644 test/notification/backend-transport.spec.ts create mode 100644 test/notification/mail-queue.spec.ts create mode 100644 test/storage/bulk-storage.spec.ts create mode 100644 test/storage/elasticsearch/aggregations.spec.ts create mode 100644 test/storage/elasticsearch/common.spec.ts create mode 100644 test/storage/elasticsearch/elasticsearch.spec.ts create mode 100644 test/storage/elasticsearch/monitoring.spec.ts create mode 100644 test/storage/elasticsearch/query.spec.ts create mode 100644 test/storage/elasticsearch/templating.spec.ts diff --git a/test/common.spec.ts b/test/common.spec.ts index cd9eca10..bc2f31a5 100644 --- a/test/common.spec.ts +++ b/test/common.spec.ts @@ -22,10 +22,12 @@ describe('Common', function () { expect(inRangeInclusive(1, [1,3])).to.be.true; expect(inRangeInclusive(2, [1,3])).to.be.true; expect(inRangeInclusive(1.1, [1,3])).to.be.true; + expect(inRangeInclusive(3, [1, 3])).to.be.true; }); it('should provide false if the given number is not in the range', function () { expect(inRangeInclusive(3.1, [1,3])).to.be.false; + expect(inRangeInclusive(0, [1, 3])).to.be.false; }); }); }); diff --git a/test/notification/backend-transport.spec.ts b/test/notification/backend-transport.spec.ts new file mode 100644 index 00000000..43155715 --- /dev/null +++ b/test/notification/backend-transport.spec.ts @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {SMTP} from '@openstapps/logger/lib/smtp'; +import {Transport} from '@openstapps/logger/lib/transport'; +import {expect} from 'chai'; +import mockedEnv from 'mocked-env'; +import {BackendTransport, isTransportWithVerification} from '../../src/notification/backend-transport'; +import sinon from 'sinon'; + +describe('Backend transport', function () { + describe('isTransportWithVerification', function () { + + it('should return false if transport is not verifiable', function () { + expect(isTransportWithVerification({} as Transport)).to.be.false; + expect(isTransportWithVerification({verify: 'foo'} as unknown as Transport)).to.be.false; + }); + + it('should return true if transport is verifiable', function () { + // a transport which has verify function should be verifiable + expect(isTransportWithVerification({verify: () => {}} as unknown as Transport)).to.be.true; + }); + }); + + describe('BackendTransport', async function () { + const sandbox = sinon.createSandbox(); + + afterEach(function () { + BackendTransport.destroy(); + sandbox.restore(); + }); + + it('should provide only one instance of the transport', function () { + // @ts-ignore + sandbox.stub(SMTP, 'getInstance').callsFake(() => { + return {}; + }); + const transport1 = BackendTransport.getTransportInstance(); + const transport2 = BackendTransport.getTransportInstance(); + expect(transport1).to.be.equal(transport2); + }); + + it('should not throw in case of error getting SMTP instance when transport not allowed', function () { + sandbox.stub(SMTP, 'getInstance').throws('Foo Error'); + const restore = mockedEnv({ + ALLOW_NO_TRANSPORT: 'true', + }); + + expect(() => BackendTransport.getTransportInstance()).to.not.throw(); + + // restore env variables + restore(); + }); + + it('should throw in case of error getting SMTP instance when transport is allowed', function () { + sandbox.stub(SMTP, 'getInstance').throws('Foo Error'); + const restore = mockedEnv({ + ALLOW_NO_TRANSPORT: undefined, + }); + + expect(() => BackendTransport.getTransportInstance()).to.throw(); + + // restore env variables + restore(); + }); + + it('should provide information that the transport if waiting for its verification', function () { + // @ts-ignore + sandbox.stub(SMTP, 'getInstance').callsFake(() => {return {verify: () => Promise.resolve(true)}}); + + expect(BackendTransport.getInstance().isWaitingForVerification()).to.be.true; + }); + + it('should provide information that the transport if not waiting for its verification after the verification is over', function () { + sinon.stub(SMTP.prototype, 'verify').resolves(true); + + expect(BackendTransport.getInstance().isWaitingForVerification()).to.be.false; + }); + }); +}); diff --git a/test/notification/mail-queue.spec.ts b/test/notification/mail-queue.spec.ts new file mode 100644 index 00000000..f6a47460 --- /dev/null +++ b/test/notification/mail-queue.spec.ts @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {Logger} from '@openstapps/logger'; +import sinon from 'sinon'; +import {MailQueue} from '../../src/notification/mail-queue'; +import {expect} from 'chai'; +import Queue from 'promise-queue'; +import {MailOptions} from 'nodemailer/lib/sendmail-transport'; +import {getTransport, TRANSPORT_SEND_RESPONSE} from '../common'; + +describe('MailQueue', async function () { + const sandbox = sinon.createSandbox(); + let clock: sinon.SinonFakeTimers; + + beforeEach(() => { + clock = sandbox.useFakeTimers(); + }); + + afterEach(function () { + clock.restore(); + sandbox.restore(); + }); + + it('should fail after maximal number of verification checks', function () { + const loggerStub = sandbox.spy(Logger, 'warn'); + const test = () => { + // @ts-ignore + new MailQueue(getTransport(false)); + // fake that VERIFICATION_TIMEOUT was reached more times (one more) than MAX_VERIFICATION_ATTEMPTS + clock.tick(MailQueue.VERIFICATION_TIMEOUT * (MailQueue.MAX_VERIFICATION_ATTEMPTS + 1)); + }; + + expect(() => test()).to.throw(); + expect(loggerStub.callCount).to.be.equal(MailQueue.MAX_VERIFICATION_ATTEMPTS); + }); + + it('should add queued mails to the queue for sending when transport is verified', async function () { + const queueAddStub = sandbox.stub(Queue.prototype, 'add'); + const numberOfMails = 3; + let transport = getTransport(false); + // @ts-ignore + const mailQueue = new MailQueue(transport); + const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'}; + for (let i = 0; i < numberOfMails; i++) { + await mailQueue.push(mail); + } + + // fake that transport is verified + transport.isVerified = () => true; + clock.tick(MailQueue.VERIFICATION_TIMEOUT); + + expect(queueAddStub.callCount).to.be.equal(numberOfMails); + }); + + it('should not add SMTP sending tasks to queue when transport is not verified', function () { + const queueAddStub = sandbox.stub(Queue.prototype, 'add'); + // @ts-ignore + const mailQueue = new MailQueue(getTransport(false)); + const mail: MailOptions = {}; + mailQueue.push(mail); + + expect(queueAddStub.called).to.be.false; + }); + + it('should add SMTP sending tasks to queue when transport is verified', function () { + const queueAddStub = sandbox.stub(Queue.prototype, 'add'); + let transport = getTransport(false); + // @ts-ignore + const mailQueue = new MailQueue(transport); + const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'}; + // fake that transport is verified + transport.isVerified = () => true; + mailQueue.push(mail); + + expect(queueAddStub.called).to.be.true; + }); + + it('should send SMTP mails when transport is verified', async function () { + let caught: any; + sandbox.stub(Queue.prototype, 'add').callsFake(async (promiseGenerator) => { + caught = await promiseGenerator(); + }); + let transport = getTransport(false); + // @ts-ignore + const mailQueue = new MailQueue(transport); + const mail: MailOptions = {from: 'Foo', subject: 'Foo Subject'}; + // fake that transport is verified + transport.isVerified = () => true; + await mailQueue.push(mail); + + expect(caught).to.be.equal(TRANSPORT_SEND_RESPONSE); + }); +}); diff --git a/test/routes/virtual-plugin-route.spec.ts b/test/routes/virtual-plugin-route.spec.ts index 8ee30595..2c0a11cf 100644 --- a/test/routes/virtual-plugin-route.spec.ts +++ b/test/routes/virtual-plugin-route.spec.ts @@ -177,16 +177,16 @@ describe('Virtual plugin routes', async function () { .post('/foo', {query: 'bar'}) .reply(200, {result: [{foo: 'bar'}, {bar: 'bar'}]}); - const fooResponse = await testApp - .post('/foo') - .set('Content-Type', 'application/json') - .set('Accept', 'application/json') - .send({query: 'foo'}); - const barResponse = await testApp - .post('/foo') - .set('Content-Type', 'application/json') - .set('Accept', 'application/json') - .send({query: 'bar'}); + const fooResponse = await testApp + .post('/foo') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send({query: 'foo'}); + const barResponse = await testApp + .post('/foo') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json') + .send({query: 'bar'}); expect(fooResponse.status).to.be.equal(OK); expect(fooResponse.body).to.be.deep.equal({result: [{foo: 'foo'}, {bar: 'foo'}]}); diff --git a/test/storage/bulk-storage.spec.ts b/test/storage/bulk-storage.spec.ts new file mode 100644 index 00000000..7420f5d9 --- /dev/null +++ b/test/storage/bulk-storage.spec.ts @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {SCBulkRequest, SCThingType} from '@openstapps/core'; +import moment from 'moment'; +import util from 'util'; +import {configFile} from '../../src/common'; +import {Bulk, BulkStorage} from '../../src/storage/bulk-storage'; +import {expect} from 'chai'; +import {ElasticsearchMock} from '../common'; +import sinon from 'sinon'; +import NodeCache from 'node-cache'; + +describe('Bulk Storage', function () { + describe('Bulk', function () { + let bulkRequest: SCBulkRequest; + beforeEach(function () { + bulkRequest = {source: 'some_source', type: SCThingType.Book}; + }); + + it('should create a bulk with the given expiration', function () { + const expiration = moment().add(3600, 'seconds').toISOString(); + bulkRequest.expiration = expiration; + + const bulk = new Bulk(bulkRequest); + + expect(bulk.expiration).to.be.equal(expiration); + expect(bulk.state).to.be.equal('in progress'); + expect(bulk.uid).to.not.be.undefined; + }); + + it('should fallback and set expiration when it is not provided', function () { + const bulk = new Bulk(bulkRequest); + + expect(bulk.expiration).to.not.be.undefined; + }); + }); + + describe('BulkStorage', async function () { + const sandbox = sinon.createSandbox(); + const bulkRequest = {source: 'some_source', type: SCThingType.Book}; + const bulk = new Bulk(bulkRequest); + bulk.uid = '123'; + let esMock: sinon.SinonStub; + let database: ElasticsearchMock; + + beforeEach(function () { + database = new ElasticsearchMock(configFile); + esMock = sandbox.stub(database, 'bulkExpired'); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should call appropriate database clean-up method on expire', async function () { + sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk); + new BulkStorage(database); + + expect(esMock.calledWith(bulk)).to.be.true; + }); + + it('should not call appropriate database clean-up method on expire if bulk\'s state is done', async function () { + bulk.state = 'done'; + sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk); + new BulkStorage(database); + + expect(esMock.called).to.be.false; + }); + + it('should throw an error if the bulk for deletion cannot be read', async function () { + sandbox.stub(BulkStorage.prototype, 'read').callsFake(async () => Promise.resolve(undefined)); + const bulkStorage = new BulkStorage(database); + + return expect(bulkStorage.delete('123')).to.be.rejected; + }); + + it('should delete a bulk', async function () { + const readStub = sandbox.stub(BulkStorage.prototype, 'read').callsFake(async () => Promise.resolve(bulk)); + let caught: any; + sandbox.stub(NodeCache.prototype, 'del').callsFake(() => caught = 123); + // force call + sandbox.stub(util, 'promisify').callsFake(() => () => {}).yields(null); + const bulkStorage = new BulkStorage(database); + + await bulkStorage.delete(bulk.uid); + + expect(readStub.called).to.be.true; + expect(caught).to.be.equal(123); + expect(esMock.called).to.be.true; + }); + + it('should read an existing bulk', async function () { + let caught: any; + sandbox.stub(NodeCache.prototype, 'get').callsFake(() => caught = 123); + // force call + sandbox.stub(util, 'promisify').callsFake(() => () => {}).yields(null); + const bulkStorage = new BulkStorage(database); + + await bulkStorage.read(bulk.uid); + + expect(caught).to.be.equal(123); + });`` + }); +}); diff --git a/test/storage/elasticsearch/aggregations.spec.ts b/test/storage/elasticsearch/aggregations.spec.ts new file mode 100644 index 00000000..ed09c177 --- /dev/null +++ b/test/storage/elasticsearch/aggregations.spec.ts @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {SCFacet, SCThingType} from '@openstapps/core'; +import {expect} from 'chai'; +import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations'; +import {AggregationResponse, AggregationSchema} from '../../../src/storage/elasticsearch/common'; + +describe('Aggregations', function () { + const schema: AggregationSchema = { + '@all': { + aggs: { + type: { + terms: { + field: 'type.raw', + size: 1000 + } + } + }, + filter: { + match_all: {} + } + }, + 'academic event': { + aggs: { + 'academicTerms.acronym': { + terms: { + field: 'academicTerms.acronym.raw', + size: 1000 + } + }, + 'catalogs.categories': { + terms: { + field: 'catalogs.categories.raw', + size: 1000 + } + }, + categories: { + terms: { + field: 'categories.raw', + size: 1000 + } + }, + 'creativeWorks.keywords': { + terms: { + field: 'creativeWorks.keywords.raw', + size: 1000 + } + }, + majors: { + terms: { + field: 'majors.raw', + size: 1000 + } + } + }, + filter: { + type: { + value: 'academic event' + } + } + }, + catalog: { + aggs: { + 'academicTerm.acronym': { + terms: { + field: 'academicTerm.acronym.raw', + size: 1000 + } + }, + categories: { + terms: { + field: 'categories.raw', + size: 1000 + } + }, + 'superCatalog.categories': { + terms: { + field: 'superCatalog.categories.raw', + size: 1000 + } + }, + 'superCatalogs.categories': { + terms: { + field: 'superCatalogs.categories.raw', + size: 1000 + } + } + }, + filter: { + type: { + value: 'catalog' + } + } + }, + person: { + aggs: { + 'homeLocations.categories': { + terms: { + field: 'homeLocations.categories.raw', + size: 1000 + } + } + }, + filter: { + type: { + value: 'person' + } + } + }, + fooType: { + terms: { + field: 'foo', + size: 123, + } + } + }; + + const aggregations: AggregationResponse = { + catalog: { + doc_count: 4, + 'superCatalogs.categories': { + buckets: [] + }, + 'academicTerm.acronym': { + buckets: [ + { + key: 'SoSe 2020', + doc_count: 2 + } + ] + }, + 'superCatalog.categories': { + buckets: [] + }, + categories: { + buckets: [ + { + key: 'foo', + doc_count: 1, + }, + { + key: 'bar', + doc_count: 3, + }, + ] + } + }, + person: { + doc_count: 13, + 'homeLocations.categories': { + buckets: [] + } + }, + 'academic event': { + doc_count: 0, + 'academicTerms.acronym': { + buckets: [] + }, + categories: { + buckets: [ + { + key: 'foobar', + doc_count: 8, + }, + { + key: 'bar', + doc_count: 2, + }, + ] + }, + 'creativeWorks.keywords': { + buckets: [] + } + }, + fooType: { + buckets: [ + { + doc_count: 321, + key: 'foo' + } + ], + }, + '@all': { + doc_count: 17, + type: { + buckets: [ + { + key: 'person', + doc_count: 13 + }, + { + key: 'catalog', + doc_count: 4 + } + ] + } + } + }; + + const expectedFacets: SCFacet[] = [ + { + buckets: [ + { + count: 13, + 'key': 'person' + }, + { + count: 4, + key: 'catalog' + } + ], + field: 'type', + }, + { + buckets: [ + { + count: 8, + key: 'foobar' + }, + { + count: 2, + key: 'bar' + } + ], + field: 'categories', + onlyOnType: SCThingType.AcademicEvent, + }, + { + buckets: [ + { + count: 2, + key: 'SoSe 2020' + } + ], + field: 'academicTerm.acronym', + onlyOnType: SCThingType.Catalog + }, + { + buckets: [ + { + count: 1, + key: 'foo' + }, + { + count: 3, + key: 'bar' + } + ], + field: 'categories', + onlyOnType: SCThingType.Catalog, + }, + { + buckets: [ + { + count: 321, + key: 'foo' + } + ], + field: 'fooType' + } + ]; + + it('should parse the aggregations providing the appropriate facets', function () { + const facets = parseAggregations(schema, aggregations); + + expect(facets).to.be.eql(expectedFacets); + }); +}); diff --git a/test/storage/elasticsearch/common.spec.ts b/test/storage/elasticsearch/common.spec.ts new file mode 100644 index 00000000..82ead6fc --- /dev/null +++ b/test/storage/elasticsearch/common.spec.ts @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + ESAggMatchAllFilter, + ESAggTypeFilter, + ESNestedAggregation, + ESTermsFilter +} from '@openstapps/core-tools/lib/mappings/aggregation-definitions'; +import { expect } from "chai"; +import { + BucketAggregation, + isBucketAggregation, isESAggMatchAllFilter, isESNestedAggregation, isESTermsFilter, + isNestedAggregation, + NestedAggregation +} from '../../../src/storage/elasticsearch/common'; + +describe('Common', function () { + const bucketAggregation: BucketAggregation = {buckets: []}; + const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}}; + const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}}; + + describe('isBucketAggregation', function () { + it('should be false for a number', function () { + expect(isBucketAggregation(123)).to.be.false; + }); + + it('should be true for a bucket aggregation', function () { + expect(isBucketAggregation(bucketAggregation)).to.be.true; + }); + }); + + describe('isNestedAggregation', function () { + it('should be false for a bucket aggregation', function () { + expect(isNestedAggregation(bucketAggregation)).to.be.false; + }); + + it('should be true for a nested aggregation', function () { + const nestedAggregation: NestedAggregation = {doc_count: 123}; + + expect(isNestedAggregation(nestedAggregation)).to.be.true; + }); + }); + + describe('isESTermsFilter', function () { + it('should be false for an elasticsearch nested aggregation', function () { + expect(isESTermsFilter(esNestedAggregation)).to.be.false; + }); + + it('should be true for an elasticsearch terms filter', function () { + expect(isESTermsFilter(esTermsFilter)).to.be.true; + }); + }); + + describe('isESNestedAggregation', function () { + it('should be false for an elasticsearch terms filter', function () { + expect(isESNestedAggregation(esTermsFilter)).to.be.false; + }); + + it('should be true for an elasticsearch nested aggregation', function () { + expect(isESNestedAggregation(esNestedAggregation)).to.be.true; + }); + }); + + describe('isESAggMatchAllFilter', function () { + it('should be false for an elasticsearch aggregation type filter', function () { + const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}}; + expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false; + }); + + it('should be true for an elasticsearch aggregation match all filter', function () { + const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}}; + expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true; + }); + }); +}); diff --git a/test/storage/elasticsearch/elasticsearch.spec.ts b/test/storage/elasticsearch/elasticsearch.spec.ts new file mode 100644 index 00000000..7907a5f8 --- /dev/null +++ b/test/storage/elasticsearch/elasticsearch.spec.ts @@ -0,0 +1,641 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {ApiResponse, Client} from '@elastic/elasticsearch'; +import {SCBook, SCBulkResponse, SCConfigFile, SCMessage, SCSearchQuery, SCThings, SCThingType} from '@openstapps/core'; +import {instance as book} from '@openstapps/core/test/resources/Book.1.json'; +import {instance as message} from '@openstapps/core/test/resources/Message.1.json'; +import {Logger} from '@openstapps/logger'; +import {SMTP} from '@openstapps/logger/lib/smtp'; +import {expect, use} from 'chai'; +import chaiAsPromised from 'chai-as-promised'; +import {SearchResponse} from 'elasticsearch'; +import mockedEnv from 'mocked-env'; +import sinon from 'sinon'; +import {configFile} from '../../../src/common'; +import {MailQueue} from '../../../src/notification/mail-queue'; +import * as aggregations from '../../../src/storage/elasticsearch/aggregations'; +import {ElasticsearchObject} from '../../../src/storage/elasticsearch/common'; +import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch'; +import * as Monitoring from '../../../src/storage/elasticsearch/monitoring'; +import * as query from '../../../src/storage/elasticsearch/query'; +import * as templating from '../../../src/storage/elasticsearch/templating'; +import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, index} from '../../common'; +import fs from 'fs'; + +use(chaiAsPromised); + +describe('Elasticsearch', function () { + // increase timeout for the suite + this.timeout(DEFAULT_TEST_TIMEOUT); + const sandbox = sinon.createSandbox(); + let checkESTemplateStub: sinon.SinonStub = sandbox.stub(templating, 'checkESTemplate'); + + before(function () { + console.log('before'); + sandbox.stub(fs, 'readFileSync').returns('{}'); + }); + after(function () { + sandbox.restore(); + }); + + describe('getElasticsearchUrl', function () { + it('should provide custom elasticsearch URL if defined', function () { + const customAddress = 'http://foo-address:9200'; + const restore = mockedEnv({ + ES_ADDR: customAddress + }); + + expect(Elasticsearch.getElasticsearchUrl()).to.be.equal(customAddress); + // restore env variables + restore(); + }); + + it('should provide local URL as fallback', function () { + const restore = mockedEnv({ + ES_ADDR: undefined + }); + + expect(Elasticsearch.getElasticsearchUrl()).to.match(/(https?:\/\/)?localhost(:\d+)?/); + // restore env variables + restore(); + }); + }); + + describe('getIndex (including getIndexUID)', function () { + const type = 'foo bar type'; + const source = 'foo_source'; + const bulk: SCBulkResponse = { + expiration: '', + source: '', + state: 'in progress', + type: SCThingType.Semester, + uid: 'bulk-uid-123-123-123' + }; + + it('should provide index UID from the provided UID', function () { + const indexUID = Elasticsearch.getIndexUID(bulk.uid); + + expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH); + // test starting and ending character + expect(indexUID[0]).to.be.equal(bulk.uid[0]); + expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]); + }); + + it('should provide index name from the provided data', function () { + expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)) + .to.be.equal(`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`); + }); + }); + + describe('removeAliasChars', function () { + it('should remove invalid characters', function () { + expect(Elasticsearch.removeAliasChars('f,o#o\\ b|ar/ * ', 'bulk-uid')).to.be.equal('foobaralias'); + }); + + it('should remove invalid starting characters', function () { + expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); + expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); + expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); + }); + + it('should replace with a placeholder in case of invalid alias', function () { + expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder'); + expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder'); + }); + + it('should work with common cases', function () { + expect(Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid')) + .to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890'); + expect(Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid')) + .to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG'); + }); + + it('should warn in case of characters that are invalid in future elasticsearch versions', function () { + const sandbox = sinon.createSandbox(); + const loggerWarnStub = sandbox.stub(Logger, 'warn'); + + expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias'); + expect(loggerWarnStub.called).to.be.true; + }); + }); + + describe('constructor', async function () { + const sandbox = sinon.createSandbox(); + afterEach(function () { + sandbox.restore(); + }); + + it('should complain (throw an error) if database in config is undefined', function () { + const config: SCConfigFile = {...configFile, internal: {...configFile.internal, database: undefined}}; + + expect(() => new Elasticsearch(config)).to.throw(Error); + }); + + it('should complain (throw an error) if database version is not a string', function () { + const config: SCConfigFile = { + ...configFile, + internal: { + ...configFile.internal, + database: { + name: 'foo', + version: 123 + } + } + }; + + expect(() => new Elasticsearch(config)).to.throw(Error); + }); + + it('should log an error in case of there is one when getting response from the elasticsearch client', async function () { + const error = Error('Foo Error'); + const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo'); + sandbox.stub(Client.prototype, 'on').yields(error); + + new Elasticsearch(configFile); + + expect(loggerErrorStub.calledWith(error)).to.be.true; + }); + + it('should log the result in the debug mode when getting response from the elasticsearch client', async function () { + const fakeResponse = {foo: 'bar'}; + const loggerLogStub = sandbox.stub(Logger, 'log'); + sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse); + + new Elasticsearch(configFile); + expect(loggerLogStub.calledWith(fakeResponse)).to.be.false; + + const restore = mockedEnv({ + 'ES_DEBUG': 'true', + }); + new Elasticsearch(configFile); + + expect(loggerLogStub.calledWith(fakeResponse)).to.be.true; + // restore env variables + restore(); + }); + + it('should force mapping update if related process env variable is not set', async function () { + const restore = mockedEnv({ + 'ES_FORCE_MAPPING_UPDATE': undefined, + }); + + new Elasticsearch(configFile); + + expect(checkESTemplateStub.calledWith(false)).to.be.true; + // restore env variables + restore(); + }); + + it('should force mapping update if related process env variable is set', async function () { + const restore = mockedEnv({ + 'ES_FORCE_MAPPING_UPDATE': 'true', + }); + + new Elasticsearch(configFile); + + expect(checkESTemplateStub.calledWith(true)).to.be.true; + // restore env variables + restore(); + }); + }); + + describe('init', async function () { + const sandbox = sinon.createSandbox(); + after(function () { + sandbox.restore(); + }); + + it('should complain (throw an error) if monitoring is set but mail queue is undefined', async function () { + const config: SCConfigFile = { + ...configFile, + internal: { + ...configFile.internal, + monitoring: { + actions: [], + watchers: [], + } + } + }; + + const es = new Elasticsearch(config); + + return expect(es.init()).to.be.rejected; + }); + + it('should setup the monitoring if there is monitoring is set and mail queue is defined', function () { + const config: SCConfigFile = { + ...configFile, + internal: { + ...configFile.internal, + monitoring: { + actions: [], + watchers: [], + } + } + }; + const monitoringSetUpStub = sandbox.stub(Monitoring, 'setUp'); + + const es = new Elasticsearch(config, new MailQueue(getTransport(false) as unknown as SMTP)); + + es.init(); + + expect(monitoringSetUpStub.called).to.be.true; + }); + }); + + describe('Operations with bundle/index', async function () { + const sandbox = sinon.createSandbox(); + let es: Elasticsearch; + const oldIndex = 'stapps_footype_foosource_oldindex'; + + beforeEach(function () { + es = new Elasticsearch(configFile); + // @ts-ignore + es.client.indices = { + // @ts-ignore + getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}), + // @ts-ignore + putTemplate: () => Promise.resolve({}), + // @ts-ignore + create: () => Promise.resolve({}), + // @ts-ignore + delete: () => Promise.resolve({}), + // @ts-ignore + exists: () => Promise.resolve({}), + // @ts-ignore + refresh: () => Promise.resolve({}), + // @ts-ignore + updateAliases: () => Promise.resolve({}) + }; + }); + + afterEach(function () { + sandbox.restore(); + }); + + describe('bulkCreated', async function () { + it('should reject (throw an error) if the connection to elasticsearch is not established', async function () { + return expect(es.bulkCreated(bulk)).to.be.rejectedWith('elasticsearch'); + }); + + it('should reject (throw an error) if the index name is not valid', async function () { + sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${index}`); + sandbox.createStubInstance(Client, {}); + await es.init(); + + return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index'); + }); + + it('should create a new index', async function () { + sandbox.stub(Elasticsearch, 'getIndex').returns(index); + const putTemplateStub = sandbox.stub(templating, 'putTemplate'); + const createStub = sandbox.stub(es.client.indices, 'create'); + await es.init(); + + await es.bulkCreated(bulk); + + expect(putTemplateStub.called).to.be.true; + expect(createStub.calledWith({index})).to.be.true; + }); + }); + + describe('bulkExpired', async function () { + const sandbox = sinon.createSandbox(); + afterEach(function () { + sandbox.restore(); + }); + it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () { + sandbox.stub(Elasticsearch, 'getIndex').returns(index); + const clientDeleteStub = sandbox.stub(es.client.indices, 'delete'); + + await es.bulkExpired({...bulk, state: 'in progress'}); + + expect(clientDeleteStub.called).to.be.true; + }); + + it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () { + sandbox.stub(Elasticsearch, 'getIndex').returns(index); + const clientDeleteStub = sandbox.stub(es.client.indices, 'delete'); + + await es.bulkExpired({...bulk, state: 'done'}); + + expect(clientDeleteStub.called).to.be.false; + }); + }); + + describe('bulkUpdated', function () { + it('should reject if the connection to elasticsearch is not established', async function () { + return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('elasticsearch'); + }); + + it('should reject if the index name is not valid', async function () { + sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${index}`); + sandbox.createStubInstance(Client, {}); + await es.init(); + + return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index'); + }); + + it('should create a new index', async function () { + const expectedRefreshActions = [ + { + add: {index: index, alias: SCThingType.Book}, + }, + { + remove: {index: oldIndex, alias: SCThingType.Book}, + } + ]; + sandbox.stub(Elasticsearch, 'getIndex').returns(index); + sandbox.stub(es, 'aliasMap').value({ + [SCThingType.Book]: { + [bulk.source]: oldIndex, + } + }); + const refreshStub = sandbox.stub(es.client.indices, 'refresh'); + const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases'); + const deleteStub = sandbox.stub(es.client.indices, 'delete'); + sandbox.stub(templating, 'putTemplate'); + await es.init(); + + await es.bulkUpdated(bulk); + + expect(refreshStub.calledWith({index})).to.be.true; + expect(updateAliasesStub.calledWith({ + body: { + actions: expectedRefreshActions + } + }) + ).to.be.true; + expect(deleteStub.called).to.be.true; + }); + }); + }); + + describe('get', async function () { + let es: Elasticsearch; + const sandbox = sinon.createSandbox(); + + before(function () { + es = new Elasticsearch(configFile); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should reject if object is not found', async function () { + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}}); + + return expect(es.get('123')).to.rejectedWith('found'); + }); + + it('should provide the thing if object is found', async function () { + const foundObject: ElasticsearchObject = {_id: '', _index: '', _score: 0, _type: '', _source: message as SCMessage}; + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: [foundObject]}}}); + + return expect(await es.get('123')).to.be.eql(message); + }); + }); + + describe('post', async function () { + let es: Elasticsearch; + const sandbox = sinon.createSandbox(); + + before(function () { + es = new Elasticsearch(configFile); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should not post if the object already exists in an index which will not be rolled over', async function () { + const oldIndex = index.replace('foosource', 'barsource'); + const object: ElasticsearchObject = {_id: '', _index: oldIndex, _score: 0, _type: '', _source: message as SCMessage}; + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: [object]}}}); + sandbox.stub(Elasticsearch, 'getIndex').returns(index); + + return expect(es.post(object._source, bulk)).to.rejectedWith('exist'); + }); + + it('should reject if there is an object creation error on the elasticsearch side', async function () { + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}}); + sandbox.stub(es.client, 'create').resolves({body: {created: false}}); + + return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation'); + }); + + it('should create a new object', async function () { + let caughtParam: any; + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}}); + // @ts-ignore + let createStub = sandbox.stub(es.client, 'create').callsFake((param) => { + caughtParam = param; + return Promise.resolve({body: { created: true }}); + }); + + await es.post(message as SCMessage, bulk); + + expect(createStub.called).to.be.true; + expect(caughtParam.body).to.be.eql({...message, creation_date: caughtParam.body.creation_date}); + }); + }); + + describe('put', async function () { + let es: Elasticsearch; + const sandbox = sinon.createSandbox(); + + before(function () { + es = new Elasticsearch(configFile); + }); + afterEach(function () { + sandbox.restore(); + }); + it('should reject to put if the object does not already exist', async function () { + const oldIndex = index.replace('foosource', 'barsource'); + const object: ElasticsearchObject = {_id: '', _index: oldIndex, _score: 0, _type: '', _source: message as SCMessage}; + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: []}}}); + + return expect(es.put(object._source)).to.rejectedWith('exist'); + }); + + it('should update the object if it already exists', async function () { + let caughtParam: any; + const oldIndex = index.replace('foosource', 'barsource'); + const object: ElasticsearchObject = {_id: '', _index: oldIndex, _score: 0, _type: '', _source: message as SCMessage}; + sandbox.stub(es.client, 'search').resolves({body:{hits: { hits: [object]}}}); + // @ts-ignore + const stubUpdate = sandbox.stub(es.client, 'update').callsFake((params) => { + caughtParam = params; + return Promise.resolve({body: { created: true }}); + }); + + await es.put(object._source); + + expect(caughtParam.body.doc).to.be.eql(object._source); + }); + }); + + describe('search', async function () { + let es: Elasticsearch; + const sandbox = sinon.createSandbox(); + const objectMessage: ElasticsearchObject = {_id: '123', _index: index, _score: 0, _type: '', _source: message as SCMessage}; + const objectBook: ElasticsearchObject = {_id: '321', _index: index, _score: 0, _type: '', _source: book as SCBook}; + const fakeEsAggregations = { + '@all': { + doc_count: 17, + type: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'person', + doc_count: 13 + }, + { + key: 'catalog', + doc_count: 4 + } + ] + } + } + }; + const fakeSearchResponse: Partial>> = { + // @ts-ignore + body: { + took: 12, + timed_out: false, + // @ts-ignore + _shards: {}, + // @ts-ignore + hits: { + hits: [ + objectMessage, + objectBook, + ], + total: 123 + }, + aggregations: fakeEsAggregations + }, + headers: {}, + // @ts-ignore + meta: {}, + // @ts-ignore + statusCode: {}, + // @ts-ignore + warnings: {} + }; + let searchStub: sinon.SinonStub; + before(function () { + es = new Elasticsearch(configFile); + }); + beforeEach(function () { + searchStub = sandbox.stub(es.client, 'search').resolves(fakeSearchResponse); + }); + afterEach(function () { + sandbox.restore(); + }); + + it('should provide appropriate data and facets', async function () { + const fakeFacets = [ + { + buckets: [ + { + count: 1, + 'key': 'foo' + }, + { + count: 1, + key: 'bar' + } + ], + field: 'type', + } + ]; + const parseAggregationsStub = sandbox.stub(aggregations, 'parseAggregations').returns(fakeFacets); + + const {data, facets} = await es.search({}); + + expect(data).to.be.eql([objectMessage._source, objectBook._source]); + expect(facets).to.be.eql(fakeFacets); + expect(parseAggregationsStub.calledWith(sinon.match.any, fakeEsAggregations)).to.be.true; + }); + + it('should provide pagination from params', async function () { + const from = 30; + const {pagination} = await es.search({from}); + + expect(pagination).to.be.eql({ + count: fakeSearchResponse.body!.hits.hits.length, + offset: from, + total: fakeSearchResponse.body!.hits.total + }); + }); + + it('should have fallback to zero if from is not given through params', async function () { + const {pagination} = await es.search({}); + + expect(pagination.offset).to.be.equal(0); + }); + + it('should build the search request properly', async function () { + const params: SCSearchQuery = { + query: 'mathematics', + from: 30, + size: 5, + sort: [ + { + type: 'ducet', + order: 'desc', + arguments: + { + field: 'name' + } + } + ], + filter: { + type: 'value', + arguments: { + field: 'type', + value: SCThingType.AcademicEvent + } + } + }; + const fakeResponse = {foo: 'bar'}; + const fakeBuildSortResponse = [fakeResponse]; + // @ts-ignore + sandbox.stub(query, 'buildQuery').returns(fakeResponse); + // @ts-ignore + sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse); + + await es.search(params); + + sandbox.assert + .calledWithMatch(searchStub, + { + body: { + aggs: es.aggregationsSchema, + query: fakeResponse, + sort: fakeBuildSortResponse + }, + from: params.from, + index: Elasticsearch.getListOfAllIndices(), + size: params.size, + } + ); + }); + }); +}); diff --git a/test/storage/elasticsearch/monitoring.spec.ts b/test/storage/elasticsearch/monitoring.spec.ts new file mode 100644 index 00000000..9cc3aef0 --- /dev/null +++ b/test/storage/elasticsearch/monitoring.spec.ts @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {ApiResponse, Client} from '@elastic/elasticsearch'; +import { + SCMonitoringConfiguration, + SCMonitoringLogAction, + SCMonitoringMailAction, + SCMonitoringWatcher, SCThings +} from '@openstapps/core'; +import {Logger} from '@openstapps/logger'; +import {SearchResponse} from 'elasticsearch'; +import {MailQueue} from '../../../src/notification/mail-queue'; +import {setUp} from '../../../src/storage/elasticsearch/monitoring'; + +import {getTransport} from '../../common'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import cron from 'node-cron'; +import * as templating from '../../../src/storage/elasticsearch/templating'; + +describe('Monitoring', async function () { + const sandbox = sinon.createSandbox(); + const logAction: SCMonitoringLogAction = { + message: 'Foo monitoring message', + prefix: 'Backend Monitoring', + type: 'log' + }; + const mailAction: SCMonitoringMailAction = { + message: 'Bar monitoring message', + recipients: ['xyz@xyz.com'], + subject: 'Backend Monitoring', + type: 'mail' + }; + let transport: any; + // @ts-ignore + let mailQueue: any; + beforeEach(async function () { + transport = getTransport(true); + mailQueue = new MailQueue(transport); + cronScheduleStub = sandbox.stub(cron, 'schedule'); + sandbox.stub(templating, 'checkESTemplate'); + }); + afterEach(async function () { + sandbox.restore(); + }); + // const sandbox = sinon.createSandbox(); + let cronScheduleStub: sinon.SinonStub + const minLengthWatcher: SCMonitoringWatcher = { + actions: [logAction, mailAction], + conditions: [ + { + length: 10, + type: 'MinimumLength' + } + ], + name: 'foo watcher', + query: {foo: 'bar'}, + triggers: [ + { + executionTime: 'monthly', + name: 'beginning of month' + }, + { + executionTime: 'daily', + name: 'every night' + } + ] + }; + const maxLengthWatcher: SCMonitoringWatcher = { + actions: [logAction, mailAction], + conditions: [ + { + length: 30, + type: 'MaximumLength' + } + ], + name: 'foo watcher', + query: {bar: 'foo'}, + triggers: [ + { + executionTime: 'hourly', + name: 'every hour' + }, + { + executionTime: 'weekly', + name: 'every week' + }, + ] + }; + const monitoringConfig: SCMonitoringConfiguration = { + actions: [logAction, mailAction], + watchers: [minLengthWatcher, maxLengthWatcher] + }; + + it('should create a schedule for each trigger', async function () { + await setUp(monitoringConfig, new Client({node: 'http://foohost:9200'}), mailQueue); + + expect(cronScheduleStub.callCount).to.be.equal(4); + }); + + it('should log errors where conditions failed', async function () { + const fakeSearchResponse: Partial>> = { + // @ts-ignore + body: { + took: 12, + timed_out: false, + // @ts-ignore + _shards: {}, + // @ts-ignore + hits: { + total: 123 + }, + }, + }; + let fakeClient = new Client({node: 'http://foohost:9200'}); + const loggerErrorStub = sandbox.stub(Logger, 'error'); + const mailQueueSpy = sinon.spy(mailQueue, 'push'); + cronScheduleStub.yields(); + sandbox.stub(fakeClient, 'search').resolves(fakeSearchResponse); + await setUp(monitoringConfig, fakeClient, mailQueue); + + expect(loggerErrorStub.callCount).to.be.equal(2); + expect(mailQueueSpy.callCount).to.be.equal(2); + }); +}); diff --git a/test/storage/elasticsearch/query.spec.ts b/test/storage/elasticsearch/query.spec.ts new file mode 100644 index 00000000..572e649b --- /dev/null +++ b/test/storage/elasticsearch/query.spec.ts @@ -0,0 +1,442 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { + SCConfigFile, + SCSearchBooleanFilter, + SCSearchFilter, + SCSearchQuery, + SCSearchSort, + SCThingType +} from '@openstapps/core'; +import { expect } from 'chai'; +import {configFile} from '../../../src/common'; +import { + ElasticsearchConfig, ESBooleanFilter, ESDucetSort, ESGeoDistanceFilter, + ESGeoDistanceSort, + ESTermFilter, + ScriptSort +} from '../../../src/storage/elasticsearch/common'; +import {buildBooleanFilter, buildFilter, buildQuery, buildSort} from '../../../src/storage/elasticsearch/query'; + +describe('Query', function () { + describe('buildBooleanFilter', function () { + const booleanFilter: SCSearchBooleanFilter = { + arguments: { + operation: 'and', + filters: [ + { + type: 'value', + arguments: { + field: 'type', + value: SCThingType.Catalog + } + }, + { + type: 'value', + arguments: { + field: 'type', + value: SCThingType.Building + } + } + ] + }, + type: 'boolean' + }; + const booleanFilters: {[key: string]: SCSearchBooleanFilter} = { + and: booleanFilter, + or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}}, + not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}}, + }; + const expectedEsFilters: Array = [ + { + term: { + 'type.raw': 'catalog' + } + }, + { + term: { + 'type.raw': 'building' + } + } + ]; + + it('should create appropriate elasticsearch "and" filter argument', function () { + const {must} = buildBooleanFilter(booleanFilters.and); + + expect(must).to.be.eql(expectedEsFilters); + }); + + it('should create appropriate elasticsearch "or" filter argument', function () { + const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or); + + expect(should).to.be.eql(expectedEsFilters); + expect(minimum_should_match).to.be.equal(1); + }); + + it('should create appropriate elasticsearch "not" filter argument', function () { + const {must_not} = buildBooleanFilter(booleanFilters.not); + + expect(must_not).to.be.eql(expectedEsFilters); + }); + }); + + describe('buildQuery', function () { + const params: SCSearchQuery = { + query: 'mathematics', + from: 30, + size: 5, + sort: [ + { + type: 'ducet', + order: 'desc', + arguments: + { + field: 'name' + } + }, + { + type: 'ducet', + order: 'desc', + arguments: + { + field: 'categories' + } + }, + ], + filter: { + type: 'value', + arguments: { + field: 'type', + value: SCThingType.AcademicEvent + } + } + }; + let esConfig: ElasticsearchConfig = { + name: 'elasticsearch', + version: '123', + query: { + minMatch: '75%', + queryType: 'dis_max', + matchBoosting: 1.3, + fuzziness: 'AUTO', + cutoffFrequency: 0.0, + tieBreaker: 0, + }, + }; + const query = { + minMatch: '75%', + queryType: 'dis_max', + matchBoosting: 1.3, + fuzziness: 'AUTO', + cutoffFrequency: 0.0, + tieBreaker: 0, + } + const config: SCConfigFile = { + ...configFile + }; + beforeEach(function () { + esConfig = { + name: 'elasticsearch', + version: '123' + }; + }); + + // TODO: check parts of received elasticsearch query for each test case + + it('should build query that includes sorting when query is undefined', function () { + expect(buildQuery(params, config, esConfig)).to.be.an('object'); + }); + + it('should build query that includes sorting when query type is query_string', function () { + esConfig.query = {...query, queryType: 'query_string'}; + + expect(buildQuery(params, config, esConfig)).to.be.an('object'); + }); + + it('should build query that includes sorting when query type is dis_max', function () { + esConfig.query = {...query, queryType: 'dis_max'}; + + expect(buildQuery(params, config, esConfig)).to.be.an('object'); + }); + + it('should build query that includes sorting when query type is dis_max', function () { + esConfig.query = {...query, queryType: 'dis_max'}; + + expect(buildQuery(params, config, esConfig)).to.be.an('object'); + }); + + it('should reject (throw an error) if provided query type is not supported', function () { + // @ts-ignore + esConfig.query = {...query, queryType: 'invalid_query_type'}; + + expect(() => buildQuery(params, config, esConfig)).to.throw('query type'); + }); + }); + + describe('buildFilter', function () { + const searchFilters: {[key: string]: SCSearchFilter} = { + value: { + type: 'value', + arguments: { + field: 'type', + value: SCThingType.Dish + } + }, + availability: { + type: 'availability', + arguments: { + time: '2017-01-30T12:05:00.000Z', + fromField: 'offers.availabilityStarts', + toField: 'offers.availabilityEnds' + } + }, + distance: { + type: 'distance', + arguments: { + distance: 1000, + field: 'geo.point.coordinates', + position: [50.123, 8.123], + } + }, + boolean: { + type: 'boolean', + arguments: { + operation: 'and', + filters: [ + { + type: 'value', + arguments: { + field: 'type', + value: SCThingType.Dish, + } + }, + { + type: 'availability', + arguments: { + fromField: 'offers.availabilityStarts', + toField: 'offers.availabilityEnds' + } + } + ] + } + }, + }; + + it('should build value filter', function () { + const filter = buildFilter(searchFilters.value); + const expectedFilter: ESTermFilter = { + term: { + 'type.raw': SCThingType.Dish + } + }; + + expect(filter).to.be.eql(expectedFilter); + }); + + it('should build availability filter', function () { + const filter = buildFilter(searchFilters.availability); + const expectedFilter: ESBooleanFilter = { + bool: { + should: [ + { + bool: { + must: [ + { + range: { + 'offers.availabilityStarts': { + lte: '2017-01-30T12:05:00.000Z' + } + } + }, + { + range: { + 'offers.availabilityEnds': { + gte: '2017-01-30T12:05:00.000Z' + } + } + } + ] + } + }, + { + bool: { + must_not: [ + { + exists: { + field: 'offers.availabilityStarts' + } + }, + { + exists: { + field: 'offers.availabilityEnds' + } + } + ] + } + } + ] + } + }; + + expect(filter).to.be.eql(expectedFilter); + }); + + it('should build distance filter', function () { + const filter = buildFilter(searchFilters.distance); + const expectedFilter: ESGeoDistanceFilter = { + geo_distance: { + distance: '1000m', + 'geo.point.coordinates': { + lat: 8.123, + lon: 50.123 + } + } + }; + + expect(filter).to.be.eql(expectedFilter); + }); + + it('should build boolean filter', function () { + const filter = buildFilter(searchFilters.boolean); + const expectedFilter: ESBooleanFilter = { + bool: { + minimum_should_match: 0, + must: [ + { + term: { + 'type.raw': 'dish' + } + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + range: { + 'offers.availabilityStarts': { + lte: 'now' + } + } + }, + { + range: { + 'offers.availabilityEnds': { + gte: 'now' + } + } + } + ] + } + }, + { + bool: { + must_not: [ + { + exists: { + field: 'offers.availabilityStarts' + } + }, + { + exists: { + field: 'offers.availabilityEnds' + } + } + ] + } + } + ] + } + } + ], + must_not: [], + should: [] + } + } + + expect(filter).to.be.eql(expectedFilter); + }); + }); + + describe('buildSort', function () { + const searchSCSearchSort: Array = [ + { + type: 'ducet', + order: 'desc', + arguments: { + field: 'name' + }, + }, + { + type: 'distance', + order: 'desc', + arguments: { + field: 'geo.point', + position: [8.123, 50.123] + }, + }, + { + type: 'price', + order: 'asc', + arguments: { + universityRole: 'student', + field: 'offers.prices', + } + }, + ]; + let sorts: Array = []; + const expectedSorts: {[key: string]: ESDucetSort | ESGeoDistanceSort | ScriptSort} = { + ducet: { + 'name.sort': 'desc' + }, + distance: { + _geo_distance: { + mode: 'avg', + order: 'desc', + unit: 'm', + 'geo.point': { + lat: 50.123, + lon: 8.123 + } + } + }, + price: { + _script: { + order: 'asc', + script: '\n // foo price sort script', + type: 'number' + } + } + }; + before(function () { + sorts = buildSort(searchSCSearchSort); + }); + + it('should build ducet sort', function () { + expect(sorts[0]).to.be.eql(expectedSorts.ducet); + }); + + it('should build distance sort', function () { + expect(sorts[1]).to.be.eql(expectedSorts.distance); + }); + + it('should build price sort', function () { + const priceSortNoScript = {...sorts[2], _script: {...(sorts[2] as ScriptSort)._script, script: (expectedSorts.price as ScriptSort)._script.script}} + expect(priceSortNoScript).to.be.eql(expectedSorts.price); + }); + }); +}); diff --git a/test/storage/elasticsearch/templating.spec.ts b/test/storage/elasticsearch/templating.spec.ts new file mode 100644 index 00000000..74140098 --- /dev/null +++ b/test/storage/elasticsearch/templating.spec.ts @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2020 StApps + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import {SCThingType} from '@openstapps/core'; +import * as mapping from '@openstapps/core-tools/lib/mapping'; +import {ElasticsearchTemplateCollection} from '@openstapps/core-tools/lib/mappings/mapping-definitions'; +import {Logger} from '@openstapps/logger'; +import {AggregationSchema} from '../../../src/storage/elasticsearch/common'; +import {checkESTemplate, refreshAllTemplates} from '../../../src/storage/elasticsearch/templating'; +import sinon from "sinon"; +import * as path from 'path'; +import * as common from '@openstapps/core-tools/lib/common'; +import {expect} from 'chai'; +import fs from 'fs'; +import fsExtra from 'fs-extra'; +import {Client} from '@elastic/elasticsearch'; + +describe('templating', function () { + describe('checkESTemplate', function () { + const sandbox = sinon.createSandbox(); + let fakeMap: { aggregations: AggregationSchema, errors: string[], mappings: ElasticsearchTemplateCollection }; + beforeEach(function () { + fakeMap = { + aggregations: { + '@all': { + aggs: { + type: { + terms: { + field: 'type.raw', + size: 1000 + } + } + }, + filter: { + match_all: {} + } + }, + }, + errors: [], + mappings: { + 'template_dish': { + mappings: { + dish: { + // @ts-ignore just mock the mapping + foo: 'mapping' + } + }, + settings: { + analysis: { + ducet_sort: { + filter: [ + 'german_phonebook' + ], + tokenizer: 'keyword', + type: 'custom' + }, + search_german: { + filter: [ + 'lowercase', + 'german_stop', + 'german_stemmer' + ], + tokenizer: 'stapps_ngram', + type: 'custom' + } + }, + max_result_window: 30000, + }, + template: 'stapps_dish*' + }, + 'template_book': { + mappings: { + book: { + // @ts-ignore just mock the mapping + foo: 'mapping' + } + }, + settings: { + analysis: { + ducet_sort: { + filter: [ + 'german_phonebook' + ], + tokenizer: 'keyword', + type: 'custom' + }, + search_german: { + filter: [ + 'lowercase', + 'german_stop', + 'german_stemmer' + ], + tokenizer: 'stapps_ngram', + type: 'custom' + } + }, + max_result_window: 30000, + }, + template: 'stapps_book*' + } + } + } + }); + afterEach(function () { + sandbox.restore(); + }); + + it('should write new templates when "force update" is true', async function () { + sandbox.stub(Logger, 'error').resolves(); + sandbox.stub(fs, 'existsSync').returns(true); + sandbox.stub(common, 'getProjectReflection'); + let caughtData: any = []; + const writeFileSyncStub = sandbox.stub(fs, 'writeFileSync'); + sandbox.stub(path, 'resolve').returns('/foo/bar'); + sandbox.stub(mapping, 'generateTemplate').returns(fakeMap); + + checkESTemplate(true); + + expect(writeFileSyncStub.callCount).to.be.gt(0); + for (let i = 0; i < writeFileSyncStub.callCount; i++) { + caughtData.push(writeFileSyncStub.getCall(i).args[1]); + } + + expect(caughtData).to.be.eql([ + JSON.stringify(fakeMap.mappings['template_dish'], null, 2), + JSON.stringify(fakeMap.mappings['template_book'], null, 2), + JSON.stringify(fakeMap.aggregations), + ]); + }); + + it('should not write new templates when "force update" is false', async function () { + sandbox.stub(Logger, 'error').resolves(); + sandbox.stub(fs, 'existsSync').returns(true); + sandbox.stub(common, 'getProjectReflection'); + const writeFileSyncStub = sandbox.stub(fs, 'writeFileSync'); + sandbox.stub(path, 'resolve').returns('/foo/bar'); + sandbox.stub(mapping, 'generateTemplate').returns(fakeMap); + + checkESTemplate(false); + + expect(writeFileSyncStub.called).to.be.false; + }); + + it('should terminate if there are errors in the map', async function () { + const processExitStub = sandbox.stub(process, 'exit'); + const fakeMapWithErrors = { + ...fakeMap, + errors: ['Foo Error'] + }; + sandbox.stub(Logger, 'error').resolves(); + sandbox.stub(fs, 'existsSync').returns(true); + sandbox.stub(common, 'getProjectReflection'); + sandbox.stub(fs, 'writeFileSync'); + sandbox.stub(path, 'resolve').returns('/foo/bar'); + sandbox.stub(mapping, 'generateTemplate').returns(fakeMapWithErrors); + + checkESTemplate(true); + + expect(processExitStub.called).to.be.true; + }); + }); + + describe('refreshAllTemplates', async function () { + const sandbox = sinon.createSandbox(); + const client = { + indices: { + putTemplate: (_template: any) => { + } + } + } + + after(function () { + sandbox.restore(); + }); + + it('should put templates for all types', async function () { + const clientPutTemplateStub = sandbox.stub(client.indices, 'putTemplate'); + sandbox.stub(fsExtra, 'readFile').resolves(Buffer.from('{"foo": "file content"}', 'utf8')); + await refreshAllTemplates(client as Client); + + for (const type of Object.values(SCThingType)) { + sinon.assert.calledWith(clientPutTemplateStub, { + body: {foo: 'file content'}, + name: `template_${type.split(' ').join('_')}` + }) + } + }); + }); +});