Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-11 12:12:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Resolve "Transition to ESLint"

Browse Source
This commit is contained in:
Thea Schöbl
2022-06-27 14:40:09 +00:00
committed by Rainer Killinger
parent ca1d2444e0
commit 418ba67d15
47 changed files with 1854 additions and 1634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
test/storage/bulk-storage.spec.ts
View File

@@ -15,6 +15,7 @@
*/
import {SCBulkRequest, SCThingType} from '@openstapps/core';
import moment from 'moment';
// eslint-disable-next-line unicorn/import-style
import util from 'util';
import {configFile} from '../../src/common';
import {Bulk, BulkStorage} from '../../src/storage/bulk-storage';
@@ -72,7 +73,7 @@ describe('Bulk Storage', function () {
expect(esMock.calledWith(bulk)).to.be.true;
});
it('should not call appropriate database clean-up method on expire if bulk\'s state is done', async function () {
it("should not call appropriate database clean-up method on expire if bulk's state is done", async function () {
bulk.state = 'done';
sandbox.stub(NodeCache.prototype, 'on').withArgs('expired', sinon.match.any).yields(123, bulk);
new BulkStorage(database);
@@ -88,11 +89,17 @@ describe('Bulk Storage', function () {
});
it('should delete a bulk', async function () {
const readStub = sandbox.stub(BulkStorage.prototype, 'read').callsFake(() => bulk);
const readStub = sandbox.stub(BulkStorage.prototype, 'read').callsFake(() => bulk);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let caught: any;
sandbox.stub(NodeCache.prototype, 'del').callsFake(() => caught = 123);
sandbox.stub(NodeCache.prototype, 'del').callsFake(() => (caught = 123));
// force call
sandbox.stub(util, 'promisify').callsFake(() => () => {}).yields(null);
sandbox
.stub(util, 'promisify')
// eslint-disable-next-line @typescript-eslint/no-empty-function,unicorn/consistent-function-scoping
.callsFake(() => () => {})
// eslint-disable-next-line unicorn/no-null
.yields(null);
const bulkStorage = new BulkStorage(database);
await bulkStorage.delete(bulk.uid);
@@ -103,15 +110,22 @@ describe('Bulk Storage', function () {
});
it('should read an existing bulk', async function () {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let caught: any;
sandbox.stub(NodeCache.prototype, 'get').callsFake(() => caught = 123);
sandbox.stub(NodeCache.prototype, 'get').callsFake(() => (caught = 123));
// force call
sandbox.stub(util, 'promisify').callsFake(() => () => {}).yields(null);
sandbox
.stub(util, 'promisify')
// eslint-disable-next-line unicorn/consistent-function-scoping,@typescript-eslint/no-empty-function
.callsFake(() => () => {})
// eslint-disable-next-line unicorn/no-null
.yields(null);
const bulkStorage = new BulkStorage(database);
await bulkStorage.read(bulk.uid);
expect(caught).to.be.equal(123);
});``
});
``;
});
});

164
test/storage/elasticsearch/aggregations.spec.ts
View File

@@ -20,23 +20,23 @@ import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elas
describe('Aggregations', function () {
const aggregations: AggregationResponse = {
catalog: {
doc_count: 4,
'catalog': {
'doc_count': 4,
'superCatalogs.categories': {
buckets: []
buckets: [],
},
'academicTerm.acronym': {
buckets: [
{
key: 'SoSe 2020',
doc_count: 2
}
]
doc_count: 2,
},
],
},
'superCatalog.categories': {
buckets: []
buckets: [],
},
categories: {
'categories': {
buckets: [
{
key: 'foo',
@@ -46,21 +46,21 @@ describe('Aggregations', function () {
key: 'bar',
doc_count: 3,
},
]
}
],
},
},
person: {
doc_count: 13,
'person': {
'doc_count': 13,
'homeLocations.categories': {
buckets: []
}
buckets: [],
},
},
'academic event': {
doc_count: 0,
'doc_count': 0,
'academicTerms.acronym': {
buckets: []
buckets: [],
},
categories: {
'categories': {
buckets: [
{
key: 'foobar',
@@ -70,18 +70,18 @@ describe('Aggregations', function () {
key: 'bar',
doc_count: 2,
},
]
],
},
'creativeWorks.keywords': {
buckets: []
}
buckets: [],
},
},
fooType: {
'fooType': {
buckets: [
{
doc_count: 321,
key: 'foo'
}
key: 'foo',
},
],
},
'@all': {
@@ -90,71 +90,71 @@ describe('Aggregations', function () {
buckets: [
{
key: 'person',
doc_count: 13
doc_count: 13,
},
{
key: 'catalog',
doc_count: 4
}
]
}
}
doc_count: 4,
},
],
},
},
};
const expectedFacets: SCFacet[] = [
{
buckets: [
{
count: 13,
'key': 'person'
},
{
count: 4,
key: 'catalog'
}
],
field: 'type',
},
{
buckets: [
{
count: 8,
key: 'foobar'
},
{
count: 2,
key: 'bar'
}
],
field: 'categories',
onlyOnType: SCThingType.AcademicEvent,
},
{
buckets: [
{
count: 2,
key: 'SoSe 2020'
}
],
field: 'academicTerm.acronym',
onlyOnType: SCThingType.Catalog
},
{
buckets: [
{
count: 1,
key: 'foo'
},
{
count: 3,
key: 'bar'
}
],
field: 'categories',
onlyOnType: SCThingType.Catalog,
},
// no fooType as it doesn't appear in the aggregation schema
];
{
buckets: [
{
count: 13,
key: 'person',
},
{
count: 4,
key: 'catalog',
},
],
field: 'type',
},
{
buckets: [
{
count: 8,
key: 'foobar',
},
{
count: 2,
key: 'bar',
},
],
field: 'categories',
onlyOnType: SCThingType.AcademicEvent,
},
{
buckets: [
{
count: 2,
key: 'SoSe 2020',
},
],
field: 'academicTerm.acronym',
onlyOnType: SCThingType.Catalog,
},
{
buckets: [
{
count: 1,
key: 'foo',
},
{
count: 3,
key: 'bar',
},
],
field: 'categories',
onlyOnType: SCThingType.Catalog,
},
// no fooType as it doesn't appear in the aggregation schema
];
it('should parse the aggregations providing the appropriate facets', function () {
const facets = parseAggregations(aggregations);

6
test/storage/elasticsearch/common.spec.ts
View File

@@ -17,15 +17,15 @@ import {
ESAggMatchAllFilter,
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter
ESTermsFilter,
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {expect} from "chai";
import {expect} from 'chai';
import {
isNestedAggregation,
isBucketAggregation,
isESTermsFilter,
isESAggMatchAllFilter,
isESNestedAggregation
isESNestedAggregation,
} from '../../../lib/storage/elasticsearch/types/guards';
import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';

209
test/storage/elasticsearch/elasticsearch.spec.ts
View File

@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any,unicorn/no-null */
/*
* Copyright (C) 2020 StApps
* This program is free software: you can redistribute it and/or modify
@@ -14,7 +15,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client} from '@elastic/elasticsearch';
import {SCBook, SCBulkResponse, SCConfigFile, SCMessage, SCSearchQuery, SCThings, SCThingType} from '@openstapps/core';
import {
SCBook,
SCBulkResponse,
SCConfigFile,
SCMessage,
SCSearchQuery,
SCThings,
SCThingType,
} from '@openstapps/core';
import {instance as book} from '@openstapps/core/test/resources/indexable/Book.1.json';
import {instance as message} from '@openstapps/core/test/resources/indexable/Message.1.json';
import {Logger} from '@openstapps/logger';
@@ -43,6 +52,7 @@ describe('Elasticsearch', function () {
const sandbox = sinon.createSandbox();
before(function () {
// eslint-disable-next-line no-console
console.log('before');
sandbox.stub(fs, 'readFileSync').returns('{}');
});
@@ -54,7 +64,7 @@ describe('Elasticsearch', function () {
it('should provide custom elasticsearch URL if defined', function () {
const customAddress = 'http://foo-address:9200';
const restore = mockedEnv({
ES_ADDR: customAddress
ES_ADDR: customAddress,
});
expect(Elasticsearch.getElasticsearchUrl()).to.be.equal(customAddress);
@@ -64,7 +74,7 @@ describe('Elasticsearch', function () {
it('should provide local URL as fallback', function () {
const restore = mockedEnv({
ES_ADDR: undefined
ES_ADDR: undefined,
});
expect(Elasticsearch.getElasticsearchUrl()).to.match(/(https?:\/\/)?localhost(:\d+)?/);
@@ -81,7 +91,7 @@ describe('Elasticsearch', function () {
source: '',
state: 'in progress',
type: SCThingType.Semester,
uid: 'bulk-uid-123-123-123'
uid: 'bulk-uid-123-123-123',
};
it('should provide index UID from the provided UID', function () {
@@ -94,8 +104,9 @@ describe('Elasticsearch', function () {
});
it('should provide index name from the provided data', function () {
expect(Elasticsearch.getIndex(type as SCThingType, source, bulk))
.to.be.equal(`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`);
expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
);
});
});
@@ -109,7 +120,7 @@ describe('Elasticsearch', function () {
});
it('should remove invalid characters', function () {
expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?al\ias>* ', 'bulk-uid')).to.be.equal('foobaralias');
expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should remove invalid starting characters', function () {
@@ -124,10 +135,12 @@ describe('Elasticsearch', function () {
});
it('should work with common cases', function () {
expect(Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'))
.to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
expect(Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'))
.to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
expect(
Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
expect(
Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
});
it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
@@ -158,16 +171,16 @@ describe('Elasticsearch', function () {
...configFile.internal,
database: {
name: 'foo',
version: 123
}
}
version: 123,
},
},
};
expect(() => new Elasticsearch(config)).to.throw(Error);
});
it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
const error = Error('Foo Error');
const error = new Error('Foo Error');
const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
sandbox.stub(Client.prototype, 'on').yields(error);
@@ -185,7 +198,7 @@ describe('Elasticsearch', function () {
expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
const restore = mockedEnv({
'ES_DEBUG': 'true',
ES_DEBUG: 'true',
});
new Elasticsearch(configFile);
@@ -208,8 +221,8 @@ describe('Elasticsearch', function () {
monitoring: {
actions: [],
watchers: [],
}
}
},
},
};
const es = new Elasticsearch(config);
@@ -225,8 +238,8 @@ describe('Elasticsearch', function () {
monitoring: {
actions: [],
watchers: [],
}
}
},
},
};
const monitoringSetUpStub = sandbox.stub(Monitoring, 'setUp');
@@ -245,22 +258,21 @@ describe('Elasticsearch', function () {
beforeEach(function () {
es = new Elasticsearch(configFile);
// @ts-ignore
es.client.indices = {
// @ts-ignore
// @ts-expect-error not assignable
getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
// @ts-ignore
// @ts-expect-error not assignable
putTemplate: () => Promise.resolve({}),
// @ts-ignore
// @ts-expect-error not assignable
create: () => Promise.resolve({}),
// @ts-ignore
// @ts-expect-error not assignable
delete: () => Promise.resolve({}),
// @ts-ignore
// @ts-expect-error not assignable
exists: () => Promise.resolve({}),
// @ts-ignore
// @ts-expect-error not assignable
refresh: () => Promise.resolve({}),
// @ts-ignore
updateAliases: () => Promise.resolve({})
// @ts-expect-error not assignable
updateAliases: () => Promise.resolve({}),
};
});
@@ -340,13 +352,13 @@ describe('Elasticsearch', function () {
},
{
remove: {index: oldIndex, alias: SCThingType.Book},
}
},
];
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(es, 'aliasMap').value({
[SCThingType.Book]: {
[bulk.source]: oldIndex,
}
},
});
const refreshStub = sandbox.stub(es.client.indices, 'refresh');
const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
@@ -357,11 +369,12 @@ describe('Elasticsearch', function () {
await es.bulkUpdated(bulk);
expect(refreshStub.calledWith({index})).to.be.true;
expect(updateAliasesStub.calledWith({
expect(
updateAliasesStub.calledWith({
body: {
actions: expectedRefreshActions
}
})
actions: expectedRefreshActions,
},
}),
).to.be.true;
expect(deleteStub.called).to.be.true;
});
@@ -392,7 +405,7 @@ describe('Elasticsearch', function () {
_index: '',
_score: 0,
_type: '',
_source: message as SCMessage
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
@@ -420,7 +433,7 @@ describe('Elasticsearch', function () {
_index: oldIndex,
_score: 0,
_type: '',
_source: message as SCMessage
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
@@ -434,7 +447,7 @@ describe('Elasticsearch', function () {
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
// return index name with different generated UID (see getIndex method)
@@ -451,18 +464,21 @@ describe('Elasticsearch', function () {
});
it('should create a new object', async function () {
let caughtParam: any;
let caughtParameter: any;
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
// @ts-ignore
let createStub = sandbox.stub(es.client, 'create').callsFake((param) => {
caughtParam = param;
// @ts-expect-error call
const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
caughtParameter = parameter;
return Promise.resolve({body: {created: true}});
});
await es.post(message as SCMessage, bulk);
expect(createStub.called).to.be.true;
expect(caughtParam.body).to.be.eql({...message, creation_date: caughtParam.body.creation_date});
expect(caughtParameter.body).to.be.eql({
...message,
creation_date: caughtParameter.body.creation_date,
});
});
});
@@ -482,32 +498,34 @@ describe('Elasticsearch', function () {
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
return expect(es.put(object._source)).to.rejectedWith('exist');
});
// noinspection JSUnusedLocalSymbols
it('should update the object if it already exists', async function () {
let caughtParam: any;
let caughtParameter: any;
const object: ElasticsearchObject<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
// @ts-ignore
const stubUpdate = sandbox.stub(es.client, 'update').callsFake((params) => {
caughtParam = params;
// @ts-expect-error unused
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
caughtParameter = parameters;
return Promise.resolve({body: {created: true}});
});
await es.put(object._source);
expect(caughtParam.body.doc).to.be.eql(object._source);
expect(caughtParameter.body.doc).to.be.eql(object._source);
});
});
@@ -519,14 +537,14 @@ describe('Elasticsearch', function () {
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage
_source: message as SCMessage,
};
const objectBook: ElasticsearchObject<SCBook> = {
_id: '321',
_index: getIndex(),
_score: 0,
_type: '',
_source: book as SCBook
_source: book as SCBook,
};
const fakeEsAggregations = {
'@all': {
@@ -537,40 +555,36 @@ describe('Elasticsearch', function () {
buckets: [
{
key: 'person',
doc_count: 13
doc_count: 13,
},
{
key: 'catalog',
doc_count: 4
}
]
}
}
doc_count: 4,
},
],
},
},
};
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
// @ts-ignore
body: {
took: 12,
timed_out: false,
// @ts-ignore
// @ts-expect-error not assignable
_shards: {},
// @ts-ignore
// @ts-expect-error not assignable
hits: {
hits: [
objectMessage,
objectBook,
],
total: 123
hits: [objectMessage, objectBook],
total: 123,
},
aggregations: fakeEsAggregations
aggregations: fakeEsAggregations,
},
headers: {},
// @ts-ignore
// @ts-expect-error not assignable
meta: {},
// @ts-ignore
// @ts-expect-error not assignable
statusCode: {},
// @ts-ignore
warnings: {}
// @ts-expect-error not assignable
warnings: {},
};
let searchStub: sinon.SinonStub;
before(function () {
@@ -593,11 +607,11 @@ describe('Elasticsearch', function () {
},
{
count: 4,
key: 'catalog'
}
key: 'catalog',
},
],
field: 'type',
}
},
];
const {data, facets} = await es.search({});
@@ -613,7 +627,7 @@ describe('Elasticsearch', function () {
expect(pagination).to.be.eql({
count: fakeSearchResponse.body!.hits.hits.length,
offset: from,
total: fakeSearchResponse.body!.hits.total
total: fakeSearchResponse.body!.hits.total,
});
});
@@ -624,7 +638,7 @@ describe('Elasticsearch', function () {
});
it('should build the search request properly', async function () {
const params: SCSearchQuery = {
const parameters: SCSearchQuery = {
query: 'mathematics',
from: 30,
size: 5,
@@ -632,42 +646,37 @@ describe('Elasticsearch', function () {
{
type: 'ducet',
order: 'desc',
arguments:
{
field: 'name'
}
}
arguments: {
field: 'name',
},
},
],
filter: {
type: 'value',
arguments: {
field: 'type',
value: SCThingType.AcademicEvent
}
}
value: SCThingType.AcademicEvent,
},
},
};
const fakeResponse = {foo: 'bar'};
const fakeBuildSortResponse = [fakeResponse];
// @ts-ignore
// @ts-expect-error not assignable
sandbox.stub(query, 'buildQuery').returns(fakeResponse);
// @ts-ignore
sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
await es.search(params);
await es.search(parameters);
sandbox.assert
.calledWithMatch(searchStub,
{
body: {
aggs: aggregations,
query: fakeResponse,
sort: fakeBuildSortResponse
},
from: params.from,
index: Elasticsearch.getListOfAllIndices(),
size: params.size,
}
);
sandbox.assert.calledWithMatch(searchStub, {
body: {
aggs: aggregations,
query: fakeResponse,
sort: fakeBuildSortResponse,
},
from: parameters.from,
index: Elasticsearch.getListOfAllIndices(),
size: parameters.size,
});
});
});
});

56
test/storage/elasticsearch/monitoring.spec.ts
View File

@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/*
* Copyright (C) 2020 StApps
* This program is free software: you can redistribute it and/or modify
@@ -18,7 +19,8 @@ import {
SCMonitoringConfiguration,
SCMonitoringLogAction,
SCMonitoringMailAction,
SCMonitoringWatcher, SCThings
SCMonitoringWatcher,
SCThings,
} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
import {SearchResponse} from 'elasticsearch';
@@ -26,7 +28,7 @@ import {MailQueue} from '../../../src/notification/mail-queue';
import {setUp} from '../../../src/storage/elasticsearch/monitoring';
import {getTransport} from '../../common';
import { expect } from 'chai';
import {expect} from 'chai';
import sinon from 'sinon';
import cron from 'node-cron';
@@ -35,16 +37,15 @@ describe('Monitoring', async function () {
const logAction: SCMonitoringLogAction = {
message: 'Foo monitoring message',
prefix: 'Backend Monitoring',
type: 'log'
type: 'log',
};
const mailAction: SCMonitoringMailAction = {
message: 'Bar monitoring message',
recipients: ['xyz@xyz.com'],
subject: 'Backend Monitoring',
type: 'mail'
type: 'mail',
};
let transport: any;
// @ts-ignore
let mailQueue: any;
beforeEach(async function () {
transport = getTransport(true);
@@ -55,52 +56,52 @@ describe('Monitoring', async function () {
sandbox.restore();
});
// const sandbox = sinon.createSandbox();
let cronScheduleStub: sinon.SinonStub
let cronScheduleStub: sinon.SinonStub;
const minLengthWatcher: SCMonitoringWatcher = {
actions: [logAction, mailAction],
conditions: [
{
length: 10,
type: 'MinimumLength'
}
type: 'MinimumLength',
},
],
name: 'foo watcher',
query: {foo: 'bar'},
triggers: [
{
executionTime: 'monthly',
name: 'beginning of month'
name: 'beginning of month',
},
{
executionTime: 'daily',
name: 'every night'
}
]
name: 'every night',
},
],
};
const maxLengthWatcher: SCMonitoringWatcher = {
actions: [logAction, mailAction],
conditions: [
{
length: 30,
type: 'MaximumLength'
}
type: 'MaximumLength',
},
],
name: 'foo watcher',
query: {bar: 'foo'},
triggers: [
{
executionTime: 'hourly',
name: 'every hour'
name: 'every hour',
},
{
executionTime: 'weekly',
name: 'every week'
name: 'every week',
},
]
],
};
const monitoringConfig: SCMonitoringConfiguration = {
actions: [logAction, mailAction],
watchers: [minLengthWatcher, maxLengthWatcher]
watchers: [minLengthWatcher, maxLengthWatcher],
};
it('should create a schedule for each trigger', async function () {
@@ -111,19 +112,18 @@ describe('Monitoring', async function () {
it('should log errors where conditions failed', async function () {
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
// @ts-ignore
body: {
took: 12,
timed_out: false,
// @ts-ignore
_shards: {},
// @ts-ignore
hits: {
total: 123
},
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
total: 123,
},
},
};
let fakeClient = new Client({node: 'http://foohost:9200'});
const fakeClient = new Client({node: 'http://foohost:9200'});
const loggerErrorStub = sandbox.stub(Logger, 'error');
const mailQueueSpy = sinon.spy(mailQueue, 'push');
cronScheduleStub.yields();

248
test/storage/elasticsearch/query.spec.ts
View File

@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any,unicorn/no-null */
/*
* Copyright (C) 2020 StApps
* This program is free software: you can redistribute it and/or modify
@@ -15,11 +16,13 @@
*/
import {
SCConfigFile,
SCSearchBooleanFilter, SCSearchDateRangeFilter,
SCSearchFilter, SCSearchNumericRangeFilter,
SCSearchBooleanFilter,
SCSearchDateRangeFilter,
SCSearchFilter,
SCSearchNumericRangeFilter,
SCSearchQuery,
SCSearchSort,
SCThingType
SCThingType,
} from '@openstapps/core';
import {expect} from 'chai';
import {
@@ -35,7 +38,12 @@ import {
ScriptSort,
} from '../../../src/storage/elasticsearch/types/elasticsearch';
import {configFile} from '../../../src/common';
import {buildBooleanFilter, buildFilter, buildQuery, buildSort} from '../../../src/storage/elasticsearch/query';
import {
buildBooleanFilter,
buildFilter,
buildQuery,
buildSort,
} from '../../../src/storage/elasticsearch/query';
describe('Query', function () {
describe('buildBooleanFilter', function () {
@@ -47,21 +55,21 @@ describe('Query', function () {
type: 'value',
arguments: {
field: 'type',
value: SCThingType.Catalog
}
value: SCThingType.Catalog,
},
},
{
type: 'value',
arguments: {
field: 'type',
value: SCThingType.Building
}
}
]
value: SCThingType.Building,
},
},
],
},
type: 'boolean'
type: 'boolean',
};
const booleanFilters: { [key: string]: SCSearchBooleanFilter } = {
const booleanFilters: {[key: string]: SCSearchBooleanFilter} = {
and: booleanFilter,
or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
@@ -69,14 +77,14 @@ describe('Query', function () {
const expectedEsFilters: Array<ESTermFilter> = [
{
term: {
'type.raw': 'catalog'
}
'type.raw': 'catalog',
},
},
{
term: {
'type.raw': 'building'
}
}
'type.raw': 'building',
},
},
];
it('should create appropriate elasticsearch "and" filter argument', function () {
@@ -100,7 +108,7 @@ describe('Query', function () {
});
describe('buildQuery', function () {
const params: SCSearchQuery = {
const parameters: SCSearchQuery = {
query: 'mathematics',
from: 30,
size: 5,
@@ -108,27 +116,25 @@ describe('Query', function () {
{
type: 'ducet',
order: 'desc',
arguments:
{
field: 'name'
}
arguments: {
field: 'name',
},
},
{
type: 'ducet',
order: 'desc',
arguments:
{
field: 'categories'
}
arguments: {
field: 'categories',
},
},
],
filter: {
type: 'value',
arguments: {
field: 'type',
value: SCThingType.AcademicEvent
}
}
value: SCThingType.AcademicEvent,
},
},
};
let esConfig: ElasticsearchConfig = {
name: 'elasticsearch',
@@ -138,7 +144,7 @@ describe('Query', function () {
queryType: 'dis_max',
matchBoosting: 1.3,
fuzziness: 'AUTO',
cutoffFrequency: 0.0,
cutoffFrequency: 0,
tieBreaker: 0,
},
};
@@ -147,59 +153,59 @@ describe('Query', function () {
queryType: 'dis_max',
matchBoosting: 1.3,
fuzziness: 'AUTO',
cutoffFrequency: 0.0,
cutoffFrequency: 0,
tieBreaker: 0,
};
const config: SCConfigFile = {
...configFile
...configFile,
};
beforeEach(function () {
esConfig = {
name: 'elasticsearch',
version: '123'
version: '123',
};
});
// TODO: check parts of received elasticsearch query for each test case
it('should build query that includes sorting when query is undefined', function () {
expect(buildQuery(params, config, esConfig)).to.be.an('object');
expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
});
it('should build query that includes sorting when query type is query_string', function () {
esConfig.query = {...query, queryType: 'query_string'};
expect(buildQuery(params, config, esConfig)).to.be.an('object');
expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
});
it('should build query that includes sorting when query type is dis_max', function () {
esConfig.query = {...query, queryType: 'dis_max'};
expect(buildQuery(params, config, esConfig)).to.be.an('object');
expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
});
it('should build query that includes sorting when query type is dis_max', function () {
esConfig.query = {...query, queryType: 'dis_max'};
expect(buildQuery(params, config, esConfig)).to.be.an('object');
expect(buildQuery(parameters, config, esConfig)).to.be.an('object');
});
it('should reject (throw an error) if provided query type is not supported', function () {
// @ts-ignore
// @ts-expect-error not assignable
esConfig.query = {...query, queryType: 'invalid_query_type'};
expect(() => buildQuery(params, config, esConfig)).to.throw('query type');
expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
});
});
describe('buildFilter', function () {
const searchFilters: { [key: string]: SCSearchFilter } = {
const searchFilters: {[key: string]: SCSearchFilter} = {
value: {
type: 'value',
arguments: {
field: 'type',
value: SCThingType.Dish
}
value: SCThingType.Dish,
},
},
distance: {
type: 'distance',
@@ -207,7 +213,7 @@ describe('Query', function () {
distance: 1000,
field: 'geo',
position: [50.123, 8.123],
}
},
},
geoPoint: {
type: 'geo',
@@ -218,9 +224,9 @@ describe('Query', function () {
coordinates: [
[50.123, 8.123],
[50.123, 8.123],
]
}
}
],
},
},
},
geoShape: {
type: 'geo',
@@ -232,9 +238,9 @@ describe('Query', function () {
coordinates: [
[50.123, 8.123],
[50.123, 8.123],
]
}
}
],
},
},
},
boolean: {
type: 'boolean',
@@ -246,16 +252,16 @@ describe('Query', function () {
arguments: {
field: 'type',
value: SCThingType.Dish,
}
},
},
{
type: 'availability',
arguments: {
field: 'offers.availabilityRange'
}
}
]
}
field: 'offers.availabilityRange',
},
},
],
},
},
};
@@ -263,8 +269,8 @@ describe('Query', function () {
const filter = buildFilter(searchFilters.value);
const expectedFilter: ESTermFilter = {
term: {
'type.raw': SCThingType.Dish
}
'type.raw': SCThingType.Dish,
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -277,28 +283,30 @@ describe('Query', function () {
range: {
price: {
relation: undefined,
}
}
},
},
};
const rawFilter: SCSearchNumericRangeFilter = {
type: 'numeric range',
arguments: {
bounds: {},
field: 'price'
}
field: 'price',
},
};
// eslint-disable-next-line unicorn/consistent-function-scoping
const setBound = (location: 'upperBound' | 'lowerBound', bound: string | null) => {
let out: number | null = null;
if (bound != null) {
if (bound != undefined) {
out = Math.random();
rawFilter.arguments.bounds[location] = {
mode: bound as 'inclusive' | 'exclusive',
limit: out,
};
// @ts-ignore implicit any
expectedFilter.range.price[`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`] = out;
expectedFilter.range.price[
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
] = out;
}
};
setBound('upperBound', upperMode);
@@ -307,9 +315,9 @@ describe('Query', function () {
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
// @ts-ignore implicit any
// @ts-expect-error implicit any
const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
// @ts-ignore implicit any
// @ts-expect-error implicit any
const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
// only one should exist at the same time
@@ -328,8 +336,8 @@ describe('Query', function () {
format: 'thisIsADummyFormat',
time_zone: 'thisIsADummyTimeZone',
relation: 'testRelation' as any,
}
}
},
},
};
const rawFilter: SCSearchDateRangeFilter = {
@@ -340,19 +348,20 @@ describe('Query', function () {
relation: 'testRelation' as any,
format: 'thisIsADummyFormat',
timeZone: 'thisIsADummyTimeZone',
}
},
};
const setBound = (location: 'upperBound' | 'lowerBound', bound: string | null) => {
let out: string | null = null;
if (bound != null) {
if (bound != undefined) {
out = `${location} ${bound} ${upperMode} ${lowerMode}`;
rawFilter.arguments.bounds[location] = {
mode: bound as 'inclusive' | 'exclusive',
limit: out,
};
// @ts-ignore implicit any
expectedFilter.range.price[`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`] = out;
expectedFilter.range.price[
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
] = out;
}
};
setBound('upperBound', upperMode);
@@ -361,9 +370,9 @@ describe('Query', function () {
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
// @ts-ignore implicit any
// @ts-expect-error implicit any
const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
// @ts-ignore implicit any
// @ts-expect-error implicit any
const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
// only one should exist at the same time
@@ -390,7 +399,7 @@ describe('Query', function () {
'offers.availabilityRange': {
gte: `test||/${scope}`,
lt: `test||+1${scope}/${scope}`,
}
},
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -411,7 +420,7 @@ describe('Query', function () {
'offers.availabilityRange': {
gte: 'test||/s',
lt: 'test||+1s/s',
}
},
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -432,7 +441,7 @@ describe('Query', function () {
'offers.availabilityRange': {
gte: `test||/d`,
lt: `test||+1d/d`,
}
},
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -452,7 +461,7 @@ describe('Query', function () {
'offers.availabilityRange': {
gte: `now/d`,
lt: `now+1d/d`,
}
},
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -463,12 +472,12 @@ describe('Query', function () {
const filter = buildFilter(searchFilters.distance);
const expectedFilter: ESGeoDistanceFilter = {
geo_distance: {
distance: '1000m',
'distance': '1000m',
'geo.point.coordinates': {
lat: 8.123,
lon: 50.123
}
}
lon: 50.123,
},
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -488,24 +497,24 @@ describe('Query', function () {
type: 'envelope',
coordinates: [
[50.123, 8.123],
[50.123, 8.123]
]
[50.123, 8.123],
],
},
},
ignore_unmapped: true,
}
'ignore_unmapped': true,
},
},
{
geo_bounding_box: {
'geo.point.coordinates': {
bottom_right: [50.123, 8.123],
top_left: [50.123, 8.123]
top_left: [50.123, 8.123],
},
ignore_unmapped: true,
'ignore_unmapped': true,
},
},
]
}
],
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -521,12 +530,12 @@ describe('Query', function () {
type: 'envelope',
coordinates: [
[50.123, 8.123],
[50.123, 8.123]
]
[50.123, 8.123],
],
},
},
ignore_unmapped: true,
}
'ignore_unmapped': true,
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -540,8 +549,8 @@ describe('Query', function () {
must: [
{
term: {
'type.raw': 'dish'
}
'type.raw': 'dish',
},
},
{
range: {
@@ -549,13 +558,13 @@ describe('Query', function () {
gte: 'now/s',
lt: 'now+1s/s',
relation: 'intersects',
}
}
}
},
},
},
],
must_not: [],
should: []
}
should: [],
},
};
expect(filter).to.be.eql(expectedFilter);
@@ -568,7 +577,7 @@ describe('Query', function () {
type: 'ducet',
order: 'desc',
arguments: {
field: 'name'
field: 'name',
},
},
{
@@ -576,14 +585,14 @@ describe('Query', function () {
order: 'desc',
arguments: {
field: 'name',
}
},
},
{
type: 'distance',
order: 'desc',
arguments: {
field: 'geo',
position: [8.123, 50.123]
position: [8.123, 50.123],
},
},
{
@@ -592,35 +601,35 @@ describe('Query', function () {
arguments: {
universityRole: 'student',
field: 'offers.prices',
}
},
},
];
let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
const expectedSorts: { [key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort } = {
const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
ducet: {
'name.sort': 'desc'
'name.sort': 'desc',
},
generic: {
'name': 'desc'
name: 'desc',
},
distance: {
_geo_distance: {
mode: 'avg',
order: 'desc',
unit: 'm',
'mode': 'avg',
'order': 'desc',
'unit': 'm',
'geo.point.coordinates': {
lat: 50.123,
lon: 8.123
}
}
lon: 8.123,
},
},
},
price: {
_script: {
order: 'asc',
script: '\n // foo price sort script',
type: 'number'
}
}
type: 'number',
},
},
};
before(function () {
sorts = buildSort(searchSCSearchSort);
@@ -641,7 +650,10 @@ describe('Query', function () {
it('should build price sort', function () {
const priceSortNoScript = {
...sorts[3],
_script: {...(sorts[3] as ScriptSort)._script, script: (expectedSorts.price as ScriptSort)._script.script}
_script: {
...(sorts[3] as ScriptSort)._script,
script: (expectedSorts.price as ScriptSort)._script.script,
},
};
expect(priceSortNoScript).to.be.eql(expectedSorts.price);
});