Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-07 14:02:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: update to of elasticsearch 8.4

Browse Source
This commit is contained in:
Thea Schöbl
2023-04-28 12:43:31 +00:00
committed by Rainer Killinger
parent 515a6eeea5
commit c9b83b5d71
42 changed files with 1843 additions and 2242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
test/common.ts
View File

@@ -16,6 +16,7 @@
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCThingType, SCUuid} from '@openstapps/core';
import {Express} from 'express';
import moment from 'moment';
import {getIndexUID} from '../src/storage/elasticsearch/util';
import {configureApp} from '../src/app';
import express from 'express';
import http from 'http';
@@ -24,7 +25,6 @@ import {MailQueue} from '../src/notification/mail-queue';
import {Bulk, BulkStorage} from '../src/storage/bulk-storage';
import getPort from 'get-port';
import {Database} from '../src/storage/database';
import {Elasticsearch} from '../src/storage/elasticsearch/elasticsearch';
import {v4} from 'uuid';
/**
@@ -147,5 +147,4 @@ export const getTransport = (verified: boolean) => {
};
};
export const getIndex = (uid?: string) =>
`stapps_footype_foosource_${uid ?? Elasticsearch.getIndexUID(v4())}`;
export const getIndex = (uid?: string) => `stapps_footype_foosource_${uid ?? getIndexUID(v4())}`;

67
test/storage/elasticsearch/aggregations.spec.ts
View File

@@ -13,13 +13,13 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {AggregateName, AggregationsMultiTermsBucket} from '@elastic/elasticsearch/lib/api/types';
import {SCFacet, SCThingType} from '@openstapps/core';
import {expect} from 'chai';
import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch';
describe('Aggregations', function () {
const aggregations: AggregationResponse = {
const aggregations: Record<AggregateName, Partial<AggregationsMultiTermsBucket>> = {
'catalog': {
'doc_count': 4,
'superCatalogs.categories': {
@@ -76,14 +76,6 @@ describe('Aggregations', function () {
buckets: [],
},
},
'fooType': {
buckets: [
{
doc_count: 321,
key: 'foo',
},
],
},
'@all': {
doc_count: 17,
type: {
@@ -102,33 +94,6 @@ describe('Aggregations', function () {
};
const expectedFacets: SCFacet[] = [
{
buckets: [
{
count: 13,
key: 'person',
},
{
count: 4,
key: 'catalog',
},
],
field: 'type',
},
{
buckets: [
{
count: 8,
key: 'foobar',
},
{
count: 2,
key: 'bar',
},
],
field: 'categories',
onlyOnType: SCThingType.AcademicEvent,
},
{
buckets: [
{
@@ -153,7 +118,33 @@ describe('Aggregations', function () {
field: 'categories',
onlyOnType: SCThingType.Catalog,
},
// no fooType as it doesn't appear in the aggregation schema
{
buckets: [
{
count: 8,
key: 'foobar',
},
{
count: 2,
key: 'bar',
},
],
field: 'categories',
onlyOnType: SCThingType.AcademicEvent,
},
{
buckets: [
{
count: 13,
key: 'person',
},
{
count: 4,
key: 'catalog',
},
],
field: 'type',
},
];
it('should parse the aggregations providing the appropriate facets', function () {

90
test/storage/elasticsearch/common.spec.ts
View File

@@ -1,90 +0,0 @@
/*
* Copyright (C) 2020 StApps
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
ESAggMatchAllFilter,
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter,
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {expect} from 'chai';
import {
isNestedAggregation,
isBucketAggregation,
isESTermsFilter,
isESAggMatchAllFilter,
isESNestedAggregation,
} from '../../../lib/storage/elasticsearch/types/guards';
import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';
describe('Common', function () {
const bucketAggregation: BucketAggregation = {buckets: []};
const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
describe('isBucketAggregation', function () {
it('should be false for a number', function () {
expect(isBucketAggregation(123)).to.be.false;
});
it('should be true for a bucket aggregation', function () {
expect(isBucketAggregation(bucketAggregation)).to.be.true;
});
});
describe('isNestedAggregation', function () {
it('should be false for a bucket aggregation', function () {
expect(isNestedAggregation(bucketAggregation)).to.be.false;
});
it('should be true for a nested aggregation', function () {
const nestedAggregation: NestedAggregation = {doc_count: 123};
expect(isNestedAggregation(nestedAggregation)).to.be.true;
});
});
describe('isESTermsFilter', function () {
it('should be false for an elasticsearch nested aggregation', function () {
expect(isESTermsFilter(esNestedAggregation)).to.be.false;
});
it('should be true for an elasticsearch terms filter', function () {
expect(isESTermsFilter(esTermsFilter)).to.be.true;
});
});
describe('isESNestedAggregation', function () {
it('should be false for an elasticsearch terms filter', function () {
expect(isESNestedAggregation(esTermsFilter)).to.be.false;
});
it('should be true for an elasticsearch nested aggregation', function () {
expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
});
});
describe('isESAggMatchAllFilter', function () {
it('should be false for an elasticsearch aggregation type filter', function () {
const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
});
it('should be true for an elasticsearch aggregation match all filter', function () {
const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
});
});
});

250
test/storage/elasticsearch/elasticsearch.spec.ts
View File

@@ -14,7 +14,14 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client} from '@elastic/elasticsearch';
import {Client, Diagnostic} from '@elastic/elasticsearch';
import Indices from '@elastic/elasticsearch/lib/api/api/indices';
import {
CreateResponse,
SearchHit,
SearchResponse,
SortCombinations,
} from '@elastic/elasticsearch/lib/api/types';
import {
SCBook,
SCBulkResponse,
@@ -30,22 +37,32 @@ import {Logger} from '@openstapps/logger';
import {SMTP} from '@openstapps/logger/lib/smtp';
import {expect, use} from 'chai';
import chaiAsPromised from 'chai-as-promised';
import {SearchResponse} from 'elasticsearch';
import mockedEnv from 'mocked-env';
import sinon from 'sinon';
import {ALL_INDICES_QUERY, parseIndexName} from '../../../src/storage/elasticsearch/util';
import * as queryModule from '../../../src/storage/elasticsearch/query/query';
import * as sortModule from '../../../src/storage/elasticsearch/query/sort';
import sinon, {SinonStub} from 'sinon';
import {getIndexUID, getThingIndexName, INDEX_UID_LENGTH} from '../../../src/storage/elasticsearch/util';
import * as utilModule from '../../../src/storage/elasticsearch/util';
import {removeInvalidAliasChars} from '../../../src/storage/elasticsearch/util/alias';
import {configFile} from '../../../src/common';
import {MailQueue} from '../../../src/notification/mail-queue';
import {aggregations} from '../../../src/storage/elasticsearch/templating';
import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch';
import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
import * as query from '../../../src/storage/elasticsearch/query';
import * as templating from '../../../src/storage/elasticsearch/templating';
import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common';
import fs from 'fs';
use(chaiAsPromised);
/**
*
*/
function searchResponse<T>(...hits: SearchHit<T>[]): SearchResponse<T> {
return {hits: {hits}, took: 0, timed_out: false, _shards: {total: 1, failed: 0, successful: 1}};
}
describe('Elasticsearch', function () {
// increase timeout for the suite
this.timeout(DEFAULT_TEST_TIMEOUT);
@@ -83,6 +100,14 @@ describe('Elasticsearch', function () {
});
});
describe('getAliasMap', function () {
it('should fail after retries', async function () {
const es = new Elasticsearch(configFile);
sandbox.stub(es.client.indices, 'getAlias').throws();
await expect(es.init({maxRetries: 1, retryInterval: 10})).to.be.rejected;
});
});
describe('getIndex (including getIndexUID)', function () {
const type = 'foo bar type';
const source = 'foo_source';
@@ -95,59 +120,63 @@ describe('Elasticsearch', function () {
};
it('should provide index UID from the provided UID', function () {
const indexUID = Elasticsearch.getIndexUID(bulk.uid);
const indexUID = getIndexUID(bulk.uid);
expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
expect(indexUID.length).to.be.equal(INDEX_UID_LENGTH);
// test starting and ending character
expect(indexUID[0]).to.be.equal(bulk.uid[0]);
expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[INDEX_UID_LENGTH - 1]);
});
it('should provide index name from the provided data', function () {
expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
expect(getThingIndexName(type as SCThingType, source, bulk)).to.be.equal(
`stapps_${type.split(' ').join('_')}_${source}_${getIndexUID(bulk.uid)}`,
);
});
it('should reject invalid index names', function () {
expect(() => parseIndexName(':)')).to.throw(SyntaxError);
});
});
describe('removeAliasChars', function () {
it('should remove spaces from both ends', function () {
expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should replace inner spaces with underscores', function () {
expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
expect(removeInvalidAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
});
it('should remove invalid characters', function () {
expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should remove invalid starting characters', function () {
expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
expect(removeInvalidAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
});
it('should replace with a placeholder in case of invalid alias', function () {
expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
expect(removeInvalidAliasChars('.', 'bulk-uid')).to.contain('placeholder');
expect(removeInvalidAliasChars('..', 'bulk-uid')).to.contain('placeholder');
});
it('should work with common cases', function () {
expect(
Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
removeInvalidAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
expect(
Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
expect(removeInvalidAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid')).to.be.equal(
'THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG',
);
});
it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
const sandbox = sinon.createSandbox();
const loggerWarnStub = sandbox.stub(Logger, 'warn');
expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
expect(removeInvalidAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
expect(loggerWarnStub.called).to.be.true;
});
});
@@ -182,7 +211,7 @@ describe('Elasticsearch', function () {
it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
const error = new Error('Foo Error');
const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
sandbox.stub(Client.prototype, 'on').yields(error);
sandbox.stub(Diagnostic.prototype, 'on').yields(error);
new Elasticsearch(configFile);
@@ -192,7 +221,7 @@ describe('Elasticsearch', function () {
it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
const fakeResponse = {foo: 'bar'};
const loggerLogStub = sandbox.stub(Logger, 'log');
sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
sandbox.stub(Diagnostic.prototype, 'on').yields(null, fakeResponse);
new Elasticsearch(configFile);
expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
@@ -254,26 +283,24 @@ describe('Elasticsearch', function () {
describe('Operations with bundle/index', async function () {
const sandbox = sinon.createSandbox();
let es: Elasticsearch;
let createStub: SinonStub;
let deleteStub: SinonStub;
let refreshStub: SinonStub;
let updateAliasesStub: SinonStub;
let existsStub: SinonStub;
const oldIndex = 'stapps_footype_foosource_oldindex';
beforeEach(function () {
sandbox
.stub(Indices.prototype, 'getAlias')
.resolves({[oldIndex]: {aliases: {[SCThingType.Book]: {}}}} as any);
sandbox.stub(Indices.prototype, 'putTemplate').resolves({} as any);
createStub = sandbox.stub(Indices.prototype, 'create').resolves({} as any);
deleteStub = sandbox.stub(Indices.prototype, 'delete').resolves({} as any);
existsStub = sandbox.stub(Indices.prototype, 'exists').resolves({} as any);
refreshStub = sandbox.stub(Indices.prototype, 'refresh').resolves({} as any);
updateAliasesStub = sandbox.stub(Indices.prototype, 'updateAliases').resolves({} as any);
es = new Elasticsearch(configFile);
es.client.indices = {
// @ts-expect-error not assignable
getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
// @ts-expect-error not assignable
putTemplate: () => Promise.resolve({}),
// @ts-expect-error not assignable
create: () => Promise.resolve({}),
// @ts-expect-error not assignable
delete: () => Promise.resolve({}),
// @ts-expect-error not assignable
exists: () => Promise.resolve({}),
// @ts-expect-error not assignable
refresh: () => Promise.resolve({}),
// @ts-expect-error not assignable
updateAliases: () => Promise.resolve({}),
};
});
afterEach(function () {
@@ -286,8 +313,8 @@ describe('Elasticsearch', function () {
});
it('should reject (throw an error) if the index name is not valid', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`);
sandbox.createStubInstance(Client, {});
sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex}`);
await es.init();
return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
@@ -295,9 +322,8 @@ describe('Elasticsearch', function () {
it('should create a new index', async function () {
const index = getIndex();
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
const putTemplateStub = sandbox.stub(templating, 'putTemplate');
const createStub = sandbox.stub(es.client.indices, 'create');
await es.init();
await es.bulkCreated(bulk);
@@ -313,21 +339,19 @@ describe('Elasticsearch', function () {
sandbox.restore();
});
it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
await es.bulkExpired({...bulk, state: 'in progress'});
expect(clientDeleteStub.called).to.be.true;
expect(deleteStub.called).to.be.true;
});
it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
await es.bulkExpired({...bulk, state: 'done'});
expect(clientDeleteStub.called).to.be.false;
expect(deleteStub.called).to.be.false;
});
});
@@ -337,13 +361,23 @@ describe('Elasticsearch', function () {
});
it('should reject if the index name is not valid', async function () {
sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`);
sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex()}`);
sandbox.createStubInstance(Client, {});
await es.init();
return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
});
it("should create templates if index doesn't exist", async function () {
await es.init();
existsStub.resolves(false);
const putTemplateSpy = sandbox.spy(templating, 'putTemplate');
await es.bulkUpdated(bulk);
expect(createStub.called).to.be.true;
expect(putTemplateSpy.called).to.be.true;
});
it('should create a new index', async function () {
const index = getIndex();
const expectedRefreshActions = [
@@ -354,15 +388,12 @@ describe('Elasticsearch', function () {
remove: {index: oldIndex, alias: SCThingType.Book},
},
];
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
sandbox.stub(es, 'aliasMap').value({
[SCThingType.Book]: {
[bulk.source]: oldIndex,
},
});
const refreshStub = sandbox.stub(es.client.indices, 'refresh');
const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
const deleteStub = sandbox.stub(es.client.indices, 'delete');
sandbox.stub(templating, 'putTemplate');
await es.init();
@@ -371,9 +402,7 @@ describe('Elasticsearch', function () {
expect(refreshStub.calledWith({index})).to.be.true;
expect(
updateAliasesStub.calledWith({
body: {
actions: expectedRefreshActions,
},
actions: expectedRefreshActions,
}),
).to.be.true;
expect(deleteStub.called).to.be.true;
@@ -394,20 +423,19 @@ describe('Elasticsearch', function () {
});
it('should reject if object is not found', async function () {
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
return expect(es.get('123')).to.rejectedWith('found');
});
it('should provide the thing if object is found', async function () {
const foundObject: ElasticsearchObject<SCMessage> = {
const foundObject: SearchHit<SCMessage> = {
_id: '',
_index: '',
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
sandbox.stub(es.client, 'search').resolves(searchResponse(foundObject));
return expect(await es.get('123')).to.be.eql(message);
});
@@ -428,56 +456,54 @@ describe('Elasticsearch', function () {
it('should not post if the object already exists in an index which will not be rolled over', async function () {
const index = getIndex();
const oldIndex = index.replace('foosource', 'barsource');
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: oldIndex,
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
return expect(es.post(object._source!, bulk)).to.rejectedWith('exist');
});
it('should not reject if the object already exists but in an index which will be rolled over', async function () {
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
// return index name with different generated UID (see getIndex method)
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist');
return expect(es.post(object._source!, bulk)).to.not.rejectedWith('exist');
});
it('should reject if there is an object creation error on the elasticsearch side', async function () {
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'create').resolves({body: {created: false}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse);
return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
});
it('should create a new object', async function () {
let caughtParameter: any;
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
// @ts-expect-error call
const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
caughtParameter = parameter;
return Promise.resolve({body: {created: true}});
return Promise.resolve({result: 'created'});
});
await es.post(message as SCMessage, bulk);
expect(createStub.called).to.be.true;
expect(caughtParameter.body).to.be.eql({
expect(caughtParameter.document).to.be.eql({
...message,
creation_date: caughtParameter.body.creation_date,
creation_date: caughtParameter.document.creation_date,
});
});
});
@@ -493,29 +519,27 @@ describe('Elasticsearch', function () {
sandbox.restore();
});
it('should reject to put if the object does not already exist', async function () {
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
sandbox.stub(es.client, 'search').resolves(searchResponse());
return expect(es.put(object._source)).to.rejectedWith('exist');
return expect(es.put(object._source!)).to.rejectedWith('exist');
});
// noinspection JSUnusedLocalSymbols
it('should update the object if it already exists', async function () {
let caughtParameter: any;
const object: ElasticsearchObject<SCMessage> = {
const object: SearchHit<SCMessage> = {
_id: '',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
// @ts-expect-error unused
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
@@ -523,7 +547,7 @@ describe('Elasticsearch', function () {
return Promise.resolve({body: {created: true}});
});
await es.put(object._source);
await es.put(object._source!);
expect(caughtParameter.body.doc).to.be.eql(object._source);
});
@@ -532,18 +556,16 @@ describe('Elasticsearch', function () {
describe('search', async function () {
let es: Elasticsearch;
const sandbox = sinon.createSandbox();
const objectMessage: ElasticsearchObject<SCMessage> = {
const objectMessage: SearchHit<SCMessage> = {
_id: '123',
_index: getIndex(),
_score: 0,
_type: '',
_source: message as SCMessage,
};
const objectBook: ElasticsearchObject<SCBook> = {
const objectBook: SearchHit<SCBook> = {
_id: '321',
_index: getIndex(),
_score: 0,
_type: '',
_source: book as SCBook,
};
const fakeEsAggregations = {
@@ -565,26 +587,16 @@ describe('Elasticsearch', function () {
},
},
};
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
body: {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
hits: [objectMessage, objectBook],
total: 123,
},
aggregations: fakeEsAggregations,
const fakeSearchResponse: SearchResponse<SCThings> = {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
hits: {
hits: [objectMessage, objectBook],
total: 123,
},
headers: {},
// @ts-expect-error not assignable
meta: {},
// @ts-expect-error not assignable
statusCode: {},
// @ts-expect-error not assignable
warnings: {},
aggregations: fakeEsAggregations,
};
let searchStub: sinon.SinonStub;
before(function () {
@@ -625,9 +637,9 @@ describe('Elasticsearch', function () {
const {pagination} = await es.search({from});
expect(pagination).to.be.eql({
count: fakeSearchResponse.body!.hits.hits.length,
count: fakeSearchResponse.hits.hits.length,
offset: from,
total: fakeSearchResponse.body!.hits.total,
total: fakeSearchResponse.hits.total,
});
});
@@ -659,22 +671,20 @@ describe('Elasticsearch', function () {
},
},
};
const fakeResponse = {foo: 'bar'};
const fakeResponse = {foo: 'bar'} as SortCombinations;
const fakeBuildSortResponse = [fakeResponse];
// @ts-expect-error not assignable
sandbox.stub(query, 'buildQuery').returns(fakeResponse);
sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
sandbox.stub(queryModule, 'buildQuery').returns(fakeResponse);
sandbox.stub(sortModule, 'buildSort').returns(fakeBuildSortResponse);
await es.search(parameters);
sandbox.assert.calledWithMatch(searchStub, {
body: {
aggs: aggregations,
query: fakeResponse,
sort: fakeBuildSortResponse,
},
aggs: aggregations,
query: fakeResponse,
sort: fakeBuildSortResponse,
from: parameters.from,
index: Elasticsearch.getListOfAllIndices(),
index: ALL_INDICES_QUERY,
size: parameters.size,
});
});

22
test/storage/elasticsearch/monitoring.spec.ts
View File

@@ -14,7 +14,8 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ApiResponse, Client} from '@elastic/elasticsearch';
import {Client} from '@elastic/elasticsearch';
import {SearchResponse} from '@elastic/elasticsearch/lib/api/types';
import {
SCMonitoringConfiguration,
SCMonitoringLogAction,
@@ -23,7 +24,6 @@ import {
SCThings,
} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
import {SearchResponse} from 'elasticsearch';
import {MailQueue} from '../../../src/notification/mail-queue';
import {setUp} from '../../../src/storage/elasticsearch/monitoring';
@@ -111,16 +111,14 @@ describe('Monitoring', async function () {
});
it('should log errors where conditions failed', async function () {
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
body: {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
total: 123,
},
const fakeSearchResponse: SearchResponse<SCThings> = {
took: 12,
timed_out: false,
// @ts-expect-error not assignable
_shards: {},
// @ts-expect-error not assignable
hits: {
total: 123,
},
};
const fakeClient = new Client({node: 'http://foohost:9200'});

113
test/storage/elasticsearch/query.spec.ts
View File

@@ -25,25 +25,14 @@ import {
SCThingType,
} from '@openstapps/core';
import {expect} from 'chai';
import {
ESDateRangeFilter,
ESRangeFilter,
ESNumericRangeFilter,
ElasticsearchConfig,
ESBooleanFilter,
ESGenericSort,
ESGeoDistanceFilter,
ESGeoDistanceSort,
ESTermFilter,
ScriptSort,
} from '../../../src/storage/elasticsearch/types/elasticsearch';
import {buildFilter} from '../../../src/storage/elasticsearch/query/filter';
import {buildBooleanFilter} from '../../../src/storage/elasticsearch/query/filters/boolean';
import {buildQuery} from '../../../src/storage/elasticsearch/query/query';
import {buildSort} from '../../../src/storage/elasticsearch/query/sort';
import {ElasticsearchConfig} from '../../../src/storage/elasticsearch/types/elasticsearch-config';
import {QueryDslSpecificQueryContainer} from '../../../src/storage/elasticsearch/types/util';
import {configFile} from '../../../src/common';
import {
buildBooleanFilter,
buildFilter,
buildQuery,
buildSort,
} from '../../../src/storage/elasticsearch/query';
import {SortCombinations} from '@elastic/elasticsearch/lib/api/types';
describe('Query', function () {
describe('buildBooleanFilter', function () {
@@ -74,7 +63,7 @@ describe('Query', function () {
or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
};
const expectedEsFilters: Array<ESTermFilter> = [
const expectedEsFilters: Array<QueryDslSpecificQueryContainer<'term'>> = [
{
term: {
'type.raw': 'catalog',
@@ -88,20 +77,20 @@ describe('Query', function () {
];
it('should create appropriate elasticsearch "and" filter argument', function () {
const {must} = buildBooleanFilter(booleanFilters.and);
const {must} = buildBooleanFilter(booleanFilters.and).bool;
expect(must).to.be.eql(expectedEsFilters);
});
it('should create appropriate elasticsearch "or" filter argument', function () {
const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or).bool;
expect(should).to.be.eql(expectedEsFilters);
expect(minimum_should_match).to.be.equal(1);
});
it('should create appropriate elasticsearch "not" filter argument', function () {
const {must_not} = buildBooleanFilter(booleanFilters.not);
const {must_not} = buildBooleanFilter(booleanFilters.not).bool;
expect(must_not).to.be.eql(expectedEsFilters);
});
@@ -196,6 +185,10 @@ describe('Query', function () {
expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
});
it('should accept other search contexts', function () {
expect(buildQuery({context: 'place', ...parameters}, config, esConfig)).to.be.an('object');
});
});
describe('buildFilter', function () {
@@ -267,7 +260,7 @@ describe('Query', function () {
it('should build value filter', function () {
const filter = buildFilter(searchFilters.value);
const expectedFilter: ESTermFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'term'> = {
term: {
'type.raw': SCThingType.Dish,
},
@@ -279,7 +272,7 @@ describe('Query', function () {
it('should build numeric range filters', function () {
for (const upperMode of ['inclusive', 'exclusive', null]) {
for (const lowerMode of ['inclusive', 'exclusive', null]) {
const expectedFilter: ESNumericRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
price: {
relation: undefined,
@@ -304,7 +297,7 @@ describe('Query', function () {
mode: bound as 'inclusive' | 'exclusive',
limit: out,
};
expectedFilter.range.price[
expectedFilter.range.price![
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
] = out;
}
@@ -312,7 +305,7 @@ describe('Query', function () {
setBound('upperBound', upperMode);
setBound('lowerBound', lowerMode);
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'term'>;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
// @ts-expect-error implicit any
@@ -330,7 +323,7 @@ describe('Query', function () {
it('should build date range filters', function () {
for (const upperMode of ['inclusive', 'exclusive', null]) {
for (const lowerMode of ['inclusive', 'exclusive', null]) {
const expectedFilter: ESDateRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
price: {
format: 'thisIsADummyFormat',
@@ -359,7 +352,7 @@ describe('Query', function () {
mode: bound as 'inclusive' | 'exclusive',
limit: out,
};
expectedFilter.range.price[
expectedFilter.range.price![
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
] = out;
}
@@ -367,7 +360,7 @@ describe('Query', function () {
setBound('upperBound', upperMode);
setBound('lowerBound', lowerMode);
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'range'>;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
// @ts-expect-error implicit any
@@ -394,7 +387,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: `test||/${scope}`,
@@ -415,7 +408,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: 'test||/s',
@@ -436,7 +429,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: `test||/d`,
@@ -456,7 +449,7 @@ describe('Query', function () {
},
});
const expectedFilter: ESRangeFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
range: {
'offers.availabilityRange': {
gte: `now/d`,
@@ -470,7 +463,7 @@ describe('Query', function () {
it('should build distance filter', function () {
const filter = buildFilter(searchFilters.distance);
const expectedFilter: ESGeoDistanceFilter = {
const expectedFilter: QueryDslSpecificQueryContainer<'geo_distance'> = {
geo_distance: {
'distance': '1000m',
'geo.point.coordinates': {
@@ -486,34 +479,18 @@ describe('Query', function () {
it('should build geo filter for shapes and points', function () {
const filter = buildFilter(searchFilters.geoPoint);
const expectedFilter = {
bool: {
minimum_should_match: 1,
should: [
{
geo_shape: {
'geo.polygon': {
relation: undefined,
shape: {
type: 'envelope',
coordinates: [
[50.123, 8.123],
[50.123, 8.123],
],
},
},
'ignore_unmapped': true,
},
geo_shape: {
'geo.polygon': {
relation: undefined,
shape: {
type: 'envelope',
coordinates: [
[50.123, 8.123],
[50.123, 8.123],
],
},
{
geo_bounding_box: {
'geo.point.coordinates': {
bottom_right: [50.123, 8.123],
top_left: [50.123, 8.123],
},
'ignore_unmapped': true,
},
},
],
},
'ignore_unmapped': true,
},
};
@@ -543,7 +520,7 @@ describe('Query', function () {
it('should build boolean filter', function () {
const filter = buildFilter(searchFilters.boolean);
const expectedFilter: ESBooleanFilter<any> = {
const expectedFilter: QueryDslSpecificQueryContainer<'bool'> = {
bool: {
minimum_should_match: 0,
must: [
@@ -604,8 +581,8 @@ describe('Query', function () {
},
},
];
let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
let sorts: SortCombinations[] = [];
const expectedSorts: {[key: string]: SortCombinations} = {
ducet: {
'name.sort': 'desc',
},
@@ -632,7 +609,7 @@ describe('Query', function () {
},
};
before(function () {
sorts = buildSort(searchSCSearchSort);
sorts = buildSort(searchSCSearchSort) as SortCombinations[];
});
it('should build ducet sort', function () {
@@ -649,10 +626,10 @@ describe('Query', function () {
it('should build price sort', function () {
const priceSortNoScript = {
...sorts[3],
...(sorts[3] as any),
_script: {
...(sorts[3] as ScriptSort)._script,
script: (expectedSorts.price as ScriptSort)._script.script,
...(sorts[3] as any)._script,
script: (expectedSorts.price as any)._script.script,
},
};
expect(priceSortNoScript).to.be.eql(expectedSorts.price);