mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-21 00:52:55 +00:00
Resolve "Transition to ESLint"
This commit is contained in:
Thea Schöbl
committed by
Rainer Killinger
Rainer Killinger
parent
ca1d2444e0
commit
418ba67d15
@@ -29,7 +29,6 @@ export type BulkOperation = 'create' | 'expired' | 'update';
|
||||
* Describes an indexing process
|
||||
*/
|
||||
export class Bulk implements SCBulkRequest {
|
||||
|
||||
/**
|
||||
* Expiration of the bulk
|
||||
*
|
||||
@@ -70,19 +69,15 @@ export class Bulk implements SCBulkRequest {
|
||||
|
||||
/**
|
||||
* Creates a new bulk process
|
||||
*
|
||||
* @param request Data needed for requesting a bulk
|
||||
*/
|
||||
constructor(request: SCBulkRequest) {
|
||||
this.uid = v4();
|
||||
this.state = 'in progress';
|
||||
|
||||
if (typeof request.expiration === 'string') {
|
||||
this.expiration = request.expiration;
|
||||
} else {
|
||||
this.expiration = moment()
|
||||
.add(1, 'hour')
|
||||
.toISOString();
|
||||
}
|
||||
this.expiration =
|
||||
typeof request.expiration === 'string' ? request.expiration : moment().add(1, 'hour').toISOString();
|
||||
// when should this process be finished
|
||||
// where does the process come from
|
||||
this.source = request.source;
|
||||
@@ -102,10 +97,10 @@ export class BulkStorage {
|
||||
|
||||
/**
|
||||
* Creates a new BulkStorage
|
||||
*
|
||||
* @param database the database that is controlled by this bulk storage
|
||||
*/
|
||||
constructor(public database: Database) {
|
||||
|
||||
// a bulk lives 60 minutes if no expiration is given
|
||||
// the cache is checked every 60 seconds
|
||||
this.cache = new NodeCache({stdTTL: 3600, checkperiod: 60});
|
||||
@@ -121,13 +116,12 @@ export class BulkStorage {
|
||||
|
||||
/**
|
||||
* Saves a bulk process and assigns to it a user-defined ttl (time-to-live)
|
||||
*
|
||||
* @param bulk the bulk process to save
|
||||
* @returns the bulk process that was saved
|
||||
*/
|
||||
private save(bulk: Bulk): Bulk {
|
||||
const expirationInSeconds = moment(bulk.expiration)
|
||||
// tslint:disable-next-line: no-magic-numbers
|
||||
.diff(moment.now()) / 1000;
|
||||
const expirationInSeconds = moment(bulk.expiration).diff(moment.now()) / 1000;
|
||||
Logger.info('Bulk expires in ', expirationInSeconds, 'seconds');
|
||||
|
||||
// save the item in the cache with it's expected expiration
|
||||
@@ -138,6 +132,7 @@ export class BulkStorage {
|
||||
|
||||
/**
|
||||
* Create and save a new bulk process
|
||||
*
|
||||
* @param bulkRequest a request for a new bulk process
|
||||
* @returns a promise that contains the new bulk process
|
||||
*/
|
||||
@@ -156,6 +151,7 @@ export class BulkStorage {
|
||||
|
||||
/**
|
||||
* Delete a bulk process
|
||||
*
|
||||
* @param uid uid of the bulk process
|
||||
* @returns a promise that contains the deleted bulk process
|
||||
*/
|
||||
@@ -163,7 +159,7 @@ export class BulkStorage {
|
||||
const bulk = this.read(uid);
|
||||
|
||||
if (typeof bulk === 'undefined') {
|
||||
throw new Error(`Bulk that should be deleted was not found. UID was "${uid}"`);
|
||||
throw new TypeError(`Bulk that should be deleted was not found. UID was "${uid}"`);
|
||||
}
|
||||
|
||||
// delete the bulk process from the cache
|
||||
@@ -177,6 +173,7 @@ export class BulkStorage {
|
||||
|
||||
/**
|
||||
* Update an old bulk process (replace it with the new one)
|
||||
*
|
||||
* @param bulk new bulk process
|
||||
* @returns an empty promise
|
||||
*/
|
||||
@@ -192,11 +189,11 @@ export class BulkStorage {
|
||||
|
||||
/**
|
||||
* Read an existing bulk process
|
||||
*
|
||||
* @param uid uid of the bulk process
|
||||
* @returns a promise that contains a bulk
|
||||
*/
|
||||
public read(uid: string): Bulk | undefined {
|
||||
return this.cache.get(uid);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -26,11 +26,11 @@ export type DatabaseConstructor = new (config: SCConfigFile, mailQueue?: MailQue
|
||||
* Defines what one database class needs to have defined
|
||||
*/
|
||||
export interface Database {
|
||||
|
||||
/**
|
||||
* Gets called if a bulk was created
|
||||
*
|
||||
* The database should
|
||||
*
|
||||
* @param bulk A bulk to be created
|
||||
*/
|
||||
bulkCreated(bulk: Bulk): Promise<void>;
|
||||
@@ -39,6 +39,7 @@ export interface Database {
|
||||
* Gets called if a bulk expires
|
||||
*
|
||||
* The database should delete all data that is associtated with this bulk
|
||||
*
|
||||
* @param bulk A bulk which data needs to be removed
|
||||
*/
|
||||
bulkExpired(bulk: Bulk): Promise<void>;
|
||||
@@ -55,6 +56,7 @@ export interface Database {
|
||||
|
||||
/**
|
||||
* Get a single document
|
||||
*
|
||||
* @param uid Unique identifier of the document
|
||||
*/
|
||||
get(uid: SCUuid): Promise<SCThings>;
|
||||
@@ -66,6 +68,7 @@ export interface Database {
|
||||
|
||||
/**
|
||||
* Add a thing to an existing bulk
|
||||
*
|
||||
* @param thing A StAppsCore thing to be added
|
||||
* @param bulk A bulk to which the thing should be added
|
||||
*/
|
||||
@@ -82,7 +85,8 @@ export interface Database {
|
||||
|
||||
/**
|
||||
* Search for things
|
||||
*
|
||||
* @param params Parameters which form a search query to search the backend data
|
||||
*/
|
||||
search(params: SCSearchQuery): Promise<SCSearchResponse>;
|
||||
search(parameters: SCSearchQuery): Promise<SCSearchResponse>;
|
||||
}
|
||||
|
||||
@@ -26,10 +26,10 @@ import {
|
||||
|
||||
/**
|
||||
* Parses elasticsearch aggregations (response from es) to facets for the app
|
||||
*
|
||||
* @param aggregationResponse - aggregations response from elasticsearch
|
||||
*/
|
||||
export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
|
||||
|
||||
const facets: SCFacet[] = [];
|
||||
|
||||
// get all names of the types an aggregation is on
|
||||
@@ -52,7 +52,7 @@ export function parseAggregations(aggregationResponse: AggregationResponse): SCF
|
||||
// this should always be true in theory...
|
||||
if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) {
|
||||
const facet: SCFacet = {
|
||||
buckets: realField.buckets.map((bucket) => {
|
||||
buckets: realField.buckets.map(bucket => {
|
||||
return {
|
||||
count: bucket.doc_count,
|
||||
key: bucket.key,
|
||||
@@ -71,7 +71,7 @@ export function parseAggregations(aggregationResponse: AggregationResponse): SCF
|
||||
// the last part here means that it is a bucket aggregation
|
||||
} else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) {
|
||||
facets.push({
|
||||
buckets: realType.buckets.map((bucket) => {
|
||||
buckets: realType.buckets.map(bucket => {
|
||||
return {
|
||||
count: bucket.doc_count,
|
||||
key: bucket.key,
|
||||
|
||||
@@ -27,7 +27,6 @@ import {
|
||||
import {Logger} from '@openstapps/logger';
|
||||
// we only have the @types package because some things type definitions are still missing from the official
|
||||
// @elastic/elasticsearch package
|
||||
// tslint:disable-next-line:no-implicit-dependencies
|
||||
import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
|
||||
import moment from 'moment';
|
||||
import {MailQueue} from '../../notification/mail-queue';
|
||||
@@ -39,7 +38,8 @@ import {buildQuery, buildSort} from './query';
|
||||
import {aggregations, putTemplate} from './templating';
|
||||
import {
|
||||
AggregationResponse,
|
||||
ElasticsearchConfig, ElasticsearchObject,
|
||||
ElasticsearchConfig,
|
||||
ElasticsearchObject,
|
||||
ElasticsearchQueryDisMaxConfig,
|
||||
ElasticsearchQueryQueryStringConfig,
|
||||
} from './types/elasticsearch';
|
||||
@@ -53,7 +53,6 @@ const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
|
||||
* A database interface for elasticsearch
|
||||
*/
|
||||
export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Length of the index UID used for generation of its name
|
||||
*/
|
||||
@@ -90,7 +89,7 @@ export class Elasticsearch implements Database {
|
||||
*/
|
||||
static getElasticsearchUrl(): string {
|
||||
// check if we have a docker link
|
||||
if (process.env.ES_ADDR !== undefined ) {
|
||||
if (process.env.ES_ADDR !== undefined) {
|
||||
return process.env.ES_ADDR;
|
||||
}
|
||||
|
||||
@@ -100,6 +99,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Gets the index name in elasticsearch for one SCThingType
|
||||
*
|
||||
* @param type SCThingType of data in the index
|
||||
* @param source source of data in the index
|
||||
* @param bulk bulk process which created this index
|
||||
@@ -115,10 +115,11 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Provides the index UID (for its name) from the bulk UID
|
||||
*
|
||||
* @param uid Bulk UID
|
||||
*/
|
||||
static getIndexUID(uid: SCUuid) {
|
||||
return uid.substring(0, Elasticsearch.INDEX_UID_LENGTH);
|
||||
return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -131,6 +132,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Checks for invalid character in alias names and removes them
|
||||
*
|
||||
* @param alias The alias name
|
||||
* @param uid The UID of the current bulk (for debugging purposes)
|
||||
*/
|
||||
@@ -140,26 +142,25 @@ export class Elasticsearch implements Database {
|
||||
// spaces are included in some types, replace them with underscores
|
||||
if (formattedAlias.includes(' ')) {
|
||||
formattedAlias = formattedAlias.trim();
|
||||
formattedAlias = formattedAlias.split(' ')
|
||||
.join('_');
|
||||
formattedAlias = formattedAlias.split(' ').join('_');
|
||||
}
|
||||
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
|
||||
['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => {
|
||||
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
|
||||
if (formattedAlias.includes(value)) {
|
||||
formattedAlias = formattedAlias.replace(value, '');
|
||||
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
|
||||
having the same alias despite having different types, as invalid characters are removed automatically.
|
||||
New alias name is "${formattedAlias}."`);
|
||||
}
|
||||
});
|
||||
['-', '_', '+'].forEach((value) => {
|
||||
}
|
||||
for (const value of ['-', '_', '+']) {
|
||||
if (formattedAlias.charAt(0) === value) {
|
||||
formattedAlias = formattedAlias.substring(1);
|
||||
formattedAlias = formattedAlias.slice(1);
|
||||
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
|
||||
alias despite having different types, as invalid characters are removed automatically.
|
||||
New alias name is "${formattedAlias}."`);
|
||||
}
|
||||
});
|
||||
}
|
||||
if (formattedAlias === '.' || formattedAlias === '..') {
|
||||
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
|
||||
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
|
||||
@@ -176,22 +177,24 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Create a new interface for elasticsearch
|
||||
*
|
||||
* @param config an assembled config file
|
||||
* @param mailQueue a mailqueue for monitoring
|
||||
*/
|
||||
constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) {
|
||||
|
||||
if (typeof config.internal.database === 'undefined'
|
||||
|| typeof config.internal.database.version !== 'string') {
|
||||
throw new Error('Database version is undefined. Check your config file');
|
||||
if (
|
||||
typeof config.internal.database === 'undefined' ||
|
||||
typeof config.internal.database.version !== 'string'
|
||||
) {
|
||||
throw new TypeError('Database version is undefined. Check your config file');
|
||||
}
|
||||
|
||||
this.client = new Client({
|
||||
node: Elasticsearch.getElasticsearchUrl(),
|
||||
});
|
||||
this.client.on(events.REQUEST, async (err: Error | null, result: ApiResponse<unknown>) => {
|
||||
if (err !== null) {
|
||||
await Logger.error(err);
|
||||
this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse<unknown>) => {
|
||||
if (error !== null) {
|
||||
await Logger.error(error);
|
||||
}
|
||||
if (process.env.ES_DEBUG === 'true') {
|
||||
Logger.log(result);
|
||||
@@ -215,18 +218,20 @@ export class Elasticsearch implements Database {
|
||||
// create a list of old indices that are not in use
|
||||
const oldIndicesToDelete: string[] = [];
|
||||
|
||||
let aliases: {
|
||||
[index: string]: {
|
||||
/**
|
||||
* Aliases of an index
|
||||
*/
|
||||
aliases: {
|
||||
[K in SCThingType]: unknown
|
||||
};
|
||||
};
|
||||
} | undefined;
|
||||
let aliases:
|
||||
| {
|
||||
[index: string]: {
|
||||
/**
|
||||
* Aliases of an index
|
||||
*/
|
||||
aliases: {
|
||||
[K in SCThingType]: unknown;
|
||||
};
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
|
||||
for(const retry of [...Array(RETRY_COUNT)].map((_, i) => i+1)) {
|
||||
for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) {
|
||||
if (typeof aliases !== 'undefined') {
|
||||
break;
|
||||
}
|
||||
@@ -241,16 +246,14 @@ export class Elasticsearch implements Database {
|
||||
}
|
||||
|
||||
if (typeof aliases === 'undefined') {
|
||||
throw Error(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
|
||||
throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
|
||||
}
|
||||
|
||||
for (const index in aliases) {
|
||||
if (aliases.hasOwnProperty(index)) {
|
||||
|
||||
const matches = indexRegex.exec(index);
|
||||
if (matches !== null) {
|
||||
const type = matches[1];
|
||||
// tslint:disable-next-line: no-magic-numbers
|
||||
const source = matches[2];
|
||||
|
||||
// check if there is an alias for the current index
|
||||
@@ -278,12 +281,13 @@ export class Elasticsearch implements Database {
|
||||
Logger.warn(`Deleted old indices: oldIndicesToDelete`);
|
||||
}
|
||||
|
||||
// tslint:disable-next-line: no-magic-numbers
|
||||
// eslint-disable-next-line unicorn/no-null
|
||||
Logger.ok(`Read alias map from elasticsearch: ${JSON.stringify(this.aliasMap, null, 2)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides an elasticsearch object using containing thing's UID
|
||||
*
|
||||
* @param uid an UID to use for the search
|
||||
* @returns an elasticsearch object containing the thing
|
||||
*/
|
||||
@@ -309,6 +313,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
|
||||
*
|
||||
* @param bulk the bulk process that was created
|
||||
*/
|
||||
public async bulkCreated(bulk: Bulk): Promise<void> {
|
||||
@@ -346,6 +351,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Should be called when a bulk process is expired. The index that was created with this bulk gets deleted
|
||||
*
|
||||
* @param bulk the bulk process that is expired
|
||||
*/
|
||||
public async bulkExpired(bulk: Bulk): Promise<void> {
|
||||
@@ -365,6 +371,7 @@ export class Elasticsearch implements Database {
|
||||
/**
|
||||
* Should be called when a bulk process is updated (replaced by a newer bulk). This will replace the old
|
||||
* index and publish all data, that was index in the new instead
|
||||
*
|
||||
* @param bulk the new bulk process that should replace the old one with same type and source
|
||||
*/
|
||||
public async bulkUpdated(bulk: Bulk): Promise<void> {
|
||||
@@ -391,6 +398,7 @@ export class Elasticsearch implements Database {
|
||||
}
|
||||
|
||||
// create the new index if it does not exists
|
||||
// eslint-disable-next-line unicorn/no-await-expression-member
|
||||
if (!(await this.client.indices.exists({index})).body) {
|
||||
// re-apply the index template before each new bulk operation
|
||||
await putTemplate(this.client, bulk.type);
|
||||
@@ -411,6 +419,7 @@ export class Elasticsearch implements Database {
|
||||
];
|
||||
|
||||
// remove our old index if it exists
|
||||
// noinspection SuspiciousTypeOfGuard
|
||||
if (typeof oldIndex === 'string') {
|
||||
actions.push({
|
||||
remove: {index: oldIndex, alias: alias},
|
||||
@@ -431,6 +440,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
// swap the index in our aliasMap
|
||||
this.aliasMap[alias][bulk.source] = index;
|
||||
// noinspection SuspiciousTypeOfGuard
|
||||
if (typeof oldIndex === 'string') {
|
||||
// delete the old index
|
||||
await this.client.indices.delete({index: oldIndex});
|
||||
@@ -441,13 +451,14 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Gets an SCThing from all indexed data
|
||||
*
|
||||
* @param uid uid of an SCThing
|
||||
*/
|
||||
public async get(uid: SCUuid): Promise<SCThings> {
|
||||
const object = await this.getObject(uid);
|
||||
|
||||
if (typeof object === 'undefined') {
|
||||
throw new Error('Item not found.');
|
||||
throw new TypeError('Item not found.');
|
||||
}
|
||||
|
||||
return object._source;
|
||||
@@ -461,7 +472,9 @@ export class Elasticsearch implements Database {
|
||||
|
||||
if (typeof monitoringConfiguration !== 'undefined') {
|
||||
if (typeof this.mailQueue === 'undefined') {
|
||||
throw new Error('Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.');
|
||||
throw new TypeError(
|
||||
'Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.',
|
||||
);
|
||||
}
|
||||
// read all watches and schedule searches on the client
|
||||
await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
|
||||
@@ -472,56 +485,55 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Add an item to an index
|
||||
*
|
||||
* @param object the SCThing to add to the index
|
||||
* @param bulk the bulk process which item belongs to
|
||||
*/
|
||||
public async post(object: SCThings, bulk: Bulk): Promise<void> {
|
||||
|
||||
// tslint:disable-next-line: completed-docs
|
||||
const obj: SCThings & { creation_date: string; } = {
|
||||
const object_: SCThings & {creation_date: string} = {
|
||||
...object,
|
||||
creation_date: moment()
|
||||
.format(),
|
||||
creation_date: moment().format(),
|
||||
};
|
||||
|
||||
const item = await this.getObject(object.uid);
|
||||
|
||||
// check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid)
|
||||
if (typeof item !== 'undefined') {
|
||||
const indexOfNew = Elasticsearch.getIndex(obj.type, bulk.source, bulk);
|
||||
const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk);
|
||||
const oldIndex = item._index;
|
||||
|
||||
// new item doesn't replace the old one
|
||||
// tslint:disable-next-line:no-magic-numbers
|
||||
if (oldIndex.substring(0, oldIndex.lastIndexOf('_'))
|
||||
!== indexOfNew.substring(0, indexOfNew.lastIndexOf('_'))) {
|
||||
if (
|
||||
oldIndex.slice(0, Math.max(0, oldIndex.lastIndexOf('_'))) !==
|
||||
indexOfNew.slice(0, Math.max(0, indexOfNew.lastIndexOf('_')))
|
||||
) {
|
||||
throw new Error(
|
||||
// tslint:disable-next-line: no-magic-numbers
|
||||
`Object "${obj.uid}" already exists. Object was: ${JSON.stringify(obj, null, 2)}`,
|
||||
// eslint-disable-next-line unicorn/no-null
|
||||
`Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// regular bulk update (item gets replaced when bulk is updated)
|
||||
const searchResponse = await this.client.create({
|
||||
body: obj,
|
||||
id: obj.uid,
|
||||
index: Elasticsearch.getIndex(obj.type, bulk.source, bulk),
|
||||
body: object_,
|
||||
id: object_.uid,
|
||||
index: Elasticsearch.getIndex(object_.type, bulk.source, bulk),
|
||||
timeout: '90s',
|
||||
type: obj.type,
|
||||
type: object_.type,
|
||||
});
|
||||
|
||||
if (!searchResponse.body.created) {
|
||||
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(obj)}`);
|
||||
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Put (update) an existing item
|
||||
*
|
||||
* @param object SCThing to put
|
||||
*/
|
||||
public async put(object: SCThings): Promise<void> {
|
||||
|
||||
const item = await this.getObject(object.uid);
|
||||
|
||||
if (typeof item !== 'undefined') {
|
||||
@@ -542,12 +554,12 @@ export class Elasticsearch implements Database {
|
||||
|
||||
/**
|
||||
* Search all indexed data
|
||||
* @param params search query
|
||||
*
|
||||
* @param parameters search query
|
||||
*/
|
||||
public async search(params: SCSearchQuery): Promise<SCSearchResponse> {
|
||||
|
||||
public async search(parameters: SCSearchQuery): Promise<SCSearchResponse> {
|
||||
if (typeof this.config.internal.database === 'undefined') {
|
||||
throw new Error('Database is undefined. You have to configure the query build');
|
||||
throw new TypeError('Database is undefined. You have to configure the query build');
|
||||
}
|
||||
|
||||
// create elasticsearch configuration out of data from database configuration
|
||||
@@ -557,23 +569,23 @@ export class Elasticsearch implements Database {
|
||||
};
|
||||
|
||||
if (typeof this.config.internal.database.query !== 'undefined') {
|
||||
esConfig.query =
|
||||
this.config.internal.database
|
||||
.query as ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
|
||||
esConfig.query = this.config.internal.database.query as
|
||||
| ElasticsearchQueryDisMaxConfig
|
||||
| ElasticsearchQueryQueryStringConfig;
|
||||
}
|
||||
|
||||
const searchRequest: RequestParams.Search = {
|
||||
body: {
|
||||
aggs: aggregations,
|
||||
query: buildQuery(params, this.config, esConfig),
|
||||
query: buildQuery(parameters, this.config, esConfig),
|
||||
},
|
||||
from: params.from,
|
||||
from: parameters.from,
|
||||
index: Elasticsearch.getListOfAllIndices(),
|
||||
size: params.size,
|
||||
size: parameters.size,
|
||||
};
|
||||
|
||||
if (typeof params.sort !== 'undefined') {
|
||||
searchRequest.body.sort = buildSort(params.sort);
|
||||
if (typeof parameters.sort !== 'undefined') {
|
||||
searchRequest.body.sort = buildSort(parameters.sort);
|
||||
}
|
||||
|
||||
// perform the search against elasticsearch
|
||||
@@ -582,7 +594,7 @@ export class Elasticsearch implements Database {
|
||||
// gather pagination information
|
||||
const pagination = {
|
||||
count: response.body.hits.hits.length,
|
||||
offset: (typeof params.from === 'number') ? params.from : 0,
|
||||
offset: typeof parameters.from === 'number' ? parameters.from : 0,
|
||||
total: response.body.hits.total,
|
||||
};
|
||||
|
||||
@@ -593,7 +605,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
// we only directly return the _source documents
|
||||
// elasticsearch provides much more information, the user shouldn't see
|
||||
const data = response.body.hits.hits.map((hit) => {
|
||||
const data = response.body.hits.hits.map(hit => {
|
||||
return hit._source; // SCThing
|
||||
});
|
||||
|
||||
|
||||
@@ -25,13 +25,13 @@ import {
|
||||
import {Logger} from '@openstapps/logger';
|
||||
// we only have the @types package because some things type definitions are still missing from the official
|
||||
// @elastic/elasticsearch package
|
||||
// tslint:disable-next-line:no-implicit-dependencies
|
||||
import {SearchResponse} from 'elasticsearch';
|
||||
import cron from 'node-cron';
|
||||
import {MailQueue} from '../../notification/mail-queue';
|
||||
|
||||
/**
|
||||
* Check if the given condition fails on the given number of results and the condition
|
||||
*
|
||||
* @param condition condition
|
||||
* @param total number of results
|
||||
*/
|
||||
@@ -48,6 +48,7 @@ function conditionFails(
|
||||
|
||||
/**
|
||||
* Check if the min condition fails
|
||||
*
|
||||
* @param minimumLength Minimal length allowed
|
||||
* @param total Number of results
|
||||
*/
|
||||
@@ -57,6 +58,7 @@ function minConditionFails(minimumLength: number, total: number) {
|
||||
|
||||
/**
|
||||
* Check if the max condition fails
|
||||
*
|
||||
* @param maximumLength Maximal length allowed
|
||||
* @param total Number of results
|
||||
*/
|
||||
@@ -66,6 +68,7 @@ function maxConditionFails(maximumLength: number, total: number) {
|
||||
|
||||
/**
|
||||
* Run all the given actions
|
||||
*
|
||||
* @param actions actions to perform
|
||||
* @param watcherName name of watcher that wants to perform them
|
||||
* @param triggerName name of trigger that triggered the watcher
|
||||
@@ -79,38 +82,39 @@ function runActions(
|
||||
total: number,
|
||||
mailQueue: MailQueue,
|
||||
) {
|
||||
|
||||
actions.forEach(async (action) => {
|
||||
if (action.type === 'log') {
|
||||
await Logger.error(
|
||||
action.prefix,
|
||||
`Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`, `Found ${total} hits instead`,
|
||||
action.message,
|
||||
);
|
||||
} else {
|
||||
await mailQueue.push({
|
||||
subject: action.subject,
|
||||
text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'
|
||||
for (const action of actions) {
|
||||
void (action.type === 'log'
|
||||
? Logger.error(
|
||||
action.prefix,
|
||||
`Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`,
|
||||
`Found ${total} hits instead`,
|
||||
action.message,
|
||||
)
|
||||
: mailQueue.push({
|
||||
subject: action.subject,
|
||||
text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'
|
||||
${action.message} Found ${total} hits instead`,
|
||||
to: action.recipients,
|
||||
});
|
||||
}
|
||||
});
|
||||
to: action.recipients,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the triggers for the configured watchers
|
||||
*
|
||||
* @param monitoringConfig configuration of the monitoring
|
||||
* @param esClient elasticsearch client
|
||||
* @param mailQueue mailQueue for mail actions
|
||||
*/
|
||||
export async function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: Client, mailQueue: MailQueue) {
|
||||
|
||||
export async function setUp(
|
||||
monitoringConfig: SCMonitoringConfiguration,
|
||||
esClient: Client,
|
||||
mailQueue: MailQueue,
|
||||
) {
|
||||
// set up Watches
|
||||
monitoringConfig.watchers.forEach((watcher) => {
|
||||
|
||||
for (const watcher of monitoringConfig.watchers) {
|
||||
// make a schedule for each trigger
|
||||
watcher.triggers.forEach((trigger) => {
|
||||
for (const trigger of watcher.triggers) {
|
||||
switch (trigger.executionTime) {
|
||||
case 'hourly':
|
||||
trigger.executionTime = '5 * * * *';
|
||||
@@ -127,21 +131,21 @@ export async function setUp(monitoringConfig: SCMonitoringConfiguration, esClien
|
||||
|
||||
cron.schedule(trigger.executionTime, async () => {
|
||||
// execute watch (search->condition->action)
|
||||
const result: ApiResponse<SearchResponse<SCThings>> =
|
||||
await esClient.search(watcher.query as RequestParams.Search);
|
||||
const result: ApiResponse<SearchResponse<SCThings>> = await esClient.search(
|
||||
watcher.query as RequestParams.Search,
|
||||
);
|
||||
|
||||
// check conditions
|
||||
const total = result.body.hits.total;
|
||||
|
||||
watcher.conditions.forEach((condition) => {
|
||||
for (const condition of watcher.conditions) {
|
||||
if (conditionFails(condition, total)) {
|
||||
runActions(watcher.actions, watcher.name, trigger.name, total, mailQueue);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Logger.log(`Scheduled ${monitoringConfig.watchers.length} watches`);
|
||||
}
|
||||
|
||||
@@ -34,7 +34,8 @@ import {
|
||||
ESFunctionScoreQuery,
|
||||
ESFunctionScoreQueryFunction,
|
||||
ESGenericRange,
|
||||
ESGenericSort, ESGeoBoundingBoxFilter,
|
||||
ESGenericSort,
|
||||
ESGeoBoundingBoxFilter,
|
||||
ESGeoDistanceFilter,
|
||||
ESGeoDistanceFilterArguments,
|
||||
ESGeoDistanceSort,
|
||||
@@ -55,19 +56,19 @@ import {
|
||||
* It is possible to use all, with the exception of < and >, of them by escaping them with a \
|
||||
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
|
||||
*
|
||||
* @param str the string to escape the characters from
|
||||
* @param string_ the string to escape the characters from
|
||||
*/
|
||||
function escapeESReservedCharacters(str: string): string {
|
||||
return str.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
|
||||
function escapeESReservedCharacters(string_: string): string {
|
||||
return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a boolean filter. Returns an elasticsearch boolean filter
|
||||
*
|
||||
* @param booleanFilter a search boolean filter for the retrieval of the data
|
||||
* @returns elasticsearch boolean arguments object
|
||||
*/
|
||||
export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<unknown> {
|
||||
|
||||
const result: ESBooleanFilterArguments<unknown> = {
|
||||
minimum_should_match: 0,
|
||||
must: [],
|
||||
@@ -76,16 +77,16 @@ export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBool
|
||||
};
|
||||
|
||||
if (booleanFilter.arguments.operation === 'and') {
|
||||
result.must = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
|
||||
result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
|
||||
}
|
||||
|
||||
if (booleanFilter.arguments.operation === 'or') {
|
||||
result.should = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
|
||||
result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
|
||||
result.minimum_should_match = 1;
|
||||
}
|
||||
|
||||
if (booleanFilter.arguments.operation === 'not') {
|
||||
result.must_not = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
|
||||
result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
|
||||
}
|
||||
|
||||
return result;
|
||||
@@ -93,22 +94,31 @@ export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBool
|
||||
|
||||
/**
|
||||
* Converts Array of Filters to elasticsearch query-syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildFilter(filter: SCSearchFilter):
|
||||
ESTermFilter | ESGeoDistanceFilter | ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter> | ESGeoShapeFilter | ESBooleanFilter<unknown> | ESRangeFilter {
|
||||
|
||||
export function buildFilter(
|
||||
filter: SCSearchFilter,
|
||||
):
|
||||
| ESTermFilter
|
||||
| ESGeoDistanceFilter
|
||||
| ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter>
|
||||
| ESGeoShapeFilter
|
||||
| ESBooleanFilter<unknown>
|
||||
| ESRangeFilter {
|
||||
switch (filter.type) {
|
||||
case 'value':
|
||||
return Array.isArray(filter.arguments.value) ? {
|
||||
terms: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
} : {
|
||||
term: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
};
|
||||
return Array.isArray(filter.arguments.value)
|
||||
? {
|
||||
terms: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
}
|
||||
: {
|
||||
term: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
};
|
||||
case 'availability':
|
||||
const scope = filter.arguments.scope?.charAt(0) ?? 's';
|
||||
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
|
||||
@@ -185,8 +195,7 @@ export function buildFilter(filter: SCSearchFilter):
|
||||
/**
|
||||
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
|
||||
*/
|
||||
// tslint:disable-next-line:ban-ts-ignore
|
||||
// @ts-ignore unfortunately, typescript is stupid and won't allow me to map this to an actual type.
|
||||
// @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type.
|
||||
ignore_unmapped: true,
|
||||
[`${filter.arguments.field}.polygon`]: {
|
||||
shape: filter.arguments.shape,
|
||||
@@ -195,8 +204,10 @@ export function buildFilter(filter: SCSearchFilter):
|
||||
},
|
||||
};
|
||||
|
||||
if ((typeof filter.arguments.spatialRelation === 'undefined' || filter.arguments.spatialRelation === 'intersects')
|
||||
&& filter.arguments.shape.type === 'envelope'
|
||||
if (
|
||||
(typeof filter.arguments.spatialRelation === 'undefined' ||
|
||||
filter.arguments.spatialRelation === 'intersects') &&
|
||||
filter.arguments.shape.type === 'envelope'
|
||||
) {
|
||||
return {
|
||||
bool: {
|
||||
@@ -208,8 +219,6 @@ export function buildFilter(filter: SCSearchFilter):
|
||||
/**
|
||||
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
|
||||
*/
|
||||
// tslint:disable-next-line:ban-ts-ignore
|
||||
// @ts-ignore unfortunately, typescript is stupid and won't allow me to map this to an actual type.
|
||||
ignore_unmapped: true,
|
||||
[`${filter.arguments.field}.point.coordinates`]: {
|
||||
top_left: filter.arguments.shape.coordinates[0],
|
||||
@@ -228,6 +237,7 @@ export function buildFilter(filter: SCSearchFilter):
|
||||
|
||||
/**
|
||||
* Builds scoring functions from boosting config
|
||||
*
|
||||
* @param boostings Backend boosting configuration for contexts and types
|
||||
* @param context The context of the app from where the search was initiated
|
||||
*/
|
||||
@@ -235,14 +245,14 @@ function buildFunctions(
|
||||
boostings: SCBackendConfigurationSearchBoostingContext,
|
||||
context: SCSearchContext | undefined,
|
||||
): ESFunctionScoreQueryFunction[] {
|
||||
|
||||
// default context
|
||||
let functions: ESFunctionScoreQueryFunction[] =
|
||||
buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]);
|
||||
let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes(
|
||||
boostings['default' as SCSearchContext],
|
||||
);
|
||||
|
||||
if (typeof context !== 'undefined' && context !== 'default') {
|
||||
// specific context provided, extend default context with additional boosts
|
||||
functions = functions.concat(buildFunctionsForBoostingTypes(boostings[context]));
|
||||
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
|
||||
}
|
||||
|
||||
return functions;
|
||||
@@ -258,7 +268,7 @@ function buildFunctionsForBoostingTypes(
|
||||
): ESFunctionScoreQueryFunction[] {
|
||||
const functions: ESFunctionScoreQueryFunction[] = [];
|
||||
|
||||
boostingTypes.forEach((boostingForOneSCType) => {
|
||||
for (const boostingForOneSCType of boostingTypes) {
|
||||
const typeFilter: ESTypeFilter = {
|
||||
type: {
|
||||
value: boostingForOneSCType.type,
|
||||
@@ -271,7 +281,6 @@ function buildFunctionsForBoostingTypes(
|
||||
});
|
||||
|
||||
if (typeof boostingForOneSCType.fields !== 'undefined') {
|
||||
|
||||
const fields = boostingForOneSCType.fields;
|
||||
|
||||
for (const fieldName in boostingForOneSCType.fields) {
|
||||
@@ -291,10 +300,7 @@ function buildFunctionsForBoostingTypes(
|
||||
functions.push({
|
||||
filter: {
|
||||
bool: {
|
||||
must: [
|
||||
typeFilter,
|
||||
termFilter,
|
||||
],
|
||||
must: [typeFilter, termFilter],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
@@ -305,24 +311,24 @@ function buildFunctionsForBoostingTypes(
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return functions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds body for Elasticsearch requests
|
||||
* @param params Parameters for querying the backend
|
||||
*
|
||||
* @param parameters Parameters for querying the backend
|
||||
* @param defaultConfig Default configuration of the backend
|
||||
* @param elasticsearchConfig Elasticsearch configuration
|
||||
* @returns ElasticsearchQuery (body of a search-request)
|
||||
*/
|
||||
export function buildQuery(
|
||||
params: SCSearchQuery,
|
||||
parameters: SCSearchQuery,
|
||||
defaultConfig: SCConfigFile,
|
||||
elasticsearchConfig: ElasticsearchConfig,
|
||||
): ESFunctionScoreQuery {
|
||||
|
||||
// if config provides an minMatch parameter we use query_string instead of match query
|
||||
let query;
|
||||
if (typeof elasticsearchConfig.query === 'undefined') {
|
||||
@@ -331,7 +337,7 @@ export function buildQuery(
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: '90%',
|
||||
query: (typeof params.query !== 'string') ? '*' : escapeESReservedCharacters(params.query),
|
||||
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'query_string') {
|
||||
@@ -340,11 +346,11 @@ export function buildQuery(
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query: (typeof params.query !== 'string') ? '*' : escapeESReservedCharacters(params.query),
|
||||
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
|
||||
if (params.query !== '*') {
|
||||
if (parameters.query !== '*') {
|
||||
query = {
|
||||
dis_max: {
|
||||
boost: 1.2,
|
||||
@@ -355,7 +361,7 @@ export function buildQuery(
|
||||
boost: elasticsearchConfig.query.matchBoosting,
|
||||
cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
|
||||
fuzziness: elasticsearchConfig.query.fuzziness,
|
||||
query: (typeof params.query !== 'string') ? '*' : params.query,
|
||||
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -364,22 +370,24 @@ export function buildQuery(
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query: (typeof params.query !== 'string') ? '*' : escapeESReservedCharacters(params.query),
|
||||
query:
|
||||
typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
|
||||
},
|
||||
},
|
||||
],
|
||||
tie_breaker: elasticsearchConfig.query.tieBreaker,
|
||||
},
|
||||
|
||||
};
|
||||
}
|
||||
} else {
|
||||
throw new Error('Unsupported query type. Check your config file and reconfigure your elasticsearch query');
|
||||
throw new Error(
|
||||
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
|
||||
);
|
||||
}
|
||||
|
||||
const functionScoreQuery: ESFunctionScoreQuery = {
|
||||
function_score: {
|
||||
functions: buildFunctions(defaultConfig.internal.boostings, params.context),
|
||||
functions: buildFunctions(defaultConfig.internal.boostings, parameters.context),
|
||||
query: {
|
||||
bool: {
|
||||
minimum_should_match: 0, // if we have no should, nothing can match
|
||||
@@ -398,8 +406,8 @@ export function buildQuery(
|
||||
mustMatch.push(query);
|
||||
}
|
||||
|
||||
if (typeof params.filter !== 'undefined') {
|
||||
mustMatch.push(buildFilter(params.filter));
|
||||
if (typeof parameters.filter !== 'undefined') {
|
||||
mustMatch.push(buildFilter(parameters.filter));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -408,13 +416,12 @@ export function buildQuery(
|
||||
|
||||
/**
|
||||
* converts query to
|
||||
*
|
||||
* @param sorts Sorting rules to apply to the data that is being queried
|
||||
* @returns an array of sort queries
|
||||
*/
|
||||
export function buildSort(
|
||||
sorts: SCSearchSort[],
|
||||
): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
|
||||
return sorts.map((sort) => {
|
||||
export function buildSort(sorts: SCSearchSort[]): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
|
||||
return sorts.map(sort => {
|
||||
switch (sort.type) {
|
||||
case 'generic':
|
||||
const esGenericSort: ESGenericSort = {};
|
||||
@@ -427,26 +434,26 @@ export function buildSort(
|
||||
|
||||
return esDucetSort;
|
||||
case 'distance':
|
||||
const args: ESGeoDistanceSortArguments = {
|
||||
const arguments_: ESGeoDistanceSortArguments = {
|
||||
mode: 'avg',
|
||||
order: sort.order,
|
||||
unit: 'm',
|
||||
};
|
||||
|
||||
args[`${sort.arguments.field}.point.coordinates`] = {
|
||||
arguments_[`${sort.arguments.field}.point.coordinates`] = {
|
||||
lat: sort.arguments.position[1],
|
||||
lon: sort.arguments.position[0],
|
||||
};
|
||||
|
||||
return {
|
||||
_geo_distance: args,
|
||||
_geo_distance: arguments_,
|
||||
};
|
||||
case 'price':
|
||||
return {
|
||||
_script: {
|
||||
order: sort.order,
|
||||
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
|
||||
type: 'number' as 'number',
|
||||
type: 'number' as const,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -459,7 +466,10 @@ export function buildSort(
|
||||
* @param universityRole User group which consumes university services
|
||||
* @param field Field in which wanted offers with prices are located
|
||||
*/
|
||||
export function buildPriceSortScript(universityRole: keyof SCSportCoursePriceGroup, field: SCThingsField): string {
|
||||
export function buildPriceSortScript(
|
||||
universityRole: keyof SCSportCoursePriceGroup,
|
||||
field: SCThingsField,
|
||||
): string {
|
||||
return `
|
||||
// initialize the sort value with the maximum
|
||||
double price = Double.MAX_VALUE;
|
||||
|
||||
@@ -15,18 +15,19 @@
|
||||
*/
|
||||
import {Client} from '@elastic/elasticsearch';
|
||||
import {SCThingType} from '@openstapps/core';
|
||||
// tslint:disable-next-line:no-implicit-dependencies
|
||||
import {AggregationSchema} from '@openstapps/es-mapping-generator/src/types/aggregation';
|
||||
// tslint:disable-next-line:no-implicit-dependencies
|
||||
import {ElasticsearchTemplateCollection} from '@openstapps/es-mapping-generator/src/types/mapping';
|
||||
import {readFileSync} from 'fs';
|
||||
import {resolve} from 'path';
|
||||
import path from 'path';
|
||||
|
||||
const mappingsPath = resolve('node_modules', '@openstapps', 'core', 'lib','mappings');
|
||||
|
||||
export const mappings = JSON.parse(readFileSync(resolve(mappingsPath, 'mappings.json'), 'utf-8')) as ElasticsearchTemplateCollection;
|
||||
export const aggregations = JSON.parse(readFileSync(resolve(mappingsPath, 'aggregations.json'), 'utf-8')) as AggregationSchema;
|
||||
const mappingsPath = path.resolve('node_modules', '@openstapps', 'core', 'lib', 'mappings');
|
||||
|
||||
export const mappings = JSON.parse(
|
||||
readFileSync(path.resolve(mappingsPath, 'mappings.json'), 'utf8'),
|
||||
) as ElasticsearchTemplateCollection;
|
||||
export const aggregations = JSON.parse(
|
||||
readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
|
||||
) as AggregationSchema;
|
||||
|
||||
/**
|
||||
* Re-applies all interfaces for every type
|
||||
@@ -44,8 +45,8 @@ export async function refreshAllTemplates(client: Client) {
|
||||
*
|
||||
* This includes applying the mapping, settings
|
||||
*
|
||||
* @param type the SCThingType of which the template should be set
|
||||
* @param client An elasticsearch client to use
|
||||
* @param type the SCThingType of which the template should be set
|
||||
*/
|
||||
export async function putTemplate(client: Client, type: SCThingType) {
|
||||
const sanitizedType = `template_${type.replace(/\s/g, '_')}`;
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
*/
|
||||
import {SCThing, SCThingType} from '@openstapps/core';
|
||||
// we only have the @types package because some things type definitions are still missing from the official
|
||||
// tslint:disable-next-line:no-implicit-dependencies
|
||||
import {NameList} from 'elasticsearch';
|
||||
// tslint:disable-next-line:no-implicit-dependencies
|
||||
import {Polygon, Position} from 'geojson';
|
||||
|
||||
/**
|
||||
@@ -75,7 +73,6 @@ export interface NestedAggregation {
|
||||
[name: string]: BucketAggregation | number;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A configuration for using the Dis Max Query
|
||||
*
|
||||
@@ -90,30 +87,35 @@ export interface ElasticsearchQueryDisMaxConfig {
|
||||
|
||||
/**
|
||||
* The maximum allowed Levenshtein Edit Distance (or number of edits)
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
|
||||
*/
|
||||
fuzziness: number | string;
|
||||
|
||||
/**
|
||||
* Increase the importance (relevance score) of a field
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
|
||||
*/
|
||||
matchBoosting: number;
|
||||
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minMatch: string;
|
||||
|
||||
/**
|
||||
* Type of the query - in this case 'dis_max' which is a union of its subqueries
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
|
||||
*/
|
||||
queryType: 'dis_max';
|
||||
|
||||
/**
|
||||
* Changes behavior of default calculation of the score when multiple results match
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
|
||||
*/
|
||||
tieBreaker: number;
|
||||
@@ -128,12 +130,14 @@ export interface ElasticsearchQueryDisMaxConfig {
|
||||
export interface ElasticsearchQueryQueryStringConfig {
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minMatch: string;
|
||||
|
||||
/**
|
||||
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
|
||||
*/
|
||||
queryType: 'query_string';
|
||||
@@ -141,6 +145,7 @@ export interface ElasticsearchQueryQueryStringConfig {
|
||||
|
||||
/**
|
||||
* A hit in an elasticsearch search result
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
|
||||
*/
|
||||
export interface ElasticsearchObject<T extends SCThing> {
|
||||
@@ -166,6 +171,7 @@ export interface ElasticsearchObject<T extends SCThing> {
|
||||
|
||||
/**
|
||||
* The document's mapping type
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
|
||||
*/
|
||||
_type: string;
|
||||
@@ -177,22 +183,25 @@ export interface ElasticsearchObject<T extends SCThing> {
|
||||
|
||||
/**
|
||||
* Used to index the same field in different ways for different purposes
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
|
||||
*/
|
||||
fields?: NameList;
|
||||
|
||||
/**
|
||||
* Used to highlight search results on one or more fields
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
|
||||
*/
|
||||
// tslint:disable-next-line: no-any
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
highlight?: any;
|
||||
|
||||
/**
|
||||
* Used in when nested/children documents match the query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
|
||||
*/
|
||||
// tslint:disable-next-line: no-any
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
inner_hits?: any;
|
||||
|
||||
/**
|
||||
@@ -246,21 +255,23 @@ export interface ElasticsearchConfig {
|
||||
/**
|
||||
* An elasticsearch term filter
|
||||
*/
|
||||
export type ESTermFilter = {
|
||||
/**
|
||||
* Definition of a term to match
|
||||
*/
|
||||
term: {
|
||||
[fieldName: string]: string;
|
||||
};
|
||||
} | {
|
||||
/**
|
||||
* Definition of terms to match (or)
|
||||
*/
|
||||
terms: {
|
||||
[fieldName: string]: string[];
|
||||
};
|
||||
};
|
||||
export type ESTermFilter =
|
||||
| {
|
||||
/**
|
||||
* Definition of a term to match
|
||||
*/
|
||||
term: {
|
||||
[fieldName: string]: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* Definition of terms to match (or)
|
||||
*/
|
||||
terms: {
|
||||
[fieldName: string]: string[];
|
||||
};
|
||||
};
|
||||
|
||||
export interface ESGenericRange<T> {
|
||||
/**
|
||||
@@ -318,7 +329,6 @@ export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<n
|
||||
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
|
||||
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
|
||||
|
||||
|
||||
/**
|
||||
* An elasticsearch type filter
|
||||
*/
|
||||
@@ -343,17 +353,19 @@ export interface ESGeoDistanceFilterArguments {
|
||||
*/
|
||||
distance: string;
|
||||
|
||||
[fieldName: string]: {
|
||||
/**
|
||||
* Latitude
|
||||
*/
|
||||
lat: number;
|
||||
[fieldName: string]:
|
||||
| {
|
||||
/**
|
||||
* Latitude
|
||||
*/
|
||||
lat: number;
|
||||
|
||||
/**
|
||||
* Longitude
|
||||
*/
|
||||
lon: number;
|
||||
} | string;
|
||||
/**
|
||||
* Longitude
|
||||
*/
|
||||
lon: number;
|
||||
}
|
||||
| string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -386,11 +398,13 @@ export interface ESEnvelope {
|
||||
|
||||
/**
|
||||
* An Elasticsearch geo bounding box filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
|
||||
*/
|
||||
export interface ESGeoBoundingBoxFilter {
|
||||
/**
|
||||
* An Elasticsearch geo bounding box filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
|
||||
*/
|
||||
geo_bounding_box: {
|
||||
@@ -410,6 +424,7 @@ export interface ESGeoBoundingBoxFilter {
|
||||
|
||||
/**
|
||||
* An Elasticsearch geo shape filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html
|
||||
*/
|
||||
export interface ESGeoShapeFilter {
|
||||
@@ -432,11 +447,13 @@ export interface ESGeoShapeFilter {
|
||||
|
||||
/**
|
||||
* Filter arguments for an elasticsearch boolean filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
|
||||
*/
|
||||
export interface ESBooleanFilterArguments<T> {
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minimum_should_match?: number;
|
||||
@@ -469,6 +486,7 @@ export interface ESBooleanFilter<T> {
|
||||
|
||||
/**
|
||||
* An elasticsearch function score query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
|
||||
*/
|
||||
export interface ESFunctionScoreQuery {
|
||||
@@ -478,6 +496,7 @@ export interface ESFunctionScoreQuery {
|
||||
function_score: {
|
||||
/**
|
||||
* Functions that compute score for query results (documents)
|
||||
*
|
||||
* @see ESFunctionScoreQueryFunction
|
||||
*/
|
||||
functions: ESFunctionScoreQueryFunction[];
|
||||
@@ -535,17 +554,19 @@ export interface ESGeoDistanceSortArguments {
|
||||
*/
|
||||
unit: 'm';
|
||||
|
||||
[field: string]: {
|
||||
/**
|
||||
* Latitude
|
||||
*/
|
||||
lat: number;
|
||||
[field: string]:
|
||||
| {
|
||||
/**
|
||||
* Latitude
|
||||
*/
|
||||
lat: number;
|
||||
|
||||
/**
|
||||
* Longitude
|
||||
*/
|
||||
lon: number;
|
||||
} | string;
|
||||
/**
|
||||
* Longitude
|
||||
*/
|
||||
lon: number;
|
||||
}
|
||||
| string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -18,12 +18,12 @@ import {
|
||||
ESAggTypeFilter,
|
||||
ESNestedAggregation,
|
||||
ESTermsFilter,
|
||||
// tslint:disable-next-line:no-implicit-dependencies we're just using the types here
|
||||
} from '@openstapps/es-mapping-generator/src/types/aggregation';
|
||||
import {BucketAggregation, NestedAggregation} from './elasticsearch';
|
||||
|
||||
/**
|
||||
* Checks if the type is a BucketAggregation
|
||||
*
|
||||
* @param agg the type to check
|
||||
*/
|
||||
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
|
||||
@@ -32,6 +32,7 @@ export function isBucketAggregation(agg: BucketAggregation | number): agg is Buc
|
||||
|
||||
/**
|
||||
* Checks if the type is a NestedAggregation
|
||||
*
|
||||
* @param agg the type to check
|
||||
*/
|
||||
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
|
||||
@@ -40,6 +41,7 @@ export function isNestedAggregation(agg: BucketAggregation | NestedAggregation):
|
||||
|
||||
/**
|
||||
* Checks if the parameter is of type ESTermsFilter
|
||||
*
|
||||
* @param agg the value to check
|
||||
*/
|
||||
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
|
||||
@@ -48,6 +50,7 @@ export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg i
|
||||
|
||||
/**
|
||||
* Checks if the parameter is of type ESTermsFilter
|
||||
*
|
||||
* @param agg the value to check
|
||||
*/
|
||||
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
|
||||
@@ -59,6 +62,8 @@ export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation):
|
||||
*
|
||||
* @param filter the filter to narrow the type of
|
||||
*/
|
||||
export function isESAggMatchAllFilter(filter: ESAggTypeFilter | ESAggMatchAllFilter): filter is ESAggMatchAllFilter {
|
||||
export function isESAggMatchAllFilter(
|
||||
filter: ESAggTypeFilter | ESAggMatchAllFilter,
|
||||
): filter is ESAggMatchAllFilter {
|
||||
return filter.hasOwnProperty('match_all');
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user