Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 00:52:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Resolve "Transition to ESLint"

Browse Source
This commit is contained in:
Thea Schöbl
2022-06-27 14:40:09 +00:00
committed by Rainer Killinger
parent ca1d2444e0
commit 418ba67d15
47 changed files with 1854 additions and 1634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/app.ts
View File

@@ -24,7 +24,7 @@ import config from 'config';
import cors from 'cors';
import {Express} from 'express';
import morgan from 'morgan';
import {join} from 'path';
import path from 'path';
import {configFile, DEFAULT_TIMEOUT, isTestEnvironment, mailer, plugins, validator} from './common';
import {getPrometheusMiddleware} from './middleware/prometheus';
import {MailQueue} from './notification/mail-queue';
@@ -43,26 +43,26 @@ import {DatabaseConstructor} from './storage/database';
/**
* Configure the backend
*/
export async function configureApp(app: Express, databases: {[name: string]: DatabaseConstructor; }) {
export async function configureApp(app: Express, databases: {[name: string]: DatabaseConstructor}) {
let integrationTestTimeout: NodeJS.Timeout;
// request loggers have to be the first middleware to be set in express
app.use(morgan('dev', {
skip: (_req, res) => {
if (process.env.NODE_ENV === 'integration-test') {
clearTimeout(integrationTestTimeout);
integrationTestTimeout = setTimeout(() => {
process.exit(1);
},
DEFAULT_TIMEOUT);
app.use(
morgan('dev', {
skip: (_request, response) => {
if (process.env.NODE_ENV === 'integration-test') {
clearTimeout(integrationTestTimeout);
integrationTestTimeout = setTimeout(() => {
process.exit(1);
}, DEFAULT_TIMEOUT);
return false;
}
return false;
}
// tslint:disable-next-line: no-magic-numbers
return res.statusCode < 400;
}, stream: process.stdout,
}));
return response.statusCode < 400;
},
stream: process.stdout,
}),
);
if (process.env.PROMETHEUS_MIDDLEWARE === 'true') {
app.use(getPrometheusMiddleware());
@@ -80,7 +80,7 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
'X-StApps-Version',
],
credentials: true,
maxAge: 1728000,
maxAge: 1_728_000,
methods: ['GET', 'POST', 'PUT', 'OPTIONS'],
optionsSuccessStatus: 204,
};
@@ -93,13 +93,13 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
app.options('*', [cors(corsOptions)]);
// only accept json as content type for all requests
app.use((req, res, next) => {
app.use((request, response, next) => {
// Only accept json as content type
if (req.is('application/json') !== 'application/json') {
if (request.is('application/json') !== 'application/json') {
// return an error in the response
const err = new SCUnsupportedMediaTypeErrorResponse(isTestEnvironment);
res.status(err.statusCode);
res.json(err);
const error = new SCUnsupportedMediaTypeErrorResponse(isTestEnvironment);
response.status(error.statusCode);
response.json(error);
return;
}
@@ -111,12 +111,12 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
bodySize += chunk.byteLength;
// when adding each chunk size to the total size, check how large it now is.
if (bodySize > configFile.backend.maxRequestBodySize) {
req.off('data', chunkGatherer);
req.off('end', endCallback);
request.off('data', chunkGatherer);
request.off('end', endCallback);
// return an error in the response
const err = new SCRequestBodyTooLargeErrorResponse(isTestEnvironment);
res.status(err.statusCode);
res.json(err);
const error = new SCRequestBodyTooLargeErrorResponse(isTestEnvironment);
response.status(error.statusCode);
response.json(error);
return;
}
@@ -125,26 +125,24 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
};
const endCallback = () => {
req.body = Buffer.concat(bodyBuffer)
.toString();
request.body = Buffer.concat(bodyBuffer).toString();
try {
req.body = JSON.parse(req.body);
request.body = JSON.parse(request.body);
next();
} catch (catchErr) {
const err = new SCSyntaxErrorResponse(catchErr.message, isTestEnvironment);
res.status(err.statusCode);
res.json(err);
} catch (error) {
const error_ = new SCSyntaxErrorResponse(error.message, isTestEnvironment);
response.status(error_.statusCode);
response.json(error_);
return;
}
};
req.on('data', chunkGatherer)
.on('end', endCallback);
request.on('data', chunkGatherer).on('end', endCallback);
});
// validate config file
await validator.addSchemas(join('node_modules', '@openstapps', 'core', 'lib', 'schema'));
await validator.addSchemas(path.join('node_modules', '@openstapps', 'core', 'lib', 'schema'));
// validate the config file
const configValidation = validator.validate(configFile, 'SCConfigFile');
@@ -161,17 +159,16 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
throw new Error('You have to configure a database');
}
const database =
new databases[config.get<string>('internal.database.name')](
configFile,
// mailQueue
typeof mailer !== 'undefined' && config.has('internal.monitoring') ? new MailQueue(mailer) : undefined,
);
const database = new databases[config.get<string>('internal.database.name')](
configFile,
// mailQueue
typeof mailer !== 'undefined' && config.has('internal.monitoring') ? new MailQueue(mailer) : undefined,
);
await database.init();
if (typeof database === 'undefined') {
throw new Error('No implementation for configured database found. Please check your configuration.');
throw new TypeError('No implementation for configured database found. Please check your configuration.');
}
Logger.ok('Validated config file successfully');
@@ -181,10 +178,7 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
app.enable('strict routing');
// make the bulk storage available to all http middlewares/routes
app.set(
'bulk',
new BulkStorage(database),
);
app.set('bulk', new BulkStorage(database));
app.set('env', process.env.NODE_ENV);
@@ -202,15 +196,15 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
);
// for plugins, as Express doesn't really want you to unregister routes (and doesn't offer any method to do so at all)
app.all('*', async (req, res, next) => {
app.all('*', async (request, response, next) => {
// if the route exists then call virtual route on the plugin that registered that route
if (plugins.has(req.originalUrl)) {
if (plugins.has(request.originalUrl)) {
try {
res.json(await virtualPluginRoute(req, plugins.get(req.originalUrl)!));
} catch (e) {
response.json(await virtualPluginRoute(request, plugins.get(request.originalUrl)!));
} catch (error) {
// in case of error send an error response
res.status(e.statusCode);
res.json(e);
response.status(error.statusCode);
response.json(error);
}
} else {
// pass to the next matching route (which is 404)
@@ -219,9 +213,9 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
});
// add a route for a missing resource (404)
app.use((_req, res) => {
app.use((_request, response) => {
const errorResponse = new SCNotFoundErrorResponse(isTestEnvironment);
res.status(errorResponse.statusCode);
res.json(errorResponse);
response.status(errorResponse.statusCode);
response.json(errorResponse);
});
}

23
src/cli.ts
View File

@@ -24,7 +24,6 @@ const app = express();
/**
* Get port from environment and store in Express.
*/
// tslint:disable-next-line: strict-boolean-expressions
const port = normalizePort(process.env.PORT || '3000');
/**
@@ -42,9 +41,9 @@ server.on('listening', onListening);
* Normalize a port into a number, string, or false.
*/
function normalizePort(value: string) {
const portNumber = parseInt(value, 10);
const portNumber = Number.parseInt(value, 10);
if (isNaN(portNumber)) {
if (Number.isNaN(portNumber)) {
// named pipe
return value;
}
@@ -60,15 +59,12 @@ function normalizePort(value: string) {
/**
* Event listener for HTTP server "error" event.
*/
// tslint:disable-next-line: completed-docs
async function onError(error: { code: string; syscall: string; }) {
async function onError(error: {code: string; syscall: string}) {
if (error.syscall !== 'listen') {
throw error;
}
const bind = typeof port === 'string'
? `Pipe ${port}`
: `Port ${port}`;
const bind = typeof port === 'string' ? `Pipe ${port}` : `Port ${port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
@@ -92,13 +88,10 @@ function onListening() {
const addr = server.address();
if (addr !== null) {
const bind = typeof addr === 'string'
? `pipe ${addr}`
: `port ${addr.port}`;
const bind = typeof addr === 'string' ? `pipe ${addr}` : `port ${addr.port}`;
Logger.ok(`Listening on ${bind}`);
} else {
// tslint:disable-next-line: no-floating-promises
Logger.error(`Failed to start binding`);
void Logger.error(`Failed to start binding`);
}
}
@@ -108,6 +101,6 @@ configureApp(app, {elasticsearch: Elasticsearch})
// After app setup listen on provided port, on all network interfaces
server.listen(port);
})
.catch((err) => {
throw err;
.catch(error => {
throw error;
});

2
src/common.ts
View File

@@ -51,4 +51,4 @@ export const coreVersion: string = configFile.backend.SCVersion;
/**
* The default timeout in milliseconds
*/
export const DEFAULT_TIMEOUT = 20000;
export const DEFAULT_TIMEOUT = 20_000;

10
src/middleware/prometheus.ts
View File

@@ -24,9 +24,9 @@ type UserOptions = Parameters<typeof expressPrometheusMiddleware>[0];
* Create and configure a new Express Prometheus Middleware instance
*
* This function tries to configure the new instance with JSON read from
* `./conf/prometheus.json`. When this fails an instance configured with
* `./conf/prometheus.json`. When this fails an instance configured with
* default options is returned.
*
*
* @returns express.Express
*/
export function getPrometheusMiddleware(): express.Express {
@@ -34,9 +34,9 @@ export function getPrometheusMiddleware(): express.Express {
let options: UserOptions = {};
try {
options = JSON.parse(fs.readFileSync(configFileName, 'utf-8'));
} catch(err) {
Logger.warn('Could not get options for Prometheus Middleware.', err);
options = JSON.parse(fs.readFileSync(configFileName, 'utf8'));
} catch (error) {
Logger.warn('Could not get options for Prometheus Middleware.', error);
}
return expressPrometheusMiddleware(options);

4
src/notification/backend-transport.ts
View File

@@ -103,8 +103,8 @@ export class BackendTransport {
if (successful) {
Logger.log('SMTP verification successful.');
}
} catch (err) {
throw err;
} catch (error) {
throw error;
} finally {
this.waitingForVerification = false;
}

13
src/notification/mail-queue.ts
View File

@@ -22,7 +22,6 @@ import Queue from 'promise-queue';
* A queue that can send mails in serial
*/
export class MailQueue {
/**
* Number of allowed verification attempts after which the initialization of transport fails
*/
@@ -52,10 +51,10 @@ export class MailQueue {
/**
* Creates a mail queue
*
* @param transport Transport which is used for sending mails
*/
constructor(private readonly transport: SMTP) {
this.queue = new Queue(1);
// this queue saves all request when the transport is not ready yet
@@ -80,7 +79,6 @@ export class MailQueue {
* Verify the given transport
*/
private checkForVerification() {
if (this.verificationCounter >= MailQueue.MAX_VERIFICATION_ATTEMPTS) {
throw new Error('Failed to initialize the SMTP transport for the mail queue');
}
@@ -94,9 +92,9 @@ export class MailQueue {
} else {
Logger.ok('Transport for mail queue was verified. We can send mails now');
// if the transport finally was verified send all our mails from the dry queue
this.dryQueue.forEach(async (mail) => {
await this.addToQueue(mail);
});
for (const mail of this.dryQueue) {
void this.addToQueue(mail);
}
}
}
@@ -106,7 +104,8 @@ export class MailQueue {
* @param mail Information required for sending a mail
*/
public async push(mail: MailOptions) {
if (!this.transport.isVerified()) { // the transport has verification, but is not verified yet
if (!this.transport.isVerified()) {
// the transport has verification, but is not verified yet
// push to a dry queue which gets pushed to the real queue when the transport is verified
this.dryQueue.push(mail);
} else {

7
src/routes/bulk-add-route.ts
View File

@@ -29,13 +29,12 @@ const bulkRouteModel = new SCBulkAddRoute();
*/
export const bulkAddRouter = createRoute<SCBulkAddRequest, SCBulkAddResponse>(
bulkRouteModel,
async (request, app, params) => {
async (request, app, parameters) => {
const bulkMemory: BulkStorage = app.get('bulk');
const bulk = bulkMemory.read(params.UID);
const bulk = bulkMemory.read(parameters.UID);
if (typeof bulk === 'undefined') {
Logger.warn(`Bulk with ${params.UID} not found.`);
Logger.warn(`Bulk with ${parameters.UID} not found.`);
throw new SCNotFoundErrorResponse(isTestEnvironment);
}

14
src/routes/bulk-done-route.ts
View File

@@ -13,7 +13,12 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCBulkDoneRequest, SCBulkDoneResponse, SCBulkDoneRoute, SCNotFoundErrorResponse} from '@openstapps/core';
import {
SCBulkDoneRequest,
SCBulkDoneResponse,
SCBulkDoneRoute,
SCNotFoundErrorResponse,
} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
import {isTestEnvironment} from '../common';
import {BulkStorage} from '../storage/bulk-storage';
@@ -29,13 +34,12 @@ const bulkDoneRouteModel = new SCBulkDoneRoute();
*/
export const bulkDoneRouter = createRoute<SCBulkDoneRequest, SCBulkDoneResponse>(
bulkDoneRouteModel,
async (_request, app, params) => {
async (_request, app, parameters) => {
const bulkMemory: BulkStorage = app.get('bulk');
const bulk = bulkMemory.read(params.UID);
const bulk = bulkMemory.read(parameters.UID);
if (typeof bulk === 'undefined') {
Logger.warn(`Bulk with ${params.UID} not found.`);
Logger.warn(`Bulk with ${parameters.UID} not found.`);
throw new SCNotFoundErrorResponse(isTestEnvironment);
}

11
src/routes/bulk-route.ts
View File

@@ -25,11 +25,8 @@ const bulkRouteModel = new SCBulkRoute();
/**
* Implementation of the bulk request route (SCBulkRoute)
*/
export const bulkRouter = createRoute<SCBulkRequest, SCBulkResponse>(
bulkRouteModel,
async (request, app) => {
const bulkMemory: BulkStorage = app.get('bulk');
export const bulkRouter = createRoute<SCBulkRequest, SCBulkResponse>(bulkRouteModel, async (request, app) => {
const bulkMemory: BulkStorage = app.get('bulk');
return bulkMemory.create(request);
},
);
return bulkMemory.create(request);
});

31
src/routes/http-types.ts
View File

@@ -14,10 +14,31 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
// the list provides option to easily implement "isHttpMethod" guard
const httpVerbs = ['get', 'post', 'put', 'delete', 'patch', 'options',
'head', 'checkout', 'copy', 'lock', 'merge', 'mkactivity', 'mkcol',
'move', 'm-search', 'notify', 'purge', 'report', 'search', 'subscribe',
'trace', 'unlock','unsubscribe'] as const;
const httpVerbs = [
'get',
'post',
'put',
'delete',
'patch',
'options',
'head',
'checkout',
'copy',
'lock',
'merge',
'mkactivity',
'mkcol',
'move',
'm-search',
'notify',
'purge',
'report',
'search',
'subscribe',
'trace',
'unlock',
'unsubscribe',
] as const;
/**
* Strings that can be used as HTTP verbs (e.g. in requests): 'get' | 'post' | 'put' | 'delete' etc.
*/
@@ -29,5 +50,5 @@ export type HTTPVerb = typeof httpVerbs[number];
* @param method A text (representing a method) to check
*/
export function isHttpMethod(method: string): method is HTTPVerb {
return (httpVerbs as unknown as string[]).indexOf(method) > -1;
return (httpVerbs as unknown as string[]).includes(method);
}

42
src/routes/multi-search-route.ts
View File

@@ -31,30 +31,28 @@ const multiSearchRouteModel = new SCMultiSearchRoute();
/**
* Implementation of the multi search route (SCMultiSearchRoute)
*/
export const multiSearchRouter = createRoute
<SCMultiSearchRequest, SCMultiSearchResponse | SCTooManyRequestsErrorResponse>(
multiSearchRouteModel,
async (request, app) => {
export const multiSearchRouter = createRoute<
SCMultiSearchRequest,
SCMultiSearchResponse | SCTooManyRequestsErrorResponse
>(multiSearchRouteModel, async (request, app) => {
const bulkMemory: BulkStorage = app.get('bulk');
const queryNames = Object.keys(request);
const bulkMemory: BulkStorage = app.get('bulk');
const queryNames = Object.keys(request);
if (queryNames.length > configFile.backend.maxMultiSearchRouteQueries) {
throw new SCTooManyRequestsErrorResponse(isTestEnvironment);
}
if (queryNames.length > configFile.backend.maxMultiSearchRouteQueries) {
throw new SCTooManyRequestsErrorResponse(isTestEnvironment);
}
// get a map of promises for each query
const searchRequests = queryNames.map(async queryName => {
return bulkMemory.database.search(request[queryName]);
});
// get a map of promises for each query
const searchRequests = queryNames.map(async (queryName) => {
return bulkMemory.database.search(request[queryName]);
});
const listOfSearchResponses = await Promise.all(searchRequests);
const listOfSearchResponses = await Promise.all(searchRequests);
const response: {[queryName: string]: SCSearchResponse} = {};
for (const [index, queryName] of queryNames.entries()) {
response[queryName] = listOfSearchResponses[index];
}
const response: { [queryName: string]: SCSearchResponse; } = {};
queryNames.forEach((queryName, index) => {
response[queryName] = listOfSearchResponses[index];
});
return response;
},
);
return response;
});

21
src/routes/plugin-register-route.ts
View File

@@ -34,8 +34,7 @@ const pluginRegisterRouteModel = new SCPluginRegisterRoute();
/**
* Implementation of the plugin registration route (SCPluginRegisterRoute)
*/
export const pluginRegisterRouter = createRoute(
pluginRegisterRouteModel, pluginRegisterHandler);
export const pluginRegisterRouter = createRoute(pluginRegisterRouteModel, pluginRegisterHandler);
/**
* Handles requests on route for registering plugins
@@ -43,8 +42,10 @@ export const pluginRegisterRouter = createRoute(
* @param request Request received for registering or unregistering a plugin
* @param _app Express application
*/
export async function pluginRegisterHandler(request: SCPluginRegisterRequest, _app: Express.Application):
Promise<SCPluginRegisterResponse> {
export async function pluginRegisterHandler(
request: SCPluginRegisterRequest,
_app: Express.Application,
): Promise<SCPluginRegisterResponse> {
switch (request.action) {
case 'add':
return addPlugin(request.plugin);
@@ -66,7 +67,7 @@ function addPlugin(plugin: SCPluginMetaData): SCPluginRegisterResponse {
deepStrictEqual(previouslyRegistered, plugin);
return {success: true};
} catch (error) {
} catch {
throw new SCPluginAlreadyRegisteredErrorResponse(
'Plugin already registered',
plugins.get(plugin.route)!,
@@ -80,8 +81,10 @@ function addPlugin(plugin: SCPluginMetaData): SCPluginRegisterResponse {
if (typeof configFile.app.features.plugins === 'undefined') {
configFile.app.features.plugins = {};
}
configFile.app.features.plugins[plugin.name] = {urlPath : plugin.route};
Logger.log(`Registered plugin (name: ${plugin.name}, address: ${plugin.address}) on the route "${plugin.route}".`);
configFile.app.features.plugins[plugin.name] = {urlPath: plugin.route};
Logger.log(
`Registered plugin (name: ${plugin.name}, address: ${plugin.address}) on the route "${plugin.route}".`,
);
return {success: true};
}
@@ -93,9 +96,7 @@ function addPlugin(plugin: SCPluginMetaData): SCPluginRegisterResponse {
*/
function removePlugin(route: string): SCPluginRegisterResponse {
if (!plugins.has(route)) {
throw new SCNotFoundErrorResponse(
isTestEnvironment,
);
throw new SCNotFoundErrorResponse(isTestEnvironment);
}
if (plugins.has(route)) {
const plugin = plugins.get(route)!;

67
src/routes/route.ts
View File

@@ -39,7 +39,8 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
routeClass: SCRoute,
handler: (
validatedBody: REQUESTTYPE,
app: Application, params: { [parameterName: string]: string; },
app: Application,
parameters: {[parameterName: string]: string},
) => Promise<RETURNTYPE>,
): Router {
// create router
@@ -54,81 +55,73 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
// check if route has a valid http verb
if (isHttpMethod(verb)) {
// create a route handler for the given HTTP method
route[verb](async (req, res) => {
route[verb](async (request, response) => {
try {
// validate request
const requestValidation = validator.validate(req.body, routeClass.requestBodyName);
const requestValidation = validator.validate(request.body, routeClass.requestBodyName);
if (requestValidation.errors.length > 0) {
const error = new SCValidationErrorResponse(
requestValidation.errors,
isTestEnvironment,
);
res.status(error.statusCode);
res.json(error);
const error = new SCValidationErrorResponse(requestValidation.errors, isTestEnvironment);
response.status(error.statusCode);
response.json(error);
await Logger.error(error);
return;
}
// hand over request to handler with path parameters
const response = await handler(req.body, req.app, req.params);
const handlerResponse = await handler(request.body, request.app, request.params);
// validate response generated by handler
const responseErrors: ValidationError[] = validator.validate(response, routeClass.responseBodyName).errors;
const responseErrors: ValidationError[] = validator.validate(
handlerResponse,
routeClass.responseBodyName,
).errors;
if (responseErrors.length > 0) {
const validationError = new SCValidationErrorResponse(
responseErrors,
isTestEnvironment,
);
const validationError = new SCValidationErrorResponse(responseErrors, isTestEnvironment);
// The validation error is not caused by faulty user input, but through an error that originates somewhere in
// the backend, therefore we use this "stacked" error.
const internalServerError = new SCInternalServerErrorResponse(
validationError,
isTestEnvironment,
);
res.status(internalServerError.statusCode);
res.json(internalServerError);
const internalServerError = new SCInternalServerErrorResponse(validationError, isTestEnvironment);
response.status(internalServerError.statusCode);
response.json(internalServerError);
await Logger.error(internalServerError);
return;
}
// set status code
res.status(routeClass.statusCodeSuccess);
response.status(routeClass.statusCodeSuccess);
// respond
res.json(response);
response.json(handlerResponse);
} catch (error) {
// if the error response is allowed on the route
if (routeClass.errorNames.some((constructorType) => error instanceof constructorType)) {
if (routeClass.errorNames.some(constructorType => error instanceof constructorType)) {
// respond with the error from the handler
res.status(error.statusCode);
res.json(error);
response.status(error.statusCode);
response.json(error);
await Logger.error(error);
} else {
// the error is not allowed so something went wrong
const internalServerError = new SCInternalServerErrorResponse(
error,
isTestEnvironment,
);
res.status(internalServerError.statusCode);
res.json(internalServerError);
const internalServerError = new SCInternalServerErrorResponse(error, isTestEnvironment);
response.status(internalServerError.statusCode);
response.json(internalServerError);
await Logger.error(error);
}
}
});
} else {
throw new Error('Invalid HTTP verb in route definition. Please check route definitions in `@openstapps/core`');
throw new Error(
'Invalid HTTP verb in route definition. Please check route definitions in `@openstapps/core`',
);
}
// return a SCMethodNotAllowedErrorResponse on all other HTTP methods
route.all((_req, res) => {
route.all((_request, response) => {
const error = new SCMethodNotAllowedErrorResponse(isTestEnvironment);
res.status(error.statusCode);
res.json(error);
response.status(error.statusCode);
response.json(error);
Logger.warn(error);
});

33
src/routes/virtual-plugin-route.ts
View File

@@ -14,11 +14,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
SCInternalServerErrorResponse,
SCPluginMetaData,
SCValidationErrorResponse,
} from '@openstapps/core';
import {SCInternalServerErrorResponse, SCPluginMetaData, SCValidationErrorResponse} from '@openstapps/core';
import {Request} from 'express';
import got from 'got';
import {configFile, isTestEnvironment, validator} from '../common';
@@ -26,35 +22,34 @@ import {configFile, isTestEnvironment, validator} from '../common';
/**
* Generic route function used to proxy actual requests to plugins
*
* @param req The request for a plugin resource
* @param request The request for a plugin resource
* @param plugin Meta data of the plugin
* @throws {SCInternalServerErrorResponse} On request/response validation or response from the plugin errors
*/
export async function virtualPluginRoute(req: Request, plugin: SCPluginMetaData): Promise<object> {
export async function virtualPluginRoute(request: Request, plugin: SCPluginMetaData): Promise<object> {
let responseBody: object;
try {
const requestValidation = validator.validate(req.body, plugin.requestSchema);
const requestValidation = validator.validate(request.body, plugin.requestSchema);
if (requestValidation.errors.length > 0) {
// noinspection ExceptionCaughtLocallyJS
throw new SCValidationErrorResponse(requestValidation.errors, isTestEnvironment);
}
// send the request to the plugin (forward the body) and save the response
const pluginResponse = await got.post(
plugin.route.replace(/^\//gi, ''),
{
prefixUrl: plugin.address,
json: req.body,
timeout: configFile.backend.externalRequestTimeout,
responseType: 'json',
},
);
const pluginResponse = await got.post(plugin.route.replace(/^\//gi, ''), {
prefixUrl: plugin.address,
json: request.body,
timeout: configFile.backend.externalRequestTimeout,
responseType: 'json',
});
responseBody = pluginResponse.body as object;
const responseValidation = validator.validate(responseBody, plugin.responseSchema);
if (responseValidation.errors.length > 0) {
// noinspection ExceptionCaughtLocallyJS
throw new SCValidationErrorResponse(responseValidation.errors, isTestEnvironment);
}
} catch (e) {
} catch (error) {
// wrap exact error inside of the internal server error response
throw new SCInternalServerErrorResponse(e, isTestEnvironment);
throw new SCInternalServerErrorResponse(error, isTestEnvironment);
}
return responseBody;

25
src/storage/bulk-storage.ts
View File

@@ -29,7 +29,6 @@ export type BulkOperation = 'create' | 'expired' | 'update';
* Describes an indexing process
*/
export class Bulk implements SCBulkRequest {
/**
* Expiration of the bulk
*
@@ -70,19 +69,15 @@ export class Bulk implements SCBulkRequest {
/**
* Creates a new bulk process
*
* @param request Data needed for requesting a bulk
*/
constructor(request: SCBulkRequest) {
this.uid = v4();
this.state = 'in progress';
if (typeof request.expiration === 'string') {
this.expiration = request.expiration;
} else {
this.expiration = moment()
.add(1, 'hour')
.toISOString();
}
this.expiration =
typeof request.expiration === 'string' ? request.expiration : moment().add(1, 'hour').toISOString();
// when should this process be finished
// where does the process come from
this.source = request.source;
@@ -102,10 +97,10 @@ export class BulkStorage {
/**
* Creates a new BulkStorage
*
* @param database the database that is controlled by this bulk storage
*/
constructor(public database: Database) {
// a bulk lives 60 minutes if no expiration is given
// the cache is checked every 60 seconds
this.cache = new NodeCache({stdTTL: 3600, checkperiod: 60});
@@ -121,13 +116,12 @@ export class BulkStorage {
/**
* Saves a bulk process and assigns to it a user-defined ttl (time-to-live)
*
* @param bulk the bulk process to save
* @returns the bulk process that was saved
*/
private save(bulk: Bulk): Bulk {
const expirationInSeconds = moment(bulk.expiration)
// tslint:disable-next-line: no-magic-numbers
.diff(moment.now()) / 1000;
const expirationInSeconds = moment(bulk.expiration).diff(moment.now()) / 1000;
Logger.info('Bulk expires in ', expirationInSeconds, 'seconds');
// save the item in the cache with it's expected expiration
@@ -138,6 +132,7 @@ export class BulkStorage {
/**
* Create and save a new bulk process
*
* @param bulkRequest a request for a new bulk process
* @returns a promise that contains the new bulk process
*/
@@ -156,6 +151,7 @@ export class BulkStorage {
/**
* Delete a bulk process
*
* @param uid uid of the bulk process
* @returns a promise that contains the deleted bulk process
*/
@@ -163,7 +159,7 @@ export class BulkStorage {
const bulk = this.read(uid);
if (typeof bulk === 'undefined') {
throw new Error(`Bulk that should be deleted was not found. UID was "${uid}"`);
throw new TypeError(`Bulk that should be deleted was not found. UID was "${uid}"`);
}
// delete the bulk process from the cache
@@ -177,6 +173,7 @@ export class BulkStorage {
/**
* Update an old bulk process (replace it with the new one)
*
* @param bulk new bulk process
* @returns an empty promise
*/
@@ -192,11 +189,11 @@ export class BulkStorage {
/**
* Read an existing bulk process
*
* @param uid uid of the bulk process
* @returns a promise that contains a bulk
*/
public read(uid: string): Bulk | undefined {
return this.cache.get(uid);
}
}

8
src/storage/database.ts
View File

@@ -26,11 +26,11 @@ export type DatabaseConstructor = new (config: SCConfigFile, mailQueue?: MailQue
* Defines what one database class needs to have defined
*/
export interface Database {
/**
* Gets called if a bulk was created
*
* The database should
*
* @param bulk A bulk to be created
*/
bulkCreated(bulk: Bulk): Promise<void>;
@@ -39,6 +39,7 @@ export interface Database {
* Gets called if a bulk expires
*
* The database should delete all data that is associtated with this bulk
*
* @param bulk A bulk which data needs to be removed
*/
bulkExpired(bulk: Bulk): Promise<void>;
@@ -55,6 +56,7 @@ export interface Database {
/**
* Get a single document
*
* @param uid Unique identifier of the document
*/
get(uid: SCUuid): Promise<SCThings>;
@@ -66,6 +68,7 @@ export interface Database {
/**
* Add a thing to an existing bulk
*
* @param thing A StAppsCore thing to be added
* @param bulk A bulk to which the thing should be added
*/
@@ -82,7 +85,8 @@ export interface Database {
/**
* Search for things
*
* @param params Parameters which form a search query to search the backend data
*/
search(params: SCSearchQuery): Promise<SCSearchResponse>;
search(parameters: SCSearchQuery): Promise<SCSearchResponse>;
}

6
src/storage/elasticsearch/aggregations.ts
View File

@@ -26,10 +26,10 @@ import {
/**
* Parses elasticsearch aggregations (response from es) to facets for the app
*
* @param aggregationResponse - aggregations response from elasticsearch
*/
export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
const facets: SCFacet[] = [];
// get all names of the types an aggregation is on
@@ -52,7 +52,7 @@ export function parseAggregations(aggregationResponse: AggregationResponse): SCF
// this should always be true in theory...
if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) {
const facet: SCFacet = {
buckets: realField.buckets.map((bucket) => {
buckets: realField.buckets.map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key,
@@ -71,7 +71,7 @@ export function parseAggregations(aggregationResponse: AggregationResponse): SCF
// the last part here means that it is a bucket aggregation
} else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) {
facets.push({
buckets: realType.buckets.map((bucket) => {
buckets: realType.buckets.map(bucket => {
return {
count: bucket.doc_count,
key: bucket.key,

146
src/storage/elasticsearch/elasticsearch.ts
View File

@@ -27,7 +27,6 @@ import {
import {Logger} from '@openstapps/logger';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
import moment from 'moment';
import {MailQueue} from '../../notification/mail-queue';
@@ -39,7 +38,8 @@ import {buildQuery, buildSort} from './query';
import {aggregations, putTemplate} from './templating';
import {
AggregationResponse,
ElasticsearchConfig, ElasticsearchObject,
ElasticsearchConfig,
ElasticsearchObject,
ElasticsearchQueryDisMaxConfig,
ElasticsearchQueryQueryStringConfig,
} from './types/elasticsearch';
@@ -53,7 +53,6 @@ const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
* A database interface for elasticsearch
*/
export class Elasticsearch implements Database {
/**
* Length of the index UID used for generation of its name
*/
@@ -90,7 +89,7 @@ export class Elasticsearch implements Database {
*/
static getElasticsearchUrl(): string {
// check if we have a docker link
if (process.env.ES_ADDR !== undefined ) {
if (process.env.ES_ADDR !== undefined) {
return process.env.ES_ADDR;
}
@@ -100,6 +99,7 @@ export class Elasticsearch implements Database {
/**
* Gets the index name in elasticsearch for one SCThingType
*
* @param type SCThingType of data in the index
* @param source source of data in the index
* @param bulk bulk process which created this index
@@ -115,10 +115,11 @@ export class Elasticsearch implements Database {
/**
* Provides the index UID (for its name) from the bulk UID
*
* @param uid Bulk UID
*/
static getIndexUID(uid: SCUuid) {
return uid.substring(0, Elasticsearch.INDEX_UID_LENGTH);
return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH));
}
/**
@@ -131,6 +132,7 @@ export class Elasticsearch implements Database {
/**
* Checks for invalid character in alias names and removes them
*
* @param alias The alias name
* @param uid The UID of the current bulk (for debugging purposes)
*/
@@ -140,26 +142,25 @@ export class Elasticsearch implements Database {
// spaces are included in some types, replace them with underscores
if (formattedAlias.includes(' ')) {
formattedAlias = formattedAlias.trim();
formattedAlias = formattedAlias.split(' ')
.join('_');
formattedAlias = formattedAlias.split(' ').join('_');
}
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => {
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
if (formattedAlias.includes(value)) {
formattedAlias = formattedAlias.replace(value, '');
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
having the same alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
});
['-', '_', '+'].forEach((value) => {
}
for (const value of ['-', '_', '+']) {
if (formattedAlias.charAt(0) === value) {
formattedAlias = formattedAlias.substring(1);
formattedAlias = formattedAlias.slice(1);
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
alias despite having different types, as invalid characters are removed automatically.
New alias name is "${formattedAlias}."`);
}
});
}
if (formattedAlias === '.' || formattedAlias === '..') {
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
@@ -176,22 +177,24 @@ export class Elasticsearch implements Database {
/**
* Create a new interface for elasticsearch
*
* @param config an assembled config file
* @param mailQueue a mailqueue for monitoring
*/
constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) {
if (typeof config.internal.database === 'undefined'
|| typeof config.internal.database.version !== 'string') {
throw new Error('Database version is undefined. Check your config file');
if (
typeof config.internal.database === 'undefined' ||
typeof config.internal.database.version !== 'string'
) {
throw new TypeError('Database version is undefined. Check your config file');
}
this.client = new Client({
node: Elasticsearch.getElasticsearchUrl(),
});
this.client.on(events.REQUEST, async (err: Error | null, result: ApiResponse<unknown>) => {
if (err !== null) {
await Logger.error(err);
this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse<unknown>) => {
if (error !== null) {
await Logger.error(error);
}
if (process.env.ES_DEBUG === 'true') {
Logger.log(result);
@@ -215,18 +218,20 @@ export class Elasticsearch implements Database {
// create a list of old indices that are not in use
const oldIndicesToDelete: string[] = [];
let aliases: {
[index: string]: {
/**
* Aliases of an index
*/
aliases: {
[K in SCThingType]: unknown
};
};
} | undefined;
let aliases:
| {
[index: string]: {
/**
* Aliases of an index
*/
aliases: {
[K in SCThingType]: unknown;
};
};
}
| undefined;
for(const retry of [...Array(RETRY_COUNT)].map((_, i) => i+1)) {
for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) {
if (typeof aliases !== 'undefined') {
break;
}
@@ -241,16 +246,14 @@ export class Elasticsearch implements Database {
}
if (typeof aliases === 'undefined') {
throw Error(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
}
for (const index in aliases) {
if (aliases.hasOwnProperty(index)) {
const matches = indexRegex.exec(index);
if (matches !== null) {
const type = matches[1];
// tslint:disable-next-line: no-magic-numbers
const source = matches[2];
// check if there is an alias for the current index
@@ -278,12 +281,13 @@ export class Elasticsearch implements Database {
Logger.warn(`Deleted old indices: oldIndicesToDelete`);
}
// tslint:disable-next-line: no-magic-numbers
// eslint-disable-next-line unicorn/no-null
Logger.ok(`Read alias map from elasticsearch: ${JSON.stringify(this.aliasMap, null, 2)}`);
}
/**
* Provides an elasticsearch object using containing thing's UID
*
* @param uid an UID to use for the search
* @returns an elasticsearch object containing the thing
*/
@@ -309,6 +313,7 @@ export class Elasticsearch implements Database {
/**
* Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
*
* @param bulk the bulk process that was created
*/
public async bulkCreated(bulk: Bulk): Promise<void> {
@@ -346,6 +351,7 @@ export class Elasticsearch implements Database {
/**
* Should be called when a bulk process is expired. The index that was created with this bulk gets deleted
*
* @param bulk the bulk process that is expired
*/
public async bulkExpired(bulk: Bulk): Promise<void> {
@@ -365,6 +371,7 @@ export class Elasticsearch implements Database {
/**
* Should be called when a bulk process is updated (replaced by a newer bulk). This will replace the old
* index and publish all data, that was index in the new instead
*
* @param bulk the new bulk process that should replace the old one with same type and source
*/
public async bulkUpdated(bulk: Bulk): Promise<void> {
@@ -391,6 +398,7 @@ export class Elasticsearch implements Database {
}
// create the new index if it does not exists
// eslint-disable-next-line unicorn/no-await-expression-member
if (!(await this.client.indices.exists({index})).body) {
// re-apply the index template before each new bulk operation
await putTemplate(this.client, bulk.type);
@@ -411,6 +419,7 @@ export class Elasticsearch implements Database {
];
// remove our old index if it exists
// noinspection SuspiciousTypeOfGuard
if (typeof oldIndex === 'string') {
actions.push({
remove: {index: oldIndex, alias: alias},
@@ -431,6 +440,7 @@ export class Elasticsearch implements Database {
// swap the index in our aliasMap
this.aliasMap[alias][bulk.source] = index;
// noinspection SuspiciousTypeOfGuard
if (typeof oldIndex === 'string') {
// delete the old index
await this.client.indices.delete({index: oldIndex});
@@ -441,13 +451,14 @@ export class Elasticsearch implements Database {
/**
* Gets an SCThing from all indexed data
*
* @param uid uid of an SCThing
*/
public async get(uid: SCUuid): Promise<SCThings> {
const object = await this.getObject(uid);
if (typeof object === 'undefined') {
throw new Error('Item not found.');
throw new TypeError('Item not found.');
}
return object._source;
@@ -461,7 +472,9 @@ export class Elasticsearch implements Database {
if (typeof monitoringConfiguration !== 'undefined') {
if (typeof this.mailQueue === 'undefined') {
throw new Error('Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.');
throw new TypeError(
'Monitoring is defined, but MailQueue is undefined. A MailQueue is obligatory for monitoring.',
);
}
// read all watches and schedule searches on the client
await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
@@ -472,56 +485,55 @@ export class Elasticsearch implements Database {
/**
* Add an item to an index
*
* @param object the SCThing to add to the index
* @param bulk the bulk process which item belongs to
*/
public async post(object: SCThings, bulk: Bulk): Promise<void> {
// tslint:disable-next-line: completed-docs
const obj: SCThings & { creation_date: string; } = {
const object_: SCThings & {creation_date: string} = {
...object,
creation_date: moment()
.format(),
creation_date: moment().format(),
};
const item = await this.getObject(object.uid);
// check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid)
if (typeof item !== 'undefined') {
const indexOfNew = Elasticsearch.getIndex(obj.type, bulk.source, bulk);
const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk);
const oldIndex = item._index;
// new item doesn't replace the old one
// tslint:disable-next-line:no-magic-numbers
if (oldIndex.substring(0, oldIndex.lastIndexOf('_'))
!== indexOfNew.substring(0, indexOfNew.lastIndexOf('_'))) {
if (
oldIndex.slice(0, Math.max(0, oldIndex.lastIndexOf('_'))) !==
indexOfNew.slice(0, Math.max(0, indexOfNew.lastIndexOf('_')))
) {
throw new Error(
// tslint:disable-next-line: no-magic-numbers
`Object "${obj.uid}" already exists. Object was: ${JSON.stringify(obj, null, 2)}`,
// eslint-disable-next-line unicorn/no-null
`Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`,
);
}
}
// regular bulk update (item gets replaced when bulk is updated)
const searchResponse = await this.client.create({
body: obj,
id: obj.uid,
index: Elasticsearch.getIndex(obj.type, bulk.source, bulk),
body: object_,
id: object_.uid,
index: Elasticsearch.getIndex(object_.type, bulk.source, bulk),
timeout: '90s',
type: obj.type,
type: object_.type,
});
if (!searchResponse.body.created) {
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(obj)}`);
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`);
}
}
/**
* Put (update) an existing item
*
* @param object SCThing to put
*/
public async put(object: SCThings): Promise<void> {
const item = await this.getObject(object.uid);
if (typeof item !== 'undefined') {
@@ -542,12 +554,12 @@ export class Elasticsearch implements Database {
/**
* Search all indexed data
* @param params search query
*
* @param parameters search query
*/
public async search(params: SCSearchQuery): Promise<SCSearchResponse> {
public async search(parameters: SCSearchQuery): Promise<SCSearchResponse> {
if (typeof this.config.internal.database === 'undefined') {
throw new Error('Database is undefined. You have to configure the query build');
throw new TypeError('Database is undefined. You have to configure the query build');
}
// create elasticsearch configuration out of data from database configuration
@@ -557,23 +569,23 @@ export class Elasticsearch implements Database {
};
if (typeof this.config.internal.database.query !== 'undefined') {
esConfig.query =
this.config.internal.database
.query as ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
esConfig.query = this.config.internal.database.query as
| ElasticsearchQueryDisMaxConfig
| ElasticsearchQueryQueryStringConfig;
}
const searchRequest: RequestParams.Search = {
body: {
aggs: aggregations,
query: buildQuery(params, this.config, esConfig),
query: buildQuery(parameters, this.config, esConfig),
},
from: params.from,
from: parameters.from,
index: Elasticsearch.getListOfAllIndices(),
size: params.size,
size: parameters.size,
};
if (typeof params.sort !== 'undefined') {
searchRequest.body.sort = buildSort(params.sort);
if (typeof parameters.sort !== 'undefined') {
searchRequest.body.sort = buildSort(parameters.sort);
}
// perform the search against elasticsearch
@@ -582,7 +594,7 @@ export class Elasticsearch implements Database {
// gather pagination information
const pagination = {
count: response.body.hits.hits.length,
offset: (typeof params.from === 'number') ? params.from : 0,
offset: typeof parameters.from === 'number' ? parameters.from : 0,
total: response.body.hits.total,
};
@@ -593,7 +605,7 @@ export class Elasticsearch implements Database {
// we only directly return the _source documents
// elasticsearch provides much more information, the user shouldn't see
const data = response.body.hits.hits.map((hit) => {
const data = response.body.hits.hits.map(hit => {
return hit._source; // SCThing
});

62
src/storage/elasticsearch/monitoring.ts
View File

@@ -25,13 +25,13 @@ import {
import {Logger} from '@openstapps/logger';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {SearchResponse} from 'elasticsearch';
import cron from 'node-cron';
import {MailQueue} from '../../notification/mail-queue';
/**
* Check if the given condition fails on the given number of results and the condition
*
* @param condition condition
* @param total number of results
*/
@@ -48,6 +48,7 @@ function conditionFails(
/**
* Check if the min condition fails
*
* @param minimumLength Minimal length allowed
* @param total Number of results
*/
@@ -57,6 +58,7 @@ function minConditionFails(minimumLength: number, total: number) {
/**
* Check if the max condition fails
*
* @param maximumLength Maximal length allowed
* @param total Number of results
*/
@@ -66,6 +68,7 @@ function maxConditionFails(maximumLength: number, total: number) {
/**
* Run all the given actions
*
* @param actions actions to perform
* @param watcherName name of watcher that wants to perform them
* @param triggerName name of trigger that triggered the watcher
@@ -79,38 +82,39 @@ function runActions(
total: number,
mailQueue: MailQueue,
) {
actions.forEach(async (action) => {
if (action.type === 'log') {
await Logger.error(
action.prefix,
`Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`, `Found ${total} hits instead`,
action.message,
);
} else {
await mailQueue.push({
subject: action.subject,
text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'
for (const action of actions) {
void (action.type === 'log'
? Logger.error(
action.prefix,
`Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'`,
`Found ${total} hits instead`,
action.message,
)
: mailQueue.push({
subject: action.subject,
text: `Watcher '${watcherName}' failed. Watcher was triggered by '${triggerName}'
${action.message} Found ${total} hits instead`,
to: action.recipients,
});
}
});
to: action.recipients,
}));
}
}
/**
* Set up the triggers for the configured watchers
*
* @param monitoringConfig configuration of the monitoring
* @param esClient elasticsearch client
* @param mailQueue mailQueue for mail actions
*/
export async function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: Client, mailQueue: MailQueue) {
export async function setUp(
monitoringConfig: SCMonitoringConfiguration,
esClient: Client,
mailQueue: MailQueue,
) {
// set up Watches
monitoringConfig.watchers.forEach((watcher) => {
for (const watcher of monitoringConfig.watchers) {
// make a schedule for each trigger
watcher.triggers.forEach((trigger) => {
for (const trigger of watcher.triggers) {
switch (trigger.executionTime) {
case 'hourly':
trigger.executionTime = '5 * * * *';
@@ -127,21 +131,21 @@ export async function setUp(monitoringConfig: SCMonitoringConfiguration, esClien
cron.schedule(trigger.executionTime, async () => {
// execute watch (search->condition->action)
const result: ApiResponse<SearchResponse<SCThings>> =
await esClient.search(watcher.query as RequestParams.Search);
const result: ApiResponse<SearchResponse<SCThings>> = await esClient.search(
watcher.query as RequestParams.Search,
);
// check conditions
const total = result.body.hits.total;
watcher.conditions.forEach((condition) => {
for (const condition of watcher.conditions) {
if (conditionFails(condition, total)) {
runActions(watcher.actions, watcher.name, trigger.name, total, mailQueue);
}
});
}
});
});
});
}
}
Logger.log(`Scheduled ${monitoringConfig.watchers.length} watches`);
}

128
src/storage/elasticsearch/query.ts
View File

@@ -34,7 +34,8 @@ import {
ESFunctionScoreQuery,
ESFunctionScoreQueryFunction,
ESGenericRange,
ESGenericSort, ESGeoBoundingBoxFilter,
ESGenericSort,
ESGeoBoundingBoxFilter,
ESGeoDistanceFilter,
ESGeoDistanceFilterArguments,
ESGeoDistanceSort,
@@ -55,19 +56,19 @@ import {
* It is possible to use all, with the exception of < and >, of them by escaping them with a \
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*
* @param str the string to escape the characters from
* @param string_ the string to escape the characters from
*/
function escapeESReservedCharacters(str: string): string {
return str.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
function escapeESReservedCharacters(string_: string): string {
return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
}
/**
* Builds a boolean filter. Returns an elasticsearch boolean filter
*
* @param booleanFilter a search boolean filter for the retrieval of the data
* @returns elasticsearch boolean arguments object
*/
export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<unknown> {
const result: ESBooleanFilterArguments<unknown> = {
minimum_should_match: 0,
must: [],
@@ -76,16 +77,16 @@ export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBool
};
if (booleanFilter.arguments.operation === 'and') {
result.must = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
}
if (booleanFilter.arguments.operation === 'or') {
result.should = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
result.minimum_should_match = 1;
}
if (booleanFilter.arguments.operation === 'not') {
result.must_not = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
}
return result;
@@ -93,22 +94,31 @@ export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBool
/**
* Converts Array of Filters to elasticsearch query-syntax
*
* @param filter A search filter for the retrieval of the data
*/
export function buildFilter(filter: SCSearchFilter):
ESTermFilter | ESGeoDistanceFilter | ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter> | ESGeoShapeFilter | ESBooleanFilter<unknown> | ESRangeFilter {
export function buildFilter(
filter: SCSearchFilter,
):
| ESTermFilter
| ESGeoDistanceFilter
| ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter>
| ESGeoShapeFilter
| ESBooleanFilter<unknown>
| ESRangeFilter {
switch (filter.type) {
case 'value':
return Array.isArray(filter.arguments.value) ? {
terms: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
} : {
term: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
};
return Array.isArray(filter.arguments.value)
? {
terms: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
}
: {
term: {
[`${filter.arguments.field}.raw`]: filter.arguments.value,
},
};
case 'availability':
const scope = filter.arguments.scope?.charAt(0) ?? 's';
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
@@ -185,8 +195,7 @@ export function buildFilter(filter: SCSearchFilter):
/**
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
*/
// tslint:disable-next-line:ban-ts-ignore
// @ts-ignore unfortunately, typescript is stupid and won't allow me to map this to an actual type.
// @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type.
ignore_unmapped: true,
[`${filter.arguments.field}.polygon`]: {
shape: filter.arguments.shape,
@@ -195,8 +204,10 @@ export function buildFilter(filter: SCSearchFilter):
},
};
if ((typeof filter.arguments.spatialRelation === 'undefined' || filter.arguments.spatialRelation === 'intersects')
&& filter.arguments.shape.type === 'envelope'
if (
(typeof filter.arguments.spatialRelation === 'undefined' ||
filter.arguments.spatialRelation === 'intersects') &&
filter.arguments.shape.type === 'envelope'
) {
return {
bool: {
@@ -208,8 +219,6 @@ export function buildFilter(filter: SCSearchFilter):
/**
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
*/
// tslint:disable-next-line:ban-ts-ignore
// @ts-ignore unfortunately, typescript is stupid and won't allow me to map this to an actual type.
ignore_unmapped: true,
[`${filter.arguments.field}.point.coordinates`]: {
top_left: filter.arguments.shape.coordinates[0],
@@ -228,6 +237,7 @@ export function buildFilter(filter: SCSearchFilter):
/**
* Builds scoring functions from boosting config
*
* @param boostings Backend boosting configuration for contexts and types
* @param context The context of the app from where the search was initiated
*/
@@ -235,14 +245,14 @@ function buildFunctions(
boostings: SCBackendConfigurationSearchBoostingContext,
context: SCSearchContext | undefined,
): ESFunctionScoreQueryFunction[] {
// default context
let functions: ESFunctionScoreQueryFunction[] =
buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]);
let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes(
boostings['default' as SCSearchContext],
);
if (typeof context !== 'undefined' && context !== 'default') {
// specific context provided, extend default context with additional boosts
functions = functions.concat(buildFunctionsForBoostingTypes(boostings[context]));
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
}
return functions;
@@ -258,7 +268,7 @@ function buildFunctionsForBoostingTypes(
): ESFunctionScoreQueryFunction[] {
const functions: ESFunctionScoreQueryFunction[] = [];
boostingTypes.forEach((boostingForOneSCType) => {
for (const boostingForOneSCType of boostingTypes) {
const typeFilter: ESTypeFilter = {
type: {
value: boostingForOneSCType.type,
@@ -271,7 +281,6 @@ function buildFunctionsForBoostingTypes(
});
if (typeof boostingForOneSCType.fields !== 'undefined') {
const fields = boostingForOneSCType.fields;
for (const fieldName in boostingForOneSCType.fields) {
@@ -291,10 +300,7 @@ function buildFunctionsForBoostingTypes(
functions.push({
filter: {
bool: {
must: [
typeFilter,
termFilter,
],
must: [typeFilter, termFilter],
should: [],
},
},
@@ -305,24 +311,24 @@ function buildFunctionsForBoostingTypes(
}
}
}
});
}
return functions;
}
/**
* Builds body for Elasticsearch requests
* @param params Parameters for querying the backend
*
* @param parameters Parameters for querying the backend
* @param defaultConfig Default configuration of the backend
* @param elasticsearchConfig Elasticsearch configuration
* @returns ElasticsearchQuery (body of a search-request)
*/
export function buildQuery(
params: SCSearchQuery,
parameters: SCSearchQuery,
defaultConfig: SCConfigFile,
elasticsearchConfig: ElasticsearchConfig,
): ESFunctionScoreQuery {
// if config provides an minMatch parameter we use query_string instead of match query
let query;
if (typeof elasticsearchConfig.query === 'undefined') {
@@ -331,7 +337,7 @@ export function buildQuery(
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: '90%',
query: (typeof params.query !== 'string') ? '*' : escapeESReservedCharacters(params.query),
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
};
} else if (elasticsearchConfig.query.queryType === 'query_string') {
@@ -340,11 +346,11 @@ export function buildQuery(
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: (typeof params.query !== 'string') ? '*' : escapeESReservedCharacters(params.query),
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
};
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
if (params.query !== '*') {
if (parameters.query !== '*') {
query = {
dis_max: {
boost: 1.2,
@@ -355,7 +361,7 @@ export function buildQuery(
boost: elasticsearchConfig.query.matchBoosting,
cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
fuzziness: elasticsearchConfig.query.fuzziness,
query: (typeof params.query !== 'string') ? '*' : params.query,
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
},
},
},
@@ -364,22 +370,24 @@ export function buildQuery(
analyzer: 'search_german',
default_field: 'name',
minimum_should_match: elasticsearchConfig.query.minMatch,
query: (typeof params.query !== 'string') ? '*' : escapeESReservedCharacters(params.query),
query:
typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
},
},
],
tie_breaker: elasticsearchConfig.query.tieBreaker,
},
};
}
} else {
throw new Error('Unsupported query type. Check your config file and reconfigure your elasticsearch query');
throw new Error(
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
);
}
const functionScoreQuery: ESFunctionScoreQuery = {
function_score: {
functions: buildFunctions(defaultConfig.internal.boostings, params.context),
functions: buildFunctions(defaultConfig.internal.boostings, parameters.context),
query: {
bool: {
minimum_should_match: 0, // if we have no should, nothing can match
@@ -398,8 +406,8 @@ export function buildQuery(
mustMatch.push(query);
}
if (typeof params.filter !== 'undefined') {
mustMatch.push(buildFilter(params.filter));
if (typeof parameters.filter !== 'undefined') {
mustMatch.push(buildFilter(parameters.filter));
}
}
@@ -408,13 +416,12 @@ export function buildQuery(
/**
* converts query to
*
* @param sorts Sorting rules to apply to the data that is being queried
* @returns an array of sort queries
*/
export function buildSort(
sorts: SCSearchSort[],
): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
return sorts.map((sort) => {
export function buildSort(sorts: SCSearchSort[]): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
return sorts.map(sort => {
switch (sort.type) {
case 'generic':
const esGenericSort: ESGenericSort = {};
@@ -427,26 +434,26 @@ export function buildSort(
return esDucetSort;
case 'distance':
const args: ESGeoDistanceSortArguments = {
const arguments_: ESGeoDistanceSortArguments = {
mode: 'avg',
order: sort.order,
unit: 'm',
};
args[`${sort.arguments.field}.point.coordinates`] = {
arguments_[`${sort.arguments.field}.point.coordinates`] = {
lat: sort.arguments.position[1],
lon: sort.arguments.position[0],
};
return {
_geo_distance: args,
_geo_distance: arguments_,
};
case 'price':
return {
_script: {
order: sort.order,
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
type: 'number' as 'number',
type: 'number' as const,
},
};
}
@@ -459,7 +466,10 @@ export function buildSort(
* @param universityRole User group which consumes university services
* @param field Field in which wanted offers with prices are located
*/
export function buildPriceSortScript(universityRole: keyof SCSportCoursePriceGroup, field: SCThingsField): string {
export function buildPriceSortScript(
universityRole: keyof SCSportCoursePriceGroup,
field: SCThingsField,
): string {
return `
// initialize the sort value with the maximum
double price = Double.MAX_VALUE;

17
src/storage/elasticsearch/templating.ts
View File

@@ -15,18 +15,19 @@
*/
import {Client} from '@elastic/elasticsearch';
import {SCThingType} from '@openstapps/core';
// tslint:disable-next-line:no-implicit-dependencies
import {AggregationSchema} from '@openstapps/es-mapping-generator/src/types/aggregation';
// tslint:disable-next-line:no-implicit-dependencies
import {ElasticsearchTemplateCollection} from '@openstapps/es-mapping-generator/src/types/mapping';
import {readFileSync} from 'fs';
import {resolve} from 'path';
import path from 'path';
const mappingsPath = resolve('node_modules', '@openstapps', 'core', 'lib','mappings');
export const mappings = JSON.parse(readFileSync(resolve(mappingsPath, 'mappings.json'), 'utf-8')) as ElasticsearchTemplateCollection;
export const aggregations = JSON.parse(readFileSync(resolve(mappingsPath, 'aggregations.json'), 'utf-8')) as AggregationSchema;
const mappingsPath = path.resolve('node_modules', '@openstapps', 'core', 'lib', 'mappings');
export const mappings = JSON.parse(
readFileSync(path.resolve(mappingsPath, 'mappings.json'), 'utf8'),
) as ElasticsearchTemplateCollection;
export const aggregations = JSON.parse(
readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
) as AggregationSchema;
/**
* Re-applies all interfaces for every type
@@ -44,8 +45,8 @@ export async function refreshAllTemplates(client: Client) {
*
* This includes applying the mapping, settings
*
* @param type the SCThingType of which the template should be set
* @param client An elasticsearch client to use
* @param type the SCThingType of which the template should be set
*/
export async function putTemplate(client: Client, type: SCThingType) {
const sanitizedType = `template_${type.replace(/\s/g, '_')}`;

103
src/storage/elasticsearch/types/elasticsearch.ts
View File

@@ -15,9 +15,7 @@
*/
import {SCThing, SCThingType} from '@openstapps/core';
// we only have the @types package because some things type definitions are still missing from the official
// tslint:disable-next-line:no-implicit-dependencies
import {NameList} from 'elasticsearch';
// tslint:disable-next-line:no-implicit-dependencies
import {Polygon, Position} from 'geojson';
/**
@@ -75,7 +73,6 @@ export interface NestedAggregation {
[name: string]: BucketAggregation | number;
}
/**
* A configuration for using the Dis Max Query
*
@@ -90,30 +87,35 @@ export interface ElasticsearchQueryDisMaxConfig {
/**
* The maximum allowed Levenshtein Edit Distance (or number of edits)
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
*/
fuzziness: number | string;
/**
* Increase the importance (relevance score) of a field
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
*/
matchBoosting: number;
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'dis_max' which is a union of its subqueries
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
*/
queryType: 'dis_max';
/**
* Changes behavior of default calculation of the score when multiple results match
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
*/
tieBreaker: number;
@@ -128,12 +130,14 @@ export interface ElasticsearchQueryDisMaxConfig {
export interface ElasticsearchQueryQueryStringConfig {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minMatch: string;
/**
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
*/
queryType: 'query_string';
@@ -141,6 +145,7 @@ export interface ElasticsearchQueryQueryStringConfig {
/**
* A hit in an elasticsearch search result
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
*/
export interface ElasticsearchObject<T extends SCThing> {
@@ -166,6 +171,7 @@ export interface ElasticsearchObject<T extends SCThing> {
/**
* The document's mapping type
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
*/
_type: string;
@@ -177,22 +183,25 @@ export interface ElasticsearchObject<T extends SCThing> {
/**
* Used to index the same field in different ways for different purposes
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
*/
fields?: NameList;
/**
* Used to highlight search results on one or more fields
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
*/
// tslint:disable-next-line: no-any
// eslint-disable-next-line @typescript-eslint/no-explicit-any
highlight?: any;
/**
* Used in when nested/children documents match the query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
*/
// tslint:disable-next-line: no-any
// eslint-disable-next-line @typescript-eslint/no-explicit-any
inner_hits?: any;
/**
@@ -246,21 +255,23 @@ export interface ElasticsearchConfig {
/**
* An elasticsearch term filter
*/
export type ESTermFilter = {
/**
* Definition of a term to match
*/
term: {
[fieldName: string]: string;
};
} | {
/**
* Definition of terms to match (or)
*/
terms: {
[fieldName: string]: string[];
};
};
export type ESTermFilter =
| {
/**
* Definition of a term to match
*/
term: {
[fieldName: string]: string;
};
}
| {
/**
* Definition of terms to match (or)
*/
terms: {
[fieldName: string]: string[];
};
};
export interface ESGenericRange<T> {
/**
@@ -318,7 +329,6 @@ export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<n
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
/**
* An elasticsearch type filter
*/
@@ -343,17 +353,19 @@ export interface ESGeoDistanceFilterArguments {
*/
distance: string;
[fieldName: string]: {
/**
* Latitude
*/
lat: number;
[fieldName: string]:
| {
/**
* Latitude
*/
lat: number;
/**
* Longitude
*/
lon: number;
} | string;
/**
* Longitude
*/
lon: number;
}
| string;
}
/**
@@ -386,11 +398,13 @@ export interface ESEnvelope {
/**
* An Elasticsearch geo bounding box filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
*/
export interface ESGeoBoundingBoxFilter {
/**
* An Elasticsearch geo bounding box filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
*/
geo_bounding_box: {
@@ -410,6 +424,7 @@ export interface ESGeoBoundingBoxFilter {
/**
* An Elasticsearch geo shape filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html
*/
export interface ESGeoShapeFilter {
@@ -432,11 +447,13 @@ export interface ESGeoShapeFilter {
/**
* Filter arguments for an elasticsearch boolean filter
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
*/
export interface ESBooleanFilterArguments<T> {
/**
* Minimal number (or percentage) of words that should match in a query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
*/
minimum_should_match?: number;
@@ -469,6 +486,7 @@ export interface ESBooleanFilter<T> {
/**
* An elasticsearch function score query
*
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
*/
export interface ESFunctionScoreQuery {
@@ -478,6 +496,7 @@ export interface ESFunctionScoreQuery {
function_score: {
/**
* Functions that compute score for query results (documents)
*
* @see ESFunctionScoreQueryFunction
*/
functions: ESFunctionScoreQueryFunction[];
@@ -535,17 +554,19 @@ export interface ESGeoDistanceSortArguments {
*/
unit: 'm';
[field: string]: {
/**
* Latitude
*/
lat: number;
[field: string]:
| {
/**
* Latitude
*/
lat: number;
/**
* Longitude
*/
lon: number;
} | string;
/**
* Longitude
*/
lon: number;
}
| string;
}
/**

9
src/storage/elasticsearch/types/guards.ts
View File

@@ -18,12 +18,12 @@ import {
ESAggTypeFilter,
ESNestedAggregation,
ESTermsFilter,
// tslint:disable-next-line:no-implicit-dependencies we're just using the types here
} from '@openstapps/es-mapping-generator/src/types/aggregation';
import {BucketAggregation, NestedAggregation} from './elasticsearch';
/**
* Checks if the type is a BucketAggregation
*
* @param agg the type to check
*/
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
@@ -32,6 +32,7 @@ export function isBucketAggregation(agg: BucketAggregation | number): agg is Buc
/**
* Checks if the type is a NestedAggregation
*
* @param agg the type to check
*/
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
@@ -40,6 +41,7 @@ export function isNestedAggregation(agg: BucketAggregation | NestedAggregation):
/**
* Checks if the parameter is of type ESTermsFilter
*
* @param agg the value to check
*/
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
@@ -48,6 +50,7 @@ export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg i
/**
* Checks if the parameter is of type ESTermsFilter
*
* @param agg the value to check
*/
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
@@ -59,6 +62,8 @@ export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation):
*
* @param filter the filter to narrow the type of
*/
export function isESAggMatchAllFilter(filter: ESAggTypeFilter | ESAggMatchAllFilter): filter is ESAggMatchAllFilter {
export function isESAggMatchAllFilter(
filter: ESAggTypeFilter | ESAggMatchAllFilter,
): filter is ESAggMatchAllFilter {
return filter.hasOwnProperty('match_all');
}