mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-21 17:12:43 +00:00
feat: more fixes
This commit is contained in:
@@ -3,7 +3,7 @@ import about from './about.js';
|
|||||||
import imprint from './imprint.js';
|
import imprint from './imprint.js';
|
||||||
import privacy from './privacy.js';
|
import privacy from './privacy.js';
|
||||||
|
|
||||||
/** @type {import('@openstapps/core').SCMap<import('@openstapps/core').SCAboutPage>} */
|
/** @type {Record<string, import('@openstapps/core').SCAboutPage>} */
|
||||||
const aboutPages = {
|
const aboutPages = {
|
||||||
'about': about,
|
'about': about,
|
||||||
'about/imprint': imprint,
|
'about/imprint': imprint,
|
||||||
|
|||||||
@@ -45,6 +45,7 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@elastic/elasticsearch": "8.10.0",
|
"@elastic/elasticsearch": "8.10.0",
|
||||||
"@openstapps/core": "workspace:*",
|
"@openstapps/core": "workspace:*",
|
||||||
|
"@openstapps/core-validator": "workspace:*",
|
||||||
"@openstapps/logger": "workspace:*",
|
"@openstapps/logger": "workspace:*",
|
||||||
"@types/body-parser": "1.19.2",
|
"@types/body-parser": "1.19.2",
|
||||||
"@types/cors": "2.8.13",
|
"@types/cors": "2.8.13",
|
||||||
|
|||||||
@@ -14,7 +14,6 @@
|
|||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
import {
|
import {
|
||||||
SCConfigFile,
|
|
||||||
SCNotFoundErrorResponse,
|
SCNotFoundErrorResponse,
|
||||||
SCRequestBodyTooLargeErrorResponse,
|
SCRequestBodyTooLargeErrorResponse,
|
||||||
SCSyntaxErrorResponse,
|
SCSyntaxErrorResponse,
|
||||||
@@ -39,7 +38,7 @@ import {virtualPluginRoute} from './routes/virtual-plugin-route.js';
|
|||||||
import {BulkStorage} from './storage/bulk-storage.js';
|
import {BulkStorage} from './storage/bulk-storage.js';
|
||||||
import {DatabaseConstructor} from './storage/database.js';
|
import {DatabaseConstructor} from './storage/database.js';
|
||||||
import {backendConfig} from './config.js';
|
import {backendConfig} from './config.js';
|
||||||
import {createValidator} from './validator.js';
|
import {validator} from './validator.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Configure the backend
|
* Configure the backend
|
||||||
@@ -143,11 +142,8 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
|
|||||||
request.on('data', chunkGatherer).on('end', endCallback);
|
request.on('data', chunkGatherer).on('end', endCallback);
|
||||||
});
|
});
|
||||||
|
|
||||||
const configFileValid = createValidator<SCConfigFile>('SCConfigFile');
|
if (!validator.validate(backendConfig, 'SCConfigFile')) {
|
||||||
if (!configFileValid(backendConfig)) {
|
throw new Error(`Validation of config file failed. Errors were: ${JSON.stringify(validator.errors)}`);
|
||||||
throw new Error(
|
|
||||||
`Validation of config file failed. Errors were: ${JSON.stringify(configFileValid.errors)}`,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if a database name was given
|
// check if a database name was given
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import {Application, Router} from 'express';
|
|||||||
import PromiseRouter from 'express-promise-router';
|
import PromiseRouter from 'express-promise-router';
|
||||||
import {isTestEnvironment} from '../common.js';
|
import {isTestEnvironment} from '../common.js';
|
||||||
import {isHttpMethod} from './http-types.js';
|
import {isHttpMethod} from './http-types.js';
|
||||||
import {createValidator} from '../validator.js';
|
import {validator} from '../validator.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a router from a route class and a handler function which implements the logic
|
* Creates a router from a route class and a handler function which implements the logic
|
||||||
@@ -44,8 +44,6 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
|
|||||||
): Router {
|
): Router {
|
||||||
// create router
|
// create router
|
||||||
const router = PromiseRouter({mergeParams: true});
|
const router = PromiseRouter({mergeParams: true});
|
||||||
const requestValidator = createValidator<REQUESTTYPE>(routeClass.requestBodyName);
|
|
||||||
const responseValidator = createValidator<RETURNTYPE>(routeClass.responseBodyName);
|
|
||||||
|
|
||||||
// create route
|
// create route
|
||||||
// the given type has no index signature so we have to cast to get the IRouteHandler when a HTTP method is given
|
// the given type has no index signature so we have to cast to get the IRouteHandler when a HTTP method is given
|
||||||
@@ -58,8 +56,8 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
|
|||||||
// create a route handler for the given HTTP method
|
// create a route handler for the given HTTP method
|
||||||
route[verb](async (request, response) => {
|
route[verb](async (request, response) => {
|
||||||
try {
|
try {
|
||||||
if (!requestValidator(request.body)) {
|
if (!validator.validate(request.body, routeClass.requestBodyName as never)) {
|
||||||
const error = new SCValidationErrorResponse(requestValidator.errors as any, isTestEnvironment);
|
const error = new SCValidationErrorResponse(validator.errors, isTestEnvironment);
|
||||||
response.status(error.statusCode);
|
response.status(error.statusCode);
|
||||||
response.json(error);
|
response.json(error);
|
||||||
await Logger.error(error);
|
await Logger.error(error);
|
||||||
@@ -67,13 +65,10 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const handlerResponse = await handler(request.body, request.app, request.params);
|
const handlerResponse = await handler(request.body as REQUESTTYPE, request.app, request.params);
|
||||||
|
|
||||||
if (!responseValidator(handlerResponse)) {
|
if (!validator.validate(handlerResponse, routeClass.responseBodyName)) {
|
||||||
const validationError = new SCValidationErrorResponse(
|
const validationError = new SCValidationErrorResponse(validator.errors, isTestEnvironment);
|
||||||
responseValidator.errors as any,
|
|
||||||
isTestEnvironment,
|
|
||||||
);
|
|
||||||
// The validation error is not caused by faulty user input, but through an error that originates somewhere in
|
// The validation error is not caused by faulty user input, but through an error that originates somewhere in
|
||||||
// the backend, therefore we use this "stacked" error.
|
// the backend, therefore we use this "stacked" error.
|
||||||
const internalServerError = new SCInternalServerErrorResponse(validationError, isTestEnvironment);
|
const internalServerError = new SCInternalServerErrorResponse(validationError, isTestEnvironment);
|
||||||
|
|||||||
@@ -13,7 +13,6 @@
|
|||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {SCInternalServerErrorResponse, SCPluginMetaData, SCValidationErrorResponse} from '@openstapps/core';
|
import {SCInternalServerErrorResponse, SCPluginMetaData, SCValidationErrorResponse} from '@openstapps/core';
|
||||||
import {Request} from 'express';
|
import {Request} from 'express';
|
||||||
import got from 'got';
|
import got from 'got';
|
||||||
@@ -31,7 +30,7 @@ export async function virtualPluginRoute(request: Request, plugin: SCPluginMetaD
|
|||||||
try {
|
try {
|
||||||
if (!validator.validate(request.body, plugin.requestSchema)) {
|
if (!validator.validate(request.body, plugin.requestSchema)) {
|
||||||
// noinspection ExceptionCaughtLocallyJS
|
// noinspection ExceptionCaughtLocallyJS
|
||||||
throw new SCValidationErrorResponse(validator.errors as any, isTestEnvironment);
|
throw new SCValidationErrorResponse(validator.errors, isTestEnvironment);
|
||||||
}
|
}
|
||||||
// send the request to the plugin (forward the body) and save the response
|
// send the request to the plugin (forward the body) and save the response
|
||||||
const response = await got.post(plugin.route.replaceAll(/^\//gi, ''), {
|
const response = await got.post(plugin.route.replaceAll(/^\//gi, ''), {
|
||||||
@@ -45,7 +44,7 @@ export async function virtualPluginRoute(request: Request, plugin: SCPluginMetaD
|
|||||||
const responseBody = response.body;
|
const responseBody = response.body;
|
||||||
if (!validator.validate(responseBody, plugin.responseSchema)) {
|
if (!validator.validate(responseBody, plugin.responseSchema)) {
|
||||||
// noinspection ExceptionCaughtLocallyJS
|
// noinspection ExceptionCaughtLocallyJS
|
||||||
throw new SCValidationErrorResponse(validator.errors as any, isTestEnvironment);
|
throw new SCValidationErrorResponse(validator.errors, isTestEnvironment);
|
||||||
}
|
}
|
||||||
return responseBody as object;
|
return responseBody as object;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ import {parseAggregations} from './aggregations.js';
|
|||||||
import * as Monitoring from './monitoring.js';
|
import * as Monitoring from './monitoring.js';
|
||||||
import {buildQuery} from './query/query.js';
|
import {buildQuery} from './query/query.js';
|
||||||
import {buildSort} from './query/sort.js';
|
import {buildSort} from './query/sort.js';
|
||||||
import {aggregations, putTemplate} from './templating.js';
|
import {putTemplate} from './templating.js';
|
||||||
import {
|
import {
|
||||||
ElasticsearchConfig,
|
ElasticsearchConfig,
|
||||||
ElasticsearchQueryDisMaxConfig,
|
ElasticsearchQueryDisMaxConfig,
|
||||||
@@ -46,6 +46,7 @@ import {
|
|||||||
} from './util/index.js';
|
} from './util/index.js';
|
||||||
import {noUndefined} from './util/no-undefined.js';
|
import {noUndefined} from './util/no-undefined.js';
|
||||||
import {retryCatch, RetryOptions} from './util/retry.js';
|
import {retryCatch, RetryOptions} from './util/retry.js';
|
||||||
|
import config from '@openstapps/core/elasticsearch-mappings.json';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A database interface for elasticsearch
|
* A database interface for elasticsearch
|
||||||
@@ -370,7 +371,7 @@ export class Elasticsearch implements Database {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const response: SearchResponse<SCThings> = await this.client.search({
|
const response: SearchResponse<SCThings> = await this.client.search({
|
||||||
aggs: aggregations,
|
aggs: config.default.search.aggs,
|
||||||
query: buildQuery(parameters, this.config, esConfig),
|
query: buildQuery(parameters, this.config, esConfig),
|
||||||
from: parameters.from,
|
from: parameters.from,
|
||||||
index: ACTIVE_INDICES_ALIAS,
|
index: ACTIVE_INDICES_ALIAS,
|
||||||
|
|||||||
@@ -15,19 +15,7 @@
|
|||||||
*/
|
*/
|
||||||
import {Client} from '@elastic/elasticsearch';
|
import {Client} from '@elastic/elasticsearch';
|
||||||
import {SCThingType} from '@openstapps/core';
|
import {SCThingType} from '@openstapps/core';
|
||||||
import type {AggregationSchema} from '@openstapps/core/lib/mappings/aggregations.json.js';
|
import config from '@openstapps/core/elasticsearch-mappings.json';
|
||||||
import type {ElasticsearchTemplateCollection} from '@openstapps/core/lib/mappings/mappings.json.js';
|
|
||||||
import {readFileSync} from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
|
|
||||||
const mappingsPath = path.resolve('node_modules', '@openstapps', 'core', 'lib', 'mappings');
|
|
||||||
|
|
||||||
export const mappings = JSON.parse(
|
|
||||||
readFileSync(path.resolve(mappingsPath, 'mappings.json'), 'utf8'),
|
|
||||||
) as ElasticsearchTemplateCollection;
|
|
||||||
export const aggregations = JSON.parse(
|
|
||||||
readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
|
|
||||||
) as AggregationSchema;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Prepares all indices
|
* Prepares all indices
|
||||||
@@ -40,7 +28,7 @@ export async function putTemplate(client: Client, type: SCThingType) {
|
|||||||
const sanitizedType = `template_${type.replaceAll(/\s/g, '_')}`;
|
const sanitizedType = `template_${type.replaceAll(/\s/g, '_')}`;
|
||||||
|
|
||||||
return client.indices.putTemplate({
|
return client.indices.putTemplate({
|
||||||
body: mappings[sanitizedType],
|
body: config.default.mappings[sanitizedType],
|
||||||
name: sanitizedType,
|
name: sanitizedType,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,25 +1,3 @@
|
|||||||
import Ajv from 'ajv';
|
import {Validator} from '@openstapps/core-validator';
|
||||||
import addFormats from 'ajv-formats';
|
|
||||||
import schema from '@openstapps/core?json-schema';
|
|
||||||
|
|
||||||
export const validator = new Ajv.default({
|
export const validator = new Validator();
|
||||||
schemas: [schema],
|
|
||||||
verbose: true,
|
|
||||||
allowUnionTypes: true,
|
|
||||||
});
|
|
||||||
addFormats.default(validator, {
|
|
||||||
formats: ['date-time', 'time', 'uuid', 'duration'],
|
|
||||||
mode: 'fast',
|
|
||||||
});
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a validator function
|
|
||||||
* @example
|
|
||||||
* import schema from '@openstapps/core#schema:SCThings'
|
|
||||||
* createValidator<SCThings>(schema)
|
|
||||||
*/
|
|
||||||
export function createValidator<T>(schemaName: string): Ajv.ValidateFunction<T> {
|
|
||||||
return validator.compile({
|
|
||||||
$ref: `#/definitions/${schemaName}`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -78,7 +78,6 @@ describe('Create route', async function () {
|
|||||||
|
|
||||||
it('should complain (throw an error) if used method is other than defined in the route creation', async function () {
|
it('should complain (throw an error) if used method is other than defined in the route creation', async function () {
|
||||||
const methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
|
const methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
|
||||||
// @ts-expect-error not assignable
|
|
||||||
sandbox.stub(validator, 'validate').returns({errors: []});
|
sandbox.stub(validator, 'validate').returns({errors: []});
|
||||||
let error: any = {};
|
let error: any = {};
|
||||||
sandbox.stub(Logger, 'warn').callsFake(error_ => {
|
sandbox.stub(Logger, 'warn').callsFake(error_ => {
|
||||||
@@ -97,7 +96,6 @@ describe('Create route', async function () {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should provide a route which returns handler response and success code', async function () {
|
it('should provide a route which returns handler response and success code', async function () {
|
||||||
// @ts-expect-error not assignable
|
|
||||||
sandbox.stub(validator, 'validate').returns({errors: []});
|
sandbox.stub(validator, 'validate').returns({errors: []});
|
||||||
const router = createRoute<any, any>(routeClass, handler);
|
const router = createRoute<any, any>(routeClass, handler);
|
||||||
app.use(router);
|
app.use(router);
|
||||||
@@ -115,7 +113,6 @@ describe('Create route', async function () {
|
|||||||
app.use(router);
|
app.use(router);
|
||||||
const startApp = supertest(app);
|
const startApp = supertest(app);
|
||||||
const validatorStub = sandbox.stub(validator, 'validate');
|
const validatorStub = sandbox.stub(validator, 'validate');
|
||||||
// @ts-expect-error not assignable
|
|
||||||
validatorStub.withArgs(body, routeClass.requestBodyName).returns({errors: [new Error('Foo Error')]});
|
validatorStub.withArgs(body, routeClass.requestBodyName).returns({errors: [new Error('Foo Error')]});
|
||||||
|
|
||||||
const response = await startApp
|
const response = await startApp
|
||||||
@@ -131,11 +128,9 @@ describe('Create route', async function () {
|
|||||||
const router = createRoute<any, any>(routeClass, handler);
|
const router = createRoute<any, any>(routeClass, handler);
|
||||||
await app.use(router);
|
await app.use(router);
|
||||||
const startApp = supertest(app);
|
const startApp = supertest(app);
|
||||||
// @ts-expect-error not assignable
|
|
||||||
const validatorStub = sandbox.stub(validator, 'validate').returns({errors: []});
|
const validatorStub = sandbox.stub(validator, 'validate').returns({errors: []});
|
||||||
validatorStub
|
validatorStub
|
||||||
.withArgs(bodySuccess, routeClass.responseBodyName)
|
.withArgs(bodySuccess, routeClass.responseBodyName)
|
||||||
// @ts-expect-error not assignable
|
|
||||||
.returns({errors: [new Error('Foo Error')]});
|
.returns({errors: [new Error('Foo Error')]});
|
||||||
|
|
||||||
const response = await startApp.post(routeClass.urlPath).send();
|
const response = await startApp.post(routeClass.urlPath).send();
|
||||||
@@ -177,7 +172,6 @@ describe('Create route', async function () {
|
|||||||
await app.use(router);
|
await app.use(router);
|
||||||
const startApp = supertest(app);
|
const startApp = supertest(app);
|
||||||
|
|
||||||
// @ts-expect-error not assignable
|
|
||||||
sandbox.stub(validator, 'validate').returns({errors: []});
|
sandbox.stub(validator, 'validate').returns({errors: []});
|
||||||
|
|
||||||
const response = await startApp.post(routeClass.urlPath).send();
|
const response = await startApp.post(routeClass.urlPath).send();
|
||||||
@@ -213,7 +207,6 @@ describe('Create route', async function () {
|
|||||||
await app.use(router);
|
await app.use(router);
|
||||||
const startApp = supertest(app);
|
const startApp = supertest(app);
|
||||||
|
|
||||||
// @ts-expect-error not assignable
|
|
||||||
sandbox.stub(validator, 'validate').returns({errors: []});
|
sandbox.stub(validator, 'validate').returns({errors: []});
|
||||||
|
|
||||||
const response = await startApp.post(routeClass.urlPath).send();
|
const response = await startApp.post(routeClass.urlPath).send();
|
||||||
|
|||||||
@@ -22,11 +22,12 @@ import got, {Options} from 'got';
|
|||||||
import nock from 'nock';
|
import nock from 'nock';
|
||||||
import sinon from 'sinon';
|
import sinon from 'sinon';
|
||||||
import {mockReq} from 'sinon-express-mock';
|
import {mockReq} from 'sinon-express-mock';
|
||||||
import {plugins, validator} from '../../src/common.js';
|
import {plugins} from '../../src/common.js';
|
||||||
import {virtualPluginRoute} from '../../src/routes/virtual-plugin-route.js';
|
import {virtualPluginRoute} from '../../src/routes/virtual-plugin-route.js';
|
||||||
import {DEFAULT_TEST_TIMEOUT, FooError} from '../common.js';
|
import {DEFAULT_TEST_TIMEOUT, FooError} from '../common.js';
|
||||||
import {registerAddRequest} from './plugin-register-route.spec.js';
|
import {registerAddRequest} from './plugin-register-route.spec.js';
|
||||||
import {testApp} from '../tests-setup.js';
|
import {testApp} from '../tests-setup.js';
|
||||||
|
import {validator} from '../../src/validator.js';
|
||||||
|
|
||||||
use(chaiAsPromised);
|
use(chaiAsPromised);
|
||||||
|
|
||||||
@@ -71,7 +72,6 @@ describe('Virtual plugin routes', async function () {
|
|||||||
// spy the post method of got
|
// spy the post method of got
|
||||||
// @ts-expect-error not assignable
|
// @ts-expect-error not assignable
|
||||||
const gotStub = sandbox.stub(got, 'post').returns({body: {}});
|
const gotStub = sandbox.stub(got, 'post').returns({body: {}});
|
||||||
// @ts-expect-error not assignable
|
|
||||||
sandbox.stub(validator, 'validate').returns({errors: []});
|
sandbox.stub(validator, 'validate').returns({errors: []});
|
||||||
const request_ = mockReq(request);
|
const request_ = mockReq(request);
|
||||||
|
|
||||||
|
|||||||
@@ -7,4 +7,5 @@ export default defineConfig({
|
|||||||
target: 'es2022',
|
target: 'es2022',
|
||||||
format: 'esm',
|
format: 'esm',
|
||||||
outDir: 'lib',
|
outDir: 'lib',
|
||||||
|
noExternal: [/\.json$/],
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ type NameOf<I extends SchemaMap[keyof SchemaMap]> = keyof IncludeProperty<Schema
|
|||||||
export class Validator {
|
export class Validator {
|
||||||
private readonly ajv: Ajv.default;
|
private readonly ajv: Ajv.default;
|
||||||
|
|
||||||
|
get errors() {
|
||||||
|
return this.ajv.errors;
|
||||||
|
}
|
||||||
|
|
||||||
constructor(additionalSchemas: AnySchema[] = []) {
|
constructor(additionalSchemas: AnySchema[] = []) {
|
||||||
this.ajv = new Ajv.default({
|
this.ajv = new Ajv.default({
|
||||||
schemas: [schema, ...additionalSchemas],
|
schemas: [schema, ...additionalSchemas],
|
||||||
@@ -36,12 +40,21 @@ export class Validator {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add additional schemas to the validator
|
||||||
|
*/
|
||||||
|
public addSchema(...schema: AnySchema[]) {
|
||||||
|
this.ajv.addSchema(schema);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates anything against a given schema name or infers schema name from object
|
* Validates anything against a given schema name or infers schema name from object
|
||||||
* @param instance Instance to validate
|
* @param instance Instance to validate
|
||||||
* @param schema Name of schema to validate instance against or the schema itself
|
* @param schema Name of schema to validate instance against or the schema itself
|
||||||
*/
|
*/
|
||||||
public validate<T>(instance: unknown, schema: NameOf<T>): instance is T {
|
public validate<T extends SchemaMap[keyof SchemaMap]>(instance: unknown, schema: NameOf<T>): instance is T;
|
||||||
return this.ajv.validate(schema as string, instance);
|
public validate(instance: unknown, schema: Ajv.Schema): boolean;
|
||||||
|
public validate(instance: unknown, schema: Ajv.Schema | string): boolean {
|
||||||
|
return this.ajv.validate(schema, instance);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,7 +32,11 @@
|
|||||||
},
|
},
|
||||||
"./schema.json": {
|
"./schema.json": {
|
||||||
"import": "./lib/schema.json",
|
"import": "./lib/schema.json",
|
||||||
"types": "./lib/schema.d.ts"
|
"types": "./lib/schema.json.d.ts"
|
||||||
|
},
|
||||||
|
"./elasticsearch-mappings.json": {
|
||||||
|
"import": "./lib/elasticsearch.json",
|
||||||
|
"types": "./lib/elasticsearch.json.d.ts"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
|
|||||||
@@ -24,6 +24,8 @@
|
|||||||
"test": "c8 mocha"
|
"test": "c8 mocha"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"ajv": "8.12.0",
|
||||||
|
"ajv-formats": "2.1.1",
|
||||||
"@elastic/elasticsearch": "8.10.0",
|
"@elastic/elasticsearch": "8.10.0",
|
||||||
"@openstapps/json-schema-generator": "workspace:*",
|
"@openstapps/json-schema-generator": "workspace:*",
|
||||||
"@openstapps/tsup-plugin": "workspace:*",
|
"@openstapps/tsup-plugin": "workspace:*",
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright (C) 2019-2021 StApps
|
|
||||||
* This program is free software: you can redistribute it and/or modify it
|
|
||||||
* under the terms of the GNU General Public License as published by the Free
|
|
||||||
* Software Foundation, version 3.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
||||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
* more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License along with
|
|
||||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
declare module 'aggregations.json' {
|
|
||||||
const value: AggregationSchema;
|
|
||||||
export default value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An elasticsearch bucket aggregation
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
|
|
||||||
*/
|
|
||||||
export interface AggregationSchema {
|
|
||||||
[aggregationName: string]: ESTermsFilter | ESNestedAggregation;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An elasticsearch terms filter
|
|
||||||
*/
|
|
||||||
export interface ESTermsFilter {
|
|
||||||
/**
|
|
||||||
* Terms filter definition
|
|
||||||
*/
|
|
||||||
terms: {
|
|
||||||
/**
|
|
||||||
* Field to apply filter to
|
|
||||||
*/
|
|
||||||
field: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Number of results
|
|
||||||
*/
|
|
||||||
size?: number;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filter that filters by name of the the field type
|
|
||||||
*/
|
|
||||||
export interface ESAggTypeFilter {
|
|
||||||
/**
|
|
||||||
* The type of the object to find
|
|
||||||
*/
|
|
||||||
term: {
|
|
||||||
/**
|
|
||||||
* The name of the type
|
|
||||||
*/
|
|
||||||
type: string;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Filter that matches everything
|
|
||||||
*/
|
|
||||||
export interface ESAggMatchAllFilter {
|
|
||||||
/**
|
|
||||||
* Filter that matches everything
|
|
||||||
*/
|
|
||||||
match_all: object;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* For nested aggregations
|
|
||||||
*/
|
|
||||||
export interface ESNestedAggregation {
|
|
||||||
/**
|
|
||||||
* Possible nested Aggregations
|
|
||||||
*/
|
|
||||||
aggs: AggregationSchema;
|
|
||||||
/**
|
|
||||||
* Possible filter for types
|
|
||||||
*/
|
|
||||||
filter: ESAggTypeFilter | ESAggMatchAllFilter;
|
|
||||||
}
|
|
||||||
127
packages/es-mapping-generator/schema/mappings.d.ts
vendored
127
packages/es-mapping-generator/schema/mappings.d.ts
vendored
@@ -1,124 +1,9 @@
|
|||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
import {IndicesPutTemplateRequest, SearchRequest} from '@elastic/elasticsearch/lib/api/types.js';
|
||||||
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types';
|
|
||||||
/*
|
|
||||||
* Copyright (C) 2019-2021 StApps
|
|
||||||
* This program is free software: you can redistribute it and/or modify it
|
|
||||||
* under the terms of the GNU General Public License as published by the Free
|
|
||||||
* Software Foundation, version 3.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
||||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
* more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License along with
|
|
||||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
import {AggregationSchema} from './aggregations';
|
|
||||||
|
|
||||||
declare module 'mappings.json' {
|
export interface ElasticsearchConfig {
|
||||||
const value: ElasticsearchTemplateCollection;
|
mappings: Record<string, IndicesPutTemplateRequest>;
|
||||||
export default value;
|
search: Partial<SearchRequest>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
declare const elasticsearchConfig: ElasticsearchConfig;
|
||||||
* Template output of the mapping generation
|
export default elasticsearchConfig;
|
||||||
*/
|
|
||||||
export interface MappingGenTemplate {
|
|
||||||
/**
|
|
||||||
* All generated aggregations
|
|
||||||
*/
|
|
||||||
aggregations: AggregationSchema;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* All errors that occurred
|
|
||||||
*/
|
|
||||||
errors: string[];
|
|
||||||
|
|
||||||
/**
|
|
||||||
* All mappings that were generated
|
|
||||||
*/
|
|
||||||
mappings: ElasticsearchTemplateCollection;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type SimpleType = MappingProperty['type'] &
|
|
||||||
('keyword' | 'float' | 'boolean' | 'date' | 'integer' | 'text');
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The Typemap is used to get the corresponding ElasticsearchDataType for a name provided by the ProjectReflection
|
|
||||||
*/
|
|
||||||
export interface ElasticsearchTypemap {
|
|
||||||
/**
|
|
||||||
* The `stringLiteral` type must always be provided
|
|
||||||
*/
|
|
||||||
stringLiteral: {
|
|
||||||
/**
|
|
||||||
* The default can be chosen freely, but must be provided
|
|
||||||
*/
|
|
||||||
default: SimpleType;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The name of the JS type, so for `number` it would be number
|
|
||||||
*/
|
|
||||||
[name: string]: {
|
|
||||||
/**
|
|
||||||
* The default ElasticsearchDataType that should be used, if no tag or only not implemented tags are found
|
|
||||||
*/
|
|
||||||
default: SimpleType;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The name of the tag, so for `@integer` it would be `integer`
|
|
||||||
*/
|
|
||||||
[name: string]: SimpleType;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The Fieldmap contains all tag names for fields and the corresponding fields
|
|
||||||
*
|
|
||||||
* The Fieldmap works in a similar fashion to the Typemap
|
|
||||||
*/
|
|
||||||
export interface ElasticsearchFieldmap {
|
|
||||||
/**
|
|
||||||
* The name of the tag, so for `@sortable` it would be `sortable`
|
|
||||||
*/
|
|
||||||
[name: string]: {
|
|
||||||
/**
|
|
||||||
* The default value if no parameter is provided
|
|
||||||
*/
|
|
||||||
default: {
|
|
||||||
/**
|
|
||||||
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}`
|
|
||||||
*
|
|
||||||
* We could also have used `default: any`, but this adds slightly more improved type-safety.
|
|
||||||
*/
|
|
||||||
[name: string]: any;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The tag parameters that will be ignored
|
|
||||||
*
|
|
||||||
* Some tag parameters might not be important for your implementation, so you can add their names here to not get
|
|
||||||
* any errors. The `default` will be used in that case.
|
|
||||||
*/
|
|
||||||
ignore: string[];
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The parameters of the tag, so for `@sortable ducet` it would be `ducet`
|
|
||||||
*/
|
|
||||||
[name: string]: {
|
|
||||||
/**
|
|
||||||
* To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}`
|
|
||||||
*
|
|
||||||
* We could also have used `default: any`, but this adds slightly more improved type-safety.
|
|
||||||
*/
|
|
||||||
[name: string]: any;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A collection of Elasticsearch Templates
|
|
||||||
*/
|
|
||||||
export type ElasticsearchTemplateCollection = Record<string, Omit<IndicesPutTemplateRequest, 'name'>>;
|
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ export class Context {
|
|||||||
this.propertyPath,
|
this.propertyPath,
|
||||||
new Map(),
|
new Map(),
|
||||||
);
|
);
|
||||||
const result = transformDefinition(derivedContext, definition);
|
const result = transformDefinition(derivedContext, definition!);
|
||||||
referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
|
referenceName ??= crypto.createHash('md5').update(JSON.stringify(result)).digest('hex');
|
||||||
|
|
||||||
this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});
|
this.generator.cache.set(referenceName, {mapping: result, dependencies: derivedContext.dependencies});
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import {ElasticsearchOptionsDSL} from '../dsl/schema.js';
|
|||||||
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
import {IndicesPutTemplateRequest, MappingProperty} from '@elastic/elasticsearch/lib/api/types.js';
|
||||||
import {MappingGenerator} from './mapping-generator.js';
|
import {MappingGenerator} from './mapping-generator.js';
|
||||||
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
|
import {getTags, INDEXABLE_TAG_NAME} from './tags.js';
|
||||||
|
import {ElasticsearchConfig} from '../../schema/mappings.js';
|
||||||
|
|
||||||
export interface GeneratorOptions {
|
export interface GeneratorOptions {
|
||||||
/**
|
/**
|
||||||
@@ -24,15 +25,16 @@ export interface GeneratorOptions {
|
|||||||
/**
|
/**
|
||||||
* Fully transform a project
|
* Fully transform a project
|
||||||
*/
|
*/
|
||||||
export function transformProject(project: JSONSchema7) {
|
export function transformProject(project: JSONSchema7): ElasticsearchConfig {
|
||||||
const context = new MappingGenerator(project, OPTIONS);
|
const context = new MappingGenerator(project, OPTIONS);
|
||||||
|
|
||||||
const results = [];
|
const results: Record<string, IndicesPutTemplateRequest> = {};
|
||||||
for (const name in project.definitions) {
|
for (const name in project.definitions) {
|
||||||
const definition = project.definitions[name];
|
const definition = project.definitions[name];
|
||||||
if (typeof definition !== 'object' || !getTags(definition).has(INDEXABLE_TAG_NAME)) continue;
|
if (typeof definition !== 'object' || !getTags(definition).has(INDEXABLE_TAG_NAME)) continue;
|
||||||
|
|
||||||
results.push(context.buildTemplate(name));
|
const [type, template] = context.buildTemplate(name);
|
||||||
|
results[type] = template;
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
mappings: results,
|
mappings: results,
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ export class MappingGenerator {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
buildTemplate(name: string): IndicesPutTemplateRequest {
|
buildTemplate(name: string): [string, IndicesPutTemplateRequest] {
|
||||||
const thingType = ((this.project.definitions![name] as JSONSchema7).properties!.type as JSONSchema7)
|
const thingType = ((this.project.definitions![name] as JSONSchema7).properties!.type as JSONSchema7)
|
||||||
.const;
|
.const;
|
||||||
if (typeof thingType !== 'string') {
|
if (typeof thingType !== 'string') {
|
||||||
@@ -71,6 +71,6 @@ export class MappingGenerator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return request;
|
return [thingType, request];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
import {transformProject} from './generator/index.js';
|
import {transformProject} from './generator/index.js';
|
||||||
import {SchemaConsumer} from '@openstapps/json-schema-generator';
|
import {SchemaConsumer} from '@openstapps/json-schema-generator';
|
||||||
|
import {readFile} from 'fs/promises';
|
||||||
|
import path from 'path';
|
||||||
|
import {fileURLToPath} from 'url';
|
||||||
|
|
||||||
|
const mappingTypes = await readFile(
|
||||||
|
path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'schema', 'mappings.d.ts'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* JSON Schema Generator plugin for Elasticsearch Mappings
|
* JSON Schema Generator plugin for Elasticsearch Mappings
|
||||||
@@ -10,6 +18,7 @@ export function elasticsearchMappingGenerator(fileName: string): [string, Schema
|
|||||||
function (schema) {
|
function (schema) {
|
||||||
return {
|
return {
|
||||||
[fileName]: JSON.stringify(transformProject(schema)),
|
[fileName]: JSON.stringify(transformProject(schema)),
|
||||||
|
[`${fileName}.d.ts`]: mappingTypes,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -24,10 +24,10 @@ export function compileSchema(path: string, tsconfig: string): [schma: JSONSchem
|
|||||||
Object.assign(fullSchema.definitions, generator.createSchema(schema).definitions);
|
Object.assign(fullSchema.definitions, generator.createSchema(schema).definitions);
|
||||||
}
|
}
|
||||||
|
|
||||||
const schemaTypes = `import {JSONSchema7} from 'json-schema';\n\nexport interface SchemaMap {\n${[
|
const schemaTypes = `import {JSONSchema7} from 'json-schema';\nimport * as index from './index.js';\n\nexport interface SchemaMap {\n${[
|
||||||
...schemaNames,
|
...schemaNames,
|
||||||
]
|
]
|
||||||
.map(schemaName => ` '${schemaName}': core.${schemaName};`)
|
.map(schemaName => ` '${schemaName}': index.${schemaName};`)
|
||||||
.join('\n')}\n}\n\nconst schema: JSONSchema7;\nexport default schema;`;
|
.join('\n')}\n}\n\nconst schema: JSONSchema7;\nexport default schema;`;
|
||||||
|
|
||||||
return [fullSchema, schemaTypes];
|
return [fullSchema, schemaTypes];
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ export function jsonSchemaPlugin(
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
[schemaName]: JSON.stringify(jsonSchema),
|
[schemaName]: JSON.stringify(jsonSchema),
|
||||||
[`${schemaName.replace(/\.json$/, '')}.d.ts`]: types,
|
[`${schemaName}.d.ts`]: types,
|
||||||
};
|
};
|
||||||
}).call(this);
|
}).call(this);
|
||||||
|
|
||||||
|
|||||||
9
pnpm-lock.yaml
generated
9
pnpm-lock.yaml
generated
@@ -58,6 +58,9 @@ importers:
|
|||||||
'@openstapps/core':
|
'@openstapps/core':
|
||||||
specifier: workspace:*
|
specifier: workspace:*
|
||||||
version: link:../../packages/core
|
version: link:../../packages/core
|
||||||
|
'@openstapps/core-validator':
|
||||||
|
specifier: workspace:*
|
||||||
|
version: link:../../packages/core-validator
|
||||||
'@openstapps/logger':
|
'@openstapps/logger':
|
||||||
specifier: workspace:*
|
specifier: workspace:*
|
||||||
version: link:../../packages/logger
|
version: link:../../packages/logger
|
||||||
@@ -1834,6 +1837,12 @@ importers:
|
|||||||
'@types/json-schema':
|
'@types/json-schema':
|
||||||
specifier: 7.0.14
|
specifier: 7.0.14
|
||||||
version: 7.0.14
|
version: 7.0.14
|
||||||
|
ajv:
|
||||||
|
specifier: 8.12.0
|
||||||
|
version: 8.12.0
|
||||||
|
ajv-formats:
|
||||||
|
specifier: 2.1.1
|
||||||
|
version: 2.1.1(ajv@8.12.0)
|
||||||
devDependencies:
|
devDependencies:
|
||||||
'@openstapps/eslint-config':
|
'@openstapps/eslint-config':
|
||||||
specifier: workspace:*
|
specifier: workspace:*
|
||||||
|
|||||||
Reference in New Issue
Block a user