feat: more fixes

backend/backend/config/f-u/about-pages/index.js

@@ -3,7 +3,7 @@ import about from './about.js';
 import imprint from './imprint.js';
 import privacy from './privacy.js';
 
-/** @type {import('@openstapps/core').SCMap<import('@openstapps/core').SCAboutPage>} */
+/** @type {Record<string, import('@openstapps/core').SCAboutPage>} */
 const aboutPages = {
   'about': about,
   'about/imprint': imprint,

backend/backend/package.json

@@ -45,6 +45,7 @@
   "dependencies": {
     "@elastic/elasticsearch": "8.10.0",
     "@openstapps/core": "workspace:*",
+    "@openstapps/core-validator": "workspace:*",
     "@openstapps/logger": "workspace:*",
     "@types/body-parser": "1.19.2",
     "@types/cors": "2.8.13",

backend/backend/src/app.ts

@@ -14,7 +14,6 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 import {
-  SCConfigFile,
   SCNotFoundErrorResponse,
   SCRequestBodyTooLargeErrorResponse,
   SCSyntaxErrorResponse,
@@ -39,7 +38,7 @@ import {virtualPluginRoute} from './routes/virtual-plugin-route.js';
 import {BulkStorage} from './storage/bulk-storage.js';
 import {DatabaseConstructor} from './storage/database.js';
 import {backendConfig} from './config.js';
-import {createValidator} from './validator.js';
+import {validator} from './validator.js';
 
 /**
  * Configure the backend
@@ -143,11 +142,8 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
     request.on('data', chunkGatherer).on('end', endCallback);
   });
 
-  const configFileValid = createValidator<SCConfigFile>('SCConfigFile');
-  if (!configFileValid(backendConfig)) {
-    throw new Error(
-      `Validation of config file failed. Errors were: ${JSON.stringify(configFileValid.errors)}`,
-    );
+  if (!validator.validate(backendConfig, 'SCConfigFile')) {
+    throw new Error(`Validation of config file failed. Errors were: ${JSON.stringify(validator.errors)}`);
   }
 
   // check if a database name was given

backend/backend/src/routes/route.ts

@@ -24,7 +24,7 @@ import {Application, Router} from 'express';
 import PromiseRouter from 'express-promise-router';
 import {isTestEnvironment} from '../common.js';
 import {isHttpMethod} from './http-types.js';
-import {createValidator} from '../validator.js';
+import {validator} from '../validator.js';
 
 /**
  * Creates a router from a route class and a handler function which implements the logic
@@ -44,8 +44,6 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
 ): Router {
   // create router
   const router = PromiseRouter({mergeParams: true});
-  const requestValidator = createValidator<REQUESTTYPE>(routeClass.requestBodyName);
-  const responseValidator = createValidator<RETURNTYPE>(routeClass.responseBodyName);
 
   // create route
   // the given type has no index signature so we have to cast to get the IRouteHandler when a HTTP method is given
@@ -58,8 +56,8 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
     // create a route handler for the given HTTP method
     route[verb](async (request, response) => {
       try {
-        if (!requestValidator(request.body)) {
-          const error = new SCValidationErrorResponse(requestValidator.errors as any, isTestEnvironment);
+        if (!validator.validate(request.body, routeClass.requestBodyName as never)) {
+          const error = new SCValidationErrorResponse(validator.errors, isTestEnvironment);
           response.status(error.statusCode);
           response.json(error);
           await Logger.error(error);
@@ -67,13 +65,10 @@ export function createRoute<REQUESTTYPE, RETURNTYPE>(
           return;
         }
 
-        const handlerResponse = await handler(request.body, request.app, request.params);
+        const handlerResponse = await handler(request.body as REQUESTTYPE, request.app, request.params);
 
-        if (!responseValidator(handlerResponse)) {
-          const validationError = new SCValidationErrorResponse(
-            responseValidator.errors as any,
-            isTestEnvironment,
-          );
+        if (!validator.validate(handlerResponse, routeClass.responseBodyName)) {
+          const validationError = new SCValidationErrorResponse(validator.errors, isTestEnvironment);
           // The validation error is not caused by faulty user input, but through an error that originates somewhere in
           // the backend, therefore we use this "stacked" error.
           const internalServerError = new SCInternalServerErrorResponse(validationError, isTestEnvironment);

backend/backend/src/routes/virtual-plugin-route.ts

@@ -13,7 +13,6 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-
 import {SCInternalServerErrorResponse, SCPluginMetaData, SCValidationErrorResponse} from '@openstapps/core';
 import {Request} from 'express';
 import got from 'got';
@@ -31,7 +30,7 @@ export async function virtualPluginRoute(request: Request, plugin: SCPluginMetaD
   try {
     if (!validator.validate(request.body, plugin.requestSchema)) {
       // noinspection ExceptionCaughtLocallyJS
-      throw new SCValidationErrorResponse(validator.errors as any, isTestEnvironment);
+      throw new SCValidationErrorResponse(validator.errors, isTestEnvironment);
     }
     // send the request to the plugin (forward the body) and save the response
     const response = await got.post(plugin.route.replaceAll(/^\//gi, ''), {
@@ -45,7 +44,7 @@ export async function virtualPluginRoute(request: Request, plugin: SCPluginMetaD
     const responseBody = response.body;
     if (!validator.validate(responseBody, plugin.responseSchema)) {
       // noinspection ExceptionCaughtLocallyJS
-      throw new SCValidationErrorResponse(validator.errors as any, isTestEnvironment);
+      throw new SCValidationErrorResponse(validator.errors, isTestEnvironment);
     }
     return responseBody as object;
   } catch (error) {

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -31,7 +31,7 @@ import {parseAggregations} from './aggregations.js';
 import * as Monitoring from './monitoring.js';
 import {buildQuery} from './query/query.js';
 import {buildSort} from './query/sort.js';
-import {aggregations, putTemplate} from './templating.js';
+import {putTemplate} from './templating.js';
 import {
   ElasticsearchConfig,
   ElasticsearchQueryDisMaxConfig,
@@ -46,6 +46,7 @@ import {
 } from './util/index.js';
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
+import config from '@openstapps/core/elasticsearch-mappings.json';
 
 /**
  * A database interface for elasticsearch
@@ -370,7 +371,7 @@ export class Elasticsearch implements Database {
     };
 
     const response: SearchResponse<SCThings> = await this.client.search({
-      aggs: aggregations,
+      aggs: config.default.search.aggs,
       query: buildQuery(parameters, this.config, esConfig),
       from: parameters.from,
       index: ACTIVE_INDICES_ALIAS,

backend/backend/src/storage/elasticsearch/templating.ts

@@ -15,19 +15,7 @@
  */
 import {Client} from '@elastic/elasticsearch';
 import {SCThingType} from '@openstapps/core';
-import type {AggregationSchema} from '@openstapps/core/lib/mappings/aggregations.json.js';
-import type {ElasticsearchTemplateCollection} from '@openstapps/core/lib/mappings/mappings.json.js';
-import {readFileSync} from 'fs';
-import path from 'path';
-
-const mappingsPath = path.resolve('node_modules', '@openstapps', 'core', 'lib', 'mappings');
-
-export const mappings = JSON.parse(
-  readFileSync(path.resolve(mappingsPath, 'mappings.json'), 'utf8'),
-) as ElasticsearchTemplateCollection;
-export const aggregations = JSON.parse(
-  readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
-) as AggregationSchema;
+import config from '@openstapps/core/elasticsearch-mappings.json';
 
 /**
  * Prepares all indices
@@ -40,7 +28,7 @@ export async function putTemplate(client: Client, type: SCThingType) {
   const sanitizedType = `template_${type.replaceAll(/\s/g, '_')}`;
 
   return client.indices.putTemplate({
-    body: mappings[sanitizedType],
+    body: config.default.mappings[sanitizedType],
     name: sanitizedType,
   });
 }

backend/backend/src/validator.ts

@@ -1,25 +1,3 @@
-import Ajv from 'ajv';
-import addFormats from 'ajv-formats';
-import schema from '@openstapps/core?json-schema';
+import {Validator} from '@openstapps/core-validator';
 
-export const validator = new Ajv.default({
-  schemas: [schema],
-  verbose: true,
-  allowUnionTypes: true,
-});
-addFormats.default(validator, {
-  formats: ['date-time', 'time', 'uuid', 'duration'],
-  mode: 'fast',
-});
-
-/**
- * Create a validator function
- * @example
- * import schema from '@openstapps/core#schema:SCThings'
- * createValidator<SCThings>(schema)
- */
-export function createValidator<T>(schemaName: string): Ajv.ValidateFunction<T> {
-  return validator.compile({
-    $ref: `#/definitions/${schemaName}`,
-  });
-}
+export const validator = new Validator();

backend/backend/test/routes/route.spec.ts

@@ -78,7 +78,6 @@ describe('Create route', async function () {
 
   it('should complain (throw an error) if used method is other than defined in the route creation', async function () {
     const methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
-    // @ts-expect-error not assignable
     sandbox.stub(validator, 'validate').returns({errors: []});
     let error: any = {};
     sandbox.stub(Logger, 'warn').callsFake(error_ => {
@@ -97,7 +96,6 @@ describe('Create route', async function () {
   });
 
   it('should provide a route which returns handler response and success code', async function () {
-    // @ts-expect-error not assignable
     sandbox.stub(validator, 'validate').returns({errors: []});
     const router = createRoute<any, any>(routeClass, handler);
     app.use(router);
@@ -115,7 +113,6 @@ describe('Create route', async function () {
     app.use(router);
     const startApp = supertest(app);
     const validatorStub = sandbox.stub(validator, 'validate');
-    // @ts-expect-error not assignable
     validatorStub.withArgs(body, routeClass.requestBodyName).returns({errors: [new Error('Foo Error')]});
 
     const response = await startApp
@@ -131,11 +128,9 @@ describe('Create route', async function () {
     const router = createRoute<any, any>(routeClass, handler);
     await app.use(router);
     const startApp = supertest(app);
-    // @ts-expect-error not assignable
     const validatorStub = sandbox.stub(validator, 'validate').returns({errors: []});
     validatorStub
       .withArgs(bodySuccess, routeClass.responseBodyName)
-      // @ts-expect-error not assignable
       .returns({errors: [new Error('Foo Error')]});
 
     const response = await startApp.post(routeClass.urlPath).send();
@@ -177,7 +172,6 @@ describe('Create route', async function () {
     await app.use(router);
     const startApp = supertest(app);
 
-    // @ts-expect-error not assignable
     sandbox.stub(validator, 'validate').returns({errors: []});
 
     const response = await startApp.post(routeClass.urlPath).send();
@@ -213,7 +207,6 @@ describe('Create route', async function () {
     await app.use(router);
     const startApp = supertest(app);
 
-    // @ts-expect-error not assignable
     sandbox.stub(validator, 'validate').returns({errors: []});
 
     const response = await startApp.post(routeClass.urlPath).send();

backend/backend/test/routes/virtual-plugin-route.spec.ts

@@ -22,11 +22,12 @@ import got, {Options} from 'got';
 import nock from 'nock';
 import sinon from 'sinon';
 import {mockReq} from 'sinon-express-mock';
-import {plugins, validator} from '../../src/common.js';
+import {plugins} from '../../src/common.js';
 import {virtualPluginRoute} from '../../src/routes/virtual-plugin-route.js';
 import {DEFAULT_TEST_TIMEOUT, FooError} from '../common.js';
 import {registerAddRequest} from './plugin-register-route.spec.js';
 import {testApp} from '../tests-setup.js';
+import {validator} from '../../src/validator.js';
 
 use(chaiAsPromised);
 
@@ -71,7 +72,6 @@ describe('Virtual plugin routes', async function () {
       // spy the post method of got
       // @ts-expect-error not assignable
       const gotStub = sandbox.stub(got, 'post').returns({body: {}});
-      // @ts-expect-error not assignable
       sandbox.stub(validator, 'validate').returns({errors: []});
       const request_ = mockReq(request);
 

backend/backend/tsup.config.ts

@@ -7,4 +7,5 @@ export default defineConfig({
   target: 'es2022',
   format: 'esm',
   outDir: 'lib',
+  noExternal: [/\.json$/],
 });