Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-19 08:02:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: update dependencies and fix resulting errors

Browse Source 
Upgraded JSON Schema from version 6 to version 7
Upgraded TypeDoc version to latest
Replaced 'jsonschema' with 'json-schema' package to better comply with 'ts-json-schema-generator'
Replace JSON Schema validation with AJV in areas where it wasn't used previously
Removed commander help output as it causes strange issues
This commit is contained in:
Wieland Schöbl
2020-02-14 11:40:53 +01:00
committed by Rainer Killinger
parent b7cdb6a9ad
commit 5330255b7e
19 changed files with 2058 additions and 1145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/cli.ts
View File

@@ -15,7 +15,7 @@
import {Logger} from '@openstapps/logger';
import {Command} from 'commander';
import {existsSync, readFileSync, writeFileSync} from 'fs';
import * as got from 'got';
import got from 'got';
import {join, resolve} from 'path';
import {exit} from 'process';
import {
@@ -158,8 +158,7 @@ commander
}
const response = await got.put(`${esAddress}_template/${template}`, {
body: result.mappings[template],
json: true,
json: result.mappings[template],
});
const HTTP_STATUS_OK = 200;
@@ -353,8 +352,3 @@ commander
});
commander.parse(process.argv);
if (commander.args.length < 1) {
commander.outputHelp();
process.exit(1);
}

92
src/common.ts
View File

@@ -15,11 +15,12 @@
import {Logger} from '@openstapps/logger';
import {existsSync, mkdir, PathLike, readFile, unlink, writeFile} from 'fs';
import {Glob} from 'glob';
import {Schema as JSONSchema, ValidationError} from 'jsonschema';
import {JSONSchema7 as JSONSchema} from 'json-schema';
import {platform} from 'os';
import {join, sep} from 'path';
import {Definition} from 'ts-json-schema-generator';
import {Application, ProjectReflection} from 'typedoc';
import {ModuleKind, ScriptTarget} from 'typescript';
import {promisify} from 'util';
import {LightweightType} from './uml/model/lightweight-type';
@@ -122,9 +123,67 @@ interface SchemaWithDefinitions extends JSONSchema {
}
/**
* An expectable error
* The validation result
*/
export interface ExpectableValidationError extends ValidationError {
export interface ValidationResult {
/**
* A list of errors that occurred
*/
errors: ValidationError[];
/**
* whether the validation was successful
*/
valid: boolean;
}
/**
* An error that occurred while validating
*
* This is a duplicate of the ValidationError in core/protocol/errors/validation because of incompatibilities
* between TypeDoc and TypeScript
*/
export interface ValidationError {
/**
* JSON schema path
*/
dataPath: string;
/**
* The instance
*/
instance: unknown;
/**
* The message
*
* Provided by https://www.npmjs.com/package/better-ajv-errors
*/
message: string;
/**
* Name of the error
*/
name: string;
/**
* Path within the Schema
*/
schemaPath: string;
/**
* Suggestion to fix the occurring error
*
* Provided by https://www.npmjs.com/package/better-ajv-errors
*/
suggestion?: string;
}
/**
* An expected error
*/
export interface ExpectedValidationError extends ValidationError {
/**
* Whether or not the error is expected
*/
@@ -132,10 +191,10 @@ export interface ExpectableValidationError extends ValidationError {
}
/**
* A map of files and their expectable validation errors
* A map of files and their expected validation errors
*/
export interface ExpectableValidationErrors {
[fileName: string]: ExpectableValidationError[];
export interface ExpectedValidationErrors {
[fileName: string]: ExpectedValidationError[];
}
/**
@@ -150,10 +209,14 @@ export function getProjectReflection(srcPath: PathLike, excludeExternals = true)
const tsconfigPath = getTsconfigPath(srcPath.toString());
// initialize new Typedoc application
const app = new Application({
const app = new Application();
app.options.setValues({
excludeExternals: excludeExternals,
ignoreCompilerErrors: false, // TODO: true
includeDeclarations: true,
module: 'commonjs',
module: ModuleKind.CommonJS,
target: ScriptTarget.Latest,
tsconfig: join(tsconfigPath, 'tsconfig.json'),
});
@@ -194,10 +257,11 @@ export function isSchemaWithDefinitions(
*/
export function isThingWithType(thing: unknown): thing is { type: string; } {
return typeof thing === 'object' &&
thing !== null &&
'type' in thing &&
typeof (thing as { type: string; }).type === 'string';
thing !== null &&
'type' in thing &&
typeof (thing as { type: unknown; }).type === 'string';
}
// tslint:enable: completed-docs
/**
@@ -233,16 +297,16 @@ export function getTsconfigPath(startPath: string): string {
}
/**
* Converts a comma seperated string into a string array
* Converts a comma separated string into a string array
*
* @param val Comma seperated string
* @param val Comma separated string
*/
export function toArray(val: string): string[] {
return val.split(',');
}
/**
* Creates the full name of a lightweight type recursivly
* Creates the full name of a lightweight type recursively
*
* @param type Type to get the full name of
*/

37
src/mapping.ts
View File

@@ -51,6 +51,9 @@ const indexableTag = 'indexable';
const aggregatableTag = 'aggregatable';
const aggregatableTagParameterGlobal = 'global';
// clamp printed object to 1000 chars to keep error messages readable
const maxErrorObjectChars = 1000;
let ignoredTagsList = ['indexable', 'validatable'];
/**
@@ -91,7 +94,7 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
}
/**
* Composes error messages, that are readable and contain a certain minumum of information
* Composes error messages, that are readable and contain a certain minimum of information
*
* @param path the path where the error took place
* @param topTypeName the name of the SCThingType
@@ -100,14 +103,27 @@ export function getAllIndexableInterfaces(projectReflection: ProjectReflection):
* @param message the error message
*/
function composeErrorMessage(path: string, topTypeName: string, typeName: string, object: string, message: string) {
const error = `At "${topTypeName}::${path.substr(0, path.length - 1)}" for ${typeName} "${object}": ${message}`;
const error = `At "${topTypeName}::${path.substr(0, path.length - 1)}" for ${typeName} "${trimString(object, maxErrorObjectChars)}": ${message}`;
errors.push(error);
if (showErrors) {
// tslint:disable-next-line:no-floating-promises
Logger.error(error);
Logger.error(error)
.then();
}
}
/**
* Trims a string to a readable size and appends "..."
*
* @param value the string to trim
* @param maxLength the maximum allowed length before it is clamped
*/
function trimString(value: string, maxLength: number): string {
return value.length > maxLength ?
`${value.substring(0, maxLength)}...` :
value;
}
/**
* Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection
*
@@ -116,6 +132,7 @@ function composeErrorMessage(path: string, topTypeName: string, typeName: string
*
* @param type the ReferenceType of a DeclarationReflection
* @param out the previous reflection, it then overrides all parameters or keeps old ones
* @param topTypeName the name of the object, with which something went wrong
* @param path the current path to the object we are in
* @param tags any tags attached to the type
*/
@@ -161,7 +178,7 @@ function getReflectionGeneric(type: ReferenceType,
function handleExternalType(ref: ReferenceType, generics: Map<string, ElasticsearchValue>,
path: string, topTypeName: string, tags: CommentTag[]): ElasticsearchValue {
for (const premap in premaps) {
if (premap === ref.name) {
if (premap === ref.name && premaps.hasOwnProperty(premap)) {
return readFieldTags(premaps[premap], path, topTypeName, tags);
}
}
@@ -231,9 +248,9 @@ function handleDeclarationReflection(decl: DeclarationReflection,
const template: ElasticsearchDynamicTemplate = {};
template[decl.name] = {
mapping: handleType(
decl.indexSignature.type,
new Map(generics), path, topTypeName,
getCommentTags(decl.indexSignature)),
decl.indexSignature.type,
new Map(generics), path, topTypeName,
getCommentTags(decl.indexSignature)),
match: '*',
match_mapping_type: '*',
path_match: `${path}*`,
@@ -619,14 +636,16 @@ export function generateTemplate(projectReflection: ProjectReflection,
.replace('"', '');
} else {
// tslint:disable-next-line:no-floating-promises
Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.');
Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.')
.then();
}
} else if (typeObject.type instanceof StringLiteralType) {
Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`);
typeName = typeObject.type.value;
} else {
// tslint:disable-next-line:no-floating-promises
Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`);
Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`)
.then();
}
// filter out

4
src/pack.ts
View File

@@ -435,9 +435,9 @@ function topologicalSort(modules: JavaScriptModule[]): JavaScriptModule[] {
// add all edges
modules.forEach((module) => {
module.dependencies.forEach((dependenciePath) => {
module.dependencies.forEach((dependencyPath) => {
// add edge from dependency to our module
edges.push([basename(dependenciePath), module.name]);
edges.push([basename(dependencyPath), module.name]);
});
nodes.push(module.name);

14
src/schema.ts
View File

@@ -13,9 +13,9 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import Ajv from 'ajv';
import {Schema as JSONSchema} from 'jsonschema';
import {JSONSchema7 as JSONSchema} from 'json-schema';
import {join} from 'path';
import {DEFAULT_CONFIG, Definition, SchemaGenerator} from 'ts-json-schema-generator';
import {Config, DEFAULT_CONFIG, Definition, SchemaGenerator} from 'ts-json-schema-generator';
import {createFormatter} from 'ts-json-schema-generator/dist/factory/formatter';
import {createParser} from 'ts-json-schema-generator/dist/factory/parser';
import {createProgram} from 'ts-json-schema-generator/dist/factory/program';
@@ -45,13 +45,11 @@ export class Converter {
*/
constructor(path: string) {
// set config for schema generator
const config = {
const config: Config = {
...DEFAULT_CONFIG,
// expose: 'exported' as any,
// jsDoc: 'extended' as any,
path: join(getTsconfigPath(path), 'tsconfig.json'),
sortProps: true,
topRef: false,
tsconfig: join(getTsconfigPath(path), 'tsconfig.json'),
type: 'SC',
};
@@ -82,7 +80,7 @@ export class Converter {
const schema: JSONSchema = this.generator.createSchema(type);
// set id of schema
schema.id = `https://core.stapps.tu-berlin.de/v${version}/lib/schema/${type}.json`;
schema.$id = `https://core.stapps.tu-berlin.de/v${version}/lib/schema/${type}.json`;
if (isSchemaWithDefinitions(schema)) {
const selfReference = {
@@ -92,7 +90,7 @@ export class Converter {
delete selfReference.$schema;
delete selfReference.definitions;
delete selfReference.id;
delete selfReference.$id;
// add self reference to definitions
schema.definitions[`SC${type}`] = {

4
src/uml/create-diagram.ts
View File

@@ -87,7 +87,7 @@ export async function createDiagram(
/**
* This will encode the plantuml code and post the code to the plantuml server
* The server will then parse the code and create a corresponding diagram
*
*
* @param modelPlantUMLCode raw PlantUML code
* @param plantUmlBaseURL PlantUML server address that shall be used
* @param outputFile filename of the output file without file extension
@@ -102,7 +102,7 @@ export async function createDiagramFromString(
const url = `${plantUmlBaseURL}/svg/${plantUMLCode}`;
let response;
try {
response = await request.get(url);
response = await request.default.get(url);
const httpOK = 200;
if (response.statusCode !== httpOK) {
await Logger.error(`Plantuml Server responded with an error.\n${response.statusMessage}`);

41
src/uml/read-definitions.ts
View File

@@ -15,9 +15,11 @@
import {Logger} from '@openstapps/logger';
import {
ArrayType,
ConditionalType,
DeclarationReflection,
IntrinsicType,
ProjectReflection,
QueryType,
ReferenceType,
ReflectionKind,
ReflectionType,
@@ -248,10 +250,43 @@ function readTypeInformation(declarationType: Type): LightweightType {
if (declarationType instanceof UnionType) {
return readAsUnionType(declarationType);
}
if (declarationType instanceof QueryType) {
return readAsQueryType(declarationType);
}
if (declarationType instanceof ConditionalType) {
return readAsConditionalType(declarationType);
}
throw new Error(`Could not read type ${declarationType.type}`);
}
/**
* Conversion method for ConditionalTypes
*
* @param _type Type to be converted
*/
function readAsConditionalType(_type: ConditionalType): LightweightType {
const returnType: LightweightType = new LightweightType();
returnType.specificationTypes = [];
returnType.name = getFullTypeName(returnType);
returnType.isUnion = true;
return returnType;
}
/**
* Conversion method for QueryTypes
*
* @param type Type to be converted
*/
function readAsQueryType(type: QueryType): LightweightType {
const out = readAsReferenceType(type.queryType);
out.isReference = true;
return out;
}
/**
* Conversion method for IntrinsicType's
*
@@ -315,10 +350,8 @@ function readAsReferenceType(type: ReferenceType): LightweightType {
// interfaces and classes in a type are a sink, since their declaration are defined elsewhere
if (
typeof tempTypeReflection.kindString !== 'undefined' &&
['Interface', 'Class', 'Enumeration', 'Type alias'].indexOf(
tempTypeReflection.kindString,
) > -1
) {
['Interface', 'Class', 'Enumeration', 'Type alias'].includes(
tempTypeReflection.kindString)) {
returnType.isReference = true;
}
}

122
src/validate.ts
View File

@@ -14,15 +14,20 @@
*/
import {asyncPool} from '@krlwlfrt/async-pool';
import {Logger} from '@openstapps/logger';
import Ajv from 'ajv';
import betterAjvErrors from 'better-ajv-errors';
import {PathLike} from 'fs';
import {Schema, Validator as JSONSchemaValidator, ValidatorResult} from 'jsonschema';
import {JSONSchema7} from 'json-schema';
import * as mustache from 'mustache';
import {basename, join, resolve} from 'path';
import {Schema} from 'ts-json-schema-generator';
import {
ExpectableValidationErrors,
ExpectedValidationErrors,
globPromisified,
isThingWithType,
readFilePromisified,
ValidationError,
ValidationResult,
writeFilePromisified,
} from './common';
@@ -30,21 +35,24 @@ import {
* StAppsCore validator
*/
export class Validator {
/**
* JSON Schema Validator
*/
private readonly ajv = Ajv({verbose: true, jsonPointers: true, extendRefs: true});
/**
* Map of schema names to schemas
*/
private readonly schemas: { [type: string]: Schema; } = {};
/**
* JSONSchema validator instance
* A wrapper function for Ajv that transforms the error into the compatible old error
*
* @param schema the schema that will be validated against
* @param instance the instance that will be validated
*/
private readonly validator: JSONSchemaValidator;
/**
* Create a new validator
*/
constructor() {
this.validator = new JSONSchemaValidator();
private ajvValidateWrapper(schema: JSONSchema7, instance: unknown): ValidationResult {
return fromAjvResult(this.ajv.validate(schema, instance), schema, instance, this.ajv);
}
/**
@@ -65,13 +73,9 @@ export class Validator {
await asyncPool(2, schemaFiles, async (file: string) => {
// read schema file
const buffer = await readFilePromisified(file);
const schema = JSON.parse(buffer.toString());
// add schema to validator
this.validator.addSchema(schema);
// add schema to map
this.schemas[basename(file, '.json')] = schema;
this.schemas[basename(file, '.json')] = JSON.parse(buffer.toString());
Logger.info(`Added ${file} to validator.`);
});
@@ -85,17 +89,17 @@ export class Validator {
* @param instance Instance to validate
* @param schema Name of schema to validate instance against or the schema itself
*/
public validate(instance: unknown, schema?: string | Schema): ValidatorResult {
public validate(instance: unknown, schema?: string | Schema): ValidationResult {
if (typeof schema === 'undefined') {
if (isThingWithType(instance)) {
// schema name can be infered from type string
// schema name can be inferred from type string
// tslint:disable-next-line: completed-docs
const schemaSuffix = (instance as { type: string; }).type.split(' ')
.map((part: string) => {
.map((part: string) => {
return part.substr(0, 1)
.toUpperCase() + part.substr(1);
.toUpperCase() + part.substr(1);
})
.join('');
.join('');
const schemaName = `SC${schemaSuffix}`;
return this.validate(instance, schemaName);
@@ -108,20 +112,61 @@ export class Validator {
throw new Error(`No schema available for ${schema}.`);
}
return this.validator.validate(instance, this.schemas[schema]);
// schema will be cached
return this.ajvValidateWrapper(this.schemas[schema], instance);
}
return this.validator.validate(instance, schema);
return this.ajvValidateWrapper(schema, instance);
}
}
/**
* Creates a ValidationResult from ajv
*
* Implemented for compatibility purposes
*
* @param result the result, now a ValidationResult
* @param schema the schema that has been validated against
* @param instance the data that has been validated
* @param ajvInstance the ajv instance with which the validation was done
*/
function fromAjvResult(
result: boolean | PromiseLike<unknown>,
schema: JSONSchema7,
instance: unknown,
ajvInstance: Ajv.Ajv,
): ValidationResult {
// tslint:disable-next-line
// @ts-ignore function can return void, which at runtime will be undefined. TS doesn't allow to assign void to undefined
const betterErrorObject: betterAjvErrors.IOutputError[] | undefined =
betterAjvErrors(schema, instance, ajvInstance.errors, {format: 'js', indent: null});
return {
errors: ajvInstance.errors?.map((ajvError, index) => {
const error: ValidationError = {
dataPath: ajvError.dataPath,
instance: instance,
message: betterErrorObject?.[index].error!,
name: ajvError.keyword,
schemaPath: ajvError.schemaPath,
suggestion: betterErrorObject?.[index].suggestion,
};
// (validationError as ValidationError).humanReadableError = betterErrorCLI?.[index] as unknown as string;
return error;
}) ?? [],
valid: typeof result === 'boolean' ? result : false,
};
}
/**
* Validate all test files in the given resources directory against schema files in the given (schema) directory
*
* @param schemaDir The directory where the JSON schema files are
* @param resourcesDir The directory where the test files are
*/
export async function validateFiles(schemaDir: string, resourcesDir: string): Promise<ExpectableValidationErrors> {
export async function validateFiles(schemaDir: string, resourcesDir: string): Promise<ExpectedValidationErrors> {
// instantiate new validator
const v = new Validator();
await v.addSchemas(schemaDir);
@@ -136,7 +181,7 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
Logger.log(`Found ${testFiles.length} file(s) to test.`);
// map of errors per file
const errors: ExpectableValidationErrors = {};
const errors: ExpectedValidationErrors = {};
// tslint:disable-next-line:no-magic-numbers - iterate over files to test
await asyncPool(2, testFiles, async (testFile: string) => {
@@ -162,13 +207,11 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
// iterate over errors
for (const error of result.errors) {
// get idx of expected error
const errorIdx = expectedErrors.indexOf(error.name);
const errorIndex = expectedErrors.indexOf(error.name);
let expected = false;
if (errorIdx >= 0) {
// remove from list of expected errors
expectedErrors.splice(errorIdx, 1);
if (errorIndex >= 0) {
expectedErrors.splice(errorIndex, 1);
expected = true;
} else {
unexpectedErrors++;
@@ -188,14 +231,14 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
await Logger.error(`Extraneous expected error '${error}' in ${testFile}.`);
errors[testFileName].push({
argument: false,
dataPath: 'undefined',
expected: false,
instance: testDescription.instance,
message: `expected error ${error} did not occur`,
instance: undefined,
// instance: testDescription.instance,
message: 'undefined',
name: `expected ${error}`,
property: 'unknown',
schema: 'undefined',
stack: 'undefined',
schemaPath: 'undefined',
suggestion: 'undefined',
});
}
} else if (unexpectedErrors === 0) {
@@ -212,7 +255,7 @@ export async function validateFiles(schemaDir: string, resourcesDir: string): Pr
* @param reportPath Path to write report to
* @param errors Errors that occurred in validation
*/
export async function writeReport(reportPath: PathLike, errors: ExpectableValidationErrors): Promise<void> {
export async function writeReport(reportPath: PathLike, errors: ExpectedValidationErrors): Promise<void> {
let buffer = await readFilePromisified(resolve(__dirname, '..', 'resources', 'file.html.mustache'));
const fileTemplate = buffer.toString();
@@ -229,15 +272,16 @@ export async function writeReport(reportPath: PathLike, errors: ExpectableValida
let fileOutput = '';
errors[fileName].forEach((error, idx) => {
fileOutput += mustache.render(errorTemplate, {
idx: idx + 1,
// tslint:disable-next-line:no-magic-numbers
instance: JSON.stringify(error.instance, null, 2),
message: error.message,
property: error.property,
// tslint:disable-next-line:no-magic-numbers
schema: JSON.stringify(error.schema, null, 2),
name: error.name,
schemaPath: error.schemaPath,
status: (error.expected) ? 'alert-success' : 'alert-danger',
suggestion: error.suggestion,
});
});