feat: add core tools

src/cli.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright (C) 2018 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import * as commander from 'commander';
+import {existsSync, readFileSync, writeFileSync} from 'fs';
+import {join, resolve} from 'path';
+import {getProjectReflection, logger, mkdirPromisified, readFilePromisifed} from './common';
+import {gatherRouteInformation, generateDocumentationForRoute, getNodeMetaInformationMap} from './routes';
+import {Converter, getValidatableTypesFromReflection} from './schema';
+import {validateFiles, writeReport} from './validate';
+
+// handle unhandled promise rejections
+process.on('unhandledRejection', (error: Error) => {
+  logger.error(error.message);
+  logger.info(error.stack);
+  process.exit(1);
+});
+
+commander
+  .version(JSON.parse(readFileSync(resolve(__dirname, '..', 'package.json')).toString()).version);
+
+commander
+  .command('routes <srcPath> <mdPath>')
+  .action(async (relativeSrcPath, relativeMdPath) => {
+    // get absolute paths
+    const srcPath = resolve(relativeSrcPath);
+    const mdPath = resolve(relativeMdPath);
+
+    // get project reflection
+    const projectReflection = getProjectReflection(srcPath);
+
+    // get information about routes
+    const routes = await gatherRouteInformation(projectReflection);
+
+    // initialize markdown output
+    let output: string = '# Routes\n\n';
+
+    // generate documentation for all routes
+    routes.forEach((routeWithMetaInformation) => {
+      output += generateDocumentationForRoute(routeWithMetaInformation, getNodeMetaInformationMap(projectReflection));
+    });
+
+    // write documentation to file
+    writeFileSync(mdPath, output);
+
+    logger.ok(`Route documentation written to ${mdPath}.`);
+  });
+
+commander
+  .command('schema <srcPath> <schemaPath>')
+  .action(async (relativeSrcPath, relativeSchemaPath) => {
+    // get absolute paths
+    const srcPath = resolve(relativeSrcPath);
+    const schemaPath = resolve(relativeSchemaPath);
+
+    // initialize new core converter
+    const coreConverter = new Converter(srcPath);
+
+    // get project reflection
+    const projectReflection = getProjectReflection(srcPath);
+
+    // get validatable types
+    const validatableTypes = getValidatableTypesFromReflection(projectReflection);
+
+    logger.info(`Found ${validatableTypes.length} type(s) to generate schemas for.`);
+
+    await mkdirPromisified(schemaPath, {
+      recursive: true,
+    });
+
+    logger.info(`Trying to find a package.json for ${srcPath}.`);
+
+    let path = srcPath;
+    // TODO: this check should be less ugly!
+    while (!existsSync(join(path, 'package.json')) && path.length > 5) {
+      path = resolve(path, '..');
+    }
+
+    const corePackageJsonPath = join(path, 'package.json');
+
+    logger.info(`Using ${corePackageJsonPath} to determine version for schemas.`);
+
+    const buffer = await readFilePromisifed(corePackageJsonPath);
+    const corePackageJson = JSON.parse(buffer.toString());
+    const coreVersion = corePackageJson.version;
+
+    logger.log(`Using ${coreVersion} as version for schemas.`);
+
+    // generate and write JSONSchema files for validatable types
+    validatableTypes.forEach((type) => {
+      const schema = coreConverter.getSchema(type, coreVersion);
+
+      const stringifiedSchema = JSON.stringify(schema, null, 2);
+
+      const file = join(schemaPath, type + '.json');
+
+      // write schema to file
+      writeFileSync(file, stringifiedSchema);
+
+      logger.info(`Generated schema for ${type} and saved to ${file}.`);
+    });
+
+    logger.ok(`Generated schemas for ${validatableTypes.length} type(s).`);
+  });
+
+commander
+  .command('validate <schemaPath> <testPath> [reportPath]')
+  .action(async (relativeSchemaPath, relativeTestPath, relativeReportPath) => {
+    // get absolute paths
+    const schemaPath = resolve(relativeSchemaPath);
+    const testPath = resolve(relativeTestPath);
+
+    const errorsPerFile = await validateFiles(schemaPath, testPath);
+
+    let unexpected = false;
+    Object.keys(errorsPerFile).forEach((file) => {
+      unexpected = unexpected || errorsPerFile[file].some((error) => !error.expected);
+    });
+
+    if (typeof relativeReportPath !== 'undefined') {
+      const reportPath = resolve(relativeReportPath);
+      await writeReport(reportPath, errorsPerFile);
+    }
+
+    if (!unexpected) {
+      logger.ok('Successfully finished validation.');
+    } else {
+      logger.error('Unexpected errors occurred during validation');
+      process.exit(1);
+    }
+  });
+
+commander
+  .parse(process.argv);
+
+if (commander.args.length < 1) {
+  commander.outputHelp();
+  process.exit(1);
+}

src/common.ts

@@ -0,0 +1,152 @@
+/*
+ * Copyright (C) 2018 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Logger} from '@openstapps/logger';
+import {mkdir, PathLike, readFile, writeFile} from 'fs';
+import * as glob from 'glob';
+import {Schema as JSONSchema, ValidationError} from 'jsonschema';
+import {Definition} from 'ts-json-schema-generator';
+import {Application, ProjectReflection} from 'typedoc';
+import {promisify} from 'util';
+
+/**
+ * Initialized logger
+ */
+export const logger = new Logger();
+
+export const globPromisfied = promisify(glob);
+export const mkdirPromisified = promisify(mkdir);
+export const readFilePromisifed = promisify(readFile);
+export const writeFilePromisified = promisify(writeFile);
+
+/**
+ * A route instance with its relevant meta information
+ */
+export interface RouteWithMetaInformation {
+  /**
+   * Description of the route
+   */
+  description: {
+    /**
+     * Short text of the description - title
+     */
+    shortText?: string;
+    /**
+     * Text of the description
+     */
+    text?: string;
+  };
+  /**
+   * Name of the route
+   */
+  name: string;
+  /**
+   * Instance of the route
+   */
+  route: {
+    errorNames: string[];
+    method: string;
+    obligatoryParameters: {
+      [k: string]: string;
+    }
+    requestBodyName: string;
+    responseBodyName: string;
+    statusCodeSuccess: number;
+    urlFragment: string;
+  };
+}
+
+/**
+ * A node with its relevant meta information
+ */
+export interface NodeWithMetaInformation {
+  /**
+   * Module the node belongs to
+   */
+  module: string;
+  /**
+   * Type of the node
+   */
+  type: string;
+}
+
+/**
+ * A map of nodes indexed by their name
+ */
+export interface NodesWithMetaInformation {
+  /**
+   * Index signature
+   */
+  [k: string]: NodeWithMetaInformation;
+}
+
+/**
+ * A schema with definitions
+ */
+interface SchemaWithDefinitions extends JSONSchema {
+  definitions: { [name: string]: Definition };
+}
+
+/**
+ * An expectable error
+ */
+export type ExpectableValidationError = ValidationError & { expected: boolean };
+
+/**
+ * A map of files and their expectable validation errors
+ */
+export interface ExpectableValidationErrors {
+  [fileName: string]: ExpectableValidationError[];
+}
+
+/**
+ * Get a project reflection from a path
+ *
+ * @param srcPath Path to get reflection from
+ */
+export function getProjectReflection(srcPath: PathLike): ProjectReflection {
+  logger.info(`Generating project reflection for ${srcPath.toString()}.`);
+
+  // initialize new Typedoc application
+  const app = new Application({
+    excludeExternals: true,
+    includeDeclarations: true,
+    module: 'commonjs',
+  });
+
+  // get input files
+  const inputFiles = app.expandInputFiles([srcPath.toString()]);
+
+  // get project reflection from input files
+  return app.convert(inputFiles);
+}
+
+/**
+ * Check if a schema has definitions
+ *
+ * @param schema Schema to check
+ */
+export function isSchemaWithDefinitions(schema: JSONSchema): schema is SchemaWithDefinitions {
+  return typeof schema.definitions !== 'undefined';
+}
+
+/**
+ * Guard method for determining if an object (a thing) has a type property with a type of string
+ *
+ * @param thing {any} Any object (thing)
+ * @returns {boolean} Is an object (a thing) with a type property with type of string
+ */
+export function isThingWithType(thing: any): thing is { type: string } {
+  return typeof thing.type === 'string';
+}

src/routes.ts

@@ -0,0 +1,181 @@
+/*
+ * Copyright (C) 2018 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {asyncPool} from 'async-pool-native/dist/async-pool';
+import humanizeString = require('humanize-string');
+import {ProjectReflection} from 'typedoc';
+import {logger, NodesWithMetaInformation, NodeWithMetaInformation} from './common';
+import {RouteWithMetaInformation} from './common';
+
+/**
+ * Gather relevant information of routes
+ *
+ * This gathers the information for all routes that implement the abstract class SCAbstractRoute.
+ * Furthermore it instantiates every route and adds it to the information.
+ *
+ * @param reflection Contents of the JSON representation which Typedoc generates
+ */
+export async function gatherRouteInformation(reflection: ProjectReflection): Promise<RouteWithMetaInformation[]> {
+  const routes: RouteWithMetaInformation[] = [];
+
+  await asyncPool(2, reflection.children, async (module: any) => {
+    if (Array.isArray(module.children) && module.children.length > 0) {
+      await asyncPool(2, module.children, (async (node: any) => {
+        if (Array.isArray(node.extendedTypes) && node.extendedTypes.length > 0) {
+          if (node.extendedTypes.some((extendedType: any) => {
+            return extendedType.name === 'SCAbstractRoute';
+          })) {
+            logger.info(`Found ${node.name} in ${module.originalName}.`);
+
+            const importedModule = await import(module.originalName);
+
+            const route = new importedModule[node.name]();
+
+            routes.push({description: node.comment, name: node.name, route});
+          }
+        }
+      }));
+    }
+  });
+
+  return routes;
+}
+
+/**
+ * Get a linked name for a node
+ *
+ * @param name Name of the node
+ * @param node Node itself
+ * @param humanize Whether to humanize the name or not
+ */
+export function getLinkedNameForNode(name: string, node: NodeWithMetaInformation, humanize: boolean = false): string {
+  let printableName = name;
+
+  if (humanize) {
+    printableName = humanizeString(name.substr(2));
+  }
+
+  let link = `[${printableName}]`;
+  link += `(${getLinkForNode(name, node)})`;
+  return link;
+}
+
+/**
+ * Get link for a node
+ *
+ * @param name Name of the node
+ * @param node Node itself
+ */
+export function getLinkForNode(name: string, node: NodeWithMetaInformation): string {
+  let link = 'https://openstapps.gitlab.io/core/';
+  const module = node.module.toLowerCase().split('/').join('_');
+
+  if (node.type === 'Type alias') {
+    link += 'modules/';
+    link += `_${module}_`;
+    link += `.html#${name.toLowerCase()}`;
+    return link;
+  }
+
+  let type = 'classes';
+  if (node.type !== 'Class') {
+    type = `${node.type.toLowerCase()}s`;
+  }
+
+  link += `${type}/`;
+  link += `_${module}_`;
+  link += `.${name.toLowerCase()}.html`;
+  return link;
+}
+
+/**
+ * Generate documentation snippet for one route
+ *
+ * @param routeWithInfo A route instance with its meta information
+ * @param nodes
+ */
+export function generateDocumentationForRoute(routeWithInfo: RouteWithMetaInformation,
+                                              nodes: NodesWithMetaInformation): string {
+  let output = '';
+
+  const route = routeWithInfo.route;
+
+  output += `## \`${route.method} ${route.urlFragment}\``;
+  output += ` ${getLinkedNameForNode(routeWithInfo.name, nodes[routeWithInfo.name], true)}\n\n`;
+
+  if (typeof routeWithInfo.description.shortText === 'string') {
+    output += `**${routeWithInfo.description.shortText}**\n\n`;
+  }
+
+  if (typeof routeWithInfo.description.text === 'string') {
+    output += `${routeWithInfo.description.text.replace('\n', '<br>')}\n\n`;
+  }
+
+  output += `### Definition
+
+| parameter | value |
+| --- | --- |
+| request | ${getLinkedNameForNode(route.requestBodyName, nodes[route.requestBodyName])} |
+| response | ${getLinkedNameForNode(route.responseBodyName, nodes[route.responseBodyName])} |
+| success code | ${route.statusCodeSuccess} |
+| errors | ${route.errorNames.map((errorName) => {
+    return getLinkedNameForNode(errorName, nodes[errorName]);
+  }).join('<br>')} |
+`;
+  if (typeof route.obligatoryParameters === 'object' && Object.keys(route.obligatoryParameters).length > 0) {
+    let parameterTable = '<table><tr><th>parameter</th><th>type</th></tr>';
+
+    Object.keys(route.obligatoryParameters).forEach((parameter) => {
+      let type = route.obligatoryParameters![parameter];
+
+      if (typeof nodes[type] !== 'undefined') {
+        type = getLinkedNameForNode(type, nodes[type]);
+      }
+
+      parameterTable += `<tr><td>${parameter}</td><td>${type}</td></tr>`;
+    });
+
+    parameterTable += '</table>';
+
+    output += `| obligatory parameters | ${parameterTable} |`;
+  }
+  output += '\n\n';
+
+  return output;
+}
+
+/**
+ * Get a map of nodes with their meta information
+ *
+ * @param projectReflection Reflection to get information from
+ */
+export function getNodeMetaInformationMap(projectReflection: ProjectReflection): NodesWithMetaInformation {
+  const nodes: NodesWithMetaInformation = {};
+
+  // iterate over modules
+  projectReflection.children.forEach((module: any) => {
+    if (Array.isArray(module.children) && module.children.length > 0) {
+      // iterate over types
+      module.children.forEach((node: any) => {
+        // add node with module and type
+        nodes[node.name] = {
+          module: module.name.substring(1, module.name.length - 1),
+          type: node.kindString,
+        };
+      });
+    }
+  });
+
+  return nodes;
+}

src/schema.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2018 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {Schema as JSONSchema} from 'jsonschema';
+import {join} from 'path';
+import {DEFAULT_CONFIG, SchemaGenerator} from 'ts-json-schema-generator';
+import {createFormatter} from 'ts-json-schema-generator/dist/factory/formatter';
+import {createParser} from 'ts-json-schema-generator/dist/factory/parser';
+import {createProgram} from 'ts-json-schema-generator/dist/factory/program';
+import {ProjectReflection} from 'typedoc';
+import * as ts from 'typescript';
+import {isSchemaWithDefinitions} from './common';
+
+/**
+ * StAppsCore converter
+ *
+ * Converts TypeScript source files to JSON schema files
+ */
+export class Converter {
+  private generator: SchemaGenerator;
+
+  /**
+   * Create a new converter
+   *
+   * @param path Path to the project
+   */
+  constructor(path: string) {
+    // set config for schema generator
+    const config = {
+      ...DEFAULT_CONFIG,
+      // expose: 'exported' as any,
+      // jsDoc: 'extended' as any,
+      path: join(path, '**/*.ts'),
+      sortProps: true,
+      topRef: false,
+      type: 'SC',
+    };
+
+    // create TypeScript program from config
+    const program: ts.Program = createProgram(config);
+
+    // create generator
+    this.generator = new SchemaGenerator(
+      program,
+      createParser(program, config),
+      createFormatter(config),
+    );
+  }
+
+  /**
+   * Get schema for specific StAppsCore type
+   *
+   * @param {string} type Type to get the schema for
+   * @param {string} version Version to set for the schema
+   * @returns {Schema} Generated schema
+   */
+  getSchema(type: string, version: string): JSONSchema {
+    // generate schema for this file/type
+    const schema: JSONSchema = this.generator.createSchema(type);
+
+    // set id of schema
+    schema.id = 'https://core.stapps.tu-berlin.de/v' + version + '/lib/schema/' + type + '.json';
+
+    if (isSchemaWithDefinitions(schema)) {
+      // add self reference to definitions
+      schema.definitions['SC' + type] = Object.assign({}, schema.properties);
+    }
+
+    return schema;
+  }
+}
+
+/**
+ * Get a list of validatable types from a reflection
+ *
+ * @param projectReflection Reflection to get validatable types from
+ */
+export function getValidatableTypesFromReflection(projectReflection: ProjectReflection): string[] {
+  const validatableTypes: string[] = [];
+
+  // iterate over modules
+  projectReflection.children.forEach((module) => {
+    if (Array.isArray(module.children) && module.children.length > 0) {
+      // iterate over types
+      module.children.forEach((type) => {
+        // check if type has annotation @validatable
+        if (typeof type.comment === 'object'
+          && Array.isArray(type.comment.tags)
+          && type.comment.tags.find((tag) => tag.tagName === 'validatable')) {
+          // add type to list
+          validatableTypes.push(type.name);
+        }
+      });
+    }
+  });
+
+  return validatableTypes;
+}

src/validate.ts

@@ -0,0 +1,248 @@
+/*
+ * Copyright (C) 2018 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {asyncPool} from 'async-pool-native/dist/async-pool';
+import {PathLike} from 'fs';
+import {Schema, Validator as JSONSchemaValidator, ValidatorResult} from 'jsonschema';
+import * as mustache from 'mustache';
+import {basename, join, resolve} from 'path';
+import {
+  ExpectableValidationErrors,
+  globPromisfied,
+  isThingWithType,
+  logger,
+  readFilePromisifed,
+  writeFilePromisified,
+} from './common';
+
+/**
+ * StAppsCore validator
+ */
+export class Validator {
+  /**
+   * Map of schema names to schemas
+   */
+  private readonly schemas: { [type: string]: Schema } = {};
+
+  /**
+   * JSONSchema validator instance
+   */
+  private readonly validator: JSONSchemaValidator;
+
+  /**
+   * Create a new validator
+   */
+  constructor() {
+    this.validator = new JSONSchemaValidator();
+  }
+
+  /**
+   * Feed the schema files to the validator
+   *
+   * @param schemaDir Path to directory that contains schema files
+   */
+  public async addSchemas(schemaDir: PathLike): Promise<string[]> {
+    const schemaFiles = await globPromisfied(join(schemaDir.toString(), '*.json'));
+
+    if (schemaFiles.length === 0) {
+      throw new Error(`No schema files in ${schemaDir.toString()}!`);
+    }
+
+    logger.log(`Adding schemas from ${schemaDir} to validator.`);
+
+    // Iterate over schema files
+    await asyncPool(2, schemaFiles, async (file) => {
+      // read schema file
+      const buffer = await readFilePromisifed(file);
+      const schema = JSON.parse(buffer.toString());
+
+      // add schema to validator
+      this.validator.addSchema(schema);
+
+      // add schema to map
+      this.schemas[basename(file, '.json')] = schema;
+
+      logger.info(`Added ${file} to validator.`);
+    });
+
+    return schemaFiles;
+  }
+
+  /**
+   * Validates anything against a given schema name
+   *
+   * @param instance Instance to validate
+   * @param schemaName Name of schema to validate instance against
+   */
+  public validate(instance: any, schemaName: string): ValidatorResult {
+    if (typeof this.schemas[schemaName] !== 'object') {
+      throw new Error(`No schema available for ${schemaName}.`);
+    }
+
+    return this.validator.validate(instance, this.schemas[schemaName]);
+  }
+
+  /**
+   * Validate an instance of a thing against the consumed schema files
+   *
+   * @param instance Instance to validate
+   * @deprecated Use [[validate]] instead
+   */
+  public validateThing<T>(instance: T): ValidatorResult {
+    if (!isThingWithType(instance)) {
+      throw new Error('Instance.type does not exist.');
+    }
+
+    const schemaName = instance.type.split(' ').map((part) => {
+      return part.substr(0, 1).toUpperCase() + part.substr(1);
+    }).join('');
+
+    return this.validate(instance, schemaName);
+  }
+}
+
+/**
+ * Validate all test files in the given resources directory against schema files in the given (schema) directory
+ *
+ * @param schemaDir The directory where the JSON schema files are
+ * @param resourcesDir The directory where the test files are
+ */
+export async function validateFiles(schemaDir: string, resourcesDir: string): Promise<ExpectableValidationErrors> {
+  // instantiate new validator
+  const v = new Validator();
+  await v.addSchemas(schemaDir);
+
+  // get list of files to test
+  const testFiles = await globPromisfied(join(resourcesDir, '*.json'));
+
+  if (testFiles.length === 0) {
+    throw new Error(`No test files in ${resourcesDir}!`);
+  }
+
+  logger.log(`Found ${testFiles.length} file(s) to test.`);
+
+  // map of errors per file
+  const errors: ExpectableValidationErrors = {};
+
+  // iterate over files to test
+  await asyncPool(2, testFiles, async (testFile) => {
+    const testFileName = basename(testFile);
+
+    const buffer = await readFilePromisifed(join(resourcesDir, testFileName));
+
+    // read test description from file
+    const testDescription = JSON.parse(buffer.toString());
+
+    // validate instance from test description
+    const result = v.validate(testDescription.instance, testDescription.schema);
+
+    // list of expected errors for this test description
+    const expectedErrors: string[] = [];
+    expectedErrors.push.apply(expectedErrors, testDescription.errorNames);
+
+    // number of unexpected errors
+    let unexpectedErrors = 0;
+
+    if (result.errors.length > 0) {
+      errors[testFileName] = [];
+
+      // iterate over errors
+      result.errors.forEach((error) => {
+        // get idx of expected error
+        const errorIdx = expectedErrors.indexOf(error.name);
+        let expected = false;
+
+        if (errorIdx >= 0) {
+          // remove from list of expected errors
+          expectedErrors.splice(errorIdx, 1);
+          expected = true;
+        } else {
+          unexpectedErrors++;
+          logger.error(`Unexpected error ${error.name} in ${testFile}`);
+        }
+
+        // add error to list of errors
+        errors[testFileName].push({
+          ...error,
+          expected,
+        });
+      });
+    }
+
+    if (expectedErrors.length > 0) {
+      expectedErrors.forEach((error) => {
+        logger.error(`Extraneous expected error '${error}' in ${testFile}.`);
+        errors[testFileName].push({
+          argument: false,
+          expected: false,
+          instance: testDescription.instance,
+          message: `expected error ${error} did not occur`,
+          name: `expected ${error}`,
+          property: 'unknown',
+          schema: undefined as any,
+        });
+      });
+    } else if (unexpectedErrors === 0) {
+      logger.info(`Successfully validated ${testFile}.`);
+    }
+  });
+
+  return errors;
+}
+
+/**
+ * Write a report for errors that occurred in validation
+ *
+ * @param reportPath Path to write report to
+ * @param errors Errors that occurred in validation
+ */
+export async function writeReport(reportPath: PathLike, errors: ExpectableValidationErrors): Promise<void> {
+  let buffer = await readFilePromisifed(resolve(__dirname, '..', 'resources', 'file.html.mustache'));
+  const fileTemplate = buffer.toString();
+
+  buffer = await readFilePromisifed(resolve(__dirname, '..', 'resources', 'error.html.mustache'));
+  const errorTemplate = buffer.toString();
+
+  let output = '';
+
+  Object.keys(errors).forEach((fileName) => {
+    let fileOutput = '';
+
+    errors[fileName].forEach((error, idx) => {
+      fileOutput += mustache.render(errorTemplate, {
+        idx: idx + 1,
+        instance: JSON.stringify(error.instance, null, 2),
+        message: error.message,
+        property: error.property,
+        schema: JSON.stringify(error.schema, null, 2),
+        status: (error.expected) ? 'alert-success' : 'alert-danger',
+      });
+    });
+
+    output += mustache.render(fileTemplate, {
+      errors: fileOutput,
+      testFile: fileName,
+    });
+  });
+
+  buffer = await readFilePromisifed(resolve(__dirname, '..', 'resources', 'report.html.mustache'));
+  const reportTemplate = buffer.toString();
+
+  await writeFilePromisified(reportPath, mustache.render(reportTemplate, {
+    report: output,
+    timestamp: (new Date()).toISOString(),
+  }));
+
+  logger.ok(`Wrote report to ${reportPath}.`);
+}