feat: replace route markdown with openapi

.gitignore

@@ -87,8 +87,11 @@ typings/
 .idea
 .vscode
 
-# ignore lib
-lib
+# ignore lib directory
+lib/
 
-# ignore docs
-docs
+# ignore docs directory
+docs/
+
+# ignore openapi resources
+openapi/

.gitlab-ci.yml

@@ -44,18 +44,18 @@ scheduled-audit:
   only:
     - schedules
 
-routes:
+openapi:
   dependencies:
     - build
   stage: test
   script:
     - npm install @openstapps/core
-    - node lib/cli.js routes node_modules/@openstapps/core/lib routes.md
-    - NUMBER_OF_LINES=$(cat routes.md | wc -l)
+    - node lib/cli.js openapi node_modules/@openstapps/core/lib openapi
+    - NUMBER_OF_LINES=$(cat openapi/openapi.json | wc -l)
     - if [ "$NUMBER_OF_LINES" -lt 100 ]; then exit 1; fi
   artifacts:
     paths:
-      - routes.md
+      - openapi/openapi.json
 
 mapping:
   dependencies:

README.md

@@ -127,12 +127,12 @@ Inside of a script in `package.json` or if the npm package is installed globally
 openstapps-core-tools validate lib/schema src/test/resources report.html
 ```
 
-## Generate documentation for routes
+## Generate openapi JSON file for routes
 
-To generate a documentation for the routes use the following command.
+To generate a openapi JSON file that represents the routes according to openapi version 3.0.3 use the following command.
 
 ```shell
-openstapps-core-tools routes PATH/TO/CORE/lib PATH/TO/ROUTES.md
+openstapps-core-tools openapi PATH/TO/CORE/lib PATH/TO/PUT/FILES/TO
 ```
 
 ## Pack definitions and implementations

package-lock.json

@@ -226,6 +226,14 @@
       "integrity": "sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg==",
       "dev": true
     },
+    "@types/fs-extra": {
+      "version": "9.0.11",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.11.tgz",
+      "integrity": "sha512-mZsifGG4QeQ7hlkhO56u7zt/ycBgGxSVsFI/6lGTU34VtwkiqrrSDgw0+ygs8kFGWcXnFQWMrzF2h7TtDFNixA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/glob": {
       "version": "7.1.4",
       "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.4.tgz",
@@ -1181,11 +1189,10 @@
       "integrity": "sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg=="
     },
     "fs-extra": {
-      "version": "9.1.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
-      "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
+      "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
       "requires": {
-        "at-least-node": "^1.0.0",
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
         "universalify": "^2.0.0"
@@ -2275,6 +2282,11 @@
         "wrappy": "1"
       }
     },
+    "openapi-types": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-9.0.3.tgz",
+      "integrity": "sha512-c4C1xAKZOvOxeSWvRY0d2XsoaZoF8M7rifxfZZCIH2mqPEQxOz8qfFx4oEpLFaE+DfDGe08HcIA/p1Bu93keLQ=="
+    },
     "os-tmpdir": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
@@ -3085,6 +3097,19 @@
         "progress": "^2.0.3",
         "shelljs": "^0.8.4",
         "typedoc-default-themes": "^0.10.2"
+      },
+      "dependencies": {
+        "fs-extra": {
+          "version": "9.1.0",
+          "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz",
+          "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==",
+          "requires": {
+            "at-least-node": "^1.0.0",
+            "graceful-fs": "^4.2.0",
+            "jsonfile": "^6.0.1",
+            "universalify": "^2.0.0"
+          }
+        }
       }
     },
     "typedoc-default-themes": {

package.json

@@ -62,6 +62,7 @@
     "humanize-string": "2.1.0",
     "json-schema": "0.3.0",
     "mustache": "4.2.0",
+    "openapi-types": "9.0.3",
     "plantuml-encoder": "1.4.0",
     "toposort": "2.0.2",
     "ts-json-schema-generator": "0.70.2",

src/cli.ts

@@ -15,10 +15,12 @@
 import {Logger} from '@openstapps/logger';
 import {Command} from 'commander';
 import {existsSync, readFileSync, writeFileSync} from 'fs';
+import {copy} from 'fs-extra';
 import got from 'got';
-import {join, resolve} from 'path';
+import {join, relative, resolve} from 'path';
 import {exit} from 'process';
 import {
+  capitalize,
   getProjectReflection,
   mkdirPromisified,
   readFilePromisified,
@@ -26,9 +28,10 @@ import {
 } from './common';
 import {generateTemplate} from './mapping';
 import {pack} from './pack';
+import {openapi3Template} from './resources/openapi-303-template';
 import {
   gatherRouteInformation,
-  generateDocumentationForRoute,
+  generateOpenAPIForRoute,
   getNodeMetaInformationMap,
 } from './routes';
 import {Converter, getValidatableTypesFromReflection} from './schema';
@@ -55,33 +58,68 @@ commander
   ).version);
 
 commander
-  .command('routes <srcPath> <mdPath>')
-  .action(async (relativeSrcPath, relativeMdPath) => {
+  .command('openapi <srcPath> <outDirPath>')
+  .action(async (relativeSrcPath, relativeOutDirPath) => {
     // get absolute paths
     const srcPath = resolve(relativeSrcPath);
-    const mdPath = resolve(relativeMdPath);
+    const outDirPath = resolve(relativeOutDirPath);
+    const outDirSchemasPath = join(outDirPath, 'schemas');
 
     // get project reflection
     const projectReflection = getProjectReflection(srcPath);
 
     // get information about routes
     const routes = await gatherRouteInformation(projectReflection);
+    routes.sort((a, b) => a.route.urlFragment.localeCompare(b.route.urlFragment));
 
-    // initialize markdown output
-    let output = '# Routes\n\n';
+    // change url path parameters to openapi notation
+    routes.forEach((routeWithMetaInformation) => {
+      routeWithMetaInformation.route.urlFragment = routeWithMetaInformation.route.urlFragment.replace(/:\w+/g, (match) => `{${match.replace(':','')}}`);
+    });
+
+    // keep openapi tags for routes that actually share url fragments
+    let tagsToKeep = routes.map((routeWithMetaInformation) => capitalize(routeWithMetaInformation.route.urlFragment.split('/')[1]));
+    tagsToKeep = tagsToKeep.filter((element, i, array) => array.indexOf(element) === i && array.lastIndexOf(element) !== i);
+
+    // initialize json output
+    const output = openapi3Template;
+
+    // names of the schemas to copy
+    const schemasToCopy: string[] = [];
 
     // generate documentation for all routes
     routes.forEach((routeWithMetaInformation) => {
-      output += generateDocumentationForRoute(
+      routeWithMetaInformation.tags = [capitalize(routeWithMetaInformation.route.urlFragment.split('/')[1])];
+
+      output.paths[routeWithMetaInformation.route.urlFragment] = generateOpenAPIForRoute(
         routeWithMetaInformation,
-        getNodeMetaInformationMap(projectReflection),
+        relative(relativeOutDirPath,outDirSchemasPath),
+        schemasToCopy,
+        tagsToKeep,
       );
     });
 
-    // write documentation to file
-    writeFileSync(mdPath, output);
+    // copy schema json schema files
+    try {
+      if (!existsSync(outDirSchemasPath)){
+        await mkdirPromisified(outDirSchemasPath, {
+          recursive: true,
+        });
+      }
+      for (const fileName of schemasToCopy) {
+        await copy(join(srcPath, 'schema', `${fileName}.json`), join(outDirSchemasPath, `${fileName}.json`));
+      }
+    } catch (error) {
+      await Logger.error(error);
+      // tslint:disable-next-line: no-magic-numbers
+      process.exit(-2);
+    }
 
-    Logger.ok(`Route documentation written to ${mdPath}.`);
+    // write openapi object to file (prettified)
+    // tslint:disable-next-line: no-magic-numbers
+    writeFileSync(join(outDirPath, 'openapi.json'), JSON.stringify(output, null, 2));
+
+    Logger.ok(`OpenAPI representation resources written to ${outDirPath} .`);
   });
 
 commander

src/common.ts

@@ -56,9 +56,9 @@ export interface RouteWithMetaInformation {
    */
   route: {
     /**
-     * Error names of a route
+     * Possible errors on a route
      */
-    errorNames: Error[];
+    errors: SCErrorResponse[];
     /**
      * Method of the route
      */
@@ -69,10 +69,18 @@ export interface RouteWithMetaInformation {
     obligatoryParameters: {
       [k: string]: string;
     };
+    /**
+     * Description of the request body
+     */
+    requestBodyDescription: string;
     /**
      * Name of the request body
      */
     requestBodyName: string;
+    /**
+     * Description of the response body
+     */
+    responseBodyDescription: string;
     /**
      * Name of the response body
      */
@@ -86,6 +94,10 @@ export interface RouteWithMetaInformation {
      */
     urlFragment: string;
   };
+  /**
+   * Possible tags/keywords the route can be associated with
+   */
+  tags?: [string];
 }
 
 /**
@@ -102,6 +114,21 @@ export interface NodeWithMetaInformation {
   type: string;
 }
 
+/**
+ * A generic error that can be returned by the backend if somethings fails during the processing of a request
+ */
+export interface SCErrorResponse extends Error {
+  /**
+   * Additional data that describes the error
+   */
+  additionalData?: unknown;
+
+  /**
+   * HTTP status code to return this error with
+   */
+  statusCode: number;
+}
+
 /**
  * A map of nodes indexed by their name
  */
@@ -345,3 +372,13 @@ export function getFullTypeName(type: LightweightType): string {
 
   return fullName;
 }
+
+/**
+ * Creates sentence cased string
+ *
+ * @param str The string to capitalize
+ */
+export function capitalize(str?: string): string {
+  // tslint:disable-next-line: newline-per-chained-call
+  return `${str?.charAt(0).toUpperCase()}${str?.slice(1).toLowerCase()}`;
+}

src/resources/openapi-303-template.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2021 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {OpenAPIV3} from 'openapi-types';
+
+export const openapi3Template: OpenAPIV3.Document = {
+    openapi: '3.0.3',
+    info: {
+      title: 'Openstapps Backend',
+      description: `# Introduction
+This is a human readable documentation of the backend OpenAPI representation.`,
+      contact: {
+        name: 'Openstapps Team',
+        url: 'https://gitlab.com/openstapps/backend',
+        email: 'app@uni-frankfurt.de',
+      },
+      license: {
+        name: 'AGPL 3.0',
+        url: 'https://www.gnu.org/licenses/agpl-3.0.en.html',
+      },
+      version: '2.0.0',
+    },
+    servers: [
+      {
+        url: 'https://mobile.server.uni-frankfurt.de:3000',
+        description: 'Production server',
+      },
+    ],
+    paths: {},
+  };

src/routes.ts

@@ -14,10 +14,11 @@
  */
 import {asyncPool} from '@krlwlfrt/async-pool/lib/async-pool';
 import {Logger} from '@openstapps/logger';
+import {OpenAPIV3} from 'openapi-types';
 import {basename, dirname, join} from 'path';
 import {ProjectReflection} from 'typedoc';
 import {Type} from 'typedoc/dist/lib/models';
-import {NodesWithMetaInformation, NodeWithMetaInformation, RouteWithMetaInformation} from './common';
+import {capitalize, NodeWithMetaInformation, RouteWithMetaInformation} from './common';
 
 /**
  * Gather relevant information of routes
@@ -55,6 +56,19 @@ export async function gatherRouteInformation(reflection: ProjectReflection): Pro
 
             const route = new importedModule[node.name]();
 
+            // tslint:disable-next-line: no-any
+            const errors = route.errorNames.map((error: any) => {
+              const scError = new importedModule[error.name]();
+              scError.name = error.name;
+
+              return scError;
+            });
+
+            route.responseBodyDescription = module.children!.find(element => element.name === route.responseBodyName)?.comment?.shortText;
+            route.requestBodyDescription = module.children!.find(element => element.name === route.requestBodyName)?.comment?.shortText;
+
+            route.errors = errors;
+
             routes.push({description: node.comment!, name: node.name, route});
           }
         }
@@ -69,28 +83,6 @@ export async function gatherRouteInformation(reflection: ProjectReflection): Pro
   return routes;
 }
 
-/**
- * Get a linked name for a node
- *
- * @param name Name of the node
- * @param node Node itself
- * @param humanize Whether to humanize the name or not
- */
-export function getLinkedNameForNode(name: string, node: NodeWithMetaInformation, humanize = false): string {
-  const humanizeString = require('humanize-string');
-
-  let printableName = name;
-
-  if (humanize) {
-    printableName = humanizeString(name.substr('SC'.length));
-  }
-
-  let link = `[${printableName}]`;
-  link += `(${getLinkForNode(name, node)})`;
-
-  return link;
-}
-
 /**
  * Get link for a node
  *
@@ -128,89 +120,85 @@ export function getLinkForNode(name: string, node: NodeWithMetaInformation): str
  * Generate documentation snippet for one route
  *
  * @param routeWithInfo A route instance with its meta information
- * @param nodes Nodes with meta information
+ * @param outDirSchemasPath Path to directory that will contain relevant schemas for the route
+ * @param schemasToCopy Schemas identified as relevant for this route
+ * @param tagsToKeep Tags / keywords that can be used for grouping routes
  */
-export function generateDocumentationForRoute(routeWithInfo: RouteWithMetaInformation,
-                                              nodes: NodesWithMetaInformation): string {
-  let output = '';
-
+ export function generateOpenAPIForRoute(routeWithInfo: RouteWithMetaInformation,
+                                     outDirSchemasPath: string,
+                                         schemasToCopy: string[],
+                                            tagsToKeep: string[]): OpenAPIV3.PathItemObject {
   const route = routeWithInfo.route;
+  const path: OpenAPIV3.PathItemObject = {};
 
-  output += `## \`${route.method} ${route.urlFragment}\``;
-  output += ` ${getLinkedNameForNode(routeWithInfo.name, nodes[routeWithInfo.name], true)}\n\n`;
+  schemasToCopy.push(route.requestBodyName, route.responseBodyName);
 
-  if (typeof routeWithInfo.description.shortText === 'string') {
-    output += `**${routeWithInfo.description.shortText}**\n\n`;
-  }
+  path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)] = {
+    summary: capitalize(routeWithInfo.description.shortText?.replace(/(Route to |Route for )/gmi, '')),
+    description: routeWithInfo.description.text,
+    requestBody: {
+      description: route.responseBodyDescription ?? undefined,
+      content: {
+        'application/json': {
+          schema: {
+            $ref: join(outDirSchemasPath, `${route.requestBodyName}.json`),
+          },
+        },
+      },
+    },
+    parameters: [{
+      name: 'X-StApps-Version',
+      in: 'header',
+      schema: {
+        type: 'string',
+        example: '2.0.0',
+      },
+      required: true,
+    }],
+    responses: {},
+    tags: routeWithInfo.tags?.filter(value => tagsToKeep.includes(value)),
+  };
 
-  if (typeof routeWithInfo.description.text === 'string') {
-    output += `${routeWithInfo.description.text.replace('\n', '<br>')}\n\n`;
-  }
+  path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)]!.responses![route.statusCodeSuccess] = {
+    description: route.responseBodyDescription,
+    content: {
+      'application/json': {
+        schema: {
+          $ref: join(outDirSchemasPath, `${route.responseBodyName}.json`),
+        },
+      },
+    },
+  };
 
-  output += `### Definition
-
-| parameter | value |
-| --- | --- |
-| request | ${getLinkedNameForNode(route.requestBodyName, nodes[route.requestBodyName])} |
-| response | ${getLinkedNameForNode(route.responseBodyName, nodes[route.responseBodyName])} |
-| success code | ${route.statusCodeSuccess} |
-| errors | ${route.errorNames
-    .map((error) => {
-      return getLinkedNameForNode(error.name, nodes[error.name]);
-    })
-    .join('<br>')} |
-`;
-  if (typeof route.obligatoryParameters === 'object' && Object.keys(route.obligatoryParameters).length > 0) {
-    let parameterTable = '<table><tr><th>parameter</th><th>type</th></tr>';
-
-    for (const parameter in route.obligatoryParameters) {
-      if (!route.obligatoryParameters.hasOwnProperty(parameter)) {
-        continue;
-      }
-
-      let type = route.obligatoryParameters![parameter];
-
-      if (typeof nodes[type] !== 'undefined') {
-        type = getLinkedNameForNode(type, nodes[type]);
-      }
-
-      parameterTable += `<tr><td>${parameter}</td><td>${type}</td></tr>`;
-    }
-
-    parameterTable += '</table>';
-
-    output += `| obligatory parameters | ${parameterTable} |`;
-  }
-  output += '\n\n';
-
-  return output;
-}
-
-/**
- * Get a map of nodes with their meta information
- *
- * @param projectReflection Reflection to get information from
- */
-export function getNodeMetaInformationMap(projectReflection: ProjectReflection): NodesWithMetaInformation {
-  const nodes: NodesWithMetaInformation = {};
-
-  if (typeof projectReflection.children === 'undefined') {
-    throw new Error('Project reflection doesn\'t contain any modules.');
-  }
-
-  // iterate over modules
-  projectReflection.children.forEach((module) => {
-    if (Array.isArray(module.children) && module.children.length > 0) {
-      // iterate over types
-      module.children.forEach((node) => {
-        // add node with module and type
-        nodes[node.name] = {
-          module: module.name.substring(1, module.name.length - 1),
-          type: node.kindString!,
-        };
-      });
-    }
+  route.errors.forEach(error => {
+    schemasToCopy.push(error.name);
+    path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)]!.responses![error.statusCode] = {
+      description: error.message ?? capitalize(error.name.replace(/([A-Z][a-z])/g,' $1')
+        .replace('SC ', '')),
+      content: {
+        'application/json': {
+          schema: {
+            $ref: join(outDirSchemasPath, `${error.name}.json`),
+          },
+        },
+      },
+    };
   });
 
-  return nodes;
+  if (typeof route.obligatoryParameters === 'object') {
+    for (const [parameter, schemaDefinition] of Object.entries(route.obligatoryParameters)) {
+      const openapiParam: OpenAPIV3.ParameterObject = {
+        in: 'path',
+        name: parameter,
+        required: true,
+        schema: {
+          // TODO make this less of a hack and search copied schemas for the first occurring definition
+          $ref: `schemas/SCSearchResponse.json#/definitions/${schemaDefinition}`,
+        },
+      };
+      path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)]?.parameters?.push(openapiParam);
+    }
+  }
+
+  return path;
 }