From f4bf7abc895f87a57fa34b2269311809f2a9413d Mon Sep 17 00:00:00 2001 From: Rainer Killinger Date: Tue, 6 Jul 2021 10:51:05 +0200 Subject: [PATCH] feat: replace route markdown with openapi --- .gitignore | 11 +- .gitlab-ci.yml | 8 +- README.md | 6 +- package-lock.json | 33 ++++- package.json | 1 + src/cli.ts | 62 +++++++-- src/common.ts | 41 +++++- src/resources/openapi-303-template.ts | 41 ++++++ src/routes.ts | 188 ++++++++++++-------------- 9 files changed, 262 insertions(+), 129 deletions(-) create mode 100644 src/resources/openapi-303-template.ts diff --git a/.gitignore b/.gitignore index 7d3c8b2a..50d2fc77 100644 --- a/.gitignore +++ b/.gitignore @@ -87,8 +87,11 @@ typings/ .idea .vscode -# ignore lib -lib +# ignore lib directory +lib/ -# ignore docs -docs +# ignore docs directory +docs/ + +# ignore openapi resources +openapi/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 701ab0f6..fe446b28 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,18 +44,18 @@ scheduled-audit: only: - schedules -routes: +openapi: dependencies: - build stage: test script: - npm install @openstapps/core - - node lib/cli.js routes node_modules/@openstapps/core/lib routes.md - - NUMBER_OF_LINES=$(cat routes.md | wc -l) + - node lib/cli.js openapi node_modules/@openstapps/core/lib openapi + - NUMBER_OF_LINES=$(cat openapi/openapi.json | wc -l) - if [ "$NUMBER_OF_LINES" -lt 100 ]; then exit 1; fi artifacts: paths: - - routes.md + - openapi/openapi.json mapping: dependencies: diff --git a/README.md b/README.md index bcc0dcfc..1d314e65 100644 --- a/README.md +++ b/README.md @@ -127,12 +127,12 @@ Inside of a script in `package.json` or if the npm package is installed globally openstapps-core-tools validate lib/schema src/test/resources report.html ``` -## Generate documentation for routes +## Generate openapi JSON file for routes -To generate a documentation for the routes use the following command. +To generate a openapi JSON file that represents the routes according to openapi version 3.0.3 use the following command. ```shell -openstapps-core-tools routes PATH/TO/CORE/lib PATH/TO/ROUTES.md +openstapps-core-tools openapi PATH/TO/CORE/lib PATH/TO/PUT/FILES/TO ``` ## Pack definitions and implementations diff --git a/package-lock.json b/package-lock.json index ae3cefe6..fb7a8950 100644 --- a/package-lock.json +++ b/package-lock.json @@ -226,6 +226,14 @@ "integrity": "sha512-yd+9qKmJxm496BOV9CMNaey8TWsikaZOwMRwPHQIjcOJM9oV+fi9ZMNw3JsVnbEEbo2gRTDnGEBv8pjyn67hNg==", "dev": true }, + "@types/fs-extra": { + "version": "9.0.11", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.11.tgz", + "integrity": "sha512-mZsifGG4QeQ7hlkhO56u7zt/ycBgGxSVsFI/6lGTU34VtwkiqrrSDgw0+ygs8kFGWcXnFQWMrzF2h7TtDFNixA==", + "requires": { + "@types/node": "*" + } + }, "@types/glob": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.4.tgz", @@ -1181,11 +1189,10 @@ "integrity": "sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg==" }, "fs-extra": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", - "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", "requires": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" @@ -2275,6 +2282,11 @@ "wrappy": "1" } }, + "openapi-types": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-9.0.3.tgz", + "integrity": "sha512-c4C1xAKZOvOxeSWvRY0d2XsoaZoF8M7rifxfZZCIH2mqPEQxOz8qfFx4oEpLFaE+DfDGe08HcIA/p1Bu93keLQ==" + }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -3085,6 +3097,19 @@ "progress": "^2.0.3", "shelljs": "^0.8.4", "typedoc-default-themes": "^0.10.2" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + } } }, "typedoc-default-themes": { diff --git a/package.json b/package.json index 60de003e..173c5c54 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "humanize-string": "2.1.0", "json-schema": "0.3.0", "mustache": "4.2.0", + "openapi-types": "9.0.3", "plantuml-encoder": "1.4.0", "toposort": "2.0.2", "ts-json-schema-generator": "0.70.2", diff --git a/src/cli.ts b/src/cli.ts index c935c76e..112c2a62 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,10 +15,12 @@ import {Logger} from '@openstapps/logger'; import {Command} from 'commander'; import {existsSync, readFileSync, writeFileSync} from 'fs'; +import {copy} from 'fs-extra'; import got from 'got'; -import {join, resolve} from 'path'; +import {join, relative, resolve} from 'path'; import {exit} from 'process'; import { + capitalize, getProjectReflection, mkdirPromisified, readFilePromisified, @@ -26,9 +28,10 @@ import { } from './common'; import {generateTemplate} from './mapping'; import {pack} from './pack'; +import {openapi3Template} from './resources/openapi-303-template'; import { gatherRouteInformation, - generateDocumentationForRoute, + generateOpenAPIForRoute, getNodeMetaInformationMap, } from './routes'; import {Converter, getValidatableTypesFromReflection} from './schema'; @@ -55,33 +58,68 @@ commander ).version); commander - .command('routes ') - .action(async (relativeSrcPath, relativeMdPath) => { + .command('openapi ') + .action(async (relativeSrcPath, relativeOutDirPath) => { // get absolute paths const srcPath = resolve(relativeSrcPath); - const mdPath = resolve(relativeMdPath); + const outDirPath = resolve(relativeOutDirPath); + const outDirSchemasPath = join(outDirPath, 'schemas'); // get project reflection const projectReflection = getProjectReflection(srcPath); // get information about routes const routes = await gatherRouteInformation(projectReflection); + routes.sort((a, b) => a.route.urlFragment.localeCompare(b.route.urlFragment)); - // initialize markdown output - let output = '# Routes\n\n'; + // change url path parameters to openapi notation + routes.forEach((routeWithMetaInformation) => { + routeWithMetaInformation.route.urlFragment = routeWithMetaInformation.route.urlFragment.replace(/:\w+/g, (match) => `{${match.replace(':','')}}`); + }); + + // keep openapi tags for routes that actually share url fragments + let tagsToKeep = routes.map((routeWithMetaInformation) => capitalize(routeWithMetaInformation.route.urlFragment.split('/')[1])); + tagsToKeep = tagsToKeep.filter((element, i, array) => array.indexOf(element) === i && array.lastIndexOf(element) !== i); + + // initialize json output + const output = openapi3Template; + + // names of the schemas to copy + const schemasToCopy: string[] = []; // generate documentation for all routes routes.forEach((routeWithMetaInformation) => { - output += generateDocumentationForRoute( + routeWithMetaInformation.tags = [capitalize(routeWithMetaInformation.route.urlFragment.split('/')[1])]; + + output.paths[routeWithMetaInformation.route.urlFragment] = generateOpenAPIForRoute( routeWithMetaInformation, - getNodeMetaInformationMap(projectReflection), + relative(relativeOutDirPath,outDirSchemasPath), + schemasToCopy, + tagsToKeep, ); }); - // write documentation to file - writeFileSync(mdPath, output); + // copy schema json schema files + try { + if (!existsSync(outDirSchemasPath)){ + await mkdirPromisified(outDirSchemasPath, { + recursive: true, + }); + } + for (const fileName of schemasToCopy) { + await copy(join(srcPath, 'schema', `${fileName}.json`), join(outDirSchemasPath, `${fileName}.json`)); + } + } catch (error) { + await Logger.error(error); + // tslint:disable-next-line: no-magic-numbers + process.exit(-2); + } - Logger.ok(`Route documentation written to ${mdPath}.`); + // write openapi object to file (prettified) + // tslint:disable-next-line: no-magic-numbers + writeFileSync(join(outDirPath, 'openapi.json'), JSON.stringify(output, null, 2)); + + Logger.ok(`OpenAPI representation resources written to ${outDirPath} .`); }); commander diff --git a/src/common.ts b/src/common.ts index 73323b80..f63609bf 100644 --- a/src/common.ts +++ b/src/common.ts @@ -56,9 +56,9 @@ export interface RouteWithMetaInformation { */ route: { /** - * Error names of a route + * Possible errors on a route */ - errorNames: Error[]; + errors: SCErrorResponse[]; /** * Method of the route */ @@ -69,10 +69,18 @@ export interface RouteWithMetaInformation { obligatoryParameters: { [k: string]: string; }; + /** + * Description of the request body + */ + requestBodyDescription: string; /** * Name of the request body */ requestBodyName: string; + /** + * Description of the response body + */ + responseBodyDescription: string; /** * Name of the response body */ @@ -86,6 +94,10 @@ export interface RouteWithMetaInformation { */ urlFragment: string; }; + /** + * Possible tags/keywords the route can be associated with + */ + tags?: [string]; } /** @@ -102,6 +114,21 @@ export interface NodeWithMetaInformation { type: string; } +/** + * A generic error that can be returned by the backend if somethings fails during the processing of a request + */ +export interface SCErrorResponse extends Error { + /** + * Additional data that describes the error + */ + additionalData?: unknown; + + /** + * HTTP status code to return this error with + */ + statusCode: number; +} + /** * A map of nodes indexed by their name */ @@ -345,3 +372,13 @@ export function getFullTypeName(type: LightweightType): string { return fullName; } + +/** + * Creates sentence cased string + * + * @param str The string to capitalize + */ +export function capitalize(str?: string): string { + // tslint:disable-next-line: newline-per-chained-call + return `${str?.charAt(0).toUpperCase()}${str?.slice(1).toLowerCase()}`; +} diff --git a/src/resources/openapi-303-template.ts b/src/resources/openapi-303-template.ts new file mode 100644 index 00000000..b27f32d9 --- /dev/null +++ b/src/resources/openapi-303-template.ts @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2021 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {OpenAPIV3} from 'openapi-types'; + +export const openapi3Template: OpenAPIV3.Document = { + openapi: '3.0.3', + info: { + title: 'Openstapps Backend', + description: `# Introduction +This is a human readable documentation of the backend OpenAPI representation.`, + contact: { + name: 'Openstapps Team', + url: 'https://gitlab.com/openstapps/backend', + email: 'app@uni-frankfurt.de', + }, + license: { + name: 'AGPL 3.0', + url: 'https://www.gnu.org/licenses/agpl-3.0.en.html', + }, + version: '2.0.0', + }, + servers: [ + { + url: 'https://mobile.server.uni-frankfurt.de:3000', + description: 'Production server', + }, + ], + paths: {}, + }; diff --git a/src/routes.ts b/src/routes.ts index 565fd01b..87893d4b 100644 --- a/src/routes.ts +++ b/src/routes.ts @@ -14,10 +14,11 @@ */ import {asyncPool} from '@krlwlfrt/async-pool/lib/async-pool'; import {Logger} from '@openstapps/logger'; +import {OpenAPIV3} from 'openapi-types'; import {basename, dirname, join} from 'path'; import {ProjectReflection} from 'typedoc'; import {Type} from 'typedoc/dist/lib/models'; -import {NodesWithMetaInformation, NodeWithMetaInformation, RouteWithMetaInformation} from './common'; +import {capitalize, NodeWithMetaInformation, RouteWithMetaInformation} from './common'; /** * Gather relevant information of routes @@ -55,6 +56,19 @@ export async function gatherRouteInformation(reflection: ProjectReflection): Pro const route = new importedModule[node.name](); + // tslint:disable-next-line: no-any + const errors = route.errorNames.map((error: any) => { + const scError = new importedModule[error.name](); + scError.name = error.name; + + return scError; + }); + + route.responseBodyDescription = module.children!.find(element => element.name === route.responseBodyName)?.comment?.shortText; + route.requestBodyDescription = module.children!.find(element => element.name === route.requestBodyName)?.comment?.shortText; + + route.errors = errors; + routes.push({description: node.comment!, name: node.name, route}); } } @@ -69,28 +83,6 @@ export async function gatherRouteInformation(reflection: ProjectReflection): Pro return routes; } -/** - * Get a linked name for a node - * - * @param name Name of the node - * @param node Node itself - * @param humanize Whether to humanize the name or not - */ -export function getLinkedNameForNode(name: string, node: NodeWithMetaInformation, humanize = false): string { - const humanizeString = require('humanize-string'); - - let printableName = name; - - if (humanize) { - printableName = humanizeString(name.substr('SC'.length)); - } - - let link = `[${printableName}]`; - link += `(${getLinkForNode(name, node)})`; - - return link; -} - /** * Get link for a node * @@ -128,89 +120,85 @@ export function getLinkForNode(name: string, node: NodeWithMetaInformation): str * Generate documentation snippet for one route * * @param routeWithInfo A route instance with its meta information - * @param nodes Nodes with meta information + * @param outDirSchemasPath Path to directory that will contain relevant schemas for the route + * @param schemasToCopy Schemas identified as relevant for this route + * @param tagsToKeep Tags / keywords that can be used for grouping routes */ -export function generateDocumentationForRoute(routeWithInfo: RouteWithMetaInformation, - nodes: NodesWithMetaInformation): string { - let output = ''; - + export function generateOpenAPIForRoute(routeWithInfo: RouteWithMetaInformation, + outDirSchemasPath: string, + schemasToCopy: string[], + tagsToKeep: string[]): OpenAPIV3.PathItemObject { const route = routeWithInfo.route; + const path: OpenAPIV3.PathItemObject = {}; - output += `## \`${route.method} ${route.urlFragment}\``; - output += ` ${getLinkedNameForNode(routeWithInfo.name, nodes[routeWithInfo.name], true)}\n\n`; + schemasToCopy.push(route.requestBodyName, route.responseBodyName); - if (typeof routeWithInfo.description.shortText === 'string') { - output += `**${routeWithInfo.description.shortText}**\n\n`; - } + path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)] = { + summary: capitalize(routeWithInfo.description.shortText?.replace(/(Route to |Route for )/gmi, '')), + description: routeWithInfo.description.text, + requestBody: { + description: route.responseBodyDescription ?? undefined, + content: { + 'application/json': { + schema: { + $ref: join(outDirSchemasPath, `${route.requestBodyName}.json`), + }, + }, + }, + }, + parameters: [{ + name: 'X-StApps-Version', + in: 'header', + schema: { + type: 'string', + example: '2.0.0', + }, + required: true, + }], + responses: {}, + tags: routeWithInfo.tags?.filter(value => tagsToKeep.includes(value)), + }; - if (typeof routeWithInfo.description.text === 'string') { - output += `${routeWithInfo.description.text.replace('\n', '
')}\n\n`; - } + path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)]!.responses![route.statusCodeSuccess] = { + description: route.responseBodyDescription, + content: { + 'application/json': { + schema: { + $ref: join(outDirSchemasPath, `${route.responseBodyName}.json`), + }, + }, + }, + }; - output += `### Definition - -| parameter | value | -| --- | --- | -| request | ${getLinkedNameForNode(route.requestBodyName, nodes[route.requestBodyName])} | -| response | ${getLinkedNameForNode(route.responseBodyName, nodes[route.responseBodyName])} | -| success code | ${route.statusCodeSuccess} | -| errors | ${route.errorNames - .map((error) => { - return getLinkedNameForNode(error.name, nodes[error.name]); - }) - .join('
')} | -`; - if (typeof route.obligatoryParameters === 'object' && Object.keys(route.obligatoryParameters).length > 0) { - let parameterTable = ''; - - for (const parameter in route.obligatoryParameters) { - if (!route.obligatoryParameters.hasOwnProperty(parameter)) { - continue; - } - - let type = route.obligatoryParameters![parameter]; - - if (typeof nodes[type] !== 'undefined') { - type = getLinkedNameForNode(type, nodes[type]); - } - - parameterTable += ``; - } - - parameterTable += '
parametertype
${parameter}${type}
'; - - output += `| obligatory parameters | ${parameterTable} |`; - } - output += '\n\n'; - - return output; -} - -/** - * Get a map of nodes with their meta information - * - * @param projectReflection Reflection to get information from - */ -export function getNodeMetaInformationMap(projectReflection: ProjectReflection): NodesWithMetaInformation { - const nodes: NodesWithMetaInformation = {}; - - if (typeof projectReflection.children === 'undefined') { - throw new Error('Project reflection doesn\'t contain any modules.'); - } - - // iterate over modules - projectReflection.children.forEach((module) => { - if (Array.isArray(module.children) && module.children.length > 0) { - // iterate over types - module.children.forEach((node) => { - // add node with module and type - nodes[node.name] = { - module: module.name.substring(1, module.name.length - 1), - type: node.kindString!, - }; - }); - } + route.errors.forEach(error => { + schemasToCopy.push(error.name); + path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)]!.responses![error.statusCode] = { + description: error.message ?? capitalize(error.name.replace(/([A-Z][a-z])/g,' $1') + .replace('SC ', '')), + content: { + 'application/json': { + schema: { + $ref: join(outDirSchemasPath, `${error.name}.json`), + }, + }, + }, + }; }); - return nodes; + if (typeof route.obligatoryParameters === 'object') { + for (const [parameter, schemaDefinition] of Object.entries(route.obligatoryParameters)) { + const openapiParam: OpenAPIV3.ParameterObject = { + in: 'path', + name: parameter, + required: true, + schema: { + // TODO make this less of a hack and search copied schemas for the first occurring definition + $ref: `schemas/SCSearchResponse.json#/definitions/${schemaDefinition}`, + }, + }; + path[(route.method.toLowerCase() as OpenAPIV3.HttpMethods)]?.parameters?.push(openapiParam); + } + } + + return path; }