From 21eeecd5ee0d68a4faa93bb70d2187ce35807b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wieland=20Sch=C3=B6bl?= Date: Thu, 12 Aug 2021 13:09:45 +0000 Subject: [PATCH] feat: add elasticsearch mappings to build Add backwards compatibility check with typescript v3.8.3 --- .gitlab-ci.yml | 14 ++- package-lock.json | 221 +++++++++++++++++++++++++++++++++++++++----- package.json | 9 +- test/guards.spec.ts | 2 + test/schema.spec.ts | 80 ++++++++++++---- 5 files changed, 279 insertions(+), 47 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69a4ef7d..4056a4d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,6 +22,15 @@ build: - lib expire_in: 1 week +build-ts-3.8.3: + tags: + - performance + stage: build + script: + - npm uninstall typescript + - npm install typescript@3.8.3 + - npm run build + docs_review: dependencies: - build @@ -89,7 +98,10 @@ mapping: - name: registry.gitlab.com/openstapps/database:master alias: elasticsearch script: - - node ./node_modules/@openstapps/core-tools/lib/cli.js put-es-templates ./src http://elasticsearch:9200/ "pattern,see,minlength,tjs-format" + - npm run mappings-integration + artifacts: + paths: + - lib package: dependencies: diff --git a/package-lock.json b/package-lock.json index 25445827..2547f30a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -471,6 +471,11 @@ "path-is-absolute": "^1.0.0" } }, + "marked": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/marked/-/marked-1.2.9.tgz", + "integrity": "sha512-H8lIX2SvyitGX+TRdtS06m1jHMijKN/XjfH6Ooii9fvxMlh8QdqBfBDkGUpMWH2kQNrtixjzYUa3SH8ROTgRRw==" + }, "ts-node": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.1.0.tgz", @@ -487,6 +492,117 @@ "source-map-support": "^0.5.17", "yn": "3.1.1" } + }, + "typedoc": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.18.0.tgz", + "integrity": "sha512-UgDQwapCGQCCdYhEQzQ+kGutmcedklilgUGf62Vw6RdI29u6FcfAXFQfRTiJEbf16aK3YnkB20ctQK1JusCRbA==", + "requires": { + "fs-extra": "^9.0.1", + "handlebars": "^4.7.6", + "highlight.js": "^10.0.0", + "lodash": "^4.17.15", + "lunr": "^2.3.8", + "marked": "^1.1.1", + "minimatch": "^3.0.0", + "progress": "^2.0.3", + "shelljs": "^0.8.4", + "typedoc-default-themes": "^0.10.2" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + } + } + }, + "typedoc-default-themes": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.10.2.tgz", + "integrity": "sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg==", + "requires": { + "lunr": "^2.3.8" + } + }, + "typescript": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", + "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==" + } + } + }, + "@openstapps/es-mapping-generator": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@openstapps/es-mapping-generator/-/es-mapping-generator-0.0.3.tgz", + "integrity": "sha512-zs30bGv00GtXvYeYgOxOyaIe43pHeb568qw/vE8wpJMS6Vqvf4e1DjwCMgHcxkFfBWhkh35+Chi9JvsZslG3jA==", + "dev": true, + "requires": { + "@openstapps/logger": "0.7.0", + "commander": "8.1.0", + "deepmerge": "4.2.2", + "flatted": "3.2.2", + "got": "11.8.2", + "typedoc": "0.18.0", + "typescript": "3.8.3" + }, + "dependencies": { + "commander": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.1.0.tgz", + "integrity": "sha512-mf45ldcuHSYShkplHHGKWb4TrmwQadxOn7v4WuhDJy0ZVoY5JFajaRDKD0PNe5qXzBX0rhovjTnP6Kz9LETcuA==", + "dev": true + }, + "flatted": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.2.tgz", + "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", + "dev": true + }, + "marked": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/marked/-/marked-1.2.9.tgz", + "integrity": "sha512-H8lIX2SvyitGX+TRdtS06m1jHMijKN/XjfH6Ooii9fvxMlh8QdqBfBDkGUpMWH2kQNrtixjzYUa3SH8ROTgRRw==", + "dev": true + }, + "typedoc": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.18.0.tgz", + "integrity": "sha512-UgDQwapCGQCCdYhEQzQ+kGutmcedklilgUGf62Vw6RdI29u6FcfAXFQfRTiJEbf16aK3YnkB20ctQK1JusCRbA==", + "dev": true, + "requires": { + "fs-extra": "^9.0.1", + "handlebars": "^4.7.6", + "highlight.js": "^10.0.0", + "lodash": "^4.17.15", + "lunr": "^2.3.8", + "marked": "^1.1.1", + "minimatch": "^3.0.0", + "progress": "^2.0.3", + "shelljs": "^0.8.4", + "typedoc-default-themes": "^0.10.2" + } + }, + "typedoc-default-themes": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.10.2.tgz", + "integrity": "sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg==", + "dev": true, + "requires": { + "lunr": "^2.3.8" + } + }, + "typescript": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", + "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", + "dev": true } } }, @@ -1919,6 +2035,7 @@ "version": "9.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, "requires": { "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", @@ -3122,9 +3239,10 @@ "dev": true }, "marked": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/marked/-/marked-1.2.9.tgz", - "integrity": "sha512-H8lIX2SvyitGX+TRdtS06m1jHMijKN/XjfH6Ooii9fvxMlh8QdqBfBDkGUpMWH2kQNrtixjzYUa3SH8ROTgRRw==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/marked/-/marked-2.1.3.tgz", + "integrity": "sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA==", + "dev": true }, "meow": { "version": "8.1.2", @@ -3662,6 +3780,32 @@ "mimic-fn": "^1.0.0" } }, + "onigasm": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/onigasm/-/onigasm-2.2.5.tgz", + "integrity": "sha512-F+th54mPc0l1lp1ZcFMyL/jTs2Tlq4SqIHKIXGZOR/VkHkF9A7Fr5rRr5+ZG/lWeRsyrClLYRq7s/yFQ/XhWCA==", + "dev": true, + "requires": { + "lru-cache": "^5.1.1" + }, + "dependencies": { + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + } + } + }, "openapi-types": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-9.1.0.tgz", @@ -4275,6 +4419,17 @@ "rechoir": "^0.6.2" } }, + "shiki": { + "version": "0.9.6", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.9.6.tgz", + "integrity": "sha512-h2y5Uq9QEWsEmi97n+BOdPOVxkOUdVunl+jVIzU9EqJ6/QbIX+U6F7TsrWZQ2xqwPgvvQaC9r7/zeegi1b48dQ==", + "dev": true, + "requires": { + "json5": "^2.2.0", + "onigasm": "^2.2.5", + "vscode-textmate": "5.2.0" + } + }, "signal-exit": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", @@ -4826,34 +4981,48 @@ } }, "typedoc": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.18.0.tgz", - "integrity": "sha512-UgDQwapCGQCCdYhEQzQ+kGutmcedklilgUGf62Vw6RdI29u6FcfAXFQfRTiJEbf16aK3YnkB20ctQK1JusCRbA==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.5.tgz", + "integrity": "sha512-uRDRmYheE5Iju9Zz0X50pTASTpBorIHFt02F5Y8Dt4eBt55h3mwk1CBSY2+EfwBxY16N4Xm7f8KXhnfFZ0AmBw==", + "dev": true, "requires": { - "fs-extra": "^9.0.1", - "handlebars": "^4.7.6", - "highlight.js": "^10.0.0", - "lodash": "^4.17.15", - "lunr": "^2.3.8", - "marked": "^1.1.1", + "glob": "^7.1.7", + "handlebars": "^4.7.7", + "lunr": "^2.3.9", + "marked": "^2.1.1", "minimatch": "^3.0.0", "progress": "^2.0.3", - "shelljs": "^0.8.4", - "typedoc-default-themes": "^0.10.2" + "shiki": "^0.9.3", + "typedoc-default-themes": "^0.12.10" + }, + "dependencies": { + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "typedoc-default-themes": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.10.2.tgz", - "integrity": "sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg==", - "requires": { - "lunr": "^2.3.8" - } + "version": "0.12.10", + "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.12.10.tgz", + "integrity": "sha512-fIS001cAYHkyQPidWXmHuhs8usjP5XVJjWB8oZGqkTowZaz3v7g3KDZeeqE82FBrmkAnIBOY3jgy7lnPnqATbA==", + "dev": true }, "typescript": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", - "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==" + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.5.tgz", + "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==", + "dev": true }, "uglify-js": { "version": "3.13.9", @@ -4913,6 +5082,12 @@ "extsprintf": "^1.2.0" } }, + "vscode-textmate": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-5.2.0.tgz", + "integrity": "sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ==", + "dev": true + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 9b524622..20cfd8bf 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "main": "./lib/index.js", "types": "./lib/index.d.ts", "scripts": { - "build": "npm run tslint && npm run compile && npm run pack && npm run schema", + "build": "npm run tslint && npm run compile && npm run pack && npm run schema && npm run mappings", "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0 && git add CHANGELOG.md && git commit -m 'docs: update changelog'", "check-configuration": "openstapps-configuration", "compile": "rimraf lib && tsc", @@ -25,6 +25,8 @@ "preversion": "npm run prepublishOnly", "push": "git push && git push origin \"v$npm_package_version\"", "schema": "node --max-old-space-size=8192 --stack-size=10240 ./node_modules/.bin/openstapps-core-tools schema src lib/schema", + "mappings": "mkdir lib/mappings && openstapps-es-mapping-generator mapping ../core/src -i minlength,pattern,see,tjs-format -m lib/mappings/mappings.json -a lib/mappings/aggregations.json", + "mappings-integration": "openstapps-es-mapping-generator put-es-templates lib/mappings/mappings.json http://elasticsearch:9200/", "test": "nyc mocha --require ts-node/register --recursive 'test/*.spec.ts'", "tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'" }, @@ -57,6 +59,7 @@ }, "devDependencies": { "@openstapps/configuration": "0.27.0", + "@openstapps/es-mapping-generator": "0.0.3", "@openstapps/logger": "0.7.0", "@testdeck/mocha": "0.1.2", "@types/chai": "4.2.21", @@ -71,8 +74,8 @@ "surge": "0.23.0", "ts-node": "9.1.1", "tslint": "6.1.3", - "typedoc": "0.18.0", - "typescript": "3.8.3" + "typedoc": "0.21.5", + "typescript": "4.3.5" }, "nyc": { "all": true, diff --git a/test/guards.spec.ts b/test/guards.spec.ts index 819b5383..d4843dd1 100644 --- a/test/guards.spec.ts +++ b/test/guards.spec.ts @@ -116,8 +116,10 @@ export class GuardsSpec { @test public isSearchResponse() { const notASearchResponse = {...GuardsSpec.searchResponse}; + // @ts-ignore delete notASearchResponse.pagination; expect(isSearchResponse(notASearchResponse)).to.be.equal(false); + // @ts-ignore delete notASearchResponse.data; expect(isSearchResponse(notASearchResponse)).to.be.equal(false); expect(isSearchResponse(null)).to.be.equal(false); diff --git a/test/schema.spec.ts b/test/schema.spec.ts index 8e2a6968..4643138e 100644 --- a/test/schema.spec.ts +++ b/test/schema.spec.ts @@ -12,21 +12,55 @@ * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ -import {getProjectReflection} from '@openstapps/core-tools/lib/common'; +import {getTsconfigPath} from '@openstapps/core-tools/lib/common'; import {validateFiles, writeReport} from '@openstapps/core-tools/lib/validate'; import {Logger} from '@openstapps/logger'; import {fail} from 'assert'; import {expect} from 'chai'; -import {mkdirSync} from 'fs'; +import {mkdirSync, PathLike} from 'fs'; import {slow, suite, test, timeout} from '@testdeck/mocha'; import {join, resolve} from 'path'; -import {DeclarationReflection, ProjectReflection} from 'typedoc'; -import {ArrayType, IntrinsicType, ReferenceType, StringLiteralType, Type, UnionType} from 'typedoc/dist/lib/models'; +import {Application, DeclarationReflection, ProjectReflection, TSConfigReader} from 'typedoc'; +import {ArrayType, IntrinsicType, ReferenceType, LiteralType, Type, UnionType} from 'typedoc/dist/lib/models'; process.on('unhandledRejection', (err) => { throw err; }); +/** + * Get a project reflection from a path + * + * @param srcPath Path to get reflection from + * @param excludeExternals Exclude external dependencies + */ +export function getProjectReflection(srcPath: PathLike, excludeExternals = true): ProjectReflection { + Logger.info(`Generating project reflection for ${srcPath.toString()}.`); + + const tsconfigPath = getTsconfigPath(srcPath.toString()); + let inputFilePath = srcPath; + if (inputFilePath === tsconfigPath) { + inputFilePath = join(tsconfigPath, 'src'); + } + + // initialize new Typedoc application + const app = new Application(); + app.options.addReader(new TSConfigReader()); + app.bootstrap({ + entryPoints: [inputFilePath.toString()], + excludeExternals: excludeExternals, + tsconfig: join(tsconfigPath, 'tsconfig.json'), + }); + + // get project reflection from input files + const result = app.convert(); + + if (typeof result === 'undefined') { + throw new Error('Project reflection could not be generated.'); + } + + return result; +} + /** * Check if type is a union type * @@ -68,8 +102,8 @@ function isIntrinsicType(type: Type): type is IntrinsicType { * * @param type Type to check */ -function isStringLiteralType(type: Type): type is StringLiteralType { - return type.type === 'stringLiteral'; +function isLiteralType(type: Type): type is LiteralType { + return type.type === 'literal'; } /** @@ -182,6 +216,22 @@ export class SchemaSpec { @test 'no property is an SCThing'() { + const handleUnionType = (type: UnionType, thingName: string, property: DeclarationReflection) => { + for (const nestedType of type.types) { + if (isIntrinsicType(nestedType) || isLiteralType(nestedType)) { + continue; + } else if (isReferenceType(nestedType)) { + expect(SchemaSpec.thingNames).not.to.contain( + nestedType.name, + `Union property '${property.name}' on type '${thingName}' contains type '${nestedType.name}'.`, + ); + } else { + // tslint:disable-next-line:max-line-length + fail(`'${thingName}'#'${property.name}' union type '${nestedType.type}' is not handled by this test!`); + } + } + } + for (const thingName of SchemaSpec.thingNames) { const thingReflection = SchemaSpec.objects[`${thingName}`]; @@ -199,8 +249,10 @@ export class SchemaSpec { } else if (isArrayType(type)) { const elementType = type.elementType; - if (isIntrinsicType(elementType)) { + if (isIntrinsicType(elementType) || isLiteralType(elementType)) { continue; + } else if (isUnionType(elementType)) { + handleUnionType(elementType, thingName, property); } else if (isReferenceType(elementType)) { expect(SchemaSpec.thingNames).not.to.contain( elementType.name, @@ -230,19 +282,7 @@ export class SchemaSpec { } } while (isReferenceType(type)); } else if (isUnionType(type)) { - for (const nestedType of type.types) { - if (isIntrinsicType(nestedType) || isStringLiteralType(nestedType)) { - continue; - } else if (isReferenceType(nestedType)) { - expect(SchemaSpec.thingNames).not.to.contain( - nestedType.name, - `Union property '${property.name}' on type '${thingName}' contains type '${nestedType.name}'.`, - ); - } else { - // tslint:disable-next-line:max-line-length - fail(`'${thingName}'#'${property.name}' union type '${nestedType.type}' is not handled by this test!`); - } - } + handleUnionType(type, thingName, property); } else { // tslint:disable-next-line:max-line-length fail(`'${thingName}'#'${property.name}' with type '${type.type}' is not handled by this test!`);