diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0e2f3ed..194c7df2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,6 +56,22 @@ routes: paths: - routes.md +mapping: + dependencies: + - build + stage: test + services: + - name: registry.gitlab.com/openstapps/database:master + alias: elasticsearch + script: + - npm install @openstapps/core + - node lib/cli.js mapping ./node_modules/@openstapps/core/src mapping.json "pattern,see,minlength" + - curl http://elasticsearch:9200/stapps --upload-file mapping.json -o response.json + - cat response.json + - grep -q "\"acknowledged\":true" response.json + # - curl --show-error --fail http://elasticsearch:9200/stapps --upload-file mapping.json + + package: dependencies: - build diff --git a/.npmignore b/.npmignore index 1038f776..6f9f66b5 100644 --- a/.npmignore +++ b/.npmignore @@ -3,6 +3,7 @@ /* # Except these files/folders !lib +lib/tsconfig.tsbuildinfo !LICENSE !package.json !package-lock.json diff --git a/README.md b/README.md index ae08463e..bcc0dcfc 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,51 @@ To pack all the different files into two distribution files - one for definition openstapps-core-tools pack ``` +## How to use the Elasticsearch Mapping generator + +The mapping generator is intended to be used by the backend directly, but it can also be used to generate these files +manually. + +### Generating the mapping files by hand + +To generate the mapping files by hand, you need a local copy of the core-tools and the core, both need to be built first. +After that you can run +``` +node lib/cli.js mapping path/to/core path/to/destination ignoredTag1,ignoredTag2,ignoredTag3 +``` +If you don't pass in any ignored tags, you will likely be prompted with errors, despite the core being absolutely correct. +This is because there are some tags that are not relevant to Elasticsearch, but the program has no direct way to tell +which ones simply lack an implementation and which ones can be ignored. Currently the ignored tags include +`minlength`, `pattern` and `see`. + +### Generating the mapping directly from another TypeScript program + +This is the more easy way, and it gives you direct access to the generated mapping as a (mostly typesafe) object. To +use it, call `generateTemplate`, However you will first need to generate a ProjectReflection of the core you are working +with. If you use the core as a dependency, you can for example use +```typescript +const map = generateTemplate(getProjectReflection(resolve('node_modules', '@openstapps', 'core', 'src')), + ignoredTags, false); +``` +to generate the mappings. Note that the result object contains both a list of errors in `map.errors` and the actual mapping +in `map.template`. You can also specify whether you want the generator to show any errors while generating the mappings +in the console in the last parameter, `true` (default) being show errors and `false` to suppress them. That said it is very +easy to replace all `type: "MISSING_PREMAP"`, `type: "PARSE_ERROR"`, `type: "TYPE_CONFLICT"` with `dynamic: true` (you +can take these exactly like written here and run a replace over the file). + +### Fixing a generated mapping by hand + +If you get errors when generating the mappings, the mappings might not work, however they will still be generated to the +programs best efforts. Most small issues can be fixed after the mapping was generated as a temporary solution and without +changing anything in the mapper's code. This however requires some understanding of how mappings work. + +The output of the program can easily reach 25.000 lines, but you can find errors quickly by searching for `MISSING_PREMAP`, +`PARSE_ERROR` and `TYPE_CONFLICT`. When you reach them you can then manually replace them with your code. + +As a last resort you can also replace all errors with dynamic types, this should get the mapping working, but it is NOT +RECOMMENDED as a fix other than using it locally. + + ## How to use the UML generator The UML Generator generates PlantUML from the project reflection of the source files. By default it will include externals, which will take considerably longer to execute, you can disable this behaviour via an option. It can help you to visually explore the data model or document a specific part. diff --git a/package-lock.json b/package-lock.json index 19169a4e..c0605c5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,6 +73,13 @@ "chalk": "2.4.2", "flatted": "2.0.0", "nodemailer": "6.2.1" + }, + "dependencies": { + "flatted": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", + "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==" + } } }, "@sindresorhus/is": { @@ -140,9 +147,9 @@ "integrity": "sha512-pGF/zvYOACZ/gLGWdQH8zSwteQS1epp68yRcVLJMgUck/MjEn/FBYmPub9pXT8C1e4a8YZfHo1CKyV8q1vKUnQ==" }, "@types/lodash": { - "version": "4.14.133", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.133.tgz", - "integrity": "sha512-/3JqnvPnY58GLzG3Y7fpphOhATV1DDZ/Ak3DQufjlRK5E4u+s0CfClfNFtAGBabw+jDGtRFbOZe+Z02ZMWCBNQ==" + "version": "4.14.135", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.135.tgz", + "integrity": "sha512-Ed+tSZ9qM1oYpi5kzdsBuOzcAIn1wDW+e8TFJ50IMJMlSopGdJgKAbhHzN6h1E1OfjlGOr2JepzEWtg9NIfoNg==" }, "@types/marked": { "version": "0.4.2", @@ -352,26 +359,31 @@ "dev": true }, "cacheable-request": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.0.0.tgz", - "integrity": "sha512-2N7AmszH/WPPpl5Z3XMw1HAP+8d+xugnKQAeKvxFZ/04dbT/CAznqwbl+7eSr3HkwdepNwtb2yx3CAMQWvG01Q==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", "requires": { "clone-response": "^1.0.2", - "get-stream": "^4.0.0", + "get-stream": "^5.1.0", "http-cache-semantics": "^4.0.0", "keyv": "^3.0.0", - "lowercase-keys": "^1.0.1", - "normalize-url": "^3.1.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", "responselike": "^1.0.2" }, "dependencies": { "get-stream": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", - "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", + "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", "requires": { "pump": "^3.0.0" } + }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" } } }, @@ -622,9 +634,9 @@ "dev": true }, "conventional-changelog-writer": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-4.0.5.tgz", - "integrity": "sha512-g/Myp4MaJ1A+f7Ai+SnVhkcWtaHk6flw0SYN7A+vQ+MTu0+gSovQWs4Pg4NtcNUcIztYQ9YHsoxHP+GGQplI7Q==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-4.0.6.tgz", + "integrity": "sha512-ou/sbrplJMM6KQpR5rKFYNVQYesFjN7WpNGdudQSWNi6X+RgyFUcSv871YBYkrUYV9EX8ijMohYVzn9RUb+4ag==", "dev": true, "requires": { "compare-func": "^1.3.1", @@ -634,17 +646,9 @@ "json-stringify-safe": "^5.0.1", "lodash": "^4.2.1", "meow": "^4.0.0", - "semver": "^5.5.0", + "semver": "^6.0.0", "split": "^1.0.0", "through2": "^3.0.0" - }, - "dependencies": { - "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", - "dev": true - } } }, "conventional-commits-filter": { @@ -658,35 +662,18 @@ } }, "conventional-commits-parser": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.0.2.tgz", - "integrity": "sha512-y5eqgaKR0F6xsBNVSQ/5cI5qIF3MojddSUi1vKIggRkqUTbkqFKH9P5YX/AT1BVZp9DtSzBTIkvjyVLotLsVog==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.0.3.tgz", + "integrity": "sha512-KaA/2EeUkO4bKjinNfGUyqPTX/6w9JGshuQRik4r/wJz7rUw3+D3fDG6sZSEqJvKILzKXFQuFkpPLclcsAuZcg==", "dev": true, "requires": { "JSONStream": "^1.0.4", - "is-text-path": "^1.0.0", + "is-text-path": "^2.0.0", "lodash": "^4.2.1", "meow": "^4.0.0", "split2": "^2.0.0", "through2": "^3.0.0", "trim-off-newlines": "^1.0.0" - }, - "dependencies": { - "is-text-path": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz", - "integrity": "sha1-Thqg+1G/vLPpJogAE5cgLBd1tm4=", - "dev": true, - "requires": { - "text-extensions": "^1.0.0" - } - }, - "text-extensions": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", - "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==", - "dev": true - } } }, "core-util-is": { @@ -803,6 +790,11 @@ "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", "dev": true }, + "deepmerge": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-3.3.0.tgz", + "integrity": "sha512-GRQOafGHwMHpjPx9iCvTgpu9NojZ49q794EEL94JVEw6VaeA8XTUyBKvAkOOjBX9oJNiV6G3P+T+tihFjo2TqA==" + }, "defer-to-connect": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.0.2.tgz", @@ -977,9 +969,9 @@ } }, "flatted": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", - "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", + "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==" }, "fs-extra": { "version": "7.0.1", @@ -1252,7 +1244,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", - "dev": true, "requires": { "pump": "^3.0.0" } @@ -1291,6 +1282,15 @@ "util-deprecate": "~1.0.1" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, "through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", @@ -1396,22 +1396,12 @@ "p-cancelable": "^1.0.0", "to-readable-stream": "^1.0.0", "url-parse-lax": "^3.0.0" - }, - "dependencies": { - "get-stream": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", - "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", - "requires": { - "pump": "^3.0.0" - } - } } }, "graceful-fs": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", - "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", + "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==" }, "growl": { "version": "1.10.5", @@ -1496,9 +1486,9 @@ } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { "version": "1.3.5", @@ -1613,6 +1603,15 @@ "has-symbols": "^1.0.0" } }, + "is-text-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-2.0.0.tgz", + "integrity": "sha512-+oDTluR6WEjdXEJMnC2z6A4FRwFoYuvShVVEGsS7ewc0UTi2QtAKMDJuL4BDEVt+5T7MjFo12RP8ghOM75oKJw==", + "dev": true, + "requires": { + "text-extensions": "^2.0.0" + } + }, "is-utf8": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", @@ -2283,9 +2282,9 @@ } }, "normalize-url": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-3.3.0.tgz", - "integrity": "sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==" + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.3.0.tgz", + "integrity": "sha512-0NLtR71o4k6GLP+mr6Ty34c5GA6CMoEsncKJxvQd8NzPxaHRJNnb5gZE8R1XF4CPIS7QPHLJ74IFszwtNVAHVQ==" }, "npm-run-path": { "version": "2.0.2", @@ -2551,9 +2550,9 @@ "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" }, "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, "progress": { @@ -2840,6 +2839,15 @@ "util-deprecate": "~1.0.1" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, "through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", @@ -2869,9 +2877,9 @@ } }, "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", + "integrity": "sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==", "dev": true, "requires": { "safe-buffer": "~5.1.0" @@ -2928,6 +2936,12 @@ "uuid": "^2.0.1" } }, + "text-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-2.0.0.tgz", + "integrity": "sha512-F91ZqLgvi1E0PdvmxMgp+gcf6q8fMH7mhdwWfzXnl1k+GbpQDmi8l7DzLC5JTASKbwpY3TfxajAUzAXcv2NmsQ==", + "dev": true + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -3005,9 +3019,9 @@ } }, "tslib": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz", - "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", "dev": true }, "tslint": { diff --git a/package.json b/package.json index 01ea6073..897298a3 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,9 @@ "ajv": "6.10.0", "chai": "4.2.0", "commander": "2.20.0", + "deepmerge": "3.3.0", "del": "4.1.1", + "flatted": "2.0.1", "glob": "7.1.4", "got": "9.6.0", "humanize-string": "2.1.0", diff --git a/src/cli.ts b/src/cli.ts index ed7dca62..3e8c13d6 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -22,6 +22,7 @@ import { readFilePromisified, toArray, } from './common'; +import {generateTemplate} from './mapping'; import {pack} from './pack'; import { gatherRouteInformation, @@ -77,6 +78,30 @@ commander Logger.ok(`Route documentation written to ${mdPath}.`); }); +commander + .command('mapping [ignoredTags]') + .action(async (relativeSrcPath, relativeMappingPath, ignoredTags) => { + // get absolute paths + const srcPath = resolve(relativeSrcPath); + const mappingPath = resolve(relativeMappingPath); + + let ignoredTagsList: string[] = []; + if (typeof ignoredTags === 'string') { + ignoredTagsList = ignoredTags.split(','); + } + + // get project reflection + const projectReflection = getProjectReflection(srcPath); + + const mapping = generateTemplate(projectReflection, ignoredTagsList); + + // write documentation to file + // tslint:disable-next-line:no-magic-numbers + writeFileSync(mappingPath, JSON.stringify(mapping.template, null, 2)); + + Logger.ok(`Elasticsearch mapping written to ${mappingPath}.`); + }); + commander .command('schema ') .action(async (relativeSrcPath, relativeSchemaPath) => { diff --git a/src/common.ts b/src/common.ts index 87ca1fee..32c9b4e7 100644 --- a/src/common.ts +++ b/src/common.ts @@ -142,6 +142,7 @@ export interface ExpectableValidationErrors { * Get a project reflection from a path * * @param srcPath Path to get reflection from + * @param excludeExternals Exclude external dependencies */ export function getProjectReflection(srcPath: PathLike, excludeExternals = true): ProjectReflection { Logger.info(`Generating project reflection for ${srcPath.toString()}.`); @@ -175,7 +176,7 @@ export function getProjectReflection(srcPath: PathLike, excludeExternals = true) } /** - * Check if a schema has definitions + * Guard method for checking if a schema has definitions * * @param schema Schema to check */ diff --git a/src/mapping.ts b/src/mapping.ts new file mode 100644 index 00000000..e82e4cb8 --- /dev/null +++ b/src/mapping.ts @@ -0,0 +1,480 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {Logger} from '@openstapps/logger'; +import * as deepmerge from 'deepmerge'; +import {stringify} from 'flatted'; +import {DeclarationReflection, ProjectReflection} from 'typedoc'; +import { + ArrayType, + CommentTag, + IntrinsicType, + ReferenceType, ReflectionType, + StringLiteralType, + Type, TypeParameterType, + UnionType, +} from 'typedoc/dist/lib/models'; +import {fieldmap} from './mappings/definitions/fieldmap'; +import {premaps} from './mappings/definitions/premap'; +import {settings} from './mappings/definitions/settings'; +import {dynamicTypes, ElasticsearchDataType, typemap} from './mappings/definitions/typemap'; +import { + ElasticsearchDynamicTemplate, + ElasticsearchObject, + ElasticsearchTemplate, + ElasticsearchValue, + ReflectionGeneric, +} from './mappings/mapping-definitions'; + +const dynamicTemplates: ElasticsearchDynamicTemplate[] = []; +let errors: string[] = []; +let showErrors = true; + +const indexableTag = 'indexable'; + +let ignoredTagsList = ['indexable', 'validatable']; + +/** + * Gets all interfaces that have an @indexable tag + * + * @param projectReflection the project reflection from which to extract the indexable interfaces + */ +export function getAllIndexableInterfaces(projectReflection: ProjectReflection): DeclarationReflection[] { + + let indexableInterfaces: DeclarationReflection[] = []; + + if (!Array.isArray(projectReflection.children) || projectReflection.children.length === 0) { + throw new Error('No DeclarationReflections found. Please check your input path'); + } + + // push all declaration reflections into one array + projectReflection.children.forEach((declarationReflection) => { + if (Array.isArray(declarationReflection.children)) { + indexableInterfaces = indexableInterfaces.concat(declarationReflection.children); + } + }); + + // filter all declaration reflections with an @indexable tag + indexableInterfaces = indexableInterfaces.filter((declarationReflection) => { + if ( + typeof declarationReflection.comment === 'undefined' || + typeof declarationReflection.comment.tags === 'undefined' + ) { + return false; + } + + return typeof declarationReflection.comment.tags.find((commentTag) => { + return commentTag.tagName === indexableTag; + }) !== 'undefined'; + }); + + return indexableInterfaces; +} + +/** + * Composes error messages, that are readable and contain a certain minumum of information + * + * @param path the path where the error took place + * @param typeName the name of the object, with which something went wrong + * @param object the object or name + * @param message the error message + */ +function composeErrorMessage(path: string, typeName: string, object: string, message: string) { + const error = `At "${path.substr(0, path.length - 1)}" for ${typeName} "${object}": ${message}`; + errors.push(error); + if (showErrors) { + // tslint:disable-next-line:no-floating-promises + Logger.error(error); + } +} + +/** + * Gets the Reflections and names for Generics in a ReferenceType of a DeclarationReflection + * + * Warning to future maintainers: The code for generics doesn't account for depth. when there is a new generic, it will + * override the previous one, if there isn't, it will just continue passing it down. + * + * @param type the ReferenceType of a DeclarationReflection + * @param out the previous reflection, it then overrides all parameters or keeps old ones + * @param path the current path to the object we are in + */ +function getReflectionGeneric(type: ReferenceType, out: ReflectionGeneric[], path: string): ReflectionGeneric[] { + if (typeof type.typeArguments !== 'undefined' + && type.reflection instanceof DeclarationReflection + && typeof type.reflection.typeParameters !== 'undefined' + && type.typeArguments.length === type.reflection.typeParameters.length) { + for (let i = 0; i < type.typeArguments.length; i++) { + let replaced = false; + for (const old of out) { + if (old.name === type.reflection.typeParameters[i].name) { + old.value = handleType(type.typeArguments[i], out, path); + replaced = true; + } + } + if (!replaced) { + out.push({ + name: type.reflection.typeParameters[i].name, + value: handleType(type.typeArguments[i], out, path), + }); + } + } + } + + return out; +} + +/** + * Handles a ReferenceType that has no value + * + * @param ref the ReferenceType + * @param generics the generics from levels above, so we can use them without having access to the parent + * @param path the current path to the object we are in + * @param tags any tags attached to the type + */ +function handleRefWithoutReflection(ref: ReferenceType, generics: ReflectionGeneric[], + path: string, tags?: CommentTag[]): ElasticsearchValue { + for (const premap in premaps) { + if (premap === ref.name) { + return readFieldTags(premaps[premap], path, tags); + } + } + + if (ref.name === 'Array') { // basically an external type, but Array is quite common, especially with generics + if (typeof ref.typeArguments === 'undefined' || typeof ref.typeArguments[0] === 'undefined') { + composeErrorMessage(path, 'Array with generics', 'array', 'Failed to parse'); + + return {type: ElasticsearchDataType.parse_error}; + } + + return readFieldTags(handleType(ref.typeArguments[0], getReflectionGeneric(ref, generics, path), path), + path, tags); + } + if (ref.name === '__type') { // empty object + return { + dynamic: 'strict', + properties: {}, + }; + } + + composeErrorMessage(path, 'external type', ref.name, 'Missing pre-map'); + + return readFieldTags({type: ElasticsearchDataType.missing_premap}, path, tags); +} + +/** + * Handles an object + * + * @param decl the DeclarationReflection of the object + * @param generics the generics from levels above, so we can use them without having access to the parent + * @param path the current path to the object we are in + */ +function handleDeclarationReflection(decl: DeclarationReflection, generics: ReflectionGeneric[], path: string): + ElasticsearchValue { + // check if we have an object referencing a generic + for (const gRefl of generics) { + if (gRefl.name === decl.name) { // if the object name is the same as the generic name + return readFieldTags(gRefl.value, path, typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); + // use the value defined by the generic + } + } + + // start the actual handling process + const out: ElasticsearchObject = { + dynamic: 'strict', + properties: {}, + }; + + let empty = true; + // first check if there are any index signatures, so for example `[name: string]: Foo` + if (typeof decl.indexSignature !== 'undefined' && typeof decl.indexSignature.parameters !== 'undefined') { + for (const param of decl.indexSignature.parameters) { + empty = false; + const template: ElasticsearchDynamicTemplate = {}; + template[decl.name] = { + mapping: handleDeclarationReflection(param as DeclarationReflection, generics, path), + match: '*', + match_mapping_type: '*', + path_match: `${path}*`, + }; + dynamicTemplates.push(template); + } + } + + // check all the children, so in this case we are dealing with an OBJECT + if (typeof decl.children !== 'undefined' && decl.children.length > 0) { + for (const child of decl.children) { + empty = false; + out.properties[child.name] = handleDeclarationReflection(child, generics, `${path}${child.name}.`); + } + } else if (decl.type instanceof Type) { // if the object is a type, so we are dealing with a PROPERTY + return handleType(decl.type, generics, path, + typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); + } else if (decl.kindString === 'Enumeration member') { + return readTypeTags(typeof decl.defaultValue, path, + typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); + } + + if (empty) { + composeErrorMessage(path, 'object', decl.name, 'Empty object'); + } + + return readFieldTags(out, path, typeof decl.comment !== 'undefined' ? decl.comment.tags : undefined); +} + +/** + * Handles UnionTypes + * + * Put into a separate function as it is a little bit more complex + * Works fairly reliable, although there are issues with primitive union types, which don't work at all (And never will) + * + * @param type the type object + * @param generics the generics from levels above, so we can use them without having access to the parent + * @param path the current path to the object we are in + */ +function handleUnionType(type: UnionType, generics: ReflectionGeneric[], path: string): ElasticsearchValue { + const list: ElasticsearchValue[] = []; + + for (const subType of type.types) { + if (subType instanceof IntrinsicType && subType.name === 'undefined') { + continue; + } + list.push(handleType(subType, generics, path)); + } + + if (list.length > 0) { + let out = list[0]; + + for (const item of list) { + out = deepmerge(out, item); + } + + return out; + } + + composeErrorMessage(path, 'Union Type', stringify(list), 'Empty union type. This is likely not a user error.'); + + return {type: ElasticsearchDataType.parse_error}; +} + +/** + * Serves as a kind of distributor for the different types, should not contain any specific code + * + * @param type the type object + * @param generics the generics from levels above, so we can use them without having access to the parent + * @param path the current path to the object we are in + * @param tags any tags attached to the type + */ +function handleType(type: Type, generics: ReflectionGeneric[], path: string, tags?: CommentTag[]): ElasticsearchValue { + // logger.log((type as any).name); + if (type instanceof ArrayType) { // array is irrelevant in Elasticsearch, so just go with the element type + return handleType(type.elementType, generics, path, tags); + } + if (type.type === 'stringLiteral') { // a string literal, usually for type + return readTypeTags(type.type, path, tags); + } + if (type instanceof IntrinsicType) { // the absolute default type, like strings + return readTypeTags(type.name, path, tags); + } + if (type instanceof UnionType) { // the union type... + return handleUnionType(type, generics, path); + } + if (type instanceof ReferenceType) { + if (typeof type.reflection !== 'undefined') { + // there is really no way to make this typesafe, every element in DeclarationReflection is optional. + return handleDeclarationReflection(type.reflection as DeclarationReflection, + getReflectionGeneric(type, generics, path), path); + } + + return handleRefWithoutReflection(type, generics, path, tags); + } + if (type instanceof TypeParameterType) { + // check if we have an object referencing a generic + for (const gRefl of generics) { + if (gRefl.name === type.name) { // if the object name is the same as the generic name + return gRefl.value; // use the value defined by the generic + } + } + composeErrorMessage(path, 'Generic', type.name, 'Missing reflection, please report!'); + + return {type: ElasticsearchDataType.parse_error}; + + } + if (type instanceof ReflectionType) { + return readFieldTags(handleDeclarationReflection(type.declaration, generics, path), path, tags); + } + + composeErrorMessage(path, 'type', stringify(type), 'Not implemented type'); + + return {type: ElasticsearchDataType.parse_error}; +} + +/** + * Reads all tags related to Elasticsearch fields from the fieldMap + * + * @param prev the previous ElasticsearchValue, for example and object + * @param path the current path to the object we are in + * @param tags tags attached to the value + */ +function readFieldTags(prev: ElasticsearchValue, path: string, tags?: CommentTag[]): ElasticsearchValue { + if (typeof tags !== 'undefined') { + for (const tag of tags) { + if (!ignoredTagsList.includes(tag.tagName)) { + if (typeof fieldmap[tag.tagName] !== 'undefined') { + if (typeof prev.fields === 'undefined') { + prev.fields = {}; + } + if (tag.text.trim() === '') { + // merge the fields + prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}; + } else if (typeof fieldmap[tag.tagName][tag.text.trim()] !== 'undefined') { + // merge the fields + prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}; + } else if (!fieldmap[tag.tagName].ignore.includes(tag.text.trim())) { + composeErrorMessage(path, 'tag', tag.tagName, `Not implemented tag param "${tag.text.trim()}"`); + } + } else { + composeErrorMessage(path, 'tag', tag.tagName, `Not implemented tag`); + } + } + } + } + + return prev; +} + +/** + * Reads all types related to Elasticsearch fields from the fieldMap + * + * @param type the type of the value + * @param path the current path to the object we are in + * @param tags tags attached to the value + */ +function readTypeTags(type: string, path: string, tags?: CommentTag[]): ElasticsearchValue { + let out: ElasticsearchValue = {type: ElasticsearchDataType.parse_error}; + + if (typeof typemap[type] !== 'undefined') { // first look if the value has a definition in the typemap + if (typeof tags !== 'undefined') { // look if there are any tags + for (let i = tags.length - 1; i >= 0; i--) { + if (!ignoredTagsList.includes(tags[i].tagName) && typeof typemap[type][tags[i].tagName] !== 'undefined') { + // if we have a tag that indicates a type + if (out.type !== ElasticsearchDataType.parse_error) { + composeErrorMessage(path, 'type', type, `Type conflict; "${typemap[type][tags[i].tagName]}" would` + + ` override "${out.type}"`); + out.type = ElasticsearchDataType.type_conflict; + continue; + } + out.type = typemap[type][tags[i].tagName]; + tags.splice(i, 1); // we need this so readFieldTags can process correctly + } + } + } + + if (out.type === ElasticsearchDataType.parse_error) { + out.type = typemap[type].default; + } + + out = readFieldTags(out, path, tags); + + return out; + } + if (dynamicTypes.includes(type)) { // Elasticsearch dynamic type + return { + dynamic: true, + properties: {}, + }; + } + + composeErrorMessage(path, 'type', type, 'Not implemented type'); + + return out; +} + +/** + * Takes a project reflection and generates an ElasticsearchTemplate from it + * + * Serves as the entry point for getting the mapping, so if you just want to get the mapping files for Elasticsearch, + * you can do so by calling this function, `RETURNED_VALUE.template` contains the mapping in a fashion that is directly + * readable by Elasticsearch. + * + * @param projectReflection a reflection of the project you want to get the ES Mappings from + * @param ignoredTags the tag names for which the error output should be suppressed + * @param showErrorOutput whether to print all errors in the command line or not + */ +export function generateTemplate(projectReflection: ProjectReflection, ignoredTags: string[], showErrorOutput = true): +// tslint:disable-next-line:completed-docs + { errors: string[]; template: ElasticsearchTemplate; } { + errors = []; + showErrors = showErrorOutput; + + ignoredTagsList = ['indexable', 'validatable']; + ignoredTagsList.push.apply(ignoredTagsList, ignoredTags); + + const indexableInterfaces = getAllIndexableInterfaces(projectReflection); + + const out: ElasticsearchTemplate = { + mappings: { + _default_: { + _source: { + excludes: [ + 'creation_date', + ], + }, + date_detection: false, + dynamic_templates: [], + properties: {}, + }, + }, + settings: settings, + template: 'stapps_*', + }; + + for (const _interface of indexableInterfaces) { + if (!Array.isArray(_interface.children) || _interface.children.length === 0) { + throw new Error('Interface needs at least some properties to be indexable'); + } + + const typeObject = _interface.children.find((declarationReflection) => { + return declarationReflection.name === 'type'; + }); + + if (typeof typeObject === 'undefined' || typeof typeObject.type === 'undefined') { + throw new Error('Interface needs a type to be indexable'); + } + + let typeName = 'INVALID_TYPE'; + if (typeObject.type instanceof ReferenceType) { + if (typeObject.type.reflection instanceof DeclarationReflection + && typeof typeObject.type.reflection.defaultValue === 'string') { + typeName = typeObject.type.reflection.defaultValue.replace('"', '') + .replace('"', ''); + } else { + // tslint:disable-next-line:no-floating-promises + Logger.error('Your input files seem to be incorrect, or there is a major bug in the mapping generator.'); + } + } else if (typeObject.type instanceof StringLiteralType) { + Logger.warn(`The interface ${_interface.name} uses a string literal as type, please use SCThingType.`); + typeName = typeObject.type.value; + } else { + // tslint:disable-next-line:no-floating-promises + Logger.error(`The interface ${_interface.name} is required to use an SCThingType as a type, please do so.`); + } + + out.mappings._default_.properties[typeName] = + handleDeclarationReflection(_interface, [], '') as ElasticsearchObject; + } + + out.mappings._default_.dynamic_templates = dynamicTemplates; + + return {template: out, errors}; +} diff --git a/src/mappings/definitions/fieldmap.ts b/src/mappings/definitions/fieldmap.ts new file mode 100644 index 00000000..9151b775 --- /dev/null +++ b/src/mappings/definitions/fieldmap.ts @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {ElasticsearchFieldmap} from '../mapping-definitions'; +import {ElasticsearchDataType} from './typemap'; + +export enum analyzers { + ducet_sort = 'ducet_sort', + search_german = 'search_german', +} + +export const fieldmap: ElasticsearchFieldmap = { + aggregatable: { + default: { + raw: { + ignore_above: 10000, + type: ElasticsearchDataType.keyword, + }, + }, + ignore: [], + }, + sortable: { + default: { + sort: { + analyzer: analyzers.ducet_sort, + fielddata: true, + type: ElasticsearchDataType.text, + }, + }, + ducet: { + sort: { + analyzer: analyzers.ducet_sort, + fielddata: true, + type: ElasticsearchDataType.text, + }, + }, + ignore: ['price'], + }, +}; diff --git a/src/mappings/definitions/premap.ts b/src/mappings/definitions/premap.ts new file mode 100644 index 00000000..9bbc6ac9 --- /dev/null +++ b/src/mappings/definitions/premap.ts @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {Logger} from '@openstapps/logger'; +import {ElasticsearchPremap, ElasticsearchValue} from '../mapping-definitions'; +import {ElasticsearchDataType} from './typemap'; + +export const premaps: ElasticsearchPremap = { + CoordinateReferenceSystem: { + dynamic: 'strict', + properties: { + properties: { + dynamic: true, + properties: {}, + }, + type: { + type: ElasticsearchDataType.keyword, + }, + }, + }, + LineString: { + dynamic: 'strict', + properties: { + coordinates: { + type: ElasticsearchDataType.float, + }, + type: { + type: ElasticsearchDataType.keyword, + }, + }, + }, + Point: { + dynamic: 'strict', + properties: { + bbox: {type: ElasticsearchDataType.float}, + coordinates: {type: ElasticsearchDataType.geo_point}, // TODO: filterable + crs: { + dynamic: 'strict', + properties: { + properties: { + dynamic: true, + properties: {}, + }, + type: {type: ElasticsearchDataType.keyword}, + }, + }, + type: {type: ElasticsearchDataType.keyword}, + }, + }, + Polygon: { // a Polygon is mapped the same way as a Point is, you can just copy & paste + dynamic: 'strict', + properties: { + bbox: {type: ElasticsearchDataType.float}, + coordinates: {type: ElasticsearchDataType.geo_point}, // TODO: filterable + crs: { + dynamic: 'strict', + properties: { + properties: { + dynamic: true, + properties: {}, + }, + type: {type: ElasticsearchDataType.keyword}, + }, + }, + type: {type: ElasticsearchDataType.keyword}, + }, + }, + 'jsonpatch.OpPatch': { + dynamic: 'strict', + properties: { + from: { + type: ElasticsearchDataType.keyword, + }, + op: { + type: ElasticsearchDataType.keyword, + }, + path: { + type: ElasticsearchDataType.keyword, + }, + value: { + dynamic: true, + properties: {}, + }, + }, + }, +}; + +/** + * Gets an ElasticsearchValue for a name + * + * @param name the name of the premap + */ +export function getPremap(name: string): ElasticsearchValue { + for (const premap in premaps) { + if (premap === name) { + return premaps[premap]; + } + } + + // tslint:disable-next-line:no-floating-promises + Logger.error(`Missing pre-map for external type ${name}`); + + return {type: ElasticsearchDataType.missing_premap}; +} diff --git a/src/mappings/definitions/settings.ts b/src/mappings/definitions/settings.ts new file mode 100644 index 00000000..fd442bd7 --- /dev/null +++ b/src/mappings/definitions/settings.ts @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {ElasticsearchSettings} from '../mapping-definitions'; + +export const settings: ElasticsearchSettings = { + analysis: { + analyzer: { + ducet_sort: { + filter: [ + 'german_phonebook', + ], + tokenizer: 'keyword', + type: 'custom', + }, + search_german: { + filter: [ + 'lowercase', + 'german_stop', + 'german_stemmer', + ], + tokenizer: 'stapps_ngram', + type: 'custom', + }, + }, + filter: { + german_phonebook: { + country: 'DE', + language: 'de', + type: 'icu_collation', + variant: '@collation=phonebook', + }, + german_stemmer: { + language: 'german', + type: 'stemmer', + }, + german_stop: { + stopwords: '_german_', + type: 'stop', + }, + }, + tokenizer: { + stapps_ngram: { + max_gram: 7, + min_gram: 4, + type: 'ngram', + }, + }, + }, + 'mapping.total_fields.limit': 10000, + max_result_window: 30000, + number_of_replicas: 0, + number_of_shards: 1, +}; diff --git a/src/mappings/definitions/typemap.ts b/src/mappings/definitions/typemap.ts new file mode 100644 index 00000000..4235788e --- /dev/null +++ b/src/mappings/definitions/typemap.ts @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {ElasticsearchTypemap} from '../mapping-definitions'; + +export enum ElasticsearchDataType { + missing_premap = 'MISSING_PREMAP', + parse_error = 'PARSE_ERROR', + type_conflict = 'TYPE_CONFLICT', + text = 'text', + keyword = 'keyword', + date = 'date', + // long = 'long', + // double = 'double', + float = 'float', + boolean = 'boolean', + ip = 'ip', + integer = 'integer', + object = 'object', + nested = 'nested', + geo_point = 'geo_point', + geo_shape = 'geo_shape', + completion = 'completion', +} + +export const typemap: ElasticsearchTypemap = { + boolean: { + default: ElasticsearchDataType.boolean, + }, + false: { + default: ElasticsearchDataType.boolean, + }, + number: { + default: ElasticsearchDataType.integer, + float: ElasticsearchDataType.float, + integer: ElasticsearchDataType.integer, + }, + string: { + default: ElasticsearchDataType.text, + keyword: ElasticsearchDataType.keyword, + text: ElasticsearchDataType.text, + }, + stringLiteral: { + default: ElasticsearchDataType.keyword, + }, + true: { + default: ElasticsearchDataType.boolean, + }, +}; + +export const dynamicTypes = ['any', 'unknown']; diff --git a/src/mappings/mapping-definitions.ts b/src/mappings/mapping-definitions.ts new file mode 100644 index 00000000..4f0635ed --- /dev/null +++ b/src/mappings/mapping-definitions.ts @@ -0,0 +1,313 @@ +/* + * Copyright (C) 2019 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {ElasticsearchDataType} from './definitions/typemap'; + +// tslint:disable:no-any + +/** + * ElasticsearchValue can be either a type or an object. + * + * Both are composed similarly, and can be the value of a propery + * of an Elasticsearch Object. + */ +export type ElasticsearchValue = ElasticsearchType | ElasticsearchObject; + +/** + * Used internally for saving a generic value contained in a reflection + */ +export interface ReflectionGeneric { + /** + * The name of the generic + * + * For example in `` the name would be 'A' + */ + name: string; + + /** + * The value of the generic + */ + value: ElasticsearchValue; +} + +/** + * The Typemap is used to get the corresponding ElasicsearchDataType for a name provided by the ProjectReflection + */ +export interface ElasticsearchTypemap { + /** + * The `stringLiteral` type must always be provided + */ + stringLiteral: { + /** + * The default can be chosen freely, but must be provided + */ + default: ElasticsearchDataType; + }; + + /** + * The name of the JS type, so for `number` it would be number + */ + [name: string]: { + /** + * The default ElasticsearchDataType that should be used, if no tag or only not implemented tags are found + */ + default: ElasticsearchDataType; + + /** + * The name of the tag, so for `@integer` it would be `integer` + */ + [name: string]: ElasticsearchDataType; + }; +} + +/** + * The representation of a `DynamicTemplate` in Elasticsearch + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/dynamic-templates.html + */ +export interface ElasticsearchDynamicTemplate { + /** + * The name of the dynamicTemplate + */ + [name: string]: { + /** + * The mapping of the template + */ + mapping: ElasticsearchValue; + + /** + * With automatic mapping, we use `path_match` more or less out of convenience and because it is least error-prone + * + * This also means that match should match all ("*") interface names (because we provide the exact path of the + * interface) + */ + match: '*'; + + /** + * With automatic mapping, we use `path_match` more or less out of convenience and because it is least error-prone + * + * This also means that match_mapping_type should match all ("*") names (because we provide the exact path of the + * interface) + */ + match_mapping_type: '*'; + + /** + * With automatic mapping, we use `path_match` more or less out of convenience and because it is least error-prone + */ + path_match: string; + }; +} + +/** + * The Fieldmap contains all tag names for fields and the corresponding fields + * + * The Fieldmap works in a similar fashion to the Typemap + */ +export interface ElasticsearchFieldmap { + /** + * The name of the tag, so for `@sortable` it would be `sortable` + */ + [name: string]: { + /** + * The default value if no parameter is provided + */ + default: { + /** + * To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName].default}` + * + * We could also have used `default: any`, but this adds slightly more improved type-safety. + */ + [name: string]: any; + }; + + /** + * The tag parameters that will be ignored + * + * Some tag parameters might not be important for your implementation, so you can add their names here to not get + * any errors. The `default` will be used in that case. + */ + ignore: string[]; + + /** + * The parameters of the tag, so for `@sortable ducet` it would be `ducet` + */ + [name: string]: { + /** + * To allow the usage of `prev.fields = {...prev.fields, ...fieldmap[tag.tagName][tag.text.trim()]}` + * + * We could also have used `default: any`, but this adds slightly more improved type-safety. + */ + [name: string]: any; + }; + }; +} + +/** + * A primitive data type + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-types.html + */ +export interface ElasticsearchType { + /** + * Fields for a type + * + * The fields are optional, they are used for things like sorting, which is not needed for every single type. + */ + fields?: { + [name: string]: any; + }; + + /** + * The type as an ElasticsearchDataType + */ + type: ElasticsearchDataType; +} + +/** + * An object data type + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/object.html + */ +export interface ElasticsearchObject { + /** + * If the object is a dynamic + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/dynamic.html + * The default should be `'strict'` + */ + dynamic: true | false | 'strict'; + + /** + * dynamic_templates for an object + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/dynamic-templates.html + * This is a more complex topic, before touching this you should really know what you are doing. + */ + dynamic_templates?: ElasticsearchDynamicTemplate[]; + + /** + * Fields for a type + * + * The fields are optional, they are used for things like sorting, which is not needed for every single type. + */ + fields?: { + [name: string]: any; + }; + + /** + * Any properties of the object + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/properties.html + */ + properties: { + /** + * Each property can be any Elasticsearch value + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-types.html + */ + [name: string]: ElasticsearchValue; + }; +} + +/** + * An Elasticsearch template + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html + * This is what you pass to Elasticsearch + */ +export interface ElasticsearchTemplate { + /** + * This is a pre-defined structure you should use for your mapping + */ + mappings: { + /** + * This mapping will be used by default for everything + */ + _default_: { + /** + * Contains the original JSON document body + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-source-field.html + */ + _source: { + /** + * Any fields that are excluded from the source + */ + excludes: string[]; + }; + + /** + * Whether Elasticsearch should automatically add date fields to objects + */ + date_detection: false; + + /** + * This is where all the dynamic templates should go + */ + dynamic_templates: ElasticsearchDynamicTemplate[]; + + /** + * This is where all the mappings should go + */ + properties: { + [name: string]: ElasticsearchObject; + }; + }; + }; + + /** + * The settings for Elasticsearch + */ + settings: ElasticsearchSettings; + + /** + * The name of the template, for referencing in Elasticsearch + */ + template: string; +} + +/** + * A representation of ElasticsearchSettings used in Mappings + */ +export interface ElasticsearchSettings { + /** + * The settings + */ + [name: string]: any; + + /** + * This is where any analyzers go + * + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/analysis-analyzers.html + */ + analysis: { + [name: string]: any; + }; +} + +/** + * A premap for a specific value in a ProjectReflection + * + * This is meant to be used for external types. To aid performance, you usually should not include external libs in the + * ProjectReflection. This means that there is no way the generator can generate a mapping for it, so you can use the + * premaps to map out a type manually. + */ +export interface ElasticsearchPremap { + /** + * The name of the type with the corresponding map + * + * So for `const a: B` the name would be `B` + */ + [name: string]: ElasticsearchValue; +}