diff --git a/Dockerfile b/Dockerfile index a0d83f26..d1393a09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM registry.gitlab.com/openstapps/projectmanagement/node -ADD . /app +ADD --chown=node:node . /app +RUN find /app -type d -print0 | xargs -0 -r chmod 775 && find /app -type f -print0 | xargs -0 -r chmod 660 WORKDIR /app EXPOSE 3000 diff --git a/README.md b/README.md index 80eaaa88..b4e1eaf7 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,34 @@ you with everything you need to run this backend. * Node.js (~10) / NPM * Docker +## Generating Elasticsearch Mapping +The mappings will be generated automatically on the first start. If there are any errors, the backend will inform you and stop +the execution, however it will do its best to complete the mappings. You can then either resolve these errors in the `core-tools` or the `core`, depending on where it originated. +If you need a quick solution, you can also take the generated output file and manually correct the errors, then rename it to `template_[coreVersion].json` +and restart the backend. This time it will take your file. The filenames and the path will be displayed in the log of the backend. + +### Manually Resolving Errors +There are multiple types of errors the backend can run into. Manual error resolving requires you to be familiar with Elasticsearch +mappings. +An error will be represented in the output through an Elasticsearch type written in CAPS. Refer to either the console output +or the `error_report_[coreVersion].txt` for more info. If you feel lucky you can try to replace every error (`"type": "MISSING_PREMAP"`, +`"type": "PARSE_ERROR"`, `"type": "TYPE_CONFLICT"`) with +```json +"dynamic": true, +"properties": {} +``` +This should ONLY be used as a temporary workaround. + +### Startup Behaviour +*This might be important if you work on the Core* + + +By default, the backend creates a local copy of the generated mappings in `src/storage/elasticsearch/templates/template_[coreVersion].json`. +On each start, it first checks if this file exists, if it does, it will just use that file and *not* generate a new mapping to cut down the time +it takes to start the backend. When you are working on the Core, you might not want to have this behaviour, you can then either delete +the generated file at each start or run the backend with the environment variable `ES_FORCE_MAPPING_UPDATE=true`. This will cause it to generate the mapping +each time starts regardless of whether there are already files there. + ## Start Database (Elasticsearch) Elasticsearch needs some configuration and plugins to be able to work with the backend. To save you some work we provide a diff --git a/package-lock.json b/package-lock.json index 68b322ea..6dc62638 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,27 +5,33 @@ "requires": true, "dependencies": { "@babel/code-frame": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", - "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz", + "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", "dev": true, "requires": { "@babel/highlight": "^7.0.0" } }, "@babel/generator": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.4.4.tgz", - "integrity": "sha512-53UOLK6TVNqKxf7RUh8NE851EHRxOOeVXKbK2bivdb+iziMyk03Sr4eaE9OELCbyZAAafAKPDwF2TPUES5QbxQ==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.5.5.tgz", + "integrity": "sha512-ETI/4vyTSxTzGnU2c49XHv2zhExkv9JHLTwDAFz85kmcwuShvYG2H08FwgIguQf4JC75CBnXAUM5PqeF4fj0nQ==", "dev": true, "requires": { - "@babel/types": "^7.4.4", + "@babel/types": "^7.5.5", "jsesc": "^2.5.1", - "lodash": "^4.17.11", + "lodash": "^4.17.13", "source-map": "^0.5.0", "trim-right": "^1.0.1" }, "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -75,15 +81,15 @@ } }, "@babel/parser": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz", - "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.5.5.tgz", + "integrity": "sha512-E5BN68cqR7dhKan1SfqgPGhQ178bkVKpXTPEXnFJBrEt8/DKRZlybmy+IgYLTeN7tp1R5Ccmbm2rBk17sHYU3g==", "dev": true }, "@babel/runtime": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.5.0.tgz", - "integrity": "sha512-2xsuyZ0R0RBFwjgae5NpXk8FcfH4qovj5cEM5VEeB7KXnKqzaisIu2HSV/mCEISolJJuR4wkViUGYujA8MH9tw==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.5.5.tgz", + "integrity": "sha512-28QvEGyQyNkB0/m2B4FU7IEZGK2NUrcMtT6BZEFALTguLk+AUT6ofsHtPk5QyjAdUkpMJ+/Em+quwz4HOt30AQ==", "dev": true, "requires": { "regenerator-runtime": "^0.13.2" @@ -101,20 +107,20 @@ } }, "@babel/traverse": { - "version": "7.4.5", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.4.5.tgz", - "integrity": "sha512-Vc+qjynwkjRmIFGxy0KYoPj4FdVDxLej89kMHFsWScq999uX+pwcX4v9mWRjW0KcAYTPAuVQl2LKP1wEVLsp+A==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.5.5.tgz", + "integrity": "sha512-MqB0782whsfffYfSjH4TM+LMjrJnhCNEDMDIjeTpl+ASaUvxcjoiVCo/sM1GhS1pHOXYfWVCYneLjMckuUxDaQ==", "dev": true, "requires": { - "@babel/code-frame": "^7.0.0", - "@babel/generator": "^7.4.4", + "@babel/code-frame": "^7.5.5", + "@babel/generator": "^7.5.5", "@babel/helper-function-name": "^7.1.0", "@babel/helper-split-export-declaration": "^7.4.4", - "@babel/parser": "^7.4.5", - "@babel/types": "^7.4.4", + "@babel/parser": "^7.5.5", + "@babel/types": "^7.5.5", "debug": "^4.1.0", "globals": "^11.1.0", - "lodash": "^4.17.11" + "lodash": "^4.17.13" }, "dependencies": { "debug": { @@ -125,18 +131,32 @@ "requires": { "ms": "^2.1.1" } + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true } } }, "@babel/types": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.4.4.tgz", - "integrity": "sha512-dOllgYdnEFOebhkKCjzSVFqw/PmmB8pH6RGOWkY4GsboQNd47b1fBThBSwlHAq9alF9vc1M3+6oqR47R50L0tQ==", + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.5.5.tgz", + "integrity": "sha512-s63F9nJioLqOlW3UkyMd+BYhXt44YuaFm/VV0VwuteqjYwRrObkU7ra9pY4wAJR3oXi8hJrMcrcJdO/HH33vtw==", "dev": true, "requires": { "esutils": "^2.0.2", - "lodash": "^4.17.11", + "lodash": "^4.17.13", "to-fast-properties": "^2.0.0" + }, + "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + } } }, "@krlwlfrt/async-pool": { @@ -167,18 +187,6 @@ "integrity": "sha512-on4MmIDgHXiuJDELPk1NFaKVUxxCFr37tm8E9yN6rAiF5Pzp/9bBfBHkoexqRiY+hk/Z04EJU9kKEb59YqJ82A==", "dev": true }, - "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", - "dev": true - }, - "semver": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", - "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==", - "dev": true - }, "tslint": { "version": "5.17.0", "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.17.0.tgz", @@ -211,13 +219,13 @@ } }, "@openstapps/core": { - "version": "0.23.1", - "resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.23.1.tgz", - "integrity": "sha512-cfd4uKn6TZuzLOeaL5PE3BECHQpgPIlLb8RZ8nJSKHUg4ZcppGkBznyk0mALBzX5fADqvSDtyswlWde2S9hetQ==", + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.25.0.tgz", + "integrity": "sha512-Xh+ub2Drns+fORaYt0K/RQ6N1MpYP3llpxEq3SBh48qPCwVBYJ4OVYaJEtSxN/Soxyk2l/7DioTBNvnYiya4MQ==", "requires": { "@types/geojson": "1.0.6", "@types/json-patch": "0.0.30", - "@types/node": "10.14.10", + "@types/node": "10.14.13", "fast-clone": "1.5.13", "http-status-codes": "1.3.2", "json-patch": "0.7.0", @@ -226,30 +234,35 @@ }, "dependencies": { "@types/node": { - "version": "10.14.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.10.tgz", - "integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q==" + "version": "10.14.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.13.tgz", + "integrity": "sha512-yN/FNNW1UYsRR1wwAoyOwqvDuLDtVXnaJTZ898XIw/Q5cCaeVAlVwvsmXLX5PuiScBYwZsZU4JYSHB3TvfdwvQ==" } } }, "@openstapps/core-tools": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@openstapps/core-tools/-/core-tools-0.7.0.tgz", - "integrity": "sha512-/N77OfZ7gWlN657cMk67VYR0XX2fZncrXF1yDBRTI5V5uytBswYadpDU3w3B8gtiqVPxPuEhONzaijHcm0rSig==", + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@openstapps/core-tools/-/core-tools-0.8.0.tgz", + "integrity": "sha512-2HaMQ3cxuhyvWRUPxED3/XOJilPq6A5nBVXzthgxpxeu5Wl8D/zD1Y7He3BIfDGgu+Pp+aDFkqvNKKe/3Hrn8Q==", "requires": { "@krlwlfrt/async-pool": "0.1.0", "@openstapps/logger": "0.3.1", "@types/glob": "7.1.1", + "@types/got": "9.4.4", "@types/mustache": "0.8.32", "@types/node": "10.14.8", "ajv": "6.10.0", "chai": "4.2.0", "commander": "2.20.0", + "deepmerge": "3.3.0", "del": "4.1.1", + "flatted": "2.0.1", "glob": "7.1.4", + "got": "9.6.0", "humanize-string": "2.1.0", "jsonschema": "1.2.4", "mustache": "3.0.1", + "plantuml-encoder": "1.2.5", "toposort": "2.0.2", "ts-json-schema-generator": "0.42.0", "ts-node": "8.2.0", @@ -268,10 +281,10 @@ "nodemailer": "6.2.1" }, "dependencies": { - "nodemailer": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.2.1.tgz", - "integrity": "sha512-TagB7iuIi9uyNgHExo8lUDq3VK5/B0BpbkcjIgNvxbtVrjNqq0DwAOTuzALPVkK76kMhTSzIgHqg8X1uklVs6g==" + "flatted": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", + "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==" } } }, @@ -280,6 +293,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.8.tgz", "integrity": "sha512-I4+DbJEhLEg4/vIy/2gkWDvXBOOtPKV9EnLhYjMoqxcRW+TTZtUftkHktz/a8suoD5mUL7m6ReLrkPvSsCQQmw==" }, + "flatted": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", + "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==" + }, "glob": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", @@ -292,23 +310,6 @@ "once": "^1.3.0", "path-is-absolute": "^1.0.0" } - }, - "nodemailer": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-5.1.1.tgz", - "integrity": "sha512-hKGCoeNdFL2W7S76J/Oucbw0/qRlfG815tENdhzcqTpSjKgAN91mFOqU2lQUflRRxFM7iZvCyaFcAR9noc/CqQ==" - }, - "ts-node": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.2.0.tgz", - "integrity": "sha512-m8XQwUurkbYqXrKqr3WHCW310utRNvV5OnRVeISeea7LoCWVcdfeB/Ntl8JYWFh+WRoUAdBgESrzKochQt7sMw==", - "requires": { - "arg": "^4.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "source-map-support": "^0.5.6", - "yn": "^3.0.0" - } } } }, @@ -323,18 +324,6 @@ "flatted": "2.0.1", "moment": "2.24.0", "nodemailer": "6.2.1" - }, - "dependencies": { - "@types/node": { - "version": "10.14.12", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz", - "integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==" - }, - "flatted": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", - "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==" - } } }, "@sindresorhus/is": { @@ -501,7 +490,6 @@ "version": "9.4.4", "resolved": "https://registry.npmjs.org/@types/got/-/got-9.4.4.tgz", "integrity": "sha512-IGAJokJRE9zNoBdY5csIwN4U5qQn+20HxC0kM+BbUdfTKIXa7bOX/pdhy23NnLBRP8Wvyhx7X5e6EHJs+4d8HA==", - "dev": true, "requires": { "@types/node": "*", "@types/tough-cookie": "*" @@ -526,9 +514,9 @@ "integrity": "sha512-MhCUjojzDhVLnZnxwPwa+rETFRDQ0ffjxYdrqOP6TBO2O0/Z64PV5tNeYApo4bc4y4frbWOrRwv/eEkXlI13Rw==" }, "@types/lodash": { - "version": "4.14.134", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.134.tgz", - "integrity": "sha512-2/O0khFUCFeDlbi7sZ7ZFRCcT812fAeOLm7Ev4KbwASkZ575TDrDcY7YyaoHdTOzKcNbfiwLYZqPmoC4wadrsw==" + "version": "4.14.135", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.135.tgz", + "integrity": "sha512-Ed+tSZ9qM1oYpi5kzdsBuOzcAIn1wDW+e8TFJ50IMJMlSopGdJgKAbhHzN6h1E1OfjlGOr2JepzEWtg9NIfoNg==" }, "@types/marked": { "version": "0.4.2", @@ -660,9 +648,9 @@ } }, "@types/superagent": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.1.tgz", - "integrity": "sha512-NetXrraTWPcdGG6IwYJhJ5esUGx8AYNiozbc1ENWEsF6BsD4JmNODJczI6Rm1xFPVp6HZESds9YCfqz4zIsM6A==", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.3.tgz", + "integrity": "sha512-vy2licJQwOXrTAe+yz9SCyUVXAkMgCeDq9VHzS5CWJyDU1g6CI4xKb4d5sCEmyucjw5sG0y4k2/afS0iv/1D0Q==", "dev": true, "requires": { "@types/cookiejar": "*", @@ -681,8 +669,7 @@ "@types/tough-cookie": { "version": "2.3.5", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", - "integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg==", - "dev": true + "integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg==" }, "@types/tz-offset": { "version": "0.0.0", @@ -1287,14 +1274,6 @@ "semver": "^6.0.0", "split": "^1.0.0", "through2": "^3.0.0" - }, - "dependencies": { - "semver": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", - "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==", - "dev": true - } } }, "conventional-commits-filter": { @@ -1386,6 +1365,14 @@ "semver": "^5.5.0", "shebang-command": "^1.2.0", "which": "^1.2.9" + }, + "dependencies": { + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + } } }, "currently-unhandled": { @@ -1480,6 +1467,11 @@ "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=" }, + "deepmerge": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-3.3.0.tgz", + "integrity": "sha512-GRQOafGHwMHpjPx9iCvTgpu9NojZ49q794EEL94JVEw6VaeA8XTUyBKvAkOOjBX9oJNiV6G3P+T+tihFjo2TqA==" + }, "default-require-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz", @@ -1534,9 +1526,10 @@ "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, "diff": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz", - "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true }, "doctrine": { "version": "0.7.2", @@ -1822,9 +1815,9 @@ } }, "flatted": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", - "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", + "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==" }, "foreground-child": { "version": "1.5.6", @@ -1849,9 +1842,9 @@ } }, "form-data": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.4.0.tgz", - "integrity": "sha512-4FinE8RfqYnNim20xDwZZE0V2kOs/AuElIjFUbPuegQSaoZM+vUT5FnwSl10KPugH4voTg1bEQlcbCG9ka75TA==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.0.tgz", + "integrity": "sha512-WXieX3G/8side6VIqx44ablyULoGruSde5PNTxoUyo5CeyAMX6nVWUd0rgist/EuX655cjhUhTo1Fo3tRYqbcA==", "dev": true, "requires": { "asynckit": "^0.4.0", @@ -2231,6 +2224,14 @@ "requires": { "meow": "^4.0.0", "semver": "^5.5.0" + }, + "dependencies": { + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + } } }, "gitconfiglocal": { @@ -2299,9 +2300,9 @@ } }, "graceful-fs": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", - "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", + "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==" }, "growl": { "version": "1.10.5", @@ -2389,6 +2390,13 @@ "setprototypeof": "1.1.1", "statuses": ">= 1.5.0 < 2", "toidentifier": "1.0.0" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + } } }, "http-status-codes": { @@ -2442,9 +2450,9 @@ } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { "version": "1.3.5", @@ -2514,9 +2522,9 @@ "dev": true }, "is-path-cwd": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.1.0.tgz", - "integrity": "sha512-Sc5j3/YnM8tDeyCsVeKlm/0p95075DyLmDEIkSgQ7mXkrOX+uTCtmQFm0CYzVyJwcCCmO3k8qfJt17SxQwB5Zw==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==" }, "is-path-in-cwd": { "version": "2.1.0", @@ -2624,14 +2632,6 @@ "@babel/types": "^7.4.0", "istanbul-lib-coverage": "^2.0.5", "semver": "^6.0.0" - }, - "dependencies": { - "semver": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", - "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==", - "dev": true - } } }, "istanbul-lib-report": { @@ -2838,9 +2838,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "lodash._reinterpolate": { "version": "3.0.0", @@ -2888,9 +2888,9 @@ } }, "lolex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.1.0.tgz", - "integrity": "sha512-BYxIEXiVq5lGIXeVHnsFzqa1TxN5acnKnPCdlZSpzm8viNEOhiigupA4vTQ9HEFQ6nLTQ9wQOgBknJgzUYQ9Aw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.2.0.tgz", + "integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==", "dev": true }, "loud-rejection": { @@ -2926,6 +2926,14 @@ "requires": { "pify": "^4.0.1", "semver": "^5.6.0" + }, + "dependencies": { + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + } } }, "make-error": { @@ -3120,12 +3128,6 @@ "ms": "^2.1.1" } }, - "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", - "dev": true - }, "find-up": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", @@ -3384,12 +3386,12 @@ "dev": true }, "nise": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.0.tgz", - "integrity": "sha512-Z3sfYEkLFzFmL8KY6xnSJLRxwQwYBjOXi/24lb62ZnZiGA0JUzGGTI6TBIgfCSMIDl9Jlu8SRmHNACLTemDHww==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.1.tgz", + "integrity": "sha512-edFWm0fsFG2n318rfEnKlTZTkjlbVOFF9XIA+fj+Ed+Qz1laYW2lobwavWoMzGrYDHH1EpiNJgDfvGnkZztR/g==", "dev": true, "requires": { - "@sinonjs/formatio": "^3.1.0", + "@sinonjs/formatio": "^3.2.1", "@sinonjs/text-encoding": "^0.7.1", "just-extend": "^4.0.2", "lolex": "^4.1.0", @@ -3430,6 +3432,11 @@ "requires": { "ms": "^2.1.1" } + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" } } }, @@ -3459,6 +3466,14 @@ "requires": { "object.getownpropertydescriptors": "^2.0.3", "semver": "^5.7.0" + }, + "dependencies": { + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + } } }, "nodemailer": { @@ -3476,6 +3491,14 @@ "resolve": "^1.10.0", "semver": "2 || 3 || 4 || 5", "validate-npm-package-license": "^3.0.1" + }, + "dependencies": { + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + } } }, "normalize-url": { @@ -3731,6 +3754,11 @@ "release-zalgo": "^1.0.0" } }, + "pako": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.3.tgz", + "integrity": "sha1-X1FbDGci4ZgpIK6ABerLC3ynPM8=" + }, "parse-github-repo-url": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/parse-github-repo-url/-/parse-github-repo-url-1.4.1.tgz", @@ -3878,6 +3906,15 @@ } } }, + "plantuml-encoder": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/plantuml-encoder/-/plantuml-encoder-1.2.5.tgz", + "integrity": "sha512-viV7Sz+BJNX/sC3iyebh2VfLyAZKuu3+JuBs2ISms8+zoTGwPqwk3/WEDw/zROmGAJ/xD4sNd8zsBw/YmTo7ng==", + "requires": { + "pako": "1.0.3", + "utf8-bytes": "0.0.1" + } + }, "prepend-file": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/prepend-file/-/prepend-file-1.3.1.tgz", @@ -3911,9 +3948,9 @@ "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" }, "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, "progress": { @@ -4044,9 +4081,9 @@ } }, "regenerator-runtime": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.2.tgz", - "integrity": "sha512-S/TQAZJO+D3m9xeN1WTI8dLKBBiRgXBlTJvbWjCThHWZj9EvHK70Ff50/tYj2J/fvBY6JtFVwRuazHN2E7M9BA==", + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz", + "integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==", "dev": true }, "release-zalgo": { @@ -4080,9 +4117,9 @@ "dev": true }, "resolve": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", - "integrity": "sha512-WL2pBDjqT6pGUNSUzMw00o4T7If+z4H2x3Gz893WoUQ5KW8Vr9txp00ykiP16VBaZF5+j/OcXJHZ9+PCvdiDKw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz", + "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==", "requires": { "path-parse": "^1.0.6" } @@ -4120,9 +4157,10 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "semver": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", - "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==" + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", + "integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==", + "dev": true }, "send": { "version": "0.17.1", @@ -4217,14 +4255,6 @@ "lolex": "^4.0.1", "nise": "^1.4.10", "supports-color": "^5.5.0" - }, - "dependencies": { - "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", - "dev": true - } } }, "sinon-express-mock": { @@ -4674,11 +4704,6 @@ "typescript": "~3.4.5" }, "dependencies": { - "commander": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", - "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==" - }, "glob": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", @@ -4709,6 +4734,13 @@ "make-error": "^1.1.1", "source-map-support": "^0.5.6", "yn": "^3.0.0" + }, + "dependencies": { + "diff": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz", + "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==" + } } }, "ts-optchain": { @@ -4743,10 +4775,10 @@ "tsutils": "^2.29.0" }, "dependencies": { - "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", "dev": true } } @@ -4769,9 +4801,9 @@ "dev": true }, "tsutils": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.14.0.tgz", - "integrity": "sha512-SmzGbB0l+8I0QwsPgjooFRaRvHLBLNYM8SeQ0k6rtNDru5sCGeLJcZdwilNndN+GysuFjF5EIYgN8GfFG6UeUw==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.14.1.tgz", + "integrity": "sha512-kiuZzD1uUA5DxGj/uxbde+ymp6VVdAxdzOIlAFbYKrPyla8/uiJ9JLBm1QsPhOm4Muj0/+cWEDP99yoCUcSl6Q==", "dev": true, "requires": { "tslib": "^1.8.1" @@ -4903,6 +4935,11 @@ "prepend-http": "^2.0.0" } }, + "utf8-bytes": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/utf8-bytes/-/utf8-bytes-0.0.1.tgz", + "integrity": "sha1-EWsCVEjJtQAIHN+/H01sbDfYg30=" + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index b8ec7dbc..496c4d69 100644 --- a/package.json +++ b/package.json @@ -24,15 +24,16 @@ "prepublishOnly": "npm ci && npm run build", "preversion": "npm run prepublishOnly", "push": "git push && git push origin \"v$npm_package_version\"", - "start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js", - "test": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true nyc mocha --require ts-node/register --ui mocha-typescript --exit 'test/**/*.ts'", + "start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true node ./lib/cli.js", + "test": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true nyc mocha --require ts-node/register --ui mocha-typescript --exit 'test/**/*.ts'", "tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'" }, "dependencies": { - "@openstapps/core": "0.23.1", - "@openstapps/core-tools": "0.7.0", + "@openstapps/core": "0.25.0", + "@openstapps/core-tools": "0.8.0", "@openstapps/logger": "0.4.0", "@types/node": "10.14.12", + "commander": "2.20.0", "config": "3.1.0", "cors": "2.8.5", "elasticsearch": "16.1.1", diff --git a/src/common.ts b/src/common.ts index c203ad7b..ba5c48c6 100644 --- a/src/common.ts +++ b/src/common.ts @@ -16,6 +16,8 @@ import {SCConfigFile, SCPluginMetaData} from '@openstapps/core'; import {Validator} from '@openstapps/core-tools/lib/validate'; import * as config from 'config'; +import {readFileSync} from 'fs'; +import {resolve} from 'path'; import {BackendTransport} from './notification/backend-transport'; /** @@ -42,3 +44,9 @@ export const isTestEnvironment = process.env.NODE_ENV !== 'production'; * Stores a ("key-value") list of plugins where key is route and value is plugin information */ export const plugins = new Map(); + +/** + * The version of the installed core + */ +export const coreVersion: string = JSON.parse((readFileSync(resolve('.', '.', 'package.json'), 'utf-8')).toString()) + .dependencies['@openstapps/core']; diff --git a/src/storage/elasticsearch/elasticsearch.ts b/src/storage/elasticsearch/elasticsearch.ts index d7fa4d8d..cd151fe8 100644 --- a/src/storage/elasticsearch/elasticsearch.ts +++ b/src/storage/elasticsearch/elasticsearch.ts @@ -39,7 +39,7 @@ import { } from './common'; import * as Monitoring from './monitoring'; import {buildQuery, buildSort} from './query'; -import {putTemplate} from './templating'; +import {checkESTemplate, putTemplate} from './templating'; /** * Matches index names such as stapps___ @@ -107,8 +107,12 @@ export class Elasticsearch implements Database { * @param bulk bulk process which created this index */ static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) { - return `stapps_${type.toLowerCase() - .replace(' ', '_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`; + let out = type.toLowerCase(); + while (out.includes(' ')) { + out = out.replace(' ', '_'); + } + + return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`; } /** @@ -134,7 +138,10 @@ export class Elasticsearch implements Database { */ static removeAliasChars(alias: string, uid: string | undefined): string { // spaces are included in some types, so throwing an error in this case would clutter up the log unnecessarily - let formattedAlias = alias.replace(' ', ''); + let formattedAlias = alias; + while (formattedAlias.includes(' ')) { + formattedAlias = formattedAlias.replace(' ', ''); + } // List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => { if (formattedAlias.includes(value)) { @@ -468,6 +475,9 @@ export class Elasticsearch implements Database { Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue); } + checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ? + process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false); + return this.getAliasMap(); } diff --git a/src/storage/elasticsearch/templates/.gitkeep b/src/storage/elasticsearch/templates/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/storage/elasticsearch/templates/address.field.template.json b/src/storage/elasticsearch/templates/address.field.template.json deleted file mode 100644 index 28f9f26f..00000000 --- a/src/storage/elasticsearch/templates/address.field.template.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "properties": { - "addressCountry": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "addressLocality": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "addressRegion": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "streetAddress": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "postalCode": { - "_fieldRef": "filterableKeyword.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/article.sc-type.template.json b/src/storage/elasticsearch/templates/article.sc-type.template.json deleted file mode 100644 index f04deafb..00000000 --- a/src/storage/elasticsearch/templates/article.sc-type.template.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "properties": { - "authors": { - "_typeRef": "person.sc-type.template.json" - }, - "publishers": { - "_typeRef": [ - "person.sc-type.template.json", - "organization.sc-type.template.json" - ] - }, - "datePublished": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "inLanguages": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "keywords": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "articleBody": { - "_fieldRef": "text.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/base.template.json b/src/storage/elasticsearch/templates/base.template.json deleted file mode 100644 index 22c2bc5a..00000000 --- a/src/storage/elasticsearch/templates/base.template.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "template": "stapps_*", - "settings": { - "max_result_window": 30000, - "mapping.total_fields.limit": 2000, - "number_of_shards": 1, - "number_of_replicas": 0, - "analysis": { - "filter": { - "german_stemmer": { - "type": "stemmer", - "language": "german" - }, - "german_stop": { - "type": "stop", - "stopwords": "_german_" - }, - "german_phonebook": { - "type": "icu_collation", - "language": "de", - "country": "DE", - "variant": "@collation=phonebook" - } - }, - "tokenizer": { - "stapps_ngram": { - "type": "ngram", - "min_gram": 4, - "max_gram": 7 - } - }, - "analyzer": { - "search_german": { - "tokenizer": "stapps_ngram", - "filter": [ - "lowercase", - "german_stop", - "german_stemmer" - ] - }, - "ducet_sort": { - "tokenizer": "keyword", - "filter": [ - "german_phonebook" - ] - } - } - } - }, - "mappings": { - "_default_": { - "properties": { - "creation_date": { - "type": "date", - "store": true - }, - "uid": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "type": { - "_fieldRef": "sortableKeyword.field.template.json" - }, - "name": { - "_fieldRef": "text.field.template.json" - }, - "alternateNames": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "categories": { - "_fieldRef": "sortableKeyword.field.template.json" - }, - "url": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "description": { - "_fieldRef": "text.field.template.json" - }, - "image": { - "_fieldRef": "filterableKeyword.field.template.json" - } - }, - "_source": { - "excludes": [ - "creation_date" - ] - }, - "date_detection": false, - "dynamic_templates": [] - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/book.sc-type.template.json b/src/storage/elasticsearch/templates/book.sc-type.template.json deleted file mode 100644 index bdf31647..00000000 --- a/src/storage/elasticsearch/templates/book.sc-type.template.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "properties": { - "authors": { - "_typeRef": "person.sc-type.template.json" - }, - "bookEdition": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "isbn": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "numberOfPages": { - "type": "integer" - }, - "publishers": { - "_typeRef": "organization.sc-type.template.json" - }, - "datePublished": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "inLanguages": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "keywords": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "articleBody": { - "_fieldRef": "text.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/catalog.sc-type.template.json b/src/storage/elasticsearch/templates/catalog.sc-type.template.json deleted file mode 100644 index 526d3610..00000000 --- a/src/storage/elasticsearch/templates/catalog.sc-type.template.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "properties": { - "level": { - "type": "integer", - "fields": { - "raw": { - "type": "integer" - } - } - }, - "semester": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "superCatalog": { - "_typeRef": "catalog.sc-type.template.json" - }, - "superCatalogs": { - "type": "nested", - "_typeRef": "catalog.sc-type.template.json" - } - } -} diff --git a/src/storage/elasticsearch/templates/date.sc-type.template.json b/src/storage/elasticsearch/templates/date.sc-type.template.json deleted file mode 100644 index 8429b131..00000000 --- a/src/storage/elasticsearch/templates/date.sc-type.template.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "properties": { - "startDate": { - "_fieldRef": "filterableDate.field.template.json" - }, - "endDate": { - "_fieldRef": "filterableDate.field.template.json" - }, - "startTime": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "endTime": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "dayOfWeek": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "place": { - "_typeRef": "place.sc-type.template.json" - }, - "performers": { - "type": "nested", - "_typeRef": "person.sc-type.template.json" - }, - "duration": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "frequency": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "frequencyMultiplier": { - "type": "float" - }, - "repeatFrequency": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "superEvent": { - "_typeRef": "event.sc-type.template.json" - }, - "exceptions": { - "_fieldRef": "filterableDate.field.template.json" - }, - "dates": { - "_fieldRef": "filterableDate.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/diff.sc-type.template.json b/src/storage/elasticsearch/templates/diff.sc-type.template.json deleted file mode 100644 index fe0163ba..00000000 --- a/src/storage/elasticsearch/templates/diff.sc-type.template.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "properties": { - "dateCreated": { - "_fieldRef": "filterableDate.field.template.json" - }, - "action": { - "_fieldRef": "filterableKeyword.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/dish.sc-type.template.json b/src/storage/elasticsearch/templates/dish.sc-type.template.json deleted file mode 100644 index 2c65219a..00000000 --- a/src/storage/elasticsearch/templates/dish.sc-type.template.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "properties": { - "availabilityStarts": { - "_fieldRef": "filterableDate.field.template.json" - }, - "availabilityEnds": { - "_fieldRef": "filterableDate.field.template.json" - }, - "offers": { - "_typeRef": "offers.sc-type.template.json" - }, - "characteristics": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "additives": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "place": { - "_typeRef": "place.sc-type.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/event.sc-type.template.json b/src/storage/elasticsearch/templates/event.sc-type.template.json deleted file mode 100644 index 39f0830f..00000000 --- a/src/storage/elasticsearch/templates/event.sc-type.template.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "properties": { - "subType": { - "_fieldRef": "sortableKeyword.field.template.json" - }, - "categories": { - "_fieldRef": "sortableKeyword.field.template.json" - }, - "previousStartDate": { - "_fieldRef": "filterableDate.field.template.json" - }, - "place": { - "_typeRef": "place.sc-type.template.json" - }, - "organizers": { - "type": "nested", - "_typeRef": "person.sc-type.template.json" - }, - "performers": { - "type": "nested", - "_typeRef": "person.sc-type.template.json" - }, - "attendees": { - "type": "nested", - "_typeRef": "person.sc-type.template.json" - }, - "catalogs": { - "type": "nested", - "_typeRef": "catalog.sc-type.template.json" - }, - "maximumParticipants": { - "type": "integer" - }, - "superEvent": { - "_typeRef": "event.sc-type.template.json" - }, - "subProperties": { - "_fieldRef": "eventSubProperties.field.template.json" - }, - "startDate": { - "_fieldRef": "filterableDate.field.template.json" - }, - "endDate": { - "_fieldRef": "filterableDate.field.template.json" - } - } -} diff --git a/src/storage/elasticsearch/templates/eventSubProperties.field.template.json b/src/storage/elasticsearch/templates/eventSubProperties.field.template.json deleted file mode 100644 index 28bc7eca..00000000 --- a/src/storage/elasticsearch/templates/eventSubProperties.field.template.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "properties": { - "id": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "semester": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "majors": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "originalCategory": { - "_fieldRef": "filterableKeyword.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/filterableDate.field.template.json b/src/storage/elasticsearch/templates/filterableDate.field.template.json deleted file mode 100644 index 9710b951..00000000 --- a/src/storage/elasticsearch/templates/filterableDate.field.template.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "date", - "fields": { - "raw": { - "type": "keyword" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/filterableKeyword.field.template.json b/src/storage/elasticsearch/templates/filterableKeyword.field.template.json deleted file mode 100644 index ec2e522f..00000000 --- a/src/storage/elasticsearch/templates/filterableKeyword.field.template.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "keyword", - "fields": { - "raw": { - "type": "keyword" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/floorplan.sc-type.template.json b/src/storage/elasticsearch/templates/floorplan.sc-type.template.json deleted file mode 100644 index ee37e92a..00000000 --- a/src/storage/elasticsearch/templates/floorplan.sc-type.template.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "properties": { - "place": { - "_typeRef": "place.sc-type.template.json" - }, - "floor": { - "_fieldRef": "filterableKeyword.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/jobs.field.template.json b/src/storage/elasticsearch/templates/jobs.field.template.json deleted file mode 100644 index f995f028..00000000 --- a/src/storage/elasticsearch/templates/jobs.field.template.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "properties": { - "jobTitle": { - "type": "text" - }, - "worksFor": { - "_typeRef": "organization.sc-type.template.json" - }, - "workLocation": { - "properties": { - "email": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "faxNumber": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "telephone": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "openingHours": { - "_fieldRef": "filterableKeyword.field.template.json" - } - } - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/offers.sc-type.template.json b/src/storage/elasticsearch/templates/offers.sc-type.template.json deleted file mode 100644 index 5163b4d1..00000000 --- a/src/storage/elasticsearch/templates/offers.sc-type.template.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "properties": { - "price": { - "type": "double" - }, - "prices": { - "type": "nested", - "properties": { - "alumni": { - "type": "double" - }, - "student": { - "type": "double" - }, - "employee": { - "type": "double" - }, - "guest": { - "type": "double" - } - } - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/organization.sc-type.template.json b/src/storage/elasticsearch/templates/organization.sc-type.template.json deleted file mode 100644 index 93a16c76..00000000 --- a/src/storage/elasticsearch/templates/organization.sc-type.template.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "properties": { - "address": { - "_fieldRef": "address.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/person.sc-type.template.json b/src/storage/elasticsearch/templates/person.sc-type.template.json deleted file mode 100644 index 6e3f54d7..00000000 --- a/src/storage/elasticsearch/templates/person.sc-type.template.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "properties": { - "honorificPrefix": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "honorificSuffix": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "givenName": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "additionalName": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "familyName": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "birthDate": { - "_fieldRef": "filterableDate.field.template.json" - }, - "gender": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "email": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "faxNumber": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "telephone": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "adress": { - "type": "text" - }, - "affiliations": { - "type": "nested", - "_typeRef": "organization.sc-type.template.json" - }, - "homeLocation": { - "_typeRef": "place.sc-type.template.json" - }, - "nationality": { - "_fieldRef": "filterableKeyword.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/place.sc-type.template.json b/src/storage/elasticsearch/templates/place.sc-type.template.json deleted file mode 100644 index 0bca00f2..00000000 --- a/src/storage/elasticsearch/templates/place.sc-type.template.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "properties": { - "subType": { - "_fieldRef": "sortableKeyword.field.template.json" - }, - "address": { - "_fieldRef": "address.field.template.json" - }, - "openingHours": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "geo": { - "properties": { - "point": { - "properties": { - "coordinates": { - "type": "geo_point" - } - } - }, - "polygon": { - "type": "geo_shape" - } - } - }, - "superPlace": { - "_typeRef": "place.sc-type.template.json" - }, - "subProperties": { - "_fieldRef": "placeSubProperties.field.template.json" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/placeSubProperties.field.template.json b/src/storage/elasticsearch/templates/placeSubProperties.field.template.json deleted file mode 100644 index f508e477..00000000 --- a/src/storage/elasticsearch/templates/placeSubProperties.field.template.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "properties": { - "floors": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "floor": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "paymentAccepted": { - "_fieldRef": "filterableKeyword.field.template.json" - }, - "roomCharacterization": { - "type": "nested", - "properties": { - "inventory": { - "properties": { - "key": { - "type": "keyword", - "fields": { - "raw": { - "type": "keyword" - } - } - }, - "value": { - "type": "integer", - "fields": { - "raw": { - "type": "integer" - } - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/prices.field.template.json b/src/storage/elasticsearch/templates/prices.field.template.json deleted file mode 100644 index e83dd296..00000000 --- a/src/storage/elasticsearch/templates/prices.field.template.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "nested", - "properties": { - "student": { - "type": "float" - }, - "employee": { - "type": "float" - }, - "guest": { - "type": "float" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/sortableKeyword.field.template.json b/src/storage/elasticsearch/templates/sortableKeyword.field.template.json deleted file mode 100644 index 32834e6b..00000000 --- a/src/storage/elasticsearch/templates/sortableKeyword.field.template.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "type": "text", - "analyzer": "search_german", - "fields": { - "raw": { - "type": "keyword", - "ignore_above": 10000 - }, - "sort": { - "fielddata": true, - "type": "text", - "analyzer": "ducet_sort" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templates/text.field.template.json b/src/storage/elasticsearch/templates/text.field.template.json deleted file mode 100644 index 6d4310c7..00000000 --- a/src/storage/elasticsearch/templates/text.field.template.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "type": "text", - "fielddata": true, - "analyzer": "search_german", - "fields": { - "raw": { - "type": "keyword" - } - } -} \ No newline at end of file diff --git a/src/storage/elasticsearch/templating.ts b/src/storage/elasticsearch/templating.ts index 94684a9c..cb01e05b 100644 --- a/src/storage/elasticsearch/templating.ts +++ b/src/storage/elasticsearch/templating.ts @@ -13,133 +13,54 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import {getProjectReflection} from '@openstapps/core-tools/lib/common'; +import {generateTemplate} from '@openstapps/core-tools/lib/mapping'; import {Logger} from '@openstapps/logger'; import {Client} from 'elasticsearch'; -import {readdir, readFile} from 'fs-extra'; +import {existsSync, writeFileSync} from 'fs'; +import {readFile} from 'fs-extra'; import {resolve} from 'path'; +import {coreVersion} from '../../common'; + +const dirPath = resolve('src', 'storage', 'elasticsearch', 'templates'); +const templatePath = resolve(dirPath, `template_${coreVersion}.json`); +const errorPath = resolve(dirPath, `failed_template_${coreVersion}.json`); +const errorReportPath = resolve(dirPath, `error_report_${coreVersion}.txt`); + +const ignoredTags = ['minlength', 'pattern', 'see']; /** - * Assembles an elasticsearch template with all resolved subType-references - * @param templateType Type used in the elasticsearch mapping - * @param templates Templates (elasticsearch mappings) - * @param inline Level of hierarchy - * @deprecated + * Check if the correct template exists */ -function assembleElasticsearchTemplate( - templateType: string, - // tslint:disable-next-line: no-any - templates: {[key: string]: any; }, - inline: number): object { - - const templateBase = JSON.parse(JSON.stringify(templates[templateType])); - - if (typeof inline !== 'undefined') { - delete templateBase.dynamic_templates; +export function checkESTemplate(forceUpdate: boolean) { + if (forceUpdate) { + Logger.warn('CAUTION: Force update of the mapping files is enabled. This causes the backend to ignore' + + ' existing mapping files on start.'); } + if (!existsSync(templatePath) || forceUpdate) { + Logger.info(`No mapping for Core version ${coreVersion} found, starting automatic mapping generation. ` + + `This may take a while.`); + const map = generateTemplate(getProjectReflection(resolve('node_modules', '@openstapps', 'core', 'src')), + ignoredTags, false); - // these have no properties to replace - const excludeBaseFields = [ - 'filterableKeyword.field.template.json', - 'sortableKeyword.field.template.json', - 'text.field.template.json', - 'filterableDate.field.template.json', - ]; + if (map.errors.length > 0) { + // tslint:disable-next-line:no-magic-numbers + writeFileSync(errorPath, JSON.stringify(map.template, null, 2)); - if (excludeBaseFields.indexOf(templateType) === -1) { + writeFileSync(errorReportPath, `ERROR REPORT FOR CORE VERSION ${coreVersion}\n${map.errors.join('\n')}`); - try { - // extend the template by the properties of the basetemplate - templateBase.properties = {...templateBase.properties, - ...templates['base.template.json'].mappings._default_.properties, - }; - } catch (e) { - // tslint:disable-next-line: no-floating-promises - Logger.error(`Failed to merge properties on: ${templateType}`); - throw e; + // tslint:disable-next-line:no-floating-promises + Logger.error(`There were errors while generating the template, and the backend cannot continue. A list of all ` + + `errors can be found at ${errorReportPath}. To resolve this` + + ` issue by hand you can go to "${errorPath}" and correct the issues manually, then move it to ${templatePath}.`); + process.exit(1); + } else { + Logger.ok('Mapping files were generated successfully.'); + writeFileSync(templatePath, JSON.stringify(map.template)); } - const fieldKeys = Object.keys(templateBase.properties); - - fieldKeys.forEach((fieldKey) => { - - const field = templateBase.properties[fieldKey]; - const keys = Object.keys(field); - - // we have subtype-references to replace - if (keys.indexOf('_typeRef') > -1) { - // if we are already inline of a superObject, we don't resolve types - if (inline > 1) { - delete templateBase.properties[fieldKey]; - } else { - // we have more than one reference - if (Array.isArray(field._typeRef)) { - let obj = {}; - field._typeRef.forEach((subType: string) => { - obj = {...obj, ...assembleElasticsearchTemplate(subType, templates, inline + 1)}; - }); - templateBase.properties[fieldKey] = obj; - } else { - templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._typeRef, templates, inline + 1); - } - } - } else if (keys.indexOf('_fieldRef') > -1) { - templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._fieldRef, templates, inline + 1); - } - }); + } else { + Logger.info(`Using existing mapping at "${templatePath}"`); } - - return templateBase; -} - -/** - * Reads all template files and returns the assembled template - */ -// TODO: check if redundant -export async function getElasticsearchTemplate(): Promise { - - // readIM all templates - const elasticsearchFolder = resolve('.', 'src', 'storage', 'elasticsearch', 'templates'); - // tslint:disable-next-line: no-any - const templates: {[key: string]: any; } = {}; - - const fileNames = await readdir(elasticsearchFolder); - - const availableTypes = fileNames.filter((fileName) => { - return Array.isArray(fileName.match(/\w*\.sc-type\.template\.json/i)); - }) - .map((fileName) => { - return fileName.substring(0, fileName.indexOf('.sc-type.template.json')); - }); - - const promises = fileNames.map(async (fileName) => { - const file = await readFile(resolve(elasticsearchFolder, fileName), 'utf8'); - - try { - templates[fileName] = JSON.parse(file.toString()); - } catch (jsonParsingError) { - await Logger.error(`Failed parsing file: ${fileName}`); - throw jsonParsingError; - } - }); - - await Promise.all(promises); - - const template = templates['base.template.json']; - - availableTypes.forEach((configType) => { - template.mappings[configType.toLowerCase()] = - assembleElasticsearchTemplate(`${configType}.sc-type.template.json`, templates, 0); - }); - - // this is like the base type (StappsCoreThing) - const baseProperties = template.mappings._default_.properties; - Object.keys(baseProperties) - .forEach((basePropertyName) => { - let field = baseProperties[basePropertyName]; - field = templates[field._fieldRef]; - template.mappings._default_.properties[basePropertyName] = field; - }); - - return template; } /** @@ -148,7 +69,7 @@ export async function getElasticsearchTemplate(): Promise { */ export async function putTemplate(client: Client): Promise { return client.indices.putTemplate({ - body: await getElasticsearchTemplate(), + body: JSON.parse((await readFile(templatePath, 'utf8')).toString()), name: 'global', }); }