Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-19 16:13:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add support for generated elasticsearch mappings

Browse Source 
Fixes #38
This commit is contained in:
Wieland Schöbl
2019-05-29 13:42:41 +02:00
committed by Rainer Killinger
parent 7e04fad28b
commit 8eab6b8531
30 changed files with 303 additions and 867 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Dockerfile
View File

@@ -1,6 +1,7 @@
FROM registry.gitlab.com/openstapps/projectmanagement/node
ADD . /app
ADD --chown=node:node . /app
RUN find /app -type d -print0 | xargs -0 -r chmod 775 && find /app -type f -print0 | xargs -0 -r chmod 660
WORKDIR /app
EXPOSE 3000

28
README.md
View File

@@ -20,6 +20,34 @@ you with everything you need to run this backend.
* Node.js (~10) / NPM
* Docker
## Generating Elasticsearch Mapping
The mappings will be generated automatically on the first start. If there are any errors, the backend will inform you and stop
the execution, however it will do its best to complete the mappings. You can then either resolve these errors in the `core-tools` or the `core`, depending on where it originated.
If you need a quick solution, you can also take the generated output file and manually correct the errors, then rename it to `template_[coreVersion].json`
and restart the backend. This time it will take your file. The filenames and the path will be displayed in the log of the backend.
### Manually Resolving Errors
There are multiple types of errors the backend can run into. Manual error resolving requires you to be familiar with Elasticsearch
mappings.
An error will be represented in the output through an Elasticsearch type written in CAPS. Refer to either the console output
or the `error_report_[coreVersion].txt` for more info. If you feel lucky you can try to replace every error (`"type": "MISSING_PREMAP"`,
`"type": "PARSE_ERROR"`, `"type": "TYPE_CONFLICT"`) with
```json
"dynamic": true,
"properties": {}
```
This should ONLY be used as a temporary workaround.
### Startup Behaviour
*This might be important if you work on the Core*
By default, the backend creates a local copy of the generated mappings in `src/storage/elasticsearch/templates/template_[coreVersion].json`.
On each start, it first checks if this file exists, if it does, it will just use that file and *not* generate a new mapping to cut down the time
it takes to start the backend. When you are working on the Core, you might not want to have this behaviour, you can then either delete
the generated file at each start or run the backend with the environment variable `ES_FORCE_MAPPING_UPDATE=true`. This will cause it to generate the mapping
each time starts regardless of whether there are already files there.
## Start Database (Elasticsearch)
Elasticsearch needs some configuration and plugins to be able to work
with the backend. To save you some work we provide a

381
package-lock.json generated
View File

@@ -5,27 +5,33 @@
"requires": true,
"dependencies": {
"@babel/code-frame": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz",
"integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==",
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz",
"integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==",
"dev": true,
"requires": {
"@babel/highlight": "^7.0.0"
}
},
"@babel/generator": {
"version": "7.4.4",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.4.4.tgz",
"integrity": "sha512-53UOLK6TVNqKxf7RUh8NE851EHRxOOeVXKbK2bivdb+iziMyk03Sr4eaE9OELCbyZAAafAKPDwF2TPUES5QbxQ==",
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.5.5.tgz",
"integrity": "sha512-ETI/4vyTSxTzGnU2c49XHv2zhExkv9JHLTwDAFz85kmcwuShvYG2H08FwgIguQf4JC75CBnXAUM5PqeF4fj0nQ==",
"dev": true,
"requires": {
"@babel/types": "^7.4.4",
"@babel/types": "^7.5.5",
"jsesc": "^2.5.1",
"lodash": "^4.17.11",
"lodash": "^4.17.13",
"source-map": "^0.5.0",
"trim-right": "^1.0.1"
},
"dependencies": {
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
"dev": true
},
"source-map": {
"version": "0.5.7",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@@ -75,15 +81,15 @@
}
},
"@babel/parser": {
"version": "7.4.5",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.4.5.tgz",
"integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==",
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.5.5.tgz",
"integrity": "sha512-E5BN68cqR7dhKan1SfqgPGhQ178bkVKpXTPEXnFJBrEt8/DKRZlybmy+IgYLTeN7tp1R5Ccmbm2rBk17sHYU3g==",
"dev": true
},
"@babel/runtime": {
"version": "7.5.0",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.5.0.tgz",
"integrity": "sha512-2xsuyZ0R0RBFwjgae5NpXk8FcfH4qovj5cEM5VEeB7KXnKqzaisIu2HSV/mCEISolJJuR4wkViUGYujA8MH9tw==",
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.5.5.tgz",
"integrity": "sha512-28QvEGyQyNkB0/m2B4FU7IEZGK2NUrcMtT6BZEFALTguLk+AUT6ofsHtPk5QyjAdUkpMJ+/Em+quwz4HOt30AQ==",
"dev": true,
"requires": {
"regenerator-runtime": "^0.13.2"
@@ -101,20 +107,20 @@
}
},
"@babel/traverse": {
"version": "7.4.5",
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.4.5.tgz",
"integrity": "sha512-Vc+qjynwkjRmIFGxy0KYoPj4FdVDxLej89kMHFsWScq999uX+pwcX4v9mWRjW0KcAYTPAuVQl2LKP1wEVLsp+A==",
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.5.5.tgz",
"integrity": "sha512-MqB0782whsfffYfSjH4TM+LMjrJnhCNEDMDIjeTpl+ASaUvxcjoiVCo/sM1GhS1pHOXYfWVCYneLjMckuUxDaQ==",
"dev": true,
"requires": {
"@babel/code-frame": "^7.0.0",
"@babel/generator": "^7.4.4",
"@babel/code-frame": "^7.5.5",
"@babel/generator": "^7.5.5",
"@babel/helper-function-name": "^7.1.0",
"@babel/helper-split-export-declaration": "^7.4.4",
"@babel/parser": "^7.4.5",
"@babel/types": "^7.4.4",
"@babel/parser": "^7.5.5",
"@babel/types": "^7.5.5",
"debug": "^4.1.0",
"globals": "^11.1.0",
"lodash": "^4.17.11"
"lodash": "^4.17.13"
},
"dependencies": {
"debug": {
@@ -125,18 +131,32 @@
"requires": {
"ms": "^2.1.1"
}
},
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
"dev": true
}
}
},
"@babel/types": {
"version": "7.4.4",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.4.4.tgz",
"integrity": "sha512-dOllgYdnEFOebhkKCjzSVFqw/PmmB8pH6RGOWkY4GsboQNd47b1fBThBSwlHAq9alF9vc1M3+6oqR47R50L0tQ==",
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.5.5.tgz",
"integrity": "sha512-s63F9nJioLqOlW3UkyMd+BYhXt44YuaFm/VV0VwuteqjYwRrObkU7ra9pY4wAJR3oXi8hJrMcrcJdO/HH33vtw==",
"dev": true,
"requires": {
"esutils": "^2.0.2",
"lodash": "^4.17.11",
"lodash": "^4.17.13",
"to-fast-properties": "^2.0.0"
},
"dependencies": {
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
"dev": true
}
}
},
"@krlwlfrt/async-pool": {
@@ -167,18 +187,6 @@
"integrity": "sha512-on4MmIDgHXiuJDELPk1NFaKVUxxCFr37tm8E9yN6rAiF5Pzp/9bBfBHkoexqRiY+hk/Z04EJU9kKEb59YqJ82A==",
"dev": true
},
"diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true
},
"semver": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz",
"integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==",
"dev": true
},
"tslint": {
"version": "5.17.0",
"resolved": "https://registry.npmjs.org/tslint/-/tslint-5.17.0.tgz",
@@ -211,13 +219,13 @@
}
},
"@openstapps/core": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.23.1.tgz",
"integrity": "sha512-cfd4uKn6TZuzLOeaL5PE3BECHQpgPIlLb8RZ8nJSKHUg4ZcppGkBznyk0mALBzX5fADqvSDtyswlWde2S9hetQ==",
"version": "0.25.0",
"resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.25.0.tgz",
"integrity": "sha512-Xh+ub2Drns+fORaYt0K/RQ6N1MpYP3llpxEq3SBh48qPCwVBYJ4OVYaJEtSxN/Soxyk2l/7DioTBNvnYiya4MQ==",
"requires": {
"@types/geojson": "1.0.6",
"@types/json-patch": "0.0.30",
"@types/node": "10.14.10",
"@types/node": "10.14.13",
"fast-clone": "1.5.13",
"http-status-codes": "1.3.2",
"json-patch": "0.7.0",
@@ -226,30 +234,35 @@
},
"dependencies": {
"@types/node": {
"version": "10.14.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.10.tgz",
"integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q=="
"version": "10.14.13",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.13.tgz",
"integrity": "sha512-yN/FNNW1UYsRR1wwAoyOwqvDuLDtVXnaJTZ898XIw/Q5cCaeVAlVwvsmXLX5PuiScBYwZsZU4JYSHB3TvfdwvQ=="
}
}
},
"@openstapps/core-tools": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/@openstapps/core-tools/-/core-tools-0.7.0.tgz",
"integrity": "sha512-/N77OfZ7gWlN657cMk67VYR0XX2fZncrXF1yDBRTI5V5uytBswYadpDU3w3B8gtiqVPxPuEhONzaijHcm0rSig==",
"version": "0.8.0",
"resolved": "https://registry.npmjs.org/@openstapps/core-tools/-/core-tools-0.8.0.tgz",
"integrity": "sha512-2HaMQ3cxuhyvWRUPxED3/XOJilPq6A5nBVXzthgxpxeu5Wl8D/zD1Y7He3BIfDGgu+Pp+aDFkqvNKKe/3Hrn8Q==",
"requires": {
"@krlwlfrt/async-pool": "0.1.0",
"@openstapps/logger": "0.3.1",
"@types/glob": "7.1.1",
"@types/got": "9.4.4",
"@types/mustache": "0.8.32",
"@types/node": "10.14.8",
"ajv": "6.10.0",
"chai": "4.2.0",
"commander": "2.20.0",
"deepmerge": "3.3.0",
"del": "4.1.1",
"flatted": "2.0.1",
"glob": "7.1.4",
"got": "9.6.0",
"humanize-string": "2.1.0",
"jsonschema": "1.2.4",
"mustache": "3.0.1",
"plantuml-encoder": "1.2.5",
"toposort": "2.0.2",
"ts-json-schema-generator": "0.42.0",
"ts-node": "8.2.0",
@@ -268,10 +281,10 @@
"nodemailer": "6.2.1"
},
"dependencies": {
"nodemailer": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.2.1.tgz",
"integrity": "sha512-TagB7iuIi9uyNgHExo8lUDq3VK5/B0BpbkcjIgNvxbtVrjNqq0DwAOTuzALPVkK76kMhTSzIgHqg8X1uklVs6g=="
"flatted": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz",
"integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg=="
}
}
},
@@ -280,6 +293,11 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.8.tgz",
"integrity": "sha512-I4+DbJEhLEg4/vIy/2gkWDvXBOOtPKV9EnLhYjMoqxcRW+TTZtUftkHktz/a8suoD5mUL7m6ReLrkPvSsCQQmw=="
},
"flatted": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
"integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg=="
},
"glob": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
@@ -292,23 +310,6 @@
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"nodemailer": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-5.1.1.tgz",
"integrity": "sha512-hKGCoeNdFL2W7S76J/Oucbw0/qRlfG815tENdhzcqTpSjKgAN91mFOqU2lQUflRRxFM7iZvCyaFcAR9noc/CqQ=="
},
"ts-node": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.2.0.tgz",
"integrity": "sha512-m8XQwUurkbYqXrKqr3WHCW310utRNvV5OnRVeISeea7LoCWVcdfeB/Ntl8JYWFh+WRoUAdBgESrzKochQt7sMw==",
"requires": {
"arg": "^4.1.0",
"diff": "^4.0.1",
"make-error": "^1.1.1",
"source-map-support": "^0.5.6",
"yn": "^3.0.0"
}
}
}
},
@@ -323,18 +324,6 @@
"flatted": "2.0.1",
"moment": "2.24.0",
"nodemailer": "6.2.1"
},
"dependencies": {
"@types/node": {
"version": "10.14.12",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.12.tgz",
"integrity": "sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg=="
},
"flatted": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
"integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg=="
}
}
},
"@sindresorhus/is": {
@@ -501,7 +490,6 @@
"version": "9.4.4",
"resolved": "https://registry.npmjs.org/@types/got/-/got-9.4.4.tgz",
"integrity": "sha512-IGAJokJRE9zNoBdY5csIwN4U5qQn+20HxC0kM+BbUdfTKIXa7bOX/pdhy23NnLBRP8Wvyhx7X5e6EHJs+4d8HA==",
"dev": true,
"requires": {
"@types/node": "*",
"@types/tough-cookie": "*"
@@ -526,9 +514,9 @@
"integrity": "sha512-MhCUjojzDhVLnZnxwPwa+rETFRDQ0ffjxYdrqOP6TBO2O0/Z64PV5tNeYApo4bc4y4frbWOrRwv/eEkXlI13Rw=="
},
"@types/lodash": {
"version": "4.14.134",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.134.tgz",
"integrity": "sha512-2/O0khFUCFeDlbi7sZ7ZFRCcT812fAeOLm7Ev4KbwASkZ575TDrDcY7YyaoHdTOzKcNbfiwLYZqPmoC4wadrsw=="
"version": "4.14.135",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.135.tgz",
"integrity": "sha512-Ed+tSZ9qM1oYpi5kzdsBuOzcAIn1wDW+e8TFJ50IMJMlSopGdJgKAbhHzN6h1E1OfjlGOr2JepzEWtg9NIfoNg=="
},
"@types/marked": {
"version": "0.4.2",
@@ -660,9 +648,9 @@
}
},
"@types/superagent": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.1.tgz",
"integrity": "sha512-NetXrraTWPcdGG6IwYJhJ5esUGx8AYNiozbc1ENWEsF6BsD4JmNODJczI6Rm1xFPVp6HZESds9YCfqz4zIsM6A==",
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.3.tgz",
"integrity": "sha512-vy2licJQwOXrTAe+yz9SCyUVXAkMgCeDq9VHzS5CWJyDU1g6CI4xKb4d5sCEmyucjw5sG0y4k2/afS0iv/1D0Q==",
"dev": true,
"requires": {
"@types/cookiejar": "*",
@@ -681,8 +669,7 @@
"@types/tough-cookie": {
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz",
"integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg==",
"dev": true
"integrity": "sha512-SCcK7mvGi3+ZNz833RRjFIxrn4gI1PPR3NtuIS+6vMkvmsGjosqTJwRt5bAEFLRz+wtJMWv8+uOnZf2hi2QXTg=="
},
"@types/tz-offset": {
"version": "0.0.0",
@@ -1287,14 +1274,6 @@
"semver": "^6.0.0",
"split": "^1.0.0",
"through2": "^3.0.0"
},
"dependencies": {
"semver": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz",
"integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==",
"dev": true
}
}
},
"conventional-commits-filter": {
@@ -1386,6 +1365,14 @@
"semver": "^5.5.0",
"shebang-command": "^1.2.0",
"which": "^1.2.9"
},
"dependencies": {
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true
}
}
},
"currently-unhandled": {
@@ -1480,6 +1467,11 @@
"resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
"integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU="
},
"deepmerge": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-3.3.0.tgz",
"integrity": "sha512-GRQOafGHwMHpjPx9iCvTgpu9NojZ49q794EEL94JVEw6VaeA8XTUyBKvAkOOjBX9oJNiV6G3P+T+tihFjo2TqA=="
},
"default-require-extensions": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz",
@@ -1534,9 +1526,10 @@
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
},
"diff": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz",
"integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q=="
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true
},
"doctrine": {
"version": "0.7.2",
@@ -1822,9 +1815,9 @@
}
},
"flatted": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz",
"integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg=="
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
"integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg=="
},
"foreground-child": {
"version": "1.5.6",
@@ -1849,9 +1842,9 @@
}
},
"form-data": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.4.0.tgz",
"integrity": "sha512-4FinE8RfqYnNim20xDwZZE0V2kOs/AuElIjFUbPuegQSaoZM+vUT5FnwSl10KPugH4voTg1bEQlcbCG9ka75TA==",
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.0.tgz",
"integrity": "sha512-WXieX3G/8side6VIqx44ablyULoGruSde5PNTxoUyo5CeyAMX6nVWUd0rgist/EuX655cjhUhTo1Fo3tRYqbcA==",
"dev": true,
"requires": {
"asynckit": "^0.4.0",
@@ -2231,6 +2224,14 @@
"requires": {
"meow": "^4.0.0",
"semver": "^5.5.0"
},
"dependencies": {
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true
}
}
},
"gitconfiglocal": {
@@ -2299,9 +2300,9 @@
}
},
"graceful-fs": {
"version": "4.1.15",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
"integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA=="
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz",
"integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg=="
},
"growl": {
"version": "1.10.5",
@@ -2389,6 +2390,13 @@
"setprototypeof": "1.1.1",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0"
},
"dependencies": {
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
}
}
},
"http-status-codes": {
@@ -2442,9 +2450,9 @@
}
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ini": {
"version": "1.3.5",
@@ -2514,9 +2522,9 @@
"dev": true
},
"is-path-cwd": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.1.0.tgz",
"integrity": "sha512-Sc5j3/YnM8tDeyCsVeKlm/0p95075DyLmDEIkSgQ7mXkrOX+uTCtmQFm0CYzVyJwcCCmO3k8qfJt17SxQwB5Zw=="
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz",
"integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ=="
},
"is-path-in-cwd": {
"version": "2.1.0",
@@ -2624,14 +2632,6 @@
"@babel/types": "^7.4.0",
"istanbul-lib-coverage": "^2.0.5",
"semver": "^6.0.0"
},
"dependencies": {
"semver": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz",
"integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==",
"dev": true
}
}
},
"istanbul-lib-report": {
@@ -2838,9 +2838,9 @@
}
},
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
},
"lodash._reinterpolate": {
"version": "3.0.0",
@@ -2888,9 +2888,9 @@
}
},
"lolex": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/lolex/-/lolex-4.1.0.tgz",
"integrity": "sha512-BYxIEXiVq5lGIXeVHnsFzqa1TxN5acnKnPCdlZSpzm8viNEOhiigupA4vTQ9HEFQ6nLTQ9wQOgBknJgzUYQ9Aw==",
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lolex/-/lolex-4.2.0.tgz",
"integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==",
"dev": true
},
"loud-rejection": {
@@ -2926,6 +2926,14 @@
"requires": {
"pify": "^4.0.1",
"semver": "^5.6.0"
},
"dependencies": {
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true
}
}
},
"make-error": {
@@ -3120,12 +3128,6 @@
"ms": "^2.1.1"
}
},
"diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true
},
"find-up": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
@@ -3384,12 +3386,12 @@
"dev": true
},
"nise": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/nise/-/nise-1.5.0.tgz",
"integrity": "sha512-Z3sfYEkLFzFmL8KY6xnSJLRxwQwYBjOXi/24lb62ZnZiGA0JUzGGTI6TBIgfCSMIDl9Jlu8SRmHNACLTemDHww==",
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/nise/-/nise-1.5.1.tgz",
"integrity": "sha512-edFWm0fsFG2n318rfEnKlTZTkjlbVOFF9XIA+fj+Ed+Qz1laYW2lobwavWoMzGrYDHH1EpiNJgDfvGnkZztR/g==",
"dev": true,
"requires": {
"@sinonjs/formatio": "^3.1.0",
"@sinonjs/formatio": "^3.2.1",
"@sinonjs/text-encoding": "^0.7.1",
"just-extend": "^4.0.2",
"lolex": "^4.1.0",
@@ -3430,6 +3432,11 @@
"requires": {
"ms": "^2.1.1"
}
},
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA=="
}
}
},
@@ -3459,6 +3466,14 @@
"requires": {
"object.getownpropertydescriptors": "^2.0.3",
"semver": "^5.7.0"
},
"dependencies": {
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true
}
}
},
"nodemailer": {
@@ -3476,6 +3491,14 @@
"resolve": "^1.10.0",
"semver": "2 || 3 || 4 || 5",
"validate-npm-package-license": "^3.0.1"
},
"dependencies": {
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true
}
}
},
"normalize-url": {
@@ -3731,6 +3754,11 @@
"release-zalgo": "^1.0.0"
}
},
"pako": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.3.tgz",
"integrity": "sha1-X1FbDGci4ZgpIK6ABerLC3ynPM8="
},
"parse-github-repo-url": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/parse-github-repo-url/-/parse-github-repo-url-1.4.1.tgz",
@@ -3878,6 +3906,15 @@
}
}
},
"plantuml-encoder": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/plantuml-encoder/-/plantuml-encoder-1.2.5.tgz",
"integrity": "sha512-viV7Sz+BJNX/sC3iyebh2VfLyAZKuu3+JuBs2ISms8+zoTGwPqwk3/WEDw/zROmGAJ/xD4sNd8zsBw/YmTo7ng==",
"requires": {
"pako": "1.0.3",
"utf8-bytes": "0.0.1"
}
},
"prepend-file": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/prepend-file/-/prepend-file-1.3.1.tgz",
@@ -3911,9 +3948,9 @@
"integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc="
},
"process-nextick-args": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
},
"progress": {
@@ -4044,9 +4081,9 @@
}
},
"regenerator-runtime": {
"version": "0.13.2",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.2.tgz",
"integrity": "sha512-S/TQAZJO+D3m9xeN1WTI8dLKBBiRgXBlTJvbWjCThHWZj9EvHK70Ff50/tYj2J/fvBY6JtFVwRuazHN2E7M9BA==",
"version": "0.13.3",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz",
"integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==",
"dev": true
},
"release-zalgo": {
@@ -4080,9 +4117,9 @@
"dev": true
},
"resolve": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz",
"integrity": "sha512-WL2pBDjqT6pGUNSUzMw00o4T7If+z4H2x3Gz893WoUQ5KW8Vr9txp00ykiP16VBaZF5+j/OcXJHZ9+PCvdiDKw==",
"version": "1.11.1",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
"integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==",
"requires": {
"path-parse": "^1.0.6"
}
@@ -4120,9 +4157,10 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA=="
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz",
"integrity": "sha512-rWYq2e5iYW+fFe/oPPtYJxYgjBm8sC4rmoGdUOgBB7VnwKt6HrL793l2voH1UlsyYZpJ4g0wfjnTEO1s1NP2eQ==",
"dev": true
},
"send": {
"version": "0.17.1",
@@ -4217,14 +4255,6 @@
"lolex": "^4.0.1",
"nise": "^1.4.10",
"supports-color": "^5.5.0"
},
"dependencies": {
"diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true
}
}
},
"sinon-express-mock": {
@@ -4674,11 +4704,6 @@
"typescript": "~3.4.5"
},
"dependencies": {
"commander": {
"version": "2.20.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
"integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ=="
},
"glob": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
@@ -4709,6 +4734,13 @@
"make-error": "^1.1.1",
"source-map-support": "^0.5.6",
"yn": "^3.0.0"
},
"dependencies": {
"diff": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz",
"integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q=="
}
}
},
"ts-optchain": {
@@ -4743,10 +4775,10 @@
"tsutils": "^2.29.0"
},
"dependencies": {
"diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true
}
}
@@ -4769,9 +4801,9 @@
"dev": true
},
"tsutils": {
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.14.0.tgz",
"integrity": "sha512-SmzGbB0l+8I0QwsPgjooFRaRvHLBLNYM8SeQ0k6rtNDru5sCGeLJcZdwilNndN+GysuFjF5EIYgN8GfFG6UeUw==",
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.14.1.tgz",
"integrity": "sha512-kiuZzD1uUA5DxGj/uxbde+ymp6VVdAxdzOIlAFbYKrPyla8/uiJ9JLBm1QsPhOm4Muj0/+cWEDP99yoCUcSl6Q==",
"dev": true,
"requires": {
"tslib": "^1.8.1"
@@ -4903,6 +4935,11 @@
"prepend-http": "^2.0.0"
}
},
"utf8-bytes": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/utf8-bytes/-/utf8-bytes-0.0.1.tgz",
"integrity": "sha1-EWsCVEjJtQAIHN+/H01sbDfYg30="
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

9
package.json
View File

@@ -24,15 +24,16 @@
"prepublishOnly": "npm ci && npm run build",
"preversion": "npm run prepublishOnly",
"push": "git push && git push origin \"v$npm_package_version\"",
"start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js",
"test": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true nyc mocha --require ts-node/register --ui mocha-typescript --exit 'test/**/*.ts'",
"start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true node ./lib/cli.js",
"test": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true nyc mocha --require ts-node/register --ui mocha-typescript --exit 'test/**/*.ts'",
"tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'"
},
"dependencies": {
"@openstapps/core": "0.23.1",
"@openstapps/core-tools": "0.7.0",
"@openstapps/core": "0.25.0",
"@openstapps/core-tools": "0.8.0",
"@openstapps/logger": "0.4.0",
"@types/node": "10.14.12",
"commander": "2.20.0",
"config": "3.1.0",
"cors": "2.8.5",
"elasticsearch": "16.1.1",

8
src/common.ts
View File

@@ -16,6 +16,8 @@
import {SCConfigFile, SCPluginMetaData} from '@openstapps/core';
import {Validator} from '@openstapps/core-tools/lib/validate';
import * as config from 'config';
import {readFileSync} from 'fs';
import {resolve} from 'path';
import {BackendTransport} from './notification/backend-transport';
/**
@@ -42,3 +44,9 @@ export const isTestEnvironment = process.env.NODE_ENV !== 'production';
* Stores a ("key-value") list of plugins where key is route and value is plugin information
*/
export const plugins = new Map<string, SCPluginMetaData>();
/**
* The version of the installed core
*/
export const coreVersion: string = JSON.parse((readFileSync(resolve('.', '.', 'package.json'), 'utf-8')).toString())
.dependencies['@openstapps/core'];

18
src/storage/elasticsearch/elasticsearch.ts
View File

@@ -39,7 +39,7 @@ import {
} from './common';
import * as Monitoring from './monitoring';
import {buildQuery, buildSort} from './query';
import {putTemplate} from './templating';
import {checkESTemplate, putTemplate} from './templating';
/**
* Matches index names such as stapps_<type>_<source>_<random suffix>
@@ -107,8 +107,12 @@ export class Elasticsearch implements Database {
* @param bulk bulk process which created this index
*/
static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) {
return `stapps_${type.toLowerCase()
.replace(' ', '_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`;
let out = type.toLowerCase();
while (out.includes(' ')) {
out = out.replace(' ', '_');
}
return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`;
}
/**
@@ -134,7 +138,10 @@ export class Elasticsearch implements Database {
*/
static removeAliasChars(alias: string, uid: string | undefined): string {
// spaces are included in some types, so throwing an error in this case would clutter up the log unnecessarily
let formattedAlias = alias.replace(' ', '');
let formattedAlias = alias;
while (formattedAlias.includes(' ')) {
formattedAlias = formattedAlias.replace(' ', '');
}
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => {
if (formattedAlias.includes(value)) {
@@ -468,6 +475,9 @@ export class Elasticsearch implements Database {
Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
}
checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ?
process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false);
return this.getAliasMap();
}

0
src/storage/elasticsearch/templates/.gitkeep Normal file
View File

19
src/storage/elasticsearch/templates/address.field.template.json
View File

@@ -1,19 +0,0 @@
{
"properties": {
"addressCountry": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"addressLocality": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"addressRegion": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"streetAddress": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"postalCode": {
"_fieldRef": "filterableKeyword.field.template.json"
}
}
}

25
src/storage/elasticsearch/templates/article.sc-type.template.json
View File

@@ -1,25 +0,0 @@
{
"properties": {
"authors": {
"_typeRef": "person.sc-type.template.json"
},
"publishers": {
"_typeRef": [
"person.sc-type.template.json",
"organization.sc-type.template.json"
]
},
"datePublished": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"inLanguages": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"keywords": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"articleBody": {
"_fieldRef": "text.field.template.json"
}
}
}

91
src/storage/elasticsearch/templates/base.template.json
View File

@@ -1,91 +0,0 @@
{
"template": "stapps_*",
"settings": {
"max_result_window": 30000,
"mapping.total_fields.limit": 2000,
"number_of_shards": 1,
"number_of_replicas": 0,
"analysis": {
"filter": {
"german_stemmer": {
"type": "stemmer",
"language": "german"
},
"german_stop": {
"type": "stop",
"stopwords": "_german_"
},
"german_phonebook": {
"type": "icu_collation",
"language": "de",
"country": "DE",
"variant": "@collation=phonebook"
}
},
"tokenizer": {
"stapps_ngram": {
"type": "ngram",
"min_gram": 4,
"max_gram": 7
}
},
"analyzer": {
"search_german": {
"tokenizer": "stapps_ngram",
"filter": [
"lowercase",
"german_stop",
"german_stemmer"
]
},
"ducet_sort": {
"tokenizer": "keyword",
"filter": [
"german_phonebook"
]
}
}
}
},
"mappings": {
"_default_": {
"properties": {
"creation_date": {
"type": "date",
"store": true
},
"uid": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"type": {
"_fieldRef": "sortableKeyword.field.template.json"
},
"name": {
"_fieldRef": "text.field.template.json"
},
"alternateNames": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"categories": {
"_fieldRef": "sortableKeyword.field.template.json"
},
"url": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"description": {
"_fieldRef": "text.field.template.json"
},
"image": {
"_fieldRef": "filterableKeyword.field.template.json"
}
},
"_source": {
"excludes": [
"creation_date"
]
},
"date_detection": false,
"dynamic_templates": []
}
}
}

31
src/storage/elasticsearch/templates/book.sc-type.template.json
View File

@@ -1,31 +0,0 @@
{
"properties": {
"authors": {
"_typeRef": "person.sc-type.template.json"
},
"bookEdition": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"isbn": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"numberOfPages": {
"type": "integer"
},
"publishers": {
"_typeRef": "organization.sc-type.template.json"
},
"datePublished": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"inLanguages": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"keywords": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"articleBody": {
"_fieldRef": "text.field.template.json"
}
}
}

22
src/storage/elasticsearch/templates/catalog.sc-type.template.json
View File

@@ -1,22 +0,0 @@
{
"properties": {
"level": {
"type": "integer",
"fields": {
"raw": {
"type": "integer"
}
}
},
"semester": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"superCatalog": {
"_typeRef": "catalog.sc-type.template.json"
},
"superCatalogs": {
"type": "nested",
"_typeRef": "catalog.sc-type.template.json"
}
}
}

47
src/storage/elasticsearch/templates/date.sc-type.template.json
View File

@@ -1,47 +0,0 @@
{
"properties": {
"startDate": {
"_fieldRef": "filterableDate.field.template.json"
},
"endDate": {
"_fieldRef": "filterableDate.field.template.json"
},
"startTime": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"endTime": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"dayOfWeek": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"place": {
"_typeRef": "place.sc-type.template.json"
},
"performers": {
"type": "nested",
"_typeRef": "person.sc-type.template.json"
},
"duration": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"frequency": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"frequencyMultiplier": {
"type": "float"
},
"repeatFrequency": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"superEvent": {
"_typeRef": "event.sc-type.template.json"
},
"exceptions": {
"_fieldRef": "filterableDate.field.template.json"
},
"dates": {
"_fieldRef": "filterableDate.field.template.json"
}
}
}

10
src/storage/elasticsearch/templates/diff.sc-type.template.json
View File

@@ -1,10 +0,0 @@
{
"properties": {
"dateCreated": {
"_fieldRef": "filterableDate.field.template.json"
},
"action": {
"_fieldRef": "filterableKeyword.field.template.json"
}
}
}

22
src/storage/elasticsearch/templates/dish.sc-type.template.json
View File

@@ -1,22 +0,0 @@
{
"properties": {
"availabilityStarts": {
"_fieldRef": "filterableDate.field.template.json"
},
"availabilityEnds": {
"_fieldRef": "filterableDate.field.template.json"
},
"offers": {
"_typeRef": "offers.sc-type.template.json"
},
"characteristics": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"additives": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"place": {
"_typeRef": "place.sc-type.template.json"
}
}
}

47
src/storage/elasticsearch/templates/event.sc-type.template.json
View File

@@ -1,47 +0,0 @@
{
"properties": {
"subType": {
"_fieldRef": "sortableKeyword.field.template.json"
},
"categories": {
"_fieldRef": "sortableKeyword.field.template.json"
},
"previousStartDate": {
"_fieldRef": "filterableDate.field.template.json"
},
"place": {
"_typeRef": "place.sc-type.template.json"
},
"organizers": {
"type": "nested",
"_typeRef": "person.sc-type.template.json"
},
"performers": {
"type": "nested",
"_typeRef": "person.sc-type.template.json"
},
"attendees": {
"type": "nested",
"_typeRef": "person.sc-type.template.json"
},
"catalogs": {
"type": "nested",
"_typeRef": "catalog.sc-type.template.json"
},
"maximumParticipants": {
"type": "integer"
},
"superEvent": {
"_typeRef": "event.sc-type.template.json"
},
"subProperties": {
"_fieldRef": "eventSubProperties.field.template.json"
},
"startDate": {
"_fieldRef": "filterableDate.field.template.json"
},
"endDate": {
"_fieldRef": "filterableDate.field.template.json"
}
}
}

16
src/storage/elasticsearch/templates/eventSubProperties.field.template.json
View File

@@ -1,16 +0,0 @@
{
"properties": {
"id": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"semester": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"majors": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"originalCategory": {
"_fieldRef": "filterableKeyword.field.template.json"
}
}
}

8
src/storage/elasticsearch/templates/filterableDate.field.template.json
View File

@@ -1,8 +0,0 @@
{
"type": "date",
"fields": {
"raw": {
"type": "keyword"
}
}
}

8
src/storage/elasticsearch/templates/filterableKeyword.field.template.json
View File

@@ -1,8 +0,0 @@
{
"type": "keyword",
"fields": {
"raw": {
"type": "keyword"
}
}
}

10
src/storage/elasticsearch/templates/floorplan.sc-type.template.json
View File

@@ -1,10 +0,0 @@
{
"properties": {
"place": {
"_typeRef": "place.sc-type.template.json"
},
"floor": {
"_fieldRef": "filterableKeyword.field.template.json"
}
}
}

26
src/storage/elasticsearch/templates/jobs.field.template.json
View File

@@ -1,26 +0,0 @@
{
"properties": {
"jobTitle": {
"type": "text"
},
"worksFor": {
"_typeRef": "organization.sc-type.template.json"
},
"workLocation": {
"properties": {
"email": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"faxNumber": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"telephone": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"openingHours": {
"_fieldRef": "filterableKeyword.field.template.json"
}
}
}
}
}

24
src/storage/elasticsearch/templates/offers.sc-type.template.json
View File

@@ -1,24 +0,0 @@
{
"properties": {
"price": {
"type": "double"
},
"prices": {
"type": "nested",
"properties": {
"alumni": {
"type": "double"
},
"student": {
"type": "double"
},
"employee": {
"type": "double"
},
"guest": {
"type": "double"
}
}
}
}
}

7
src/storage/elasticsearch/templates/organization.sc-type.template.json
View File

@@ -1,7 +0,0 @@
{
"properties": {
"address": {
"_fieldRef": "address.field.template.json"
}
}
}

47
src/storage/elasticsearch/templates/person.sc-type.template.json
View File

@@ -1,47 +0,0 @@
{
"properties": {
"honorificPrefix": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"honorificSuffix": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"givenName": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"additionalName": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"familyName": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"birthDate": {
"_fieldRef": "filterableDate.field.template.json"
},
"gender": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"email": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"faxNumber": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"telephone": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"adress": {
"type": "text"
},
"affiliations": {
"type": "nested",
"_typeRef": "organization.sc-type.template.json"
},
"homeLocation": {
"_typeRef": "place.sc-type.template.json"
},
"nationality": {
"_fieldRef": "filterableKeyword.field.template.json"
}
}
}

33
src/storage/elasticsearch/templates/place.sc-type.template.json
View File

@@ -1,33 +0,0 @@
{
"properties": {
"subType": {
"_fieldRef": "sortableKeyword.field.template.json"
},
"address": {
"_fieldRef": "address.field.template.json"
},
"openingHours": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"geo": {
"properties": {
"point": {
"properties": {
"coordinates": {
"type": "geo_point"
}
}
},
"polygon": {
"type": "geo_shape"
}
}
},
"superPlace": {
"_typeRef": "place.sc-type.template.json"
},
"subProperties": {
"_fieldRef": "placeSubProperties.field.template.json"
}
}
}

38
src/storage/elasticsearch/templates/placeSubProperties.field.template.json
View File

@@ -1,38 +0,0 @@
{
"properties": {
"floors": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"floor": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"paymentAccepted": {
"_fieldRef": "filterableKeyword.field.template.json"
},
"roomCharacterization": {
"type": "nested",
"properties": {
"inventory": {
"properties": {
"key": {
"type": "keyword",
"fields": {
"raw": {
"type": "keyword"
}
}
},
"value": {
"type": "integer",
"fields": {
"raw": {
"type": "integer"
}
}
}
}
}
}
}
}
}

14
src/storage/elasticsearch/templates/prices.field.template.json
View File

@@ -1,14 +0,0 @@
{
"type": "nested",
"properties": {
"student": {
"type": "float"
},
"employee": {
"type": "float"
},
"guest": {
"type": "float"
}
}
}

15
src/storage/elasticsearch/templates/sortableKeyword.field.template.json
View File

@@ -1,15 +0,0 @@
{
"type": "text",
"analyzer": "search_german",
"fields": {
"raw": {
"type": "keyword",
"ignore_above": 10000
},
"sort": {
"fielddata": true,
"type": "text",
"analyzer": "ducet_sort"
}
}
}

10
src/storage/elasticsearch/templates/text.field.template.json
View File

@@ -1,10 +0,0 @@
{
"type": "text",
"fielddata": true,
"analyzer": "search_german",
"fields": {
"raw": {
"type": "keyword"
}
}
}

153
src/storage/elasticsearch/templating.ts
View File

@@ -13,133 +13,54 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {getProjectReflection} from '@openstapps/core-tools/lib/common';
import {generateTemplate} from '@openstapps/core-tools/lib/mapping';
import {Logger} from '@openstapps/logger';
import {Client} from 'elasticsearch';
import {readdir, readFile} from 'fs-extra';
import {existsSync, writeFileSync} from 'fs';
import {readFile} from 'fs-extra';
import {resolve} from 'path';
import {coreVersion} from '../../common';
const dirPath = resolve('src', 'storage', 'elasticsearch', 'templates');
const templatePath = resolve(dirPath, `template_${coreVersion}.json`);
const errorPath = resolve(dirPath, `failed_template_${coreVersion}.json`);
const errorReportPath = resolve(dirPath, `error_report_${coreVersion}.txt`);
const ignoredTags = ['minlength', 'pattern', 'see'];
/**
* Assembles an elasticsearch template with all resolved subType-references
* @param templateType Type used in the elasticsearch mapping
* @param templates Templates (elasticsearch mappings)
* @param inline Level of hierarchy
* @deprecated
* Check if the correct template exists
*/
function assembleElasticsearchTemplate(
templateType: string,
// tslint:disable-next-line: no-any
templates: {[key: string]: any; },
inline: number): object {
const templateBase = JSON.parse(JSON.stringify(templates[templateType]));
if (typeof inline !== 'undefined') {
delete templateBase.dynamic_templates;
export function checkESTemplate(forceUpdate: boolean) {
if (forceUpdate) {
Logger.warn('CAUTION: Force update of the mapping files is enabled. This causes the backend to ignore' +
' existing mapping files on start.');
}
if (!existsSync(templatePath) || forceUpdate) {
Logger.info(`No mapping for Core version ${coreVersion} found, starting automatic mapping generation. ` +
`This may take a while.`);
const map = generateTemplate(getProjectReflection(resolve('node_modules', '@openstapps', 'core', 'src')),
ignoredTags, false);
// these have no properties to replace
const excludeBaseFields = [
'filterableKeyword.field.template.json',
'sortableKeyword.field.template.json',
'text.field.template.json',
'filterableDate.field.template.json',
];
if (map.errors.length > 0) {
// tslint:disable-next-line:no-magic-numbers
writeFileSync(errorPath, JSON.stringify(map.template, null, 2));
if (excludeBaseFields.indexOf(templateType) === -1) {
writeFileSync(errorReportPath, `ERROR REPORT FOR CORE VERSION ${coreVersion}\n${map.errors.join('\n')}`);
try {
// extend the template by the properties of the basetemplate
templateBase.properties = {...templateBase.properties,
...templates['base.template.json'].mappings._default_.properties,
};
} catch (e) {
// tslint:disable-next-line: no-floating-promises
Logger.error(`Failed to merge properties on: ${templateType}`);
throw e;
// tslint:disable-next-line:no-floating-promises
Logger.error(`There were errors while generating the template, and the backend cannot continue. A list of all ` +
`errors can be found at ${errorReportPath}. To resolve this` +
` issue by hand you can go to "${errorPath}" and correct the issues manually, then move it to ${templatePath}.`);
process.exit(1);
} else {
Logger.ok('Mapping files were generated successfully.');
writeFileSync(templatePath, JSON.stringify(map.template));
}
const fieldKeys = Object.keys(templateBase.properties);
fieldKeys.forEach((fieldKey) => {
const field = templateBase.properties[fieldKey];
const keys = Object.keys(field);
// we have subtype-references to replace
if (keys.indexOf('_typeRef') > -1) {
// if we are already inline of a superObject, we don't resolve types
if (inline > 1) {
delete templateBase.properties[fieldKey];
} else {
// we have more than one reference
if (Array.isArray(field._typeRef)) {
let obj = {};
field._typeRef.forEach((subType: string) => {
obj = {...obj, ...assembleElasticsearchTemplate(subType, templates, inline + 1)};
});
templateBase.properties[fieldKey] = obj;
} else {
templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._typeRef, templates, inline + 1);
}
}
} else if (keys.indexOf('_fieldRef') > -1) {
templateBase.properties[fieldKey] = assembleElasticsearchTemplate(field._fieldRef, templates, inline + 1);
}
});
} else {
Logger.info(`Using existing mapping at "${templatePath}"`);
}
return templateBase;
}
/**
* Reads all template files and returns the assembled template
*/
// TODO: check if redundant
export async function getElasticsearchTemplate(): Promise<object> {
// readIM all templates
const elasticsearchFolder = resolve('.', 'src', 'storage', 'elasticsearch', 'templates');
// tslint:disable-next-line: no-any
const templates: {[key: string]: any; } = {};
const fileNames = await readdir(elasticsearchFolder);
const availableTypes = fileNames.filter((fileName) => {
return Array.isArray(fileName.match(/\w*\.sc-type\.template\.json/i));
})
.map((fileName) => {
return fileName.substring(0, fileName.indexOf('.sc-type.template.json'));
});
const promises = fileNames.map(async (fileName) => {
const file = await readFile(resolve(elasticsearchFolder, fileName), 'utf8');
try {
templates[fileName] = JSON.parse(file.toString());
} catch (jsonParsingError) {
await Logger.error(`Failed parsing file: ${fileName}`);
throw jsonParsingError;
}
});
await Promise.all(promises);
const template = templates['base.template.json'];
availableTypes.forEach((configType) => {
template.mappings[configType.toLowerCase()] =
assembleElasticsearchTemplate(`${configType}.sc-type.template.json`, templates, 0);
});
// this is like the base type (StappsCoreThing)
const baseProperties = template.mappings._default_.properties;
Object.keys(baseProperties)
.forEach((basePropertyName) => {
let field = baseProperties[basePropertyName];
field = templates[field._fieldRef];
template.mappings._default_.properties[basePropertyName] = field;
});
return template;
}
/**
@@ -148,7 +69,7 @@ export async function getElasticsearchTemplate(): Promise<object> {
*/
export async function putTemplate(client: Client): Promise<void> {
return client.indices.putTemplate({
body: await getElasticsearchTemplate(),
body: JSON.parse((await readFile(templatePath, 'utf8')).toString()),
name: 'global',
});
}