diff --git a/README.md b/README.md index d259efde..8760d875 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,14 @@ environment. At the time only SMTP is being implemented. The backend wouldn't st you can set `ALLOW_NO_TRANSPORT=true`. To set up an SMTP configuration have a look at [@openstapps/logger](https://gitlab.com/openstapps/logger). +The list of environment variables includes: +* `NODE_ENV` when set to `production`, there will be a reduced amount of output from the logger +* `PORT` when this is not set, the backend will default to port 3000 +* `ES_ADDR` the Elasticsearch address, if not set it will default the Elasticsearch address to `http://localhost:9200` +* `ES_FORCE_MAPPING_UPDATE` when this variable is set to `true`, the backend will always generate a new Elasticsearch mapping from the core regardless of whether there is already a version present. This should only really be used when you are working on the core. +* `ALLOW_NO_TRANSPORT` if set to true, the backend will allow starting without an Email configured that receives critical errors. +* `ES_DEBUG` setting this to `true` will result in Elasticsearch logging to be **VERY** extensive, in almost all situation this should no be enabled. + ## Config files Each university can have it's specific config for the general backend and app and for all databases. diff --git a/package-lock.json b/package-lock.json index 4829b1b5..9e23756b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -159,6 +159,42 @@ } } }, + "@elastic/elasticsearch": { + "version": "5.6.20", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-5.6.20.tgz", + "integrity": "sha512-dOybHGWMo6J/rx/ad9IbROKYq5aQIOd0RrickbqrIU7pjg0J3qSIL4SN027jz1vdrtOcap7yovA+HUplrEXOog==", + "requires": { + "debug": "^4.1.1", + "decompress-response": "^4.2.0", + "into-stream": "^5.1.0", + "ms": "^2.1.1", + "once": "^1.4.0", + "pump": "^3.0.0" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, + "mimic-response": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.0.0.tgz", + "integrity": "sha512-8ilDoEapqA4uQ3TwS0jakGONKXVJqpy+RpM+3b7pLdOjghCrEiGp9SRkFbUHAmZW9vdnrENWHjaweIoTIJExSQ==" + } + } + }, "@krlwlfrt/async-pool": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/@krlwlfrt/async-pool/-/async-pool-0.1.0.tgz", @@ -288,11 +324,29 @@ } } }, + "@types/fs-extra": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.1.0.tgz", + "integrity": "sha512-AInn5+UBFIK9FK5xc9yP5e3TQSPNNgjHByqYcj9g5elVBnDQcQL7PlO1CIRy2gWlbwK7UPYqi7vRvFA44dCmYQ==", + "requires": { + "@types/node": "*" + } + }, "@types/node": { "version": "10.14.8", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.8.tgz", "integrity": "sha512-I4+DbJEhLEg4/vIy/2gkWDvXBOOtPKV9EnLhYjMoqxcRW+TTZtUftkHktz/a8suoD5mUL7m6ReLrkPvSsCQQmw==" }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, "glob": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", @@ -305,6 +359,45 @@ "once": "^1.3.0", "path-is-absolute": "^1.0.0" } + }, + "marked": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-0.4.0.tgz", + "integrity": "sha512-tMsdNBgOsrUophCAFQl0XPe6Zqk/uy9gnue+jIIKhykO51hxyu6uNx7zBPy0+y/WKYVZZMspV9YeXLNdKk+iYw==" + }, + "typedoc": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.14.2.tgz", + "integrity": "sha512-aEbgJXV8/KqaVhcedT7xG6d2r+mOvB5ep3eIz1KuB5sc4fDYXcepEEMdU7XSqLFO5hVPu0nllHi1QxX2h/QlpQ==", + "requires": { + "@types/fs-extra": "^5.0.3", + "@types/handlebars": "^4.0.38", + "@types/highlight.js": "^9.12.3", + "@types/lodash": "^4.14.110", + "@types/marked": "^0.4.0", + "@types/minimatch": "3.0.3", + "@types/shelljs": "^0.8.0", + "fs-extra": "^7.0.0", + "handlebars": "^4.0.6", + "highlight.js": "^9.13.1", + "lodash": "^4.17.10", + "marked": "^0.4.0", + "minimatch": "^3.0.0", + "progress": "^2.0.0", + "shelljs": "^0.8.2", + "typedoc-default-themes": "^0.5.0", + "typescript": "3.2.x" + } + }, + "typedoc-default-themes": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.5.0.tgz", + "integrity": "sha1-bcJDPnjti+qOiHo6zeLzF4W9Yic=" + }, + "typescript": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.2.4.tgz", + "integrity": "sha512-0RNDbSdEokBeEAkgNbxJ+BLwSManFy9TeXz8uW+48j/xhEXv1ePME60olyzw2XzUqUBNAYFeJadIqAgNqIACwg==" } } }, @@ -426,9 +519,9 @@ } }, "@types/elasticsearch": { - "version": "5.0.34", - "resolved": "https://registry.npmjs.org/@types/elasticsearch/-/elasticsearch-5.0.34.tgz", - "integrity": "sha512-WZ0ZvO//5ZdNV6BpOzRH+bs7kJfRM00Mg9+o4wj1nUv9VVltzNhpiBXdx+3guPKPSR8TgSpMxL58IvCsBe+dag==", + "version": "5.0.35", + "resolved": "https://registry.npmjs.org/@types/elasticsearch/-/elasticsearch-5.0.35.tgz", + "integrity": "sha512-oqQylLukPuPtQWPKT7NeTZ23sVNAF9GYAJCMIexxO/Rjmtdv95I1hdfORp7r+eukyqQ3Y5xUS6ZA5mPkKl6JwA==", "dev": true }, "@types/events": { @@ -509,9 +602,9 @@ "integrity": "sha512-MhCUjojzDhVLnZnxwPwa+rETFRDQ0ffjxYdrqOP6TBO2O0/Z64PV5tNeYApo4bc4y4frbWOrRwv/eEkXlI13Rw==" }, "@types/lodash": { - "version": "4.14.135", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.135.tgz", - "integrity": "sha512-Ed+tSZ9qM1oYpi5kzdsBuOzcAIn1wDW+e8TFJ50IMJMlSopGdJgKAbhHzN6h1E1OfjlGOr2JepzEWtg9NIfoNg==" + "version": "4.14.149", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", + "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ==" }, "@types/marked": { "version": "0.4.2", @@ -618,9 +711,9 @@ } }, "@types/shelljs": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.5.tgz", - "integrity": "sha512-bZgjwIWu9gHCjirKJoOlLzGi5N0QgZ5t7EXEuoqyWCHTuSddURXo3FOBYDyRPNOWzZ6NbkLvZnVkn483Y/tvcQ==", + "version": "0.8.6", + "resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.6.tgz", + "integrity": "sha512-svx2eQS268awlppL/P8wgDLBrsDXdKznABHJcuqXyWpSKJgE1s2clXlBvAwbO/lehTmG06NtEWJRkAk4tAgenA==", "requires": { "@types/glob": "*", "@types/node": "*" @@ -712,14 +805,6 @@ "integrity": "sha1-anmQQ3ynNtXhKI25K9MmbV9csqo=", "dev": true }, - "agentkeepalive": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", - "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", - "requires": { - "humanize-ms": "^1.2.1" - } - }, "ajv": { "version": "6.10.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", @@ -740,7 +825,8 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true }, "ansi-styles": { "version": "3.2.1", @@ -832,6 +918,15 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "dev": true }, + "backbone": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/backbone/-/backbone-1.4.0.tgz", + "integrity": "sha512-RLmDrRXkVdouTg38jcgHhyQ/2zjg7a8E6sz2zxfz21Hh17xDJYUHBZimVIt5fUyS8vbfpeSmTL3gUjTEvUV3qQ==", + "dev": true, + "requires": { + "underscore": ">=1.8.3" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1324,8 +1419,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cors": { "version": "2.8.5", @@ -1563,40 +1657,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "elasticsearch": { - "version": "16.1.1", - "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.1.1.tgz", - "integrity": "sha512-OF2fIjcTPfq/4Tj6k4/SZr2IIlfWlBBQoy/em225mfevYFW1abN3nyXKWldXGV+eWI6LWNqB8lb3hAP4d6Rh/Q==", - "requires": { - "agentkeepalive": "^3.4.1", - "chalk": "^1.0.0", - "lodash": "^4.17.10" - }, - "dependencies": { - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, "emoji-regex": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", @@ -1863,6 +1923,44 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "requires": { + "inherits": "^2.0.1", + "readable-stream": "^2.0.0" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "fs-extra": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.0.1.tgz", @@ -2306,9 +2404,9 @@ "dev": true }, "handlebars": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", - "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", + "version": "4.7.2", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.2.tgz", + "integrity": "sha512-4PwqDL2laXtTWZghzzCtunQUTLbo31pcCJrd/B/9JP8XbhVzpS5ZXuKqlOzsd1rtcaLo4KqAn8nl8mkknS4MHw==", "requires": { "neo-async": "^2.6.0", "optimist": "^0.6.1", @@ -2325,14 +2423,6 @@ "function-bind": "^1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -2360,9 +2450,9 @@ "dev": true }, "highlight.js": { - "version": "9.15.8", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.15.8.tgz", - "integrity": "sha512-RrapkKQWwE+wKdF73VsOa2RQdIoO3mxwJ4P8mhbI6KYJUraUHRKM5w5zQQKXNk0xNL4UVRdulV9SBJcmzJNzVA==" + "version": "9.18.0", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.18.0.tgz", + "integrity": "sha512-A97kI1KAUzKoAiEoaGcf2O9YPS8nbDTCRFokaaeBhnqjQTvbAuAJrQMm21zw8s8xzaMtCQBtgbyGXLGxdxQyqQ==" }, "hosted-git-info": { "version": "2.7.1", @@ -2399,14 +2489,6 @@ "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-1.3.2.tgz", "integrity": "sha512-nDUtj0ltIt08tGi2VWSpSzNNFye0v3YSe9lX3lIqLTuVvvRiYCvs4QQBSHo0eomFYw1wlUuofurUAlTm+vHnXg==" }, - "humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", - "requires": { - "ms": "^2.0.0" - } - }, "humanize-string": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/humanize-string/-/humanize-string-2.1.0.tgz", @@ -2460,6 +2542,15 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz", "integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw==" }, + "into-stream": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-5.1.0.tgz", + "integrity": "sha512-cbDhb8qlxKMxPBk/QxTtYg1DQ4CwXmadu7quG3B7nrJsgSncEreF2kwWKZFdnjc/lSNNIkFPsjI7SM0Cx/QXPw==", + "requires": { + "from2": "^2.3.0", + "p-is-promise": "^2.0.0" + } + }, "invert-kv": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz", @@ -2684,6 +2775,12 @@ "handlebars": "^4.1.2" } }, + "jquery": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.4.1.tgz", + "integrity": "sha512-36+AdBzCL+y6qjw5Tx7HgzeGCzC81MDDgaUP8ld2zhx58HdqXGoBd+tHdrBMiyjGQs0Hxs/MLZTu/eHNJJuWPw==", + "dev": true + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2833,9 +2930,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "lodash._reinterpolate": { "version": "3.0.0", @@ -2913,6 +3010,12 @@ "yallist": "^2.1.2" } }, + "lunr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz", + "integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==", + "dev": true + }, "make-dir": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", @@ -2952,9 +3055,10 @@ "dev": true }, "marked": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.4.0.tgz", - "integrity": "sha512-tMsdNBgOsrUophCAFQl0XPe6Zqk/uy9gnue+jIIKhykO51hxyu6uNx7zBPy0+y/WKYVZZMspV9YeXLNdKk+iYw==" + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.0.tgz", + "integrity": "sha512-MyUe+T/Pw4TZufHkzAfDj6HarCBWia2y27/bhuYkTaiUnfDYFnCP3KUN+9oM7Wi6JA2rymtVYbQu3spE0GCmxQ==", + "dev": true }, "media-typer": { "version": "0.3.0", @@ -3201,87 +3305,12 @@ "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", "dev": true }, - "execa": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", - "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", - "dev": true, - "requires": { - "cross-spawn": "^5.0.1", - "get-stream": "^3.0.0", - "is-stream": "^1.1.0", - "npm-run-path": "^2.0.0", - "p-finally": "^1.0.0", - "signal-exit": "^3.0.0", - "strip-eof": "^1.0.0" - }, - "dependencies": { - "cross-spawn": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", - "dev": true, - "requires": { - "lru-cache": "^4.0.1", - "shebang-command": "^1.2.0", - "which": "^1.2.9" - } - } - } - }, "get-caller-file": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", "dev": true }, - "get-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", - "dev": true - }, - "invert-kv": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", - "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=", - "dev": true - }, - "lcid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", - "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", - "dev": true, - "requires": { - "invert-kv": "^1.0.0" - } - }, - "mem": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", - "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=", - "dev": true, - "requires": { - "mimic-fn": "^1.0.0" - } - }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "dev": true - }, - "os-locale": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz", - "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==", - "dev": true, - "requires": { - "execa": "^0.7.0", - "lcid": "^1.0.0", - "mem": "^1.1.0" - } - }, "require-main-filename": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", @@ -3295,16 +3324,16 @@ "dev": true }, "yargs": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.0.tgz", - "integrity": "sha512-NwW69J42EsCSanF8kyn5upxvjp5ds+t3+udGBeTbFnERA+lF541DDpMawzo4z6W/QrzNM18D+BPMiOBibnFV5A==", + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.1.tgz", + "integrity": "sha512-PRU7gJrJaXv3q3yQZ/+/X6KBswZiaQ+zOmdprZcouPYtQgvNU35i+68M4b1ZHLZtYFT5QObFLV+ZkmJYcwKdiw==", "dev": true, "requires": { "cliui": "^4.0.0", "decamelize": "^1.1.1", "find-up": "^2.1.0", "get-caller-file": "^1.0.1", - "os-locale": "^2.0.0", + "os-locale": "^3.1.0", "require-directory": "^2.1.1", "require-main-filename": "^1.0.1", "set-blocking": "^2.0.0", @@ -3705,8 +3734,7 @@ "p-is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz", - "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", - "dev": true + "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==" }, "p-limit": { "version": "1.3.0", @@ -3945,8 +3973,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "progress": { "version": "2.0.3", @@ -4437,6 +4464,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, "requires": { "ansi-regex": "^2.0.0" } @@ -4846,58 +4874,54 @@ } }, "typedoc": { - "version": "0.14.2", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.14.2.tgz", - "integrity": "sha512-aEbgJXV8/KqaVhcedT7xG6d2r+mOvB5ep3eIz1KuB5sc4fDYXcepEEMdU7XSqLFO5hVPu0nllHi1QxX2h/QlpQ==", + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.16.7.tgz", + "integrity": "sha512-sNrLlaZ/aZHxA2rCURGf3g5YabGVwrujiwC6SCV/rgx3LFfZh+goUCatAAyTEDk7evuu6pJ0APGDSde1mSYegw==", + "dev": true, "requires": { - "@types/fs-extra": "^5.0.3", - "@types/handlebars": "^4.0.38", - "@types/highlight.js": "^9.12.3", - "@types/lodash": "^4.14.110", - "@types/marked": "^0.4.0", "@types/minimatch": "3.0.3", - "@types/shelljs": "^0.8.0", - "fs-extra": "^7.0.0", - "handlebars": "^4.0.6", - "highlight.js": "^9.13.1", - "lodash": "^4.17.10", - "marked": "^0.4.0", + "fs-extra": "^8.1.0", + "handlebars": "^4.7.2", + "highlight.js": "^9.17.1", + "lodash": "^4.17.15", + "marked": "^0.8.0", "minimatch": "^3.0.0", - "progress": "^2.0.0", - "shelljs": "^0.8.2", - "typedoc-default-themes": "^0.5.0", - "typescript": "3.2.x" + "progress": "^2.0.3", + "shelljs": "^0.8.3", + "typedoc-default-themes": "^0.7.2", + "typescript": "3.7.x" }, "dependencies": { - "@types/fs-extra": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.1.0.tgz", - "integrity": "sha512-AInn5+UBFIK9FK5xc9yP5e3TQSPNNgjHByqYcj9g5elVBnDQcQL7PlO1CIRy2gWlbwK7UPYqi7vRvFA44dCmYQ==", - "requires": { - "@types/node": "*" - } - }, "fs-extra": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", - "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, "requires": { - "graceful-fs": "^4.1.2", + "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "typescript": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.2.4.tgz", - "integrity": "sha512-0RNDbSdEokBeEAkgNbxJ+BLwSManFy9TeXz8uW+48j/xhEXv1ePME60olyzw2XzUqUBNAYFeJadIqAgNqIACwg==" + "version": "3.7.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.5.tgz", + "integrity": "sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==", + "dev": true } } }, "typedoc-default-themes": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.5.0.tgz", - "integrity": "sha1-bcJDPnjti+qOiHo6zeLzF4W9Yic=" + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.7.2.tgz", + "integrity": "sha512-fiFKlFO6VTqjcno8w6WpTsbCgXmfPHVjnLfYkmByZE7moaz+E2DSpAT+oHtDHv7E0BM5kAhPrHJELP2J2Y2T9A==", + "dev": true, + "requires": { + "backbone": "^1.4.0", + "jquery": "^3.4.1", + "lunr": "^2.3.8", + "underscore": "^1.9.1" + } }, "typescript": { "version": "3.5.3", @@ -4911,15 +4935,29 @@ "integrity": "sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==" }, "uglify-js": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", - "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "version": "3.7.6", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.6.tgz", + "integrity": "sha512-yYqjArOYSxvqeeiYH2VGjZOqq6SVmhxzaPjJC1W2F9e+bqvFL9QXQ2osQuKUFjM2hGjKG2YclQnRKWQSt/nOTQ==", "optional": true, "requires": { - "commander": "~2.20.0", + "commander": "~2.20.3", "source-map": "~0.6.1" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "optional": true + } } }, + "underscore": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.2.tgz", + "integrity": "sha512-D39qtimx0c1fI3ya1Lnhk3E9nONswSKhnffBI0gME9C99fYOkNi04xs8K6pePLhvl1frbDemkaBQ5ikWllR2HQ==", + "dev": true + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -4959,8 +4997,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", diff --git a/package.json b/package.json index 8060fca3..45599e04 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'" }, "dependencies": { + "@elastic/elasticsearch": "5.6.20", "@openstapps/core": "0.31.0", "@openstapps/core-tools": "0.11.0", "@openstapps/logger": "0.4.0", @@ -36,7 +37,6 @@ "commander": "2.20.0", "config": "3.1.0", "cors": "2.8.5", - "elasticsearch": "16.1.1", "express": "4.17.1", "express-promise-router": "3.0.3", "fs-extra": "8.0.1", @@ -59,7 +59,7 @@ "@types/chai-as-promised": "7.1.0", "@types/config": "0.0.34", "@types/cors": "2.8.5", - "@types/elasticsearch": "5.0.34", + "@types/elasticsearch": "5.0.35", "@types/express": "4.17.0", "@types/fs-extra": "7.0.0", "@types/got": "9.4.4", @@ -72,19 +72,19 @@ "@types/sinon-express-mock": "1.3.7", "@types/supertest": "2.0.7", "@types/uuid": "3.4.5", - "chai-as-promised": "7.1.1", "chai": "4.2.0", + "chai-as-promised": "7.1.1", "conventional-changelog-cli": "2.0.21", - "mocha-typescript": "1.1.17", "mocha": "6.1.4", + "mocha-typescript": "1.1.17", "nyc": "14.1.1", "prepend-file-cli": "1.0.6", "rimraf": "2.6.3", - "sinon-express-mock": "2.2.0", "sinon": "7.3.2", + "sinon-express-mock": "2.2.0", "supertest": "4.0.2", "tslint": "5.18.0", - "typedoc": "0.14.2", + "typedoc": "0.16.7", "typescript": "3.5.3" } } diff --git a/src/storage/elasticsearch/aggregations.ts b/src/storage/elasticsearch/aggregations.ts index d0c4d845..396c08d2 100644 --- a/src/storage/elasticsearch/aggregations.ts +++ b/src/storage/elasticsearch/aggregations.ts @@ -14,10 +14,10 @@ * along with this program. If not, see . */ import {SCFacet, SCThingType} from '@openstapps/core'; -import {AggregationSchema} from '@openstapps/core-tools/lib/mappings/aggregation-definitions'; import {readFileSync} from 'fs'; import { AggregationResponse, + AggregationSchema, isBucketAggregation, isESAggMatchAllFilter, isESNestedAggregation, diff --git a/src/storage/elasticsearch/common.ts b/src/storage/elasticsearch/common.ts index 1d57d55b..61125869 100644 --- a/src/storage/elasticsearch/common.ts +++ b/src/storage/elasticsearch/common.ts @@ -13,14 +13,16 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import {SCThingType} from '@openstapps/core'; +import {SCThings, SCThingType} from '@openstapps/core'; import {SCThing} from '@openstapps/core'; import { ESAggMatchAllFilter, - ESAggTypeFilter, - ESNestedAggregation, + ESAggTypeFilter, ESNestedAggregation, ESTermsFilter, } from '@openstapps/core-tools/lib/mappings/aggregation-definitions'; +// we only have the @types package because some things type definitions are still missing from the official +// @elastic/elasticsearch package +// tslint:disable-next-line:no-implicit-dependencies import {NameList} from 'elasticsearch'; /** @@ -71,6 +73,21 @@ export function isBucketAggregation(agg: BucketAggregation | number): agg is Buc return typeof agg !== 'number'; } +/** + * A response that contains info about whether the item exists plus the object itself + */ +export interface ItemExistsResponse { + /** + * Whether the item exists + */ + exists: boolean; + + /** + * The object if it exists + */ + object?: ElasticsearchObject; +} + /** * An aggregation that contains more aggregations nested inside */ @@ -94,6 +111,14 @@ export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): return typeof (agg as BucketAggregation).buckets === 'undefined'; } +/** + * An elasticsearch bucket aggregation + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html + */ +export interface AggregationSchema { + [aggregationName: string]: ESTermsFilter | ESNestedAggregation; +} + /** * A configuration for using the Dis Max Query * diff --git a/src/storage/elasticsearch/elasticsearch.ts b/src/storage/elasticsearch/elasticsearch.ts index e2f22002..1a951d5a 100644 --- a/src/storage/elasticsearch/elasticsearch.ts +++ b/src/storage/elasticsearch/elasticsearch.ts @@ -13,6 +13,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch'; import { SCBulkResponse, SCConfigFile, @@ -23,19 +24,23 @@ import { SCThingType, SCUuid, } from '@openstapps/core'; -import {AggregationSchema} from '@openstapps/core-tools/lib/mappings/aggregation-definitions'; import {Logger} from '@openstapps/logger'; -import * as ES from 'elasticsearch'; +// we only have the @types package because some things type definitions are still missing from the official +// @elastic/elasticsearch package +// tslint:disable-next-line:no-implicit-dependencies +import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch'; import * as moment from 'moment'; import {MailQueue} from '../../notification/mail-queue'; import {Bulk} from '../bulk-storage'; import {Database} from '../database'; import {buildAggregations, parseAggregations} from './aggregations'; import { + AggregationResponse, + AggregationSchema, ElasticsearchConfig, - ElasticsearchObject, ElasticsearchQueryDisMaxConfig, ElasticsearchQueryQueryStringConfig, + ItemExistsResponse, } from './common'; import * as Monitoring from './monitoring'; import {buildQuery, buildSort} from './query'; @@ -75,7 +80,7 @@ export class Elasticsearch implements Database { /** * Elasticsearch client */ - client: ES.Client; + client: Client; /** * Queue of mails to be sent @@ -92,12 +97,12 @@ export class Elasticsearch implements Database { */ static getElasticsearchUrl(): string { // check if we have a docker link - if (process.env.ES_PORT_9200_TCP_ADDR !== undefined && process.env.ES_PORT_9200_TCP_PORT !== undefined) { - return `${process.env.ES_PORT_9200_TCP_ADDR}:${process.env.ES_PORT_9200_TCP_PORT}`; + if (process.env.ES_ADDR !== undefined ) { + return process.env.ES_ADDR; } // default - return 'localhost:9200'; + return 'http://localhost:9200'; } /** @@ -146,28 +151,28 @@ export class Elasticsearch implements Database { ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => { if (formattedAlias.includes(value)) { formattedAlias = formattedAlias.replace(value, ''); - Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks ` - + `having the same alias despite having different types, as invalid characters are removed automatically. ` + - `New alias name is "${formattedAlias}."`); + Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks + having the same alias despite having different types, as invalid characters are removed automatically. + New alias name is "${formattedAlias}."`); } }); ['-', '_', '+'].forEach((value) => { if (formattedAlias.charAt(0) === value) { formattedAlias = formattedAlias.substring(1); - Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks ` - + `having the same alias despite having different types, as invalid characters are removed automatically. ` + - `New alias name is "${formattedAlias}."`); + Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same + alias despite having different types, as invalid characters are removed automatically. + New alias name is "${formattedAlias}."`); } }); if (formattedAlias === '.' || formattedAlias === '..') { - Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using ` + - `another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`); + Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using + another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`); return 'alias_placeholder'; } if (formattedAlias.includes(':')) { - Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future ` + - `Elasticsearch versions!`); + Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future + Elasticsearch versions!`); } return formattedAlias; @@ -185,28 +190,24 @@ export class Elasticsearch implements Database { throw new Error('Database version is undefined. Check your config file'); } - const options: ES.ConfigOptions = { - apiVersion: config.internal.database.version, - host: Elasticsearch.getElasticsearchUrl(), - // enable verbose logging for all request to elasticsearch - log: process.env.ES_DEBUG === 'true' ? 'trace' : 'error', - }; + this.client = new Client({ + node: Elasticsearch.getElasticsearchUrl(), + }); + this.client.on(events.REQUEST, async (err: Error | null, result: ApiResponse) => { + if (err !== null) { + await Logger.error(err); + } + if (process.env.ES_DEBUG === 'true') { + Logger.log(result); + } + }); - this.client = new ES.Client(options); this.aliasMap = {}; this.ready = false; - checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ? - process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false); - this.aggregationsSchema = buildAggregations(); this.mailQueue = mailQueue; - - /*refreshAllTemplates(this.client) - .then(() => { - // noop - });*/ } /** @@ -214,9 +215,8 @@ export class Elasticsearch implements Database { * * Returns Elasticsearch Object if it exists */ - // tslint:disable-next-line: completed-docs - private async doesItemExist(object: SCThings): Promise<{exists: boolean; object?: ElasticsearchObject; }> { - const searchResponse = await this.client.search({ + private async doesItemExist(object: SCThings): Promise { + const searchResponse: ApiResponse> = await this.client.search({ body: { query: { term: { @@ -231,10 +231,10 @@ export class Elasticsearch implements Database { size: 1, }); - if (searchResponse.hits.total > 1) { + if (searchResponse.body.hits.total > 1) { return { exists: true, - object: searchResponse.hits.hits[0], + object: searchResponse.body.hits.hits[0], }; } @@ -264,7 +264,7 @@ export class Elasticsearch implements Database { }; try { - aliases = await this.client.indices.getAlias({}); + aliases = (await this.client.indices.getAlias({})).body; } catch (error) { await Logger.error('Failed getting alias map:', error); setTimeout(async () => { @@ -335,7 +335,7 @@ export class Elasticsearch implements Database { if (!indexRegex.test(index)) { throw new Error( - `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. + `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. Make sure to set the bulk "source" and "type" to names consisting of the characters above.`, ); } @@ -363,7 +363,7 @@ export class Elasticsearch implements Database { if (bulk.state !== 'done') { Logger.info('deleting obsolete index', index); - return this.client.indices.delete({index}); + await this.client.indices.delete({index}); } } @@ -390,13 +390,13 @@ export class Elasticsearch implements Database { if (!indexRegex.test(index)) { throw new Error( - `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. + `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. Make sure to set the bulk "source" and "type" to names consisting of the characters above.`, ); } // create the new index if it does not exists - if (!(await this.client.indices.exists({index}))) { + if (!(await this.client.indices.exists({index})).body) { // re-apply the index template before each new bulk operation await putTemplate(this.client, bulk.type); await this.client.indices.create({ @@ -408,7 +408,8 @@ export class Elasticsearch implements Database { const oldIndex: string = this.aliasMap[alias][bulk.source]; // add our new index to the alias - const actions: ES.IndicesUpdateAliasesParamsAction[] = [ + // this was type safe with @types/elasticsearch, the new package however provides no type definitions + const actions: IndicesUpdateAliasesParamsAction[] = [ { add: {index: index, alias: alias}, }, @@ -423,7 +424,7 @@ export class Elasticsearch implements Database { // refresh the index (fsync changes) await this.client.indices.refresh({ - index, + index: index, }); // execute our alias actions @@ -448,7 +449,7 @@ export class Elasticsearch implements Database { * @param uid uid of an SCThing */ public async get(uid: SCUuid): Promise { - const searchResponse = await this.client.search({ + const searchResponse: ApiResponse> = await this.client.search({ body: { query: { term: { @@ -460,13 +461,13 @@ export class Elasticsearch implements Database { }); // get data from response - const hits = searchResponse.hits.hits; + const hits = searchResponse.body.hits.hits; if (hits.length !== 1) { throw new Error('No unique item found.'); } - return hits[0]._source as SCThings; + return hits[0]._source; } /** @@ -483,6 +484,9 @@ export class Elasticsearch implements Database { Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue); } + checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ? + process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false); + return this.getAliasMap(); } @@ -494,7 +498,7 @@ export class Elasticsearch implements Database { public async post(object: SCThings, bulk: Bulk): Promise { // tslint:disable-next-line: completed-docs - const obj: SCThings & {creation_date: string; } = { + const obj: SCThings & { creation_date: string; } = { ...object, creation_date: moment() .format(), @@ -510,10 +514,10 @@ export class Elasticsearch implements Database { // new item doesn't replace the old one if (oldIndex.substring(0, oldIndex.length - Elasticsearch.INDEX_UID_LENGTH + 1) !== indexOfNew.substring(0, indexOfNew.length - Elasticsearch.INDEX_UID_LENGTH + 1)) { - throw new Error( - // tslint:disable-next-line: no-magic-numbers - `Object "${obj.uid}" already exists. Object was: ${JSON.stringify(obj, null, 2)}`, - ); + throw new Error( + // tslint:disable-next-line: no-magic-numbers + `Object "${obj.uid}" already exists. Object was: ${JSON.stringify(obj, null, 2)}`, + ); } } @@ -526,7 +530,7 @@ export class Elasticsearch implements Database { type: obj.type, }); - if (!searchResponse.created) { + if (!searchResponse.body.created) { throw new Error(`Object creation Error: Instance was: ${JSON.stringify(obj)}`); } } @@ -535,12 +539,12 @@ export class Elasticsearch implements Database { * Put (update) an existing item * @param object SCThing to put */ - public async put(object: SCThings) { + public async put(object: SCThings): Promise { const itemMeta = await this.doesItemExist(object); if (itemMeta.exists && typeof itemMeta.object !== 'undefined') { - return this.client.update({ + await this.client.update({ body: { doc: object, }, @@ -548,6 +552,8 @@ export class Elasticsearch implements Database { index: itemMeta.object._index, type: object.type.toLowerCase(), }); + + return; } throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.'); @@ -581,7 +587,7 @@ export class Elasticsearch implements Database { .query as ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig; } - const searchRequest: ES.SearchParams = { + const searchRequest: RequestParams.Search = { body: { aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes) query: buildQuery(params, this.config, esConfig), @@ -596,31 +602,31 @@ export class Elasticsearch implements Database { } // perform the search against elasticsearch - const response = await this.client.search(searchRequest); + const response: ApiResponse> = await this.client.search(searchRequest); // gather pagination information const pagination = { - count: response.hits.hits.length, + count: response.body.hits.hits.length, offset: (typeof params.from === 'number') ? params.from : 0, - total: response.hits.total, + total: response.body.hits.total, }; // gather statistics about this search const stats = { - time: response.took, + time: response.body.took, }; // we only directly return the _source documents // elasticsearch provides much more information, the user shouldn't see - const data = response.hits.hits.map((hit) => { + const data = response.body.hits.hits.map((hit) => { return hit._source; // SCThing }); let facets: SCFacet[] = []; // read the aggregations from elasticsearch and parse them to facets by our configuration - if (typeof response.aggregations !== 'undefined') { - facets = parseAggregations(this.aggregationsSchema, response.aggregations); + if (typeof response.body.aggregations !== 'undefined') { + facets = parseAggregations(this.aggregationsSchema, response.body.aggregations as AggregationResponse); } return { diff --git a/src/storage/elasticsearch/monitoring.ts b/src/storage/elasticsearch/monitoring.ts index 7d5c64c5..5eb4e498 100644 --- a/src/storage/elasticsearch/monitoring.ts +++ b/src/storage/elasticsearch/monitoring.ts @@ -13,15 +13,20 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch'; import { SCMonitoringConfiguration, SCMonitoringLogAction, SCMonitoringMailAction, SCMonitoringMaximumLengthCondition, SCMonitoringMinimumLengthCondition, + SCThings, } from '@openstapps/core'; import {Logger} from '@openstapps/logger'; -import * as ES from 'elasticsearch'; +// we only have the @types package because some things type definitions are still missing from the official +// @elastic/elasticsearch package +// tslint:disable-next-line:no-implicit-dependencies +import {SearchResponse} from 'elasticsearch'; import * as cron from 'node-cron'; import {MailQueue} from '../../notification/mail-queue'; @@ -47,7 +52,7 @@ function conditionFails( * @param total Number of results */ function minConditionFails(minimumLength: number, total: number) { - return typeof minimumLength === 'number' && minimumLength > total; + return minimumLength > total; } /** @@ -99,7 +104,7 @@ export function runActions( * @param esClient elasticsearch client * @param mailQueue mailQueue for mail actions */ -export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES.Client, mailQueue: MailQueue) { +export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: Client, mailQueue: MailQueue) { // set up Watches monitoringConfig.watchers.forEach((watcher) => { @@ -122,10 +127,11 @@ export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES. cron.schedule(trigger.executionTime, async () => { // execute watch (search->condition->action) - const result = await esClient.search(watcher.query as ES.SearchParams); + const result: ApiResponse> = + await esClient.search(watcher.query as RequestParams.Search); // check conditions - const total = result.hits.total; + const total = result.body.hits.total; watcher.conditions.forEach((condition) => { if (conditionFails(condition, total)) { diff --git a/src/storage/elasticsearch/templating.ts b/src/storage/elasticsearch/templating.ts index d6963262..49ac71a3 100644 --- a/src/storage/elasticsearch/templating.ts +++ b/src/storage/elasticsearch/templating.ts @@ -13,11 +13,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import {Client} from '@elastic/elasticsearch'; import {SCThingType} from '@openstapps/core'; import {getProjectReflection} from '@openstapps/core-tools/lib/common'; import {generateTemplate} from '@openstapps/core-tools/lib/mapping'; import {Logger} from '@openstapps/logger'; -import {Client} from 'elasticsearch'; import {existsSync, writeFileSync} from 'fs'; import {readFile} from 'fs-extra'; import {resolve} from 'path';