Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-09 19:22:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: use new Elasticsearch package

Browse Source
This commit is contained in:
Wieland Schöbl
2020-01-21 11:10:29 +01:00
committed by Rainer Killinger
parent 54301ae8fb
commit 1bad092185
8 changed files with 368 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -72,6 +72,14 @@ environment. At the time only SMTP is being implemented. The backend wouldn't st
you can set `ALLOW_NO_TRANSPORT=true`. To set up an SMTP configuration have a look at you can set `ALLOW_NO_TRANSPORT=true`. To set up an SMTP configuration have a look at
[@openstapps/logger](https://gitlab.com/openstapps/logger). [@openstapps/logger](https://gitlab.com/openstapps/logger).
The list of environment variables includes:
* `NODE_ENV` when set to `production`, there will be a reduced amount of output from the logger
* `PORT` when this is not set, the backend will default to port 3000
* `ES_ADDR` the Elasticsearch address, if not set it will default the Elasticsearch address to `http://localhost:9200`
* `ES_FORCE_MAPPING_UPDATE` when this variable is set to `true`, the backend will always generate a new Elasticsearch mapping from the core regardless of whether there is already a version present. This should only really be used when you are working on the core.
* `ALLOW_NO_TRANSPORT` if set to true, the backend will allow starting without an Email configured that receives critical errors.
* `ES_DEBUG` setting this to `true` will result in Elasticsearch logging to be **VERY** extensive, in almost all situation this should no be enabled.
## Config files ## Config files
Each university can have it's specific config for the general backend and app and for all databases. Each university can have it's specific config for the general backend and app and for all databases.

451
package-lock.json generated
View File

@@ -159,6 +159,42 @@
} }
} }
}, },
"@elastic/elasticsearch": {
"version": "5.6.20",
"resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-5.6.20.tgz",
"integrity": "sha512-dOybHGWMo6J/rx/ad9IbROKYq5aQIOd0RrickbqrIU7pjg0J3qSIL4SN027jz1vdrtOcap7yovA+HUplrEXOog==",
"requires": {
"debug": "^4.1.1",
"decompress-response": "^4.2.0",
"into-stream": "^5.1.0",
"ms": "^2.1.1",
"once": "^1.4.0",
"pump": "^3.0.0"
},
"dependencies": {
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"decompress-response": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
"integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
"requires": {
"mimic-response": "^2.0.0"
}
},
"mimic-response": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.0.0.tgz",
"integrity": "sha512-8ilDoEapqA4uQ3TwS0jakGONKXVJqpy+RpM+3b7pLdOjghCrEiGp9SRkFbUHAmZW9vdnrENWHjaweIoTIJExSQ=="
}
}
},
"@krlwlfrt/async-pool": { "@krlwlfrt/async-pool": {
"version": "0.1.0", "version": "0.1.0",
"resolved": "https://registry.npmjs.org/@krlwlfrt/async-pool/-/async-pool-0.1.0.tgz", "resolved": "https://registry.npmjs.org/@krlwlfrt/async-pool/-/async-pool-0.1.0.tgz",
@@ -288,11 +324,29 @@
} }
} }
}, },
"@types/fs-extra": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.1.0.tgz",
"integrity": "sha512-AInn5+UBFIK9FK5xc9yP5e3TQSPNNgjHByqYcj9g5elVBnDQcQL7PlO1CIRy2gWlbwK7UPYqi7vRvFA44dCmYQ==",
"requires": {
"@types/node": "*"
}
},
"@types/node": { "@types/node": {
"version": "10.14.8", "version": "10.14.8",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.8.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.8.tgz",
"integrity": "sha512-I4+DbJEhLEg4/vIy/2gkWDvXBOOtPKV9EnLhYjMoqxcRW+TTZtUftkHktz/a8suoD5mUL7m6ReLrkPvSsCQQmw==" "integrity": "sha512-I4+DbJEhLEg4/vIy/2gkWDvXBOOtPKV9EnLhYjMoqxcRW+TTZtUftkHktz/a8suoD5mUL7m6ReLrkPvSsCQQmw=="
}, },
"fs-extra": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
"integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==",
"requires": {
"graceful-fs": "^4.1.2",
"jsonfile": "^4.0.0",
"universalify": "^0.1.0"
}
},
"glob": { "glob": {
"version": "7.1.4", "version": "7.1.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
@@ -305,6 +359,45 @@
"once": "^1.3.0", "once": "^1.3.0",
"path-is-absolute": "^1.0.0" "path-is-absolute": "^1.0.0"
} }
},
"marked": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/marked/-/marked-0.4.0.tgz",
"integrity": "sha512-tMsdNBgOsrUophCAFQl0XPe6Zqk/uy9gnue+jIIKhykO51hxyu6uNx7zBPy0+y/WKYVZZMspV9YeXLNdKk+iYw=="
},
"typedoc": {
"version": "0.14.2",
"resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.14.2.tgz",
"integrity": "sha512-aEbgJXV8/KqaVhcedT7xG6d2r+mOvB5ep3eIz1KuB5sc4fDYXcepEEMdU7XSqLFO5hVPu0nllHi1QxX2h/QlpQ==",
"requires": {
"@types/fs-extra": "^5.0.3",
"@types/handlebars": "^4.0.38",
"@types/highlight.js": "^9.12.3",
"@types/lodash": "^4.14.110",
"@types/marked": "^0.4.0",
"@types/minimatch": "3.0.3",
"@types/shelljs": "^0.8.0",
"fs-extra": "^7.0.0",
"handlebars": "^4.0.6",
"highlight.js": "^9.13.1",
"lodash": "^4.17.10",
"marked": "^0.4.0",
"minimatch": "^3.0.0",
"progress": "^2.0.0",
"shelljs": "^0.8.2",
"typedoc-default-themes": "^0.5.0",
"typescript": "3.2.x"
}
},
"typedoc-default-themes": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.5.0.tgz",
"integrity": "sha1-bcJDPnjti+qOiHo6zeLzF4W9Yic="
},
"typescript": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.2.4.tgz",
"integrity": "sha512-0RNDbSdEokBeEAkgNbxJ+BLwSManFy9TeXz8uW+48j/xhEXv1ePME60olyzw2XzUqUBNAYFeJadIqAgNqIACwg=="
} }
} }
}, },
@@ -426,9 +519,9 @@
} }
}, },
"@types/elasticsearch": { "@types/elasticsearch": {
"version": "5.0.34", "version": "5.0.35",
"resolved": "https://registry.npmjs.org/@types/elasticsearch/-/elasticsearch-5.0.34.tgz", "resolved": "https://registry.npmjs.org/@types/elasticsearch/-/elasticsearch-5.0.35.tgz",
"integrity": "sha512-WZ0ZvO//5ZdNV6BpOzRH+bs7kJfRM00Mg9+o4wj1nUv9VVltzNhpiBXdx+3guPKPSR8TgSpMxL58IvCsBe+dag==", "integrity": "sha512-oqQylLukPuPtQWPKT7NeTZ23sVNAF9GYAJCMIexxO/Rjmtdv95I1hdfORp7r+eukyqQ3Y5xUS6ZA5mPkKl6JwA==",
"dev": true "dev": true
}, },
"@types/events": { "@types/events": {
@@ -509,9 +602,9 @@
"integrity": "sha512-MhCUjojzDhVLnZnxwPwa+rETFRDQ0ffjxYdrqOP6TBO2O0/Z64PV5tNeYApo4bc4y4frbWOrRwv/eEkXlI13Rw==" "integrity": "sha512-MhCUjojzDhVLnZnxwPwa+rETFRDQ0ffjxYdrqOP6TBO2O0/Z64PV5tNeYApo4bc4y4frbWOrRwv/eEkXlI13Rw=="
}, },
"@types/lodash": { "@types/lodash": {
"version": "4.14.135", "version": "4.14.149",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.135.tgz", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz",
"integrity": "sha512-Ed+tSZ9qM1oYpi5kzdsBuOzcAIn1wDW+e8TFJ50IMJMlSopGdJgKAbhHzN6h1E1OfjlGOr2JepzEWtg9NIfoNg==" "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ=="
}, },
"@types/marked": { "@types/marked": {
"version": "0.4.2", "version": "0.4.2",
@@ -618,9 +711,9 @@
} }
}, },
"@types/shelljs": { "@types/shelljs": {
"version": "0.8.5", "version": "0.8.6",
"resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.5.tgz", "resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.6.tgz",
"integrity": "sha512-bZgjwIWu9gHCjirKJoOlLzGi5N0QgZ5t7EXEuoqyWCHTuSddURXo3FOBYDyRPNOWzZ6NbkLvZnVkn483Y/tvcQ==", "integrity": "sha512-svx2eQS268awlppL/P8wgDLBrsDXdKznABHJcuqXyWpSKJgE1s2clXlBvAwbO/lehTmG06NtEWJRkAk4tAgenA==",
"requires": { "requires": {
"@types/glob": "*", "@types/glob": "*",
"@types/node": "*" "@types/node": "*"
@@ -712,14 +805,6 @@
"integrity": "sha1-anmQQ3ynNtXhKI25K9MmbV9csqo=", "integrity": "sha1-anmQQ3ynNtXhKI25K9MmbV9csqo=",
"dev": true "dev": true
}, },
"agentkeepalive": {
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
"integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
"requires": {
"humanize-ms": "^1.2.1"
}
},
"ajv": { "ajv": {
"version": "6.10.0", "version": "6.10.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
@@ -740,7 +825,8 @@
"ansi-regex": { "ansi-regex": {
"version": "2.1.1", "version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
"dev": true
}, },
"ansi-styles": { "ansi-styles": {
"version": "3.2.1", "version": "3.2.1",
@@ -832,6 +918,15 @@
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
"dev": true "dev": true
}, },
"backbone": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/backbone/-/backbone-1.4.0.tgz",
"integrity": "sha512-RLmDrRXkVdouTg38jcgHhyQ/2zjg7a8E6sz2zxfz21Hh17xDJYUHBZimVIt5fUyS8vbfpeSmTL3gUjTEvUV3qQ==",
"dev": true,
"requires": {
"underscore": ">=1.8.3"
}
},
"balanced-match": { "balanced-match": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
@@ -1324,8 +1419,7 @@
"core-util-is": { "core-util-is": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
"dev": true
}, },
"cors": { "cors": {
"version": "2.8.5", "version": "2.8.5",
@@ -1563,40 +1657,6 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
}, },
"elasticsearch": {
"version": "16.1.1",
"resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.1.1.tgz",
"integrity": "sha512-OF2fIjcTPfq/4Tj6k4/SZr2IIlfWlBBQoy/em225mfevYFW1abN3nyXKWldXGV+eWI6LWNqB8lb3hAP4d6Rh/Q==",
"requires": {
"agentkeepalive": "^3.4.1",
"chalk": "^1.0.0",
"lodash": "^4.17.10"
},
"dependencies": {
"ansi-styles": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
},
"chalk": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"requires": {
"ansi-styles": "^2.2.1",
"escape-string-regexp": "^1.0.2",
"has-ansi": "^2.0.0",
"strip-ansi": "^3.0.0",
"supports-color": "^2.0.0"
}
},
"supports-color": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
}
}
},
"emoji-regex": { "emoji-regex": {
"version": "7.0.3", "version": "7.0.3",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
@@ -1863,6 +1923,44 @@
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
}, },
"from2": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
"integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=",
"requires": {
"inherits": "^2.0.1",
"readable-stream": "^2.0.0"
},
"dependencies": {
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"fs-extra": { "fs-extra": {
"version": "8.0.1", "version": "8.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.0.1.tgz", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.0.1.tgz",
@@ -2306,9 +2404,9 @@
"dev": true "dev": true
}, },
"handlebars": { "handlebars": {
"version": "4.1.2", "version": "4.7.2",
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.2.tgz",
"integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", "integrity": "sha512-4PwqDL2laXtTWZghzzCtunQUTLbo31pcCJrd/B/9JP8XbhVzpS5ZXuKqlOzsd1rtcaLo4KqAn8nl8mkknS4MHw==",
"requires": { "requires": {
"neo-async": "^2.6.0", "neo-async": "^2.6.0",
"optimist": "^0.6.1", "optimist": "^0.6.1",
@@ -2325,14 +2423,6 @@
"function-bind": "^1.1.1" "function-bind": "^1.1.1"
} }
}, },
"has-ansi": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
"integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
"requires": {
"ansi-regex": "^2.0.0"
}
},
"has-flag": { "has-flag": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -2360,9 +2450,9 @@
"dev": true "dev": true
}, },
"highlight.js": { "highlight.js": {
"version": "9.15.8", "version": "9.18.0",
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.15.8.tgz", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.18.0.tgz",
"integrity": "sha512-RrapkKQWwE+wKdF73VsOa2RQdIoO3mxwJ4P8mhbI6KYJUraUHRKM5w5zQQKXNk0xNL4UVRdulV9SBJcmzJNzVA==" "integrity": "sha512-A97kI1KAUzKoAiEoaGcf2O9YPS8nbDTCRFokaaeBhnqjQTvbAuAJrQMm21zw8s8xzaMtCQBtgbyGXLGxdxQyqQ=="
}, },
"hosted-git-info": { "hosted-git-info": {
"version": "2.7.1", "version": "2.7.1",
@@ -2399,14 +2489,6 @@
"resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-1.3.2.tgz", "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-1.3.2.tgz",
"integrity": "sha512-nDUtj0ltIt08tGi2VWSpSzNNFye0v3YSe9lX3lIqLTuVvvRiYCvs4QQBSHo0eomFYw1wlUuofurUAlTm+vHnXg==" "integrity": "sha512-nDUtj0ltIt08tGi2VWSpSzNNFye0v3YSe9lX3lIqLTuVvvRiYCvs4QQBSHo0eomFYw1wlUuofurUAlTm+vHnXg=="
}, },
"humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
"requires": {
"ms": "^2.0.0"
}
},
"humanize-string": { "humanize-string": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/humanize-string/-/humanize-string-2.1.0.tgz", "resolved": "https://registry.npmjs.org/humanize-string/-/humanize-string-2.1.0.tgz",
@@ -2460,6 +2542,15 @@
"resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.2.0.tgz",
"integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw==" "integrity": "sha512-mT34yGKMNceBQUoVn7iCDKDntA7SC6gycMAWzGx1z/CMCTV7b2AAtXlo3nRyHZ1FelRkQbQjprHSYGwzLtkVbw=="
}, },
"into-stream": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/into-stream/-/into-stream-5.1.0.tgz",
"integrity": "sha512-cbDhb8qlxKMxPBk/QxTtYg1DQ4CwXmadu7quG3B7nrJsgSncEreF2kwWKZFdnjc/lSNNIkFPsjI7SM0Cx/QXPw==",
"requires": {
"from2": "^2.3.0",
"p-is-promise": "^2.0.0"
}
},
"invert-kv": { "invert-kv": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
@@ -2684,6 +2775,12 @@
"handlebars": "^4.1.2" "handlebars": "^4.1.2"
} }
}, },
"jquery": {
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/jquery/-/jquery-3.4.1.tgz",
"integrity": "sha512-36+AdBzCL+y6qjw5Tx7HgzeGCzC81MDDgaUP8ld2zhx58HdqXGoBd+tHdrBMiyjGQs0Hxs/MLZTu/eHNJJuWPw==",
"dev": true
},
"js-tokens": { "js-tokens": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -2833,9 +2930,9 @@
} }
}, },
"lodash": { "lodash": {
"version": "4.17.11", "version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
}, },
"lodash._reinterpolate": { "lodash._reinterpolate": {
"version": "3.0.0", "version": "3.0.0",
@@ -2913,6 +3010,12 @@
"yallist": "^2.1.2" "yallist": "^2.1.2"
} }
}, },
"lunr": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz",
"integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==",
"dev": true
},
"make-dir": { "make-dir": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
@@ -2952,9 +3055,10 @@
"dev": true "dev": true
}, },
"marked": { "marked": {
"version": "0.4.0", "version": "0.8.0",
"resolved": "https://registry.npmjs.org/marked/-/marked-0.4.0.tgz", "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.0.tgz",
"integrity": "sha512-tMsdNBgOsrUophCAFQl0XPe6Zqk/uy9gnue+jIIKhykO51hxyu6uNx7zBPy0+y/WKYVZZMspV9YeXLNdKk+iYw==" "integrity": "sha512-MyUe+T/Pw4TZufHkzAfDj6HarCBWia2y27/bhuYkTaiUnfDYFnCP3KUN+9oM7Wi6JA2rymtVYbQu3spE0GCmxQ==",
"dev": true
}, },
"media-typer": { "media-typer": {
"version": "0.3.0", "version": "0.3.0",
@@ -3201,87 +3305,12 @@
"integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
"dev": true "dev": true
}, },
"execa": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
"integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
"dev": true,
"requires": {
"cross-spawn": "^5.0.1",
"get-stream": "^3.0.0",
"is-stream": "^1.1.0",
"npm-run-path": "^2.0.0",
"p-finally": "^1.0.0",
"signal-exit": "^3.0.0",
"strip-eof": "^1.0.0"
},
"dependencies": {
"cross-spawn": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
"integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
"dev": true,
"requires": {
"lru-cache": "^4.0.1",
"shebang-command": "^1.2.0",
"which": "^1.2.9"
}
}
}
},
"get-caller-file": { "get-caller-file": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
"integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==",
"dev": true "dev": true
}, },
"get-stream": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
"integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
"dev": true
},
"invert-kv": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
"integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=",
"dev": true
},
"lcid": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
"integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=",
"dev": true,
"requires": {
"invert-kv": "^1.0.0"
}
},
"mem": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz",
"integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=",
"dev": true,
"requires": {
"mimic-fn": "^1.0.0"
}
},
"mimic-fn": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
"integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==",
"dev": true
},
"os-locale": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz",
"integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==",
"dev": true,
"requires": {
"execa": "^0.7.0",
"lcid": "^1.0.0",
"mem": "^1.1.0"
}
},
"require-main-filename": { "require-main-filename": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz",
@@ -3295,16 +3324,16 @@
"dev": true "dev": true
}, },
"yargs": { "yargs": {
"version": "11.1.0", "version": "11.1.1",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.0.tgz", "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.1.tgz",
"integrity": "sha512-NwW69J42EsCSanF8kyn5upxvjp5ds+t3+udGBeTbFnERA+lF541DDpMawzo4z6W/QrzNM18D+BPMiOBibnFV5A==", "integrity": "sha512-PRU7gJrJaXv3q3yQZ/+/X6KBswZiaQ+zOmdprZcouPYtQgvNU35i+68M4b1ZHLZtYFT5QObFLV+ZkmJYcwKdiw==",
"dev": true, "dev": true,
"requires": { "requires": {
"cliui": "^4.0.0", "cliui": "^4.0.0",
"decamelize": "^1.1.1", "decamelize": "^1.1.1",
"find-up": "^2.1.0", "find-up": "^2.1.0",
"get-caller-file": "^1.0.1", "get-caller-file": "^1.0.1",
"os-locale": "^2.0.0", "os-locale": "^3.1.0",
"require-directory": "^2.1.1", "require-directory": "^2.1.1",
"require-main-filename": "^1.0.1", "require-main-filename": "^1.0.1",
"set-blocking": "^2.0.0", "set-blocking": "^2.0.0",
@@ -3705,8 +3734,7 @@
"p-is-promise": { "p-is-promise": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz", "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz",
"integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg=="
"dev": true
}, },
"p-limit": { "p-limit": {
"version": "1.3.0", "version": "1.3.0",
@@ -3945,8 +3973,7 @@
"process-nextick-args": { "process-nextick-args": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
"dev": true
}, },
"progress": { "progress": {
"version": "2.0.3", "version": "2.0.3",
@@ -4437,6 +4464,7 @@
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"dev": true,
"requires": { "requires": {
"ansi-regex": "^2.0.0" "ansi-regex": "^2.0.0"
} }
@@ -4846,58 +4874,54 @@
} }
}, },
"typedoc": { "typedoc": {
"version": "0.14.2", "version": "0.16.7",
"resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.14.2.tgz", "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.16.7.tgz",
"integrity": "sha512-aEbgJXV8/KqaVhcedT7xG6d2r+mOvB5ep3eIz1KuB5sc4fDYXcepEEMdU7XSqLFO5hVPu0nllHi1QxX2h/QlpQ==", "integrity": "sha512-sNrLlaZ/aZHxA2rCURGf3g5YabGVwrujiwC6SCV/rgx3LFfZh+goUCatAAyTEDk7evuu6pJ0APGDSde1mSYegw==",
"dev": true,
"requires": { "requires": {
"@types/fs-extra": "^5.0.3",
"@types/handlebars": "^4.0.38",
"@types/highlight.js": "^9.12.3",
"@types/lodash": "^4.14.110",
"@types/marked": "^0.4.0",
"@types/minimatch": "3.0.3", "@types/minimatch": "3.0.3",
"@types/shelljs": "^0.8.0", "fs-extra": "^8.1.0",
"fs-extra": "^7.0.0", "handlebars": "^4.7.2",
"handlebars": "^4.0.6", "highlight.js": "^9.17.1",
"highlight.js": "^9.13.1", "lodash": "^4.17.15",
"lodash": "^4.17.10", "marked": "^0.8.0",
"marked": "^0.4.0",
"minimatch": "^3.0.0", "minimatch": "^3.0.0",
"progress": "^2.0.0", "progress": "^2.0.3",
"shelljs": "^0.8.2", "shelljs": "^0.8.3",
"typedoc-default-themes": "^0.5.0", "typedoc-default-themes": "^0.7.2",
"typescript": "3.2.x" "typescript": "3.7.x"
}, },
"dependencies": { "dependencies": {
"@types/fs-extra": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.1.0.tgz",
"integrity": "sha512-AInn5+UBFIK9FK5xc9yP5e3TQSPNNgjHByqYcj9g5elVBnDQcQL7PlO1CIRy2gWlbwK7UPYqi7vRvFA44dCmYQ==",
"requires": {
"@types/node": "*"
}
},
"fs-extra": { "fs-extra": {
"version": "7.0.1", "version": "8.1.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
"integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
"dev": true,
"requires": { "requires": {
"graceful-fs": "^4.1.2", "graceful-fs": "^4.2.0",
"jsonfile": "^4.0.0", "jsonfile": "^4.0.0",
"universalify": "^0.1.0" "universalify": "^0.1.0"
} }
}, },
"typescript": { "typescript": {
"version": "3.2.4", "version": "3.7.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.2.4.tgz", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.5.tgz",
"integrity": "sha512-0RNDbSdEokBeEAkgNbxJ+BLwSManFy9TeXz8uW+48j/xhEXv1ePME60olyzw2XzUqUBNAYFeJadIqAgNqIACwg==" "integrity": "sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==",
"dev": true
} }
} }
}, },
"typedoc-default-themes": { "typedoc-default-themes": {
"version": "0.5.0", "version": "0.7.2",
"resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.5.0.tgz", "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.7.2.tgz",
"integrity": "sha1-bcJDPnjti+qOiHo6zeLzF4W9Yic=" "integrity": "sha512-fiFKlFO6VTqjcno8w6WpTsbCgXmfPHVjnLfYkmByZE7moaz+E2DSpAT+oHtDHv7E0BM5kAhPrHJELP2J2Y2T9A==",
"dev": true,
"requires": {
"backbone": "^1.4.0",
"jquery": "^3.4.1",
"lunr": "^2.3.8",
"underscore": "^1.9.1"
}
}, },
"typescript": { "typescript": {
"version": "3.5.3", "version": "3.5.3",
@@ -4911,15 +4935,29 @@
"integrity": "sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==" "integrity": "sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ=="
}, },
"uglify-js": { "uglify-js": {
"version": "3.6.0", "version": "3.7.6",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.7.6.tgz",
"integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", "integrity": "sha512-yYqjArOYSxvqeeiYH2VGjZOqq6SVmhxzaPjJC1W2F9e+bqvFL9QXQ2osQuKUFjM2hGjKG2YclQnRKWQSt/nOTQ==",
"optional": true, "optional": true,
"requires": { "requires": {
"commander": "~2.20.0", "commander": "~2.20.3",
"source-map": "~0.6.1" "source-map": "~0.6.1"
},
"dependencies": {
"commander": {
"version": "2.20.3",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
"optional": true
}
} }
}, },
"underscore": {
"version": "1.9.2",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.2.tgz",
"integrity": "sha512-D39qtimx0c1fI3ya1Lnhk3E9nONswSKhnffBI0gME9C99fYOkNi04xs8K6pePLhvl1frbDemkaBQ5ikWllR2HQ==",
"dev": true
},
"universalify": { "universalify": {
"version": "0.1.2", "version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -4959,8 +4997,7 @@
"util-deprecate": { "util-deprecate": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
"dev": true
}, },
"utils-merge": { "utils-merge": {
"version": "1.0.1", "version": "1.0.1",

12
package.json
View File

@@ -29,6 +29,7 @@
"tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'" "tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'"
}, },
"dependencies": { "dependencies": {
"@elastic/elasticsearch": "5.6.20",
"@openstapps/core": "0.31.0", "@openstapps/core": "0.31.0",
"@openstapps/core-tools": "0.11.0", "@openstapps/core-tools": "0.11.0",
"@openstapps/logger": "0.4.0", "@openstapps/logger": "0.4.0",
@@ -36,7 +37,6 @@
"commander": "2.20.0", "commander": "2.20.0",
"config": "3.1.0", "config": "3.1.0",
"cors": "2.8.5", "cors": "2.8.5",
"elasticsearch": "16.1.1",
"express": "4.17.1", "express": "4.17.1",
"express-promise-router": "3.0.3", "express-promise-router": "3.0.3",
"fs-extra": "8.0.1", "fs-extra": "8.0.1",
@@ -59,7 +59,7 @@
"@types/chai-as-promised": "7.1.0", "@types/chai-as-promised": "7.1.0",
"@types/config": "0.0.34", "@types/config": "0.0.34",
"@types/cors": "2.8.5", "@types/cors": "2.8.5",
"@types/elasticsearch": "5.0.34", "@types/elasticsearch": "5.0.35",
"@types/express": "4.17.0", "@types/express": "4.17.0",
"@types/fs-extra": "7.0.0", "@types/fs-extra": "7.0.0",
"@types/got": "9.4.4", "@types/got": "9.4.4",
@@ -72,19 +72,19 @@
"@types/sinon-express-mock": "1.3.7", "@types/sinon-express-mock": "1.3.7",
"@types/supertest": "2.0.7", "@types/supertest": "2.0.7",
"@types/uuid": "3.4.5", "@types/uuid": "3.4.5",
"chai-as-promised": "7.1.1",
"chai": "4.2.0", "chai": "4.2.0",
"chai-as-promised": "7.1.1",
"conventional-changelog-cli": "2.0.21", "conventional-changelog-cli": "2.0.21",
"mocha-typescript": "1.1.17",
"mocha": "6.1.4", "mocha": "6.1.4",
"mocha-typescript": "1.1.17",
"nyc": "14.1.1", "nyc": "14.1.1",
"prepend-file-cli": "1.0.6", "prepend-file-cli": "1.0.6",
"rimraf": "2.6.3", "rimraf": "2.6.3",
"sinon-express-mock": "2.2.0",
"sinon": "7.3.2", "sinon": "7.3.2",
"sinon-express-mock": "2.2.0",
"supertest": "4.0.2", "supertest": "4.0.2",
"tslint": "5.18.0", "tslint": "5.18.0",
"typedoc": "0.14.2", "typedoc": "0.16.7",
"typescript": "3.5.3" "typescript": "3.5.3"
} }
} }

2
src/storage/elasticsearch/aggregations.ts
View File

@@ -14,10 +14,10 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
import {SCFacet, SCThingType} from '@openstapps/core'; import {SCFacet, SCThingType} from '@openstapps/core';
import {AggregationSchema} from '@openstapps/core-tools/lib/mappings/aggregation-definitions';
import {readFileSync} from 'fs'; import {readFileSync} from 'fs';
import { import {
AggregationResponse, AggregationResponse,
AggregationSchema,
isBucketAggregation, isBucketAggregation,
isESAggMatchAllFilter, isESAggMatchAllFilter,
isESNestedAggregation, isESNestedAggregation,

31
src/storage/elasticsearch/common.ts
View File

@@ -13,14 +13,16 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
import {SCThingType} from '@openstapps/core'; import {SCThings, SCThingType} from '@openstapps/core';
import {SCThing} from '@openstapps/core'; import {SCThing} from '@openstapps/core';
import { import {
ESAggMatchAllFilter, ESAggMatchAllFilter,
ESAggTypeFilter, ESAggTypeFilter, ESNestedAggregation,
ESNestedAggregation,
ESTermsFilter, ESTermsFilter,
} from '@openstapps/core-tools/lib/mappings/aggregation-definitions'; } from '@openstapps/core-tools/lib/mappings/aggregation-definitions';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {NameList} from 'elasticsearch'; import {NameList} from 'elasticsearch';
/** /**
@@ -71,6 +73,21 @@ export function isBucketAggregation(agg: BucketAggregation | number): agg is Buc
return typeof agg !== 'number'; return typeof agg !== 'number';
} }
/**
* A response that contains info about whether the item exists plus the object itself
*/
export interface ItemExistsResponse {
/**
* Whether the item exists
*/
exists: boolean;
/**
* The object if it exists
*/
object?: ElasticsearchObject<SCThings>;
}
/** /**
* An aggregation that contains more aggregations nested inside * An aggregation that contains more aggregations nested inside
*/ */
@@ -94,6 +111,14 @@ export function isNestedAggregation(agg: BucketAggregation | NestedAggregation):
return typeof (agg as BucketAggregation).buckets === 'undefined'; return typeof (agg as BucketAggregation).buckets === 'undefined';
} }
/**
* An elasticsearch bucket aggregation
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
*/
export interface AggregationSchema {
[aggregationName: string]: ESTermsFilter | ESNestedAggregation;
}
/** /**
* A configuration for using the Dis Max Query * A configuration for using the Dis Max Query
* *

132
src/storage/elasticsearch/elasticsearch.ts
View File

@@ -13,6 +13,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch';
import { import {
SCBulkResponse, SCBulkResponse,
SCConfigFile, SCConfigFile,
@@ -23,19 +24,23 @@ import {
SCThingType, SCThingType,
SCUuid, SCUuid,
} from '@openstapps/core'; } from '@openstapps/core';
import {AggregationSchema} from '@openstapps/core-tools/lib/mappings/aggregation-definitions';
import {Logger} from '@openstapps/logger'; import {Logger} from '@openstapps/logger';
import * as ES from 'elasticsearch'; // we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
import * as moment from 'moment'; import * as moment from 'moment';
import {MailQueue} from '../../notification/mail-queue'; import {MailQueue} from '../../notification/mail-queue';
import {Bulk} from '../bulk-storage'; import {Bulk} from '../bulk-storage';
import {Database} from '../database'; import {Database} from '../database';
import {buildAggregations, parseAggregations} from './aggregations'; import {buildAggregations, parseAggregations} from './aggregations';
import { import {
AggregationResponse,
AggregationSchema,
ElasticsearchConfig, ElasticsearchConfig,
ElasticsearchObject,
ElasticsearchQueryDisMaxConfig, ElasticsearchQueryDisMaxConfig,
ElasticsearchQueryQueryStringConfig, ElasticsearchQueryQueryStringConfig,
ItemExistsResponse,
} from './common'; } from './common';
import * as Monitoring from './monitoring'; import * as Monitoring from './monitoring';
import {buildQuery, buildSort} from './query'; import {buildQuery, buildSort} from './query';
@@ -75,7 +80,7 @@ export class Elasticsearch implements Database {
/** /**
* Elasticsearch client * Elasticsearch client
*/ */
client: ES.Client; client: Client;
/** /**
* Queue of mails to be sent * Queue of mails to be sent
@@ -92,12 +97,12 @@ export class Elasticsearch implements Database {
*/ */
static getElasticsearchUrl(): string { static getElasticsearchUrl(): string {
// check if we have a docker link // check if we have a docker link
if (process.env.ES_PORT_9200_TCP_ADDR !== undefined && process.env.ES_PORT_9200_TCP_PORT !== undefined) { if (process.env.ES_ADDR !== undefined ) {
return `${process.env.ES_PORT_9200_TCP_ADDR}:${process.env.ES_PORT_9200_TCP_PORT}`; return process.env.ES_ADDR;
} }
// default // default
return 'localhost:9200'; return 'http://localhost:9200';
} }
/** /**
@@ -146,28 +151,28 @@ export class Elasticsearch implements Database {
['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => { ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#'].forEach((value) => {
if (formattedAlias.includes(value)) { if (formattedAlias.includes(value)) {
formattedAlias = formattedAlias.replace(value, ''); formattedAlias = formattedAlias.replace(value, '');
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks ` Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
+ `having the same alias despite having different types, as invalid characters are removed automatically. ` + having the same alias despite having different types, as invalid characters are removed automatically.
`New alias name is "${formattedAlias}."`); New alias name is "${formattedAlias}."`);
} }
}); });
['-', '_', '+'].forEach((value) => { ['-', '_', '+'].forEach((value) => {
if (formattedAlias.charAt(0) === value) { if (formattedAlias.charAt(0) === value) {
formattedAlias = formattedAlias.substring(1); formattedAlias = formattedAlias.substring(1);
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks ` Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
+ `having the same alias despite having different types, as invalid characters are removed automatically. ` + alias despite having different types, as invalid characters are removed automatically.
`New alias name is "${formattedAlias}."`); New alias name is "${formattedAlias}."`);
} }
}); });
if (formattedAlias === '.' || formattedAlias === '..') { if (formattedAlias === '.' || formattedAlias === '..') {
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using ` + Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
`another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`); another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
return 'alias_placeholder'; return 'alias_placeholder';
} }
if (formattedAlias.includes(':')) { if (formattedAlias.includes(':')) {
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future ` + Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
`Elasticsearch versions!`); Elasticsearch versions!`);
} }
return formattedAlias; return formattedAlias;
@@ -185,28 +190,24 @@ export class Elasticsearch implements Database {
throw new Error('Database version is undefined. Check your config file'); throw new Error('Database version is undefined. Check your config file');
} }
const options: ES.ConfigOptions = { this.client = new Client({
apiVersion: config.internal.database.version, node: Elasticsearch.getElasticsearchUrl(),
host: Elasticsearch.getElasticsearchUrl(), });
// enable verbose logging for all request to elasticsearch this.client.on(events.REQUEST, async (err: Error | null, result: ApiResponse<unknown>) => {
log: process.env.ES_DEBUG === 'true' ? 'trace' : 'error', if (err !== null) {
}; await Logger.error(err);
}
if (process.env.ES_DEBUG === 'true') {
Logger.log(result);
}
});
this.client = new ES.Client(options);
this.aliasMap = {}; this.aliasMap = {};
this.ready = false; this.ready = false;
checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ?
process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false);
this.aggregationsSchema = buildAggregations(); this.aggregationsSchema = buildAggregations();
this.mailQueue = mailQueue; this.mailQueue = mailQueue;
/*refreshAllTemplates(this.client)
.then(() => {
// noop
});*/
} }
/** /**
@@ -214,9 +215,8 @@ export class Elasticsearch implements Database {
* *
* Returns Elasticsearch Object if it exists * Returns Elasticsearch Object if it exists
*/ */
// tslint:disable-next-line: completed-docs private async doesItemExist(object: SCThings): Promise<ItemExistsResponse> {
private async doesItemExist(object: SCThings): Promise<{exists: boolean; object?: ElasticsearchObject<SCThings>; }> { const searchResponse: ApiResponse<SearchResponse<SCThings>> = await this.client.search({
const searchResponse = await this.client.search<SCThings>({
body: { body: {
query: { query: {
term: { term: {
@@ -231,10 +231,10 @@ export class Elasticsearch implements Database {
size: 1, size: 1,
}); });
if (searchResponse.hits.total > 1) { if (searchResponse.body.hits.total > 1) {
return { return {
exists: true, exists: true,
object: searchResponse.hits.hits[0], object: searchResponse.body.hits.hits[0],
}; };
} }
@@ -264,7 +264,7 @@ export class Elasticsearch implements Database {
}; };
try { try {
aliases = await this.client.indices.getAlias({}); aliases = (await this.client.indices.getAlias({})).body;
} catch (error) { } catch (error) {
await Logger.error('Failed getting alias map:', error); await Logger.error('Failed getting alias map:', error);
setTimeout(async () => { setTimeout(async () => {
@@ -335,7 +335,7 @@ export class Elasticsearch implements Database {
if (!indexRegex.test(index)) { if (!indexRegex.test(index)) {
throw new Error( throw new Error(
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`, Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
); );
} }
@@ -363,7 +363,7 @@ export class Elasticsearch implements Database {
if (bulk.state !== 'done') { if (bulk.state !== 'done') {
Logger.info('deleting obsolete index', index); Logger.info('deleting obsolete index', index);
return this.client.indices.delete({index}); await this.client.indices.delete({index});
} }
} }
@@ -390,13 +390,13 @@ export class Elasticsearch implements Database {
if (!indexRegex.test(index)) { if (!indexRegex.test(index)) {
throw new Error( throw new Error(
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`, Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
); );
} }
// create the new index if it does not exists // create the new index if it does not exists
if (!(await this.client.indices.exists({index}))) { if (!(await this.client.indices.exists({index})).body) {
// re-apply the index template before each new bulk operation // re-apply the index template before each new bulk operation
await putTemplate(this.client, bulk.type); await putTemplate(this.client, bulk.type);
await this.client.indices.create({ await this.client.indices.create({
@@ -408,7 +408,8 @@ export class Elasticsearch implements Database {
const oldIndex: string = this.aliasMap[alias][bulk.source]; const oldIndex: string = this.aliasMap[alias][bulk.source];
// add our new index to the alias // add our new index to the alias
const actions: ES.IndicesUpdateAliasesParamsAction[] = [ // this was type safe with @types/elasticsearch, the new package however provides no type definitions
const actions: IndicesUpdateAliasesParamsAction[] = [
{ {
add: {index: index, alias: alias}, add: {index: index, alias: alias},
}, },
@@ -423,7 +424,7 @@ export class Elasticsearch implements Database {
// refresh the index (fsync changes) // refresh the index (fsync changes)
await this.client.indices.refresh({ await this.client.indices.refresh({
index, index: index,
}); });
// execute our alias actions // execute our alias actions
@@ -448,7 +449,7 @@ export class Elasticsearch implements Database {
* @param uid uid of an SCThing * @param uid uid of an SCThing
*/ */
public async get(uid: SCUuid): Promise<SCThings> { public async get(uid: SCUuid): Promise<SCThings> {
const searchResponse = await this.client.search({ const searchResponse: ApiResponse<SearchResponse<SCThings>> = await this.client.search({
body: { body: {
query: { query: {
term: { term: {
@@ -460,13 +461,13 @@ export class Elasticsearch implements Database {
}); });
// get data from response // get data from response
const hits = searchResponse.hits.hits; const hits = searchResponse.body.hits.hits;
if (hits.length !== 1) { if (hits.length !== 1) {
throw new Error('No unique item found.'); throw new Error('No unique item found.');
} }
return hits[0]._source as SCThings; return hits[0]._source;
} }
/** /**
@@ -483,6 +484,9 @@ export class Elasticsearch implements Database {
Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue); Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
} }
checkESTemplate(typeof process.env.ES_FORCE_MAPPING_UPDATE !== 'undefined' ?
process.env.ES_FORCE_MAPPING_UPDATE === 'true' : false);
return this.getAliasMap(); return this.getAliasMap();
} }
@@ -494,7 +498,7 @@ export class Elasticsearch implements Database {
public async post(object: SCThings, bulk: Bulk): Promise<void> { public async post(object: SCThings, bulk: Bulk): Promise<void> {
// tslint:disable-next-line: completed-docs // tslint:disable-next-line: completed-docs
const obj: SCThings & {creation_date: string; } = { const obj: SCThings & { creation_date: string; } = {
...object, ...object,
creation_date: moment() creation_date: moment()
.format(), .format(),
@@ -510,10 +514,10 @@ export class Elasticsearch implements Database {
// new item doesn't replace the old one // new item doesn't replace the old one
if (oldIndex.substring(0, oldIndex.length - Elasticsearch.INDEX_UID_LENGTH + 1) if (oldIndex.substring(0, oldIndex.length - Elasticsearch.INDEX_UID_LENGTH + 1)
!== indexOfNew.substring(0, indexOfNew.length - Elasticsearch.INDEX_UID_LENGTH + 1)) { !== indexOfNew.substring(0, indexOfNew.length - Elasticsearch.INDEX_UID_LENGTH + 1)) {
throw new Error( throw new Error(
// tslint:disable-next-line: no-magic-numbers // tslint:disable-next-line: no-magic-numbers
`Object "${obj.uid}" already exists. Object was: ${JSON.stringify(obj, null, 2)}`, `Object "${obj.uid}" already exists. Object was: ${JSON.stringify(obj, null, 2)}`,
); );
} }
} }
@@ -526,7 +530,7 @@ export class Elasticsearch implements Database {
type: obj.type, type: obj.type,
}); });
if (!searchResponse.created) { if (!searchResponse.body.created) {
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(obj)}`); throw new Error(`Object creation Error: Instance was: ${JSON.stringify(obj)}`);
} }
} }
@@ -535,12 +539,12 @@ export class Elasticsearch implements Database {
* Put (update) an existing item * Put (update) an existing item
* @param object SCThing to put * @param object SCThing to put
*/ */
public async put(object: SCThings) { public async put(object: SCThings): Promise<void> {
const itemMeta = await this.doesItemExist(object); const itemMeta = await this.doesItemExist(object);
if (itemMeta.exists && typeof itemMeta.object !== 'undefined') { if (itemMeta.exists && typeof itemMeta.object !== 'undefined') {
return this.client.update({ await this.client.update({
body: { body: {
doc: object, doc: object,
}, },
@@ -548,6 +552,8 @@ export class Elasticsearch implements Database {
index: itemMeta.object._index, index: itemMeta.object._index,
type: object.type.toLowerCase(), type: object.type.toLowerCase(),
}); });
return;
} }
throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.'); throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.');
@@ -581,7 +587,7 @@ export class Elasticsearch implements Database {
.query as ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig; .query as ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
} }
const searchRequest: ES.SearchParams = { const searchRequest: RequestParams.Search = {
body: { body: {
aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes) aggs: this.aggregationsSchema, // use cached version of aggregations (they only change if config changes)
query: buildQuery(params, this.config, esConfig), query: buildQuery(params, this.config, esConfig),
@@ -596,31 +602,31 @@ export class Elasticsearch implements Database {
} }
// perform the search against elasticsearch // perform the search against elasticsearch
const response = await this.client.search<SCThings>(searchRequest); const response: ApiResponse<SearchResponse<SCThings>> = await this.client.search(searchRequest);
// gather pagination information // gather pagination information
const pagination = { const pagination = {
count: response.hits.hits.length, count: response.body.hits.hits.length,
offset: (typeof params.from === 'number') ? params.from : 0, offset: (typeof params.from === 'number') ? params.from : 0,
total: response.hits.total, total: response.body.hits.total,
}; };
// gather statistics about this search // gather statistics about this search
const stats = { const stats = {
time: response.took, time: response.body.took,
}; };
// we only directly return the _source documents // we only directly return the _source documents
// elasticsearch provides much more information, the user shouldn't see // elasticsearch provides much more information, the user shouldn't see
const data = response.hits.hits.map((hit) => { const data = response.body.hits.hits.map((hit) => {
return hit._source; // SCThing return hit._source; // SCThing
}); });
let facets: SCFacet[] = []; let facets: SCFacet[] = [];
// read the aggregations from elasticsearch and parse them to facets by our configuration // read the aggregations from elasticsearch and parse them to facets by our configuration
if (typeof response.aggregations !== 'undefined') { if (typeof response.body.aggregations !== 'undefined') {
facets = parseAggregations(this.aggregationsSchema, response.aggregations); facets = parseAggregations(this.aggregationsSchema, response.body.aggregations as AggregationResponse);
} }
return { return {

16
src/storage/elasticsearch/monitoring.ts
View File

@@ -13,15 +13,20 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch';
import { import {
SCMonitoringConfiguration, SCMonitoringConfiguration,
SCMonitoringLogAction, SCMonitoringLogAction,
SCMonitoringMailAction, SCMonitoringMailAction,
SCMonitoringMaximumLengthCondition, SCMonitoringMaximumLengthCondition,
SCMonitoringMinimumLengthCondition, SCMonitoringMinimumLengthCondition,
SCThings,
} from '@openstapps/core'; } from '@openstapps/core';
import {Logger} from '@openstapps/logger'; import {Logger} from '@openstapps/logger';
import * as ES from 'elasticsearch'; // we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {SearchResponse} from 'elasticsearch';
import * as cron from 'node-cron'; import * as cron from 'node-cron';
import {MailQueue} from '../../notification/mail-queue'; import {MailQueue} from '../../notification/mail-queue';
@@ -47,7 +52,7 @@ function conditionFails(
* @param total Number of results * @param total Number of results
*/ */
function minConditionFails(minimumLength: number, total: number) { function minConditionFails(minimumLength: number, total: number) {
return typeof minimumLength === 'number' && minimumLength > total; return minimumLength > total;
} }
/** /**
@@ -99,7 +104,7 @@ export function runActions(
* @param esClient elasticsearch client * @param esClient elasticsearch client
* @param mailQueue mailQueue for mail actions * @param mailQueue mailQueue for mail actions
*/ */
export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES.Client, mailQueue: MailQueue) { export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: Client, mailQueue: MailQueue) {
// set up Watches // set up Watches
monitoringConfig.watchers.forEach((watcher) => { monitoringConfig.watchers.forEach((watcher) => {
@@ -122,10 +127,11 @@ export function setUp(monitoringConfig: SCMonitoringConfiguration, esClient: ES.
cron.schedule(trigger.executionTime, async () => { cron.schedule(trigger.executionTime, async () => {
// execute watch (search->condition->action) // execute watch (search->condition->action)
const result = await esClient.search(watcher.query as ES.SearchParams); const result: ApiResponse<SearchResponse<SCThings>> =
await esClient.search(watcher.query as RequestParams.Search);
// check conditions // check conditions
const total = result.hits.total; const total = result.body.hits.total;
watcher.conditions.forEach((condition) => { watcher.conditions.forEach((condition) => {
if (conditionFails(condition, total)) { if (conditionFails(condition, total)) {

2
src/storage/elasticsearch/templating.ts
View File

@@ -13,11 +13,11 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
import {Client} from '@elastic/elasticsearch';
import {SCThingType} from '@openstapps/core'; import {SCThingType} from '@openstapps/core';
import {getProjectReflection} from '@openstapps/core-tools/lib/common'; import {getProjectReflection} from '@openstapps/core-tools/lib/common';
import {generateTemplate} from '@openstapps/core-tools/lib/mapping'; import {generateTemplate} from '@openstapps/core-tools/lib/mapping';
import {Logger} from '@openstapps/logger'; import {Logger} from '@openstapps/logger';
import {Client} from 'elasticsearch';
import {existsSync, writeFileSync} from 'fs'; import {existsSync, writeFileSync} from 'fs';
import {readFile} from 'fs-extra'; import {readFile} from 'fs-extra';
import {resolve} from 'path'; import {resolve} from 'path';