diff --git a/README.md b/README.md index 2897a208..ceb956fb 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,10 @@ you with everything you need to run this backend. # Local usage for development purposes ## Requirements -* Elasticsearch (5.6) +* Elasticsearch (8.4) + - [ICU analysis plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html) + - OR Docker * Node.js (~14) / NPM -* Docker ### Startup Behaviour @@ -34,7 +35,7 @@ with the backend. To save you some work we provide a [docker image](https://gitlab.com/openstapps/database) which only needs to be executed to work with the backend. -Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:master` +Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:latest` Elasticsearch should be running at port 9200 now. If you have problems with getting elasticsearch to work, have a look in the diff --git a/config/elasticsearch.ts b/config/elasticsearch.ts index cd07f20c..ece20b0b 100644 --- a/config/elasticsearch.ts +++ b/config/elasticsearch.ts @@ -1,6 +1,6 @@ // tslint:disable:no-default-export // tslint:disable:no-magic-numbers -import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch'; +import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch-config'; /** * This is the default configuration for elasticsearch (a database) @@ -19,13 +19,13 @@ const config: ElasticsearchConfigFile = { internal: { database: { name: 'elasticsearch', - version: '5.6', + version: '8.4', query: { minMatch: '75%', queryType: 'dis_max', matchBoosting: 1.3, fuzziness: 'AUTO', - cutoffFrequency: 0.0, + cutoffFrequency: 0, tieBreaker: 0, }, }, diff --git a/integration-test.yml b/integration-test.yml index 7c2a6930..1ee6e824 100644 --- a/integration-test.yml +++ b/integration-test.yml @@ -15,7 +15,7 @@ services: elasticsearch: ports: - "9200:9200" - image: "registry.gitlab.com/openstapps/database:master" + image: "registry.gitlab.com/openstapps/database:latest" apicli: image: "registry.gitlab.com/openstapps/api/cli:latest" diff --git a/package-lock.json b/package-lock.json index 6fc93652..bc679176 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,45 +5,45 @@ "requires": true, "dependencies": { "@ampproject/remapping": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", - "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", + "integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==", "dev": true, "requires": { - "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/gen-mapping": "^0.3.0", "@jridgewell/trace-mapping": "^0.3.9" } }, "@babel/code-frame": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", - "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.21.4.tgz", + "integrity": "sha512-LYvhNKfwWSPpocw8GI7gpK2nq3HSDuEPC/uSYaALSJu9xjsalaaYFOq0Pwt5KmVqwEbZlDu81aLXwBOmD/Fv9g==", "requires": { "@babel/highlight": "^7.18.6" } }, "@babel/compat-data": { - "version": "7.20.14", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.14.tgz", - "integrity": "sha512-0YpKHD6ImkWMEINCyDAD0HLLUH/lPCefG8ld9it8DJB2wnApraKuhgYTvTY1z7UFIfBTGy5LwncZ+5HWWGbhFw==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.21.4.tgz", + "integrity": "sha512-/DYyDpeCfaVinT40FPGdkkb+lYSKvsVuMjDAG7jPOWWiM1ibOaB9CXJAlc4d1QpP/U2q2P9jbrSlClKSErd55g==", "dev": true }, "@babel/core": { - "version": "7.20.12", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz", - "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.21.4.tgz", + "integrity": "sha512-qt/YV149Jman/6AfmlxJ04LMIu8bMoyl3RB91yTFrxQmgbrSvQMy7cI8Q62FHx1t8wJ8B5fu0UDoLwHAhUo1QA==", "dev": true, "requires": { - "@ampproject/remapping": "^2.1.0", - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.20.7", - "@babel/helper-compilation-targets": "^7.20.7", - "@babel/helper-module-transforms": "^7.20.11", - "@babel/helpers": "^7.20.7", - "@babel/parser": "^7.20.7", + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.21.4", + "@babel/generator": "^7.21.4", + "@babel/helper-compilation-targets": "^7.21.4", + "@babel/helper-module-transforms": "^7.21.2", + "@babel/helpers": "^7.21.0", + "@babel/parser": "^7.21.4", "@babel/template": "^7.20.7", - "@babel/traverse": "^7.20.12", - "@babel/types": "^7.20.7", + "@babel/traverse": "^7.21.4", + "@babel/types": "^7.21.4", "convert-source-map": "^1.7.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -60,37 +60,49 @@ } }, "@babel/generator": { - "version": "7.20.14", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.20.14.tgz", - "integrity": "sha512-AEmuXHdcD3A52HHXxaTmYlb8q/xMEhoRP67B3T4Oq7lbmSoqroMZzjnGj3+i1io3pdnF8iBYVu4Ilj+c4hBxYg==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.21.4.tgz", + "integrity": "sha512-NieM3pVIYW2SwGzKoqfPrQsf4xGs9M9AIG3ThppsSRmO+m7eQhmI6amajKMUeIO37wFfsvnvcxQFx6x6iqxDnA==", "dev": true, "requires": { - "@babel/types": "^7.20.7", + "@babel/types": "^7.21.4", "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", "jsesc": "^2.5.1" }, "dependencies": { - "@jridgewell/gen-mapping": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", - "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz", + "integrity": "sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==", "dev": true, "requires": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" } } } }, "@babel/helper-compilation-targets": { - "version": "7.20.7", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.7.tgz", - "integrity": "sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.21.4.tgz", + "integrity": "sha512-Fa0tTuOXZ1iL8IeDFUWCzjZcn+sJGd9RZdH9esYVjEejGmzf+FFYQpMi/kZUk2kPy/q1H3/GPw7np8qar/stfg==", "dev": true, "requires": { - "@babel/compat-data": "^7.20.5", - "@babel/helper-validator-option": "^7.18.6", + "@babel/compat-data": "^7.21.4", + "@babel/helper-validator-option": "^7.21.0", "browserslist": "^4.21.3", "lru-cache": "^5.1.1", "semver": "^6.3.0" @@ -126,13 +138,13 @@ "dev": true }, "@babel/helper-function-name": { - "version": "7.19.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz", - "integrity": "sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w==", + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.21.0.tgz", + "integrity": "sha512-HfK1aMRanKHpxemaY2gqBmL04iAPOPRj7DxtNbiDOrJK+gdwkiNRVpCpUJYbUT+aZyemKN8brqTOxzCaG6ExRg==", "dev": true, "requires": { - "@babel/template": "^7.18.10", - "@babel/types": "^7.19.0" + "@babel/template": "^7.20.7", + "@babel/types": "^7.21.0" } }, "@babel/helper-hoist-variables": { @@ -145,18 +157,18 @@ } }, "@babel/helper-module-imports": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", - "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.21.4.tgz", + "integrity": "sha512-orajc5T2PsRYUN3ZryCEFeMDYwyw09c/pZeaQEZPH0MpKzSvn3e0uXsDBu3k03VI+9DBiRo+l22BfKTpKwa/Wg==", "dev": true, "requires": { - "@babel/types": "^7.18.6" + "@babel/types": "^7.21.4" } }, "@babel/helper-module-transforms": { - "version": "7.20.11", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.20.11.tgz", - "integrity": "sha512-uRy78kN4psmji1s2QtbtcCSaj/LILFDp0f/ymhpQH5QY3nljUZCaNWz9X1dEj/8MBdBEFECs7yRhKn8i7NjZgg==", + "version": "7.21.2", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.21.2.tgz", + "integrity": "sha512-79yj2AR4U/Oqq/WOV7Lx6hUjau1Zfo4cI+JLAVYeMV5XIlbOhmjEk5ulbTc9fMpmlojzZHkUUxAiK+UKn+hNQQ==", "dev": true, "requires": { "@babel/helper-environment-visitor": "^7.18.9", @@ -165,8 +177,8 @@ "@babel/helper-split-export-declaration": "^7.18.6", "@babel/helper-validator-identifier": "^7.19.1", "@babel/template": "^7.20.7", - "@babel/traverse": "^7.20.10", - "@babel/types": "^7.20.7" + "@babel/traverse": "^7.21.2", + "@babel/types": "^7.21.2" } }, "@babel/helper-simple-access": { @@ -199,20 +211,20 @@ "integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==" }, "@babel/helper-validator-option": { - "version": "7.18.6", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", - "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.21.0.tgz", + "integrity": "sha512-rmL/B8/f0mKS2baE9ZpyTcTavvEuWhTTW8amjzXNvYG4AwBsqTLikfXsEofsJEfKHf+HQVQbFOHy6o+4cnC/fQ==", "dev": true }, "@babel/helpers": { - "version": "7.20.13", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.20.13.tgz", - "integrity": "sha512-nzJ0DWCL3gB5RCXbUO3KIMMsBY2Eqbx8mBpKGE/02PgyRQFcPQLbkQ1vyy596mZLaP+dAfD+R4ckASzNVmW3jg==", + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.21.0.tgz", + "integrity": "sha512-XXve0CBtOW0pd7MRzzmoyuSj0e3SEzj8pgyFxnTT1NJZL38BD1MK7yYrm8yefRPIDvNNe14xR4FdbHwpInD4rA==", "dev": true, "requires": { "@babel/template": "^7.20.7", - "@babel/traverse": "^7.20.13", - "@babel/types": "^7.20.7" + "@babel/traverse": "^7.21.0", + "@babel/types": "^7.21.0" } }, "@babel/highlight": { @@ -272,9 +284,9 @@ } }, "@babel/parser": { - "version": "7.20.13", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.20.13.tgz", - "integrity": "sha512-gFDLKMfpiXCsjt4za2JA9oTMn70CeseCehb11kRZgvd7+F67Hih3OHOK24cRrWECJ/ljfPGac6ygXAs/C8kIvw==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.21.4.tgz", + "integrity": "sha512-alVJj7k7zIxqBZ7BTRhz0IqJFxW1VJbm6N8JbcYhQ186df9ZBPbZBmWSqAMXwHGsCJdYks7z/voa3ibiS5bCIw==", "dev": true }, "@babel/template": { @@ -289,19 +301,19 @@ } }, "@babel/traverse": { - "version": "7.20.13", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.20.13.tgz", - "integrity": "sha512-kMJXfF0T6DIS9E8cgdLCSAL+cuCK+YEZHWiLK0SXpTo8YRj5lpJu3CDNKiIBCne4m9hhTIqUg6SYTAI39tAiVQ==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.21.4.tgz", + "integrity": "sha512-eyKrRHKdyZxqDm+fV1iqL9UAHMoIg0nDaGqfIOd8rKH17m5snv7Gn4qgjBoFfLz9APvjFU/ICT00NVCv1Epp8Q==", "dev": true, "requires": { - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.20.7", + "@babel/code-frame": "^7.21.4", + "@babel/generator": "^7.21.4", "@babel/helper-environment-visitor": "^7.18.9", - "@babel/helper-function-name": "^7.19.0", + "@babel/helper-function-name": "^7.21.0", "@babel/helper-hoist-variables": "^7.18.6", "@babel/helper-split-export-declaration": "^7.18.6", - "@babel/parser": "^7.20.13", - "@babel/types": "^7.20.7", + "@babel/parser": "^7.21.4", + "@babel/types": "^7.21.4", "debug": "^4.1.0", "globals": "^11.1.0" }, @@ -315,9 +327,9 @@ } }, "@babel/types": { - "version": "7.20.7", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.20.7.tgz", - "integrity": "sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg==", + "version": "7.21.4", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.21.4.tgz", + "integrity": "sha512-rU2oY501qDxE8Pyo7i/Orqma4ziCOrby0/9mvbDUGEfvZjb279Nk9k19e2fiCxHbRRpY2ZyrgW1eq22mvmOIzA==", "dev": true, "requires": { "@babel/helper-string-parser": "^7.19.4", @@ -334,17 +346,25 @@ } }, "@elastic/elasticsearch": { - "version": "5.6.22", - "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-5.6.22.tgz", - "integrity": "sha512-jg5VnRSFUQREi4gPQ773nKb3t4IaUsdAGJr5AAtOL2Mj2RSVDFKclPONjEMItcsBxDWdhDWZfhaFC/ymjCEQeA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.4.0.tgz", + "integrity": "sha512-0QZDBePnb5a+d76zjlMYq96IDf0AOuGP7JHugFUYlYwTC7rZvROuZSpoUsvpUjNH2CzMqWgNLIekIR6EHRMIQA==", "requires": { - "debug": "^4.1.1", - "decompress-response": "^4.2.0", - "into-stream": "^5.1.0", - "ms": "^2.1.1", - "once": "^1.4.0", - "pump": "^3.0.0", - "secure-json-parse": "^2.1.0" + "@elastic/transport": "^8.2.0", + "tslib": "^2.4.0" + } + }, + "@elastic/transport": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.3.1.tgz", + "integrity": "sha512-jv/Yp2VLvv5tSMEOF8iGrtL2YsYHbpf4s+nDsItxUTLFTzuJGpnsB/xBlfsoT2kAYEnWHiSJuqrbRcpXEI/SEQ==", + "requires": { + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^5.5.1" } }, "@es-joy/jsdoccomment": { @@ -512,19 +532,20 @@ "dev": true }, "@jridgewell/gen-mapping": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", - "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", "dev": true, "requires": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" } }, "@jridgewell/resolve-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", - "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", + "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==" }, "@jridgewell/set-array": { "version": "1.1.2", @@ -533,9 +554,9 @@ "dev": true }, "@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", - "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" }, "@jridgewell/trace-mapping": { "version": "0.3.9", @@ -607,13 +628,31 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", "dev": true + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } } } }, "@openstapps/core": { - "version": "0.74.0", - "resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.74.0.tgz", - "integrity": "sha512-kiW5pwCmDNFmXCEdappJ8cmrUvoQnp0ICllg2PtZwNW0GL4yIIlwFRsGZeTkyPU4khOp3Vw7PZ9d0EKi76f97Q==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@openstapps/core/-/core-1.0.1.tgz", + "integrity": "sha512-+JlycoMcj+QIaXicOZYlNU07XlDc3zRIydYEoLGueAfkTbKt0ap5FWbNL+Hz8ve3ApQP2Hj+4FuU8H6QyLA0vQ==", "requires": { "@openstapps/core-tools": "0.34.0", "@types/geojson": "1.0.6", @@ -658,11 +697,12 @@ } }, "@openstapps/es-mapping-generator": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@openstapps/es-mapping-generator/-/es-mapping-generator-0.4.0.tgz", - "integrity": "sha512-vGnVrbZDj+H+r7ncJk+gqowIO322wanK9eRtyabMA4HyzeLSO9eXpaPruG0HWXGEbpr1rBwAkq/224xipCVzmg==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@openstapps/es-mapping-generator/-/es-mapping-generator-0.6.0.tgz", + "integrity": "sha512-MOMt5AWHqR53PeU8lUR5JpyjpTkOKgeWqhNsuD2Xa7fUFzv3qeIZ9xkUYNnOgo9FpSEyuHDVv659s61tewfIeA==", "dev": true, "requires": { + "@elastic/elasticsearch": "8.4.0", "@openstapps/logger": "1.1.1", "commander": "9.5.0", "deepmerge": "4.2.2", @@ -926,12 +966,6 @@ "@types/node": "*" } }, - "@types/elasticsearch": { - "version": "5.0.40", - "resolved": "https://registry.npmjs.org/@types/elasticsearch/-/elasticsearch-5.0.40.tgz", - "integrity": "sha512-lhnbkC0XorAD7Dt7X+94cXUSHEdDNnEVk/DgFLHgIZQNhixV631Lj4+KpXunTT5rCHyj9RqK3TfO7QrOiwEeUQ==", - "dev": true - }, "@types/express": { "version": "4.17.16", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.16.tgz", @@ -945,14 +979,15 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.33", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.33.tgz", - "integrity": "sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==", + "version": "4.17.34", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.34.tgz", + "integrity": "sha512-fvr49XlCGoUj2Pp730AItckfjat4WNb0lb3kfrLWffd+RLeoGAMsq7UOy04PAPtoL01uKwcp6u8nhzpgpDYr3w==", "dev": true, "requires": { "@types/node": "*", "@types/qs": "*", - "@types/range-parser": "*" + "@types/range-parser": "*", + "@types/send": "*" } }, "@types/geojson": { @@ -984,9 +1019,9 @@ } }, "@types/mime": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", - "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", + "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", "dev": true }, "@types/minimist": { @@ -1068,10 +1103,20 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "@types/send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.1.tgz", + "integrity": "sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==", + "dev": true, + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, "@types/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.1.tgz", + "integrity": "sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==", "dev": true, "requires": { "@types/mime": "*", @@ -1079,9 +1124,9 @@ } }, "@types/sinon": { - "version": "10.0.13", - "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.13.tgz", - "integrity": "sha512-UVjDqJblVNQYvVNUsj0PuYYw0ELRmgt1Nt5Vk0pT5f16ROGfcKJY8o1HVuMOJOpD727RrGB9EGvoaTQE5tgxZQ==", + "version": "10.0.14", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.14.tgz", + "integrity": "sha512-mn72up6cjaMyMuaPaa/AwKf6WtsSRysQC7wxFkCm1XcOKXPM1z+5Y4H5wjIVBz4gdAkjvZxVVfjA6ba1nHr5WQ==", "dev": true, "requires": { "@types/sinonjs__fake-timers": "*" @@ -1104,9 +1149,9 @@ "dev": true }, "@types/superagent": { - "version": "4.1.16", - "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.16.tgz", - "integrity": "sha512-tLfnlJf6A5mB6ddqF159GqcDizfzbMUB1/DeT59/wBNqzRTNNKsaw79A/1TZ84X+f/EwWH8FeuSkjlCLyqS/zQ==", + "version": "4.1.17", + "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.17.tgz", + "integrity": "sha512-FFK/rRjNy24U6J1BvQkaNWu2ohOIF/kxRQXRsbT141YQODcOcZjzlcc4DGdI2SkTa0rhmF+X14zu6ICjCGIg+w==", "dev": true, "requires": { "@types/cookiejar": "*", @@ -1307,12 +1352,12 @@ } }, "agentkeepalive": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", - "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.3.0.tgz", + "integrity": "sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==", "requires": { "debug": "^4.1.0", - "depd": "^1.1.2", + "depd": "^2.0.0", "humanize-ms": "^1.2.1" } }, @@ -1392,18 +1437,6 @@ "requires": { "delegates": "^1.0.0", "readable-stream": "^3.6.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } } }, "arg": { @@ -1472,6 +1505,13 @@ "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", "requires": { "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } } }, "better-ajv-errors": { @@ -1524,11 +1564,6 @@ "ms": "2.0.0" } }, - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" - }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -1585,6 +1620,14 @@ "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", "dev": true }, + "busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "requires": { + "streamsearch": "^1.1.0" + } + }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1686,9 +1729,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001449", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001449.tgz", - "integrity": "sha512-CPB+UL9XMT/Av+pJxCKGhdx+yg1hzplvFJQlJ2n68PyQGMz9L/E2zCyLdOL8uasbouTUgnPl+y0tccI/se+BEw==", + "version": "1.0.30001481", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001481.tgz", + "integrity": "sha512-KCqHwRnaa1InZBtqXzP98LPg0ajCVujMKjqKDhZEthIpAsJl/YEIa3YvXjGXPVqzZVguccuu7ga9KOE1J9rKPQ==", "dev": true }, "chai": { @@ -1756,9 +1799,9 @@ "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" }, "ci-info": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.7.1.tgz", - "integrity": "sha512-4jYS4MOAaCIStSRwiuxc4B8MYhIe676yO1sYGzARnjXkWpmzZMMYxY6zu8WYWDhSuth5zhrQ1rhNSibyyvv4/w==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", + "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", "dev": true }, "clean-regexp": { @@ -1797,13 +1840,6 @@ "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", "requires": { "mimic-response": "^1.0.0" - }, - "dependencies": { - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" - } } }, "color-convert": { @@ -1890,13 +1926,6 @@ "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "requires": { "safe-buffer": "5.2.1" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" - } } }, "content-type": { @@ -2123,7 +2152,8 @@ "core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true }, "cors": { "version": "2.8.5", @@ -2139,6 +2169,15 @@ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.1" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2206,11 +2245,18 @@ } }, "decompress-response": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", - "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "requires": { - "mimic-response": "^2.0.0" + "mimic-response": "^3.1.0" + }, + "dependencies": { + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + } } }, "deep-eql": { @@ -2280,9 +2326,9 @@ "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "destroy": { "version": "1.2.0", @@ -2335,9 +2381,9 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "electron-to-chromium": { - "version": "1.4.284", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", - "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", + "version": "1.4.376", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.376.tgz", + "integrity": "sha512-TFeOKd98TpJzRHkr4Aorn16QkMnuCQuGAE6IZ0wYF+qkbSfMPqjplvRppR02tMUpVxZz8nyBNvVm9lIZsqrbPQ==", "dev": true }, "emoji-regex": { @@ -2648,9 +2694,9 @@ } }, "eslint-scope": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", - "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.0.tgz", + "integrity": "sha512-DYj5deGlHBfMt15J7rdtyKNq/Nqlv5KfU4iodrQ019XESsRnwXH9KAE0y3cwtUHDo2ob7CypAnCqefh6vioWRw==", "requires": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" @@ -2672,18 +2718,18 @@ } }, "eslint-visitor-keys": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", - "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==" + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz", + "integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==" }, "espree": { - "version": "9.4.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", - "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "version": "9.5.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz", + "integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==", "requires": { "acorn": "^8.8.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.3.0" + "eslint-visitor-keys": "^3.4.0" } }, "esprima": { @@ -2693,9 +2739,9 @@ "dev": true }, "esquery": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", - "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", "requires": { "estraverse": "^5.1.0" } @@ -2769,20 +2815,10 @@ "ms": "2.0.0" } }, - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" - }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" } } }, @@ -2980,15 +3016,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" }, - "from2": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", - "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", - "requires": { - "inherits": "^2.0.1", - "readable-stream": "^2.0.0" - } - }, "fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -3090,6 +3117,36 @@ "yargs": "^16.2.0" }, "dependencies": { + "readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, "through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", @@ -3241,27 +3298,12 @@ "lowercase-keys": "^2.0.0", "p-cancelable": "^2.0.0", "responselike": "^2.0.0" - }, - "dependencies": { - "decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "requires": { - "mimic-response": "^3.1.0" - } - }, - "mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" - } } }, "graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "grapheme-splitter": { "version": "1.0.4", @@ -3366,6 +3408,11 @@ } } }, + "hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==" + }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -3387,13 +3434,6 @@ "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" - }, - "dependencies": { - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" - } } }, "http-proxy-agent": { @@ -3514,15 +3554,6 @@ "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", "dev": true }, - "into-stream": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-5.1.1.tgz", - "integrity": "sha512-krrAJ7McQxGGmvaYbB7Q1mcA+cRwg9Ij2RfWIeVesNBgVDZmzY/Fa4IpZUT3bmdRzMzdf/mzltCG2Dq99IZGBA==", - "requires": { - "from2": "^2.3.0", - "p-is-promise": "^3.0.0" - } - }, "ip": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", @@ -3549,18 +3580,18 @@ } }, "is-builtin-module": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.0.tgz", - "integrity": "sha512-phDA4oSGt7vl1n5tJvTWooWWAsXLY+2xCnxNqvKhGEzujg+A43wPlPOyDg3C8XQHN+6k/JTQWJ/j0dQh/qr+Hw==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", + "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==", "dev": true, "requires": { "builtin-modules": "^3.3.0" } }, "is-core-module": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", - "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", "dev": true, "requires": { "has": "^1.0.3" @@ -3657,7 +3688,8 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true }, "isexe": { "version": "2.0.0", @@ -3757,9 +3789,9 @@ } }, "js-sdsl": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.3.0.tgz", - "integrity": "sha512-mifzlm2+5nZ+lEcLJMoBK0/IH/bDg8XnJfd/Wq6IP+xoCjLZsTOnV2QpxlVbX9bMnkl5PdEjNtBJ9Cj1NjifhQ==" + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz", + "integrity": "sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg==" }, "js-tokens": { "version": "4.0.0", @@ -3994,9 +4026,9 @@ "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" }, "lru-cache": { - "version": "7.14.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.14.1.tgz", - "integrity": "sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==" + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==" }, "lunr": { "version": "2.3.9", @@ -4236,9 +4268,9 @@ } }, "mimic-response": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", - "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" }, "min-indent": { "version": "1.0.1", @@ -4255,9 +4287,9 @@ } }, "minimist": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", - "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "dev": true }, "minimist-options": { @@ -4499,11 +4531,6 @@ "ms": "2.0.0" } }, - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" - }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -4670,9 +4697,9 @@ } }, "node-releases": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.8.tgz", - "integrity": "sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==", + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", + "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==", "dev": true }, "nodemailer": { @@ -4947,11 +4974,6 @@ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==" }, - "p-is-promise": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz", - "integrity": "sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ==" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -5173,7 +5195,8 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true }, "process-on-spawn": { "version": "1.0.0", @@ -5434,17 +5457,13 @@ } }, "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" } }, "readdirp": { @@ -8796,9 +8815,9 @@ } }, "regexp-tree": { - "version": "0.1.24", - "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.24.tgz", - "integrity": "sha512-s2aEVuLhvnVJW6s/iPgEGK6R+/xngd2jNQ+xy4bXNDKxZKJH6jpPHY6kVeVv1IeLCHgswRj+Kl3ELaDjG6V1iw==", + "version": "0.1.25", + "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.25.tgz", + "integrity": "sha512-szcL3aqw+vEeuxhL1AMYRyeMP+goYF5I/guaH10uJX5xbGyeQeNPPneaj3ZWVmGLCDxrVaaYekkr5R12gk4dJw==", "dev": true }, "regexpp": { @@ -8833,12 +8852,12 @@ "dev": true }, "resolve": { - "version": "1.22.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", - "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", "dev": true, "requires": { - "is-core-module": "^2.9.0", + "is-core-module": "^2.11.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" } @@ -8908,9 +8927,9 @@ } }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safe-regex": { "version": "2.1.1", @@ -8922,9 +8941,9 @@ } }, "safe-stable-stringify": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz", - "integrity": "sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==" + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" }, "safer-buffer": { "version": "2.1.2", @@ -8937,9 +8956,9 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" }, "semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz", + "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==", "requires": { "lru-cache": "^6.0.0" }, @@ -8988,11 +9007,6 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" } } - }, - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" } } }, @@ -9170,9 +9184,9 @@ } }, "spdx-correct": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", - "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "requires": { "spdx-expression-parse": "^3.0.0", @@ -9196,9 +9210,9 @@ } }, "spdx-license-ids": { - "version": "3.0.12", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.12.tgz", - "integrity": "sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==", + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.13.tgz", + "integrity": "sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w==", "dev": true }, "split": { @@ -9217,19 +9231,6 @@ "dev": true, "requires": { "readable-stream": "^3.0.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } } }, "sprintf-js": { @@ -9251,6 +9252,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" }, + "streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" + }, "string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -9262,11 +9268,11 @@ } }, "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "requires": { - "safe-buffer": "~5.1.0" + "safe-buffer": "~5.2.0" } }, "strip-ansi": { @@ -9361,12 +9367,9 @@ }, "dependencies": { "minipass": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.0.0.tgz", - "integrity": "sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw==", - "requires": { - "yallist": "^4.0.0" - } + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==" } } }, @@ -9453,19 +9456,6 @@ "dev": true, "requires": { "readable-stream": "3" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } } }, "tmp": { @@ -9527,9 +9517,9 @@ "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" }, "typescript": { - "version": "4.9.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", - "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==" + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==" } } }, @@ -9559,10 +9549,9 @@ "integrity": "sha512-crvloFKZlPIysdVcP7Ej1w4HijBx7NmLdeorqfxOvt87DcUIbhKV4ZaSgCL+IQ+zzTgDx5zDuNHRvUbTIr9aqw==" }, "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" }, "tsutils": { "version": "3.21.0", @@ -9571,6 +9560,14 @@ "dev": true, "requires": { "tslib": "^1.8.1" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true + } } }, "type-check": { @@ -9632,9 +9629,9 @@ } }, "marked": { - "version": "4.2.12", - "resolved": "https://registry.npmjs.org/marked/-/marked-4.2.12.tgz", - "integrity": "sha512-yr8hSKa3Fv4D3jdZmtMMPghgVt6TWbk86WQaWhDloQjRSQhMMYCAro7jP7VDJrjjdV8pxVxMssXS8B8Y5DZ5aw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", "dev": true }, "minimatch": { @@ -9670,6 +9667,14 @@ "dev": true, "optional": true }, + "undici": { + "version": "5.22.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.0.tgz", + "integrity": "sha512-fR9RXCc+6Dxav4P9VV/sp5w3eFiSdOjJYsbtWfd4s5L5C4ogyuVpdKIVHeW0vV1MloM65/f7W45nR9ZxwVdyiA==", + "requires": { + "busboy": "^1.6.0" + } + }, "unique-filename": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz", @@ -9697,9 +9702,9 @@ "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "update-browserslist-db": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", - "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", + "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==", "dev": true, "requires": { "escalade": "^3.1.1", @@ -9775,9 +9780,9 @@ } }, "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.1.tgz", + "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", "dev": true }, "wide-align": { diff --git a/package.json b/package.json index 08ca195c..ad28e044 100644 --- a/package.json +++ b/package.json @@ -27,14 +27,14 @@ "start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js", "start-debug": "STAPPS_LOG_LEVEL=31 NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js --require ts-node/register", "test": "npm run test-unit && npm run test-integration", - "test-unit": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'", + "test-unit": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'", "test-integration": "docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --build --abort-on-container-exit --exit-code-from apicli", "lint": "eslint -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/", "lint:fix": "eslint --fix -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/" }, "dependencies": { - "@elastic/elasticsearch": "5.6.22", - "@openstapps/core": "0.74.0", + "@elastic/elasticsearch": "8.4.0", + "@openstapps/core": "1.0.1", "@openstapps/core-tools": "0.34.0", "@openstapps/logger": "1.1.1", "@types/node": "14.18.36", @@ -57,14 +57,13 @@ }, "devDependencies": { "@openstapps/configuration": "0.34.0", - "@openstapps/es-mapping-generator": "0.4.0", + "@openstapps/es-mapping-generator": "0.6.0", "@openstapps/eslint-config": "1.1.0", "@testdeck/mocha": "0.3.3", "@types/chai": "4.3.4", "@types/chai-as-promised": "7.1.5", "@types/config": "3.3.0", "@types/cors": "2.8.13", - "@types/elasticsearch": "5.0.40", "@types/express": "4.17.16", "@types/geojson": "1.0.6", "@types/mocha": "10.0.1", @@ -80,6 +79,7 @@ "chai": "4.3.7", "chai-as-promised": "7.1.1", "conventional-changelog-cli": "2.2.2", + "cross-env": "7.0.3", "eslint": "8.33.0", "eslint-config-prettier": "8.6.0", "eslint-plugin-jsdoc": "39.7.4", diff --git a/src/app.ts b/src/app.ts index 6c98497e..0a4c024f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -64,6 +64,7 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat }), ); + /* istanbul ignore if */ if (process.env.PROMETHEUS_MIDDLEWARE === 'true') { app.use(getPrometheusMiddleware()); } @@ -142,7 +143,10 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat }); // validate config file - await validator.addSchemas(path.join('node_modules', '@openstapps', 'core', 'lib', 'schema')); + await validator.addSchemas( + // eslint-disable-next-line unicorn/prefer-module + path.join(path.dirname(require.resolve('@openstapps/core/package.json')), 'lib', 'schema'), + ); // validate the config file const configValidation = validator.validate(configFile, 'SCConfigFile'); diff --git a/src/storage/elasticsearch/aggregations.ts b/src/storage/elasticsearch/aggregations.ts index 5b01da46..0df62027 100644 --- a/src/storage/elasticsearch/aggregations.ts +++ b/src/storage/elasticsearch/aggregations.ts @@ -13,73 +13,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import {SCFacet, SCThingType} from '@openstapps/core'; -import {aggregations} from './templating'; -import {AggregationResponse} from './types/elasticsearch'; import { - isBucketAggregation, - isESAggMatchAllFilter, - isESNestedAggregation, - isESTermsFilter, - isNestedAggregation, -} from './types/guards'; + AggregateName, + AggregationsAggregate, + AggregationsFiltersAggregate, + AggregationsMultiTermsBucket, +} from '@elastic/elasticsearch/lib/api/types'; +import {SCFacet, SCThingType} from '@openstapps/core'; /** * Parses elasticsearch aggregations (response from es) to facets for the app * * @param aggregationResponse - aggregations response from elasticsearch */ -export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] { +export function parseAggregations( + aggregationResponse: Record, +): SCFacet[] { const facets: SCFacet[] = []; - // get all names of the types an aggregation is on - for (const typeName in aggregations) { - if (aggregations.hasOwnProperty(typeName) && aggregationResponse.hasOwnProperty(typeName)) { - // the type object from the schema - const type = aggregations[typeName]; - // the "real" type object from the response - const realType = aggregationResponse[typeName]; + for (const aggregateName in aggregationResponse) { + const aggregation = aggregationResponse[aggregateName] as AggregationsMultiTermsBucket; + const type = aggregateName === '@all' ? {} : {onlyOnType: aggregateName as SCThingType}; - // both conditions must apply, else we have an error somewhere - if (isESNestedAggregation(type) && isNestedAggregation(realType)) { - for (const fieldName in type.aggs) { - if (type.aggs.hasOwnProperty(fieldName) && realType.hasOwnProperty(fieldName)) { - // the field object from the schema - const field = type.aggs[fieldName]; - // the "real" field object from the response - const realField = realType[fieldName]; + for (const field in aggregation) { + const fieldAggregate = aggregation[field] as AggregationsFiltersAggregate; + if (typeof fieldAggregate !== 'object') continue; - // this should always be true in theory... - if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) { - const facet: SCFacet = { - buckets: realField.buckets.map(bucket => { - return { - count: bucket.doc_count, - key: bucket.key, - }; - }), - field: fieldName, - }; - // if it's not for all types then create the appropriate field and set the type name - if (!isESAggMatchAllFilter(type.filter)) { - facet.onlyOnType = type.filter.type.value as SCThingType; - } - facets.push(facet); - } - } - } - // the last part here means that it is a bucket aggregation - } else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) { - facets.push({ - buckets: realType.buckets.map(bucket => { - return { - count: bucket.doc_count, - key: bucket.key, - }; - }), - field: typeName, - }); - } + const buckets = Object.values(fieldAggregate.buckets).map(bucket => { + return { + count: bucket.doc_count, + key: bucket.key as string, + }; + }); + if (buckets.length === 0) continue; + + facets.push({ + buckets, + field, + ...type, + }); } } diff --git a/src/storage/elasticsearch/elasticsearch.ts b/src/storage/elasticsearch/elasticsearch.ts index b9e76958..02ca1e2e 100644 --- a/src/storage/elasticsearch/elasticsearch.ts +++ b/src/storage/elasticsearch/elasticsearch.ts @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 StApps + * Copyright (C) 2022 StApps * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the @@ -13,58 +13,47 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch'; +import {Client, events} from '@elastic/elasticsearch'; import { - SCBulkResponse, - SCConfigFile, - SCFacet, - SCSearchQuery, - SCSearchResponse, - SCThings, - SCThingType, - SCUuid, -} from '@openstapps/core'; + AggregateName, + AggregationsMultiTermsBucket, + IndicesGetAliasResponse, + IndicesUpdateAliasesAction, + SearchHit, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core'; import {Logger} from '@openstapps/logger'; -// we only have the @types package because some things type definitions are still missing from the official -// @elastic/elasticsearch package -import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch'; import moment from 'moment'; import {MailQueue} from '../../notification/mail-queue'; import {Bulk} from '../bulk-storage'; import {Database} from '../database'; import {parseAggregations} from './aggregations'; import * as Monitoring from './monitoring'; -import {buildQuery, buildSort} from './query'; +import {buildQuery} from './query/query'; +import {buildSort} from './query/sort'; import {aggregations, putTemplate} from './templating'; import { - AggregationResponse, ElasticsearchConfig, - ElasticsearchObject, ElasticsearchQueryDisMaxConfig, ElasticsearchQueryQueryStringConfig, -} from './types/elasticsearch'; - -/** - * Matches index names such as stapps___ - */ -const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/; +} from './types/elasticsearch-config'; +import {ALL_INDICES_QUERY, getThingIndexName, parseIndexName, VALID_INDEX_REGEX} from './util'; +import {removeInvalidAliasChars} from './util/alias'; +import {noUndefined} from './util/no-undefined'; +import {retryCatch, RetryOptions} from './util/retry'; /** * A database interface for elasticsearch */ export class Elasticsearch implements Database { - /** - * Length of the index UID used for generation of its name - */ - static readonly INDEX_UID_LENGTH = 8; - /** * Holds a map of all elasticsearch indices that are available to search */ aliasMap: { - // each scType has a alias which can contain multiple sources + // each scType has an alias which can contain multiple sources [scType: string]: { - // each source is assigned a index name in elasticsearch + // each source is assigned an index name in elasticsearch [source: string]: string; }; }; @@ -97,89 +86,11 @@ export class Elasticsearch implements Database { return 'http://localhost:9200'; } - /** - * Gets the index name in elasticsearch for one SCThingType - * - * @param type SCThingType of data in the index - * @param source source of data in the index - * @param bulk bulk process which created this index - */ - static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) { - let out = type.toLowerCase(); - while (out.includes(' ')) { - out = out.replace(' ', '_'); - } - - return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`; - } - - /** - * Provides the index UID (for its name) from the bulk UID - * - * @param uid Bulk UID - */ - static getIndexUID(uid: SCUuid) { - return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH)); - } - - /** - * Generates a string which matches all indices - */ - static getListOfAllIndices(): string { - // map each SC type in upper camel case - return 'stapps_*_*_*'; - } - - /** - * Checks for invalid character in alias names and removes them - * - * @param alias The alias name - * @param uid The UID of the current bulk (for debugging purposes) - */ - static removeAliasChars(alias: string, uid: string | undefined): string { - let formattedAlias = alias; - - // spaces are included in some types, replace them with underscores - if (formattedAlias.includes(' ')) { - formattedAlias = formattedAlias.trim(); - formattedAlias = formattedAlias.split(' ').join('_'); - } - // List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html - for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) { - if (formattedAlias.includes(value)) { - formattedAlias = formattedAlias.replace(value, ''); - Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks - having the same alias despite having different types, as invalid characters are removed automatically. - New alias name is "${formattedAlias}."`); - } - } - for (const value of ['-', '_', '+']) { - if (formattedAlias.charAt(0) === value) { - formattedAlias = formattedAlias.slice(1); - Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same - alias despite having different types, as invalid characters are removed automatically. - New alias name is "${formattedAlias}."`); - } - } - if (formattedAlias === '.' || formattedAlias === '..') { - Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using - another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`); - - return 'alias_placeholder'; - } - if (formattedAlias.includes(':')) { - Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future - Elasticsearch versions!`); - } - - return formattedAlias; - } - /** * Create a new interface for elasticsearch * * @param config an assembled config file - * @param mailQueue a mailqueue for monitoring + * @param mailQueue a mail queue for monitoring */ constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) { if ( @@ -192,7 +103,7 @@ export class Elasticsearch implements Database { this.client = new Client({ node: Elasticsearch.getElasticsearchUrl(), }); - this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse) => { + this.client.diagnostic.on(events.REQUEST, async (error: Error | null, result: unknown) => { if (error !== null) { await Logger.error(error); } @@ -210,73 +121,40 @@ export class Elasticsearch implements Database { /** * Gets a map which contains each alias and all indices that are associated with each alias */ - private async getAliasMap() { - // delay after which alias map will be fetched again - const RETRY_INTERVAL = 5000; - // maximum number of retries - const RETRY_COUNT = 3; - // create a list of old indices that are not in use - const oldIndicesToDelete: string[] = []; - - let aliases: - | { - [index: string]: { - /** - * Aliases of an index - */ - aliases: { - [K in SCThingType]: unknown; - }; - }; - } - | undefined; - - for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) { - if (typeof aliases !== 'undefined') { - break; - } - try { - const aliasResponse = await this.client.indices.getAlias({}); - aliases = aliasResponse.body; - } catch (error) { + private async getAliasMap(retryOptions: Partial> = {}) { + const aliasResponse = await retryCatch({ + maxRetries: 10, + retryInterval: 2000, + doAction: () => this.client.indices.getAlias(), + onFailedAttempt: (attempt, error, {maxRetries, retryInterval}) => { Logger.warn('Failed getting alias map:', error); - Logger.warn(`Retrying in ${RETRY_INTERVAL} milliseconds. (${retry} of ${RETRY_COUNT})`); - await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL)); - } - } + Logger.warn(`Retrying in ${retryInterval} milliseconds. (${attempt} of ${maxRetries})`); + }, + onFail: ({maxRetries}) => { + throw new TypeError(`Failed to retrieve alias map after ${maxRetries} attempts!`); + }, + ...retryOptions, + }); - if (typeof aliases === 'undefined') { - throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`); - } + const aliases = Object.entries(aliasResponse) + .filter(([index]) => !index.startsWith('.')) + .map(([index, alias]) => ({ + index, + alias, + ...parseIndexName(index), + })); - for (const index in aliases) { - if (aliases.hasOwnProperty(index)) { - const matches = indexRegex.exec(index); - if (matches !== null) { - const type = matches[1]; - const source = matches[2]; - - // check if there is an alias for the current index - // check that alias equals type - const hasAlias = type in aliases[index].aliases; - if (hasAlias) { - if (typeof this.aliasMap[type] === 'undefined') { - this.aliasMap[type] = {}; - } - this.aliasMap[type][source] = index; - } else { - oldIndicesToDelete.push(index); - } - } - } + for (const {type, index, source} of aliases.filter(({type, alias}) => type in alias.aliases)) { + this.aliasMap[type] = this.aliasMap[type] || {}; + this.aliasMap[type][source] = index; } this.ready = true; - // delete old indices that are not used in any alias - if (oldIndicesToDelete.length > 0) { + const unusedIndices = aliases.filter(({type, alias}) => !(type in alias.aliases)).map(({index}) => index); + if (unusedIndices.length > 0) { await this.client.indices.delete({ - index: oldIndicesToDelete, + index: unusedIndices, }); Logger.warn(`Deleted old indices: oldIndicesToDelete`); } @@ -291,8 +169,8 @@ export class Elasticsearch implements Database { * @param uid an UID to use for the search * @returns an elasticsearch object containing the thing */ - private async getObject(uid: SCUuid): Promise | undefined> { - const searchResponse: ApiResponse> = await this.client.search({ + private async getObject(uid: SCUuid): Promise | undefined> { + const searchResponse = await this.client.search({ body: { query: { term: { @@ -303,43 +181,44 @@ export class Elasticsearch implements Database { }, }, from: 0, - index: Elasticsearch.getListOfAllIndices(), + index: ALL_INDICES_QUERY, size: 1, }); // return data from response - return searchResponse.body.hits.hits[0]; + return searchResponse.hits.hits[0]; } - /** - * Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index - * - * @param bulk the bulk process that was created - */ - public async bulkCreated(bulk: Bulk): Promise { - // if our es instance is not ready yet, we cannot serve this request + private async prepareBulkWrite(bulk: Bulk): Promise<{index: string; alias: string}> { if (!this.ready) { throw new Error('No connection to elasticsearch established yet.'); } - // index name for elasticsearch - const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk); - - // there already is an index with this type and source. We will index the new one and switch the alias to it - // the old one is deleted - const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid); + const index = getThingIndexName(bulk.type, bulk.source, bulk); + const alias = removeInvalidAliasChars(bulk.type, bulk.uid); if (typeof this.aliasMap[alias] === 'undefined') { this.aliasMap[alias] = {}; } - if (!indexRegex.test(index)) { + if (!VALID_INDEX_REGEX.test(index)) { throw new Error( `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. Make sure to set the bulk "source" and "type" to names consisting of the characters above.`, ); } + return {index, alias}; + } + + /** + * Should be called, when a new bulk was created. Creates a new index and applies the mapping to the index + * + * @param bulk the bulk process that was created + */ + public async bulkCreated(bulk: Bulk): Promise { + const {index} = await this.prepareBulkWrite(bulk); + // re-apply the index template before each new bulk operation await putTemplate(this.client, bulk.type); await this.client.indices.create({ @@ -355,8 +234,7 @@ export class Elasticsearch implements Database { * @param bulk the bulk process that is expired */ public async bulkExpired(bulk: Bulk): Promise { - // index name for elasticsearch - const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk); + const index: string = getThingIndexName(bulk.type, bulk.source, bulk); Logger.info('Bulk expired. Deleting index', index); @@ -375,31 +253,11 @@ export class Elasticsearch implements Database { * @param bulk the new bulk process that should replace the old one with same type and source */ public async bulkUpdated(bulk: Bulk): Promise { - // if our es instance is not ready yet, we cannot serve this request - if (!this.ready) { - throw new Error('No connection to elasticsearch established yet.'); - } + const {index, alias} = await this.prepareBulkWrite(bulk); - // index name for elasticsearch - const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk); - - // alias for the indices - const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid); - - if (typeof this.aliasMap[alias] === 'undefined') { - this.aliasMap[alias] = {}; - } - - if (!indexRegex.test(index)) { - throw new Error( - `Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers. - Make sure to set the bulk "source" and "type" to names consisting of the characters above.`, - ); - } - - // create the new index if it does not exists + // create the new index if it does not exist // eslint-disable-next-line unicorn/no-await-expression-member - if (!(await this.client.indices.exists({index})).body) { + if (!(await this.client.indices.exists({index}))) { // re-apply the index template before each new bulk operation await putTemplate(this.client, bulk.type); await this.client.indices.create({ @@ -412,7 +270,7 @@ export class Elasticsearch implements Database { // add our new index to the alias // this was type safe with @types/elasticsearch, the new package however provides no type definitions - const actions: IndicesUpdateAliasesParamsAction[] = [ + const actions: IndicesUpdateAliasesAction[] = [ { add: {index: index, alias: alias}, }, @@ -427,16 +285,10 @@ export class Elasticsearch implements Database { } // refresh the index (fsync changes) - await this.client.indices.refresh({ - index: index, - }); + await this.client.indices.refresh({index}); // execute our alias actions - await this.client.indices.updateAliases({ - body: { - actions, - }, - }); + await this.client.indices.updateAliases({actions}); // swap the index in our aliasMap this.aliasMap[alias][bulk.source] = index; @@ -457,7 +309,7 @@ export class Elasticsearch implements Database { public async get(uid: SCUuid): Promise { const object = await this.getObject(uid); - if (typeof object === 'undefined') { + if (typeof object?._source === 'undefined') { throw new TypeError('Item not found.'); } @@ -467,7 +319,7 @@ export class Elasticsearch implements Database { /** * Initialize the elasticsearch database (call all needed methods) */ - public async init(): Promise { + public async init(retryOptions: Partial> = {}): Promise { const monitoringConfiguration = this.config.internal.monitoring; if (typeof monitoringConfiguration !== 'undefined') { @@ -480,7 +332,7 @@ export class Elasticsearch implements Database { await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue); } - return this.getAliasMap(); + return this.getAliasMap(retryOptions); } /** @@ -490,7 +342,7 @@ export class Elasticsearch implements Database { * @param bulk the bulk process which item belongs to */ public async post(object: SCThings, bulk: Bulk): Promise { - const object_: SCThings & {creation_date: string} = { + const thing: SCThings & {creation_date: string} = { ...object, creation_date: moment().format(), }; @@ -499,7 +351,7 @@ export class Elasticsearch implements Database { // check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid) if (typeof item !== 'undefined') { - const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk); + const indexOfNew = getThingIndexName(thing.type, bulk.source, bulk); const oldIndex = item._index; // new item doesn't replace the old one @@ -509,22 +361,23 @@ export class Elasticsearch implements Database { ) { throw new Error( // eslint-disable-next-line unicorn/no-null - `Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`, + `Object "${thing.uid}" already exists. Object was: ${JSON.stringify(thing, null, 2)}`, ); } } // regular bulk update (item gets replaced when bulk is updated) - const searchResponse = await this.client.create({ - body: object_, - id: object_.uid, - index: Elasticsearch.getIndex(object_.type, bulk.source, bulk), + const searchResponse = await this.client.create({ + document: thing, + id: thing.uid, + index: getThingIndexName(thing.type, bulk.source, bulk), timeout: '90s', - type: object_.type, }); - if (!searchResponse.body.created) { - throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`); + if (searchResponse.result !== 'created') { + throw new Error( + `Object creation Error (${searchResponse.result}: Instance was: ${JSON.stringify(thing)}`, + ); } } @@ -543,7 +396,6 @@ export class Elasticsearch implements Database { }, id: object.uid, index: item._index, - type: object.type.toLowerCase(), }); return; @@ -562,65 +414,46 @@ export class Elasticsearch implements Database { throw new TypeError('Database is undefined. You have to configure the query build'); } - // create elasticsearch configuration out of data from database configuration const esConfig: ElasticsearchConfig = { name: this.config.internal.database.name as 'elasticsearch', version: this.config.internal.database.version as string, - }; - - if (typeof this.config.internal.database.query !== 'undefined') { - esConfig.query = this.config.internal.database.query as + query: this.config.internal.database.query as | ElasticsearchQueryDisMaxConfig - | ElasticsearchQueryQueryStringConfig; - } + | ElasticsearchQueryQueryStringConfig + | undefined, + }; - const searchRequest: RequestParams.Search = { - body: { - aggs: aggregations, - query: buildQuery(parameters, this.config, esConfig), - }, + const query = { + aggs: aggregations, + query: buildQuery(parameters, this.config, esConfig), from: parameters.from, - index: Elasticsearch.getListOfAllIndices(), + index: ALL_INDICES_QUERY, size: parameters.size, + sort: typeof parameters.sort !== 'undefined' ? buildSort(parameters.sort) : undefined, }; - - if (typeof parameters.sort !== 'undefined') { - searchRequest.body.sort = buildSort(parameters.sort); - } - - // perform the search against elasticsearch - const response: ApiResponse> = await this.client.search(searchRequest); - - // gather pagination information - const pagination = { - count: response.body.hits.hits.length, - offset: typeof parameters.from === 'number' ? parameters.from : 0, - total: response.body.hits.total, - }; - - // gather statistics about this search - const stats = { - time: response.body.took, - }; - - // we only directly return the _source documents - // elasticsearch provides much more information, the user shouldn't see - const data = response.body.hits.hits.map(hit => { - return hit._source; // SCThing - }); - - let facets: SCFacet[] = []; - - // read the aggregations from elasticsearch and parse them to facets by our configuration - if (typeof response.body.aggregations !== 'undefined') { - facets = parseAggregations(response.body.aggregations as AggregationResponse); - } + const response: SearchResponse = await this.client.search(query); return { - data, - facets, - pagination, - stats, + data: response.hits.hits + .map(hit => { + // we only directly return the _source documents + // elasticsearch provides much more information, the user shouldn't see + return hit._source; + }) + .filter(noUndefined), + facets: + typeof response.aggregations !== 'undefined' + ? parseAggregations(response.aggregations as Record) + : [], + pagination: { + count: response.hits.hits.length, + offset: typeof parameters.from === 'number' ? parameters.from : 0, + total: + typeof response.hits.total === 'number' ? response.hits.total : response.hits.total?.value ?? 0, + }, + stats: { + time: response.took, + }, }; } } diff --git a/src/storage/elasticsearch/monitoring.ts b/src/storage/elasticsearch/monitoring.ts index 2af1b283..be7aba54 100644 --- a/src/storage/elasticsearch/monitoring.ts +++ b/src/storage/elasticsearch/monitoring.ts @@ -13,7 +13,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch'; +import {Client} from '@elastic/elasticsearch'; +import {SearchRequest} from '@elastic/elasticsearch/lib/api/types'; import { SCMonitoringConfiguration, SCMonitoringLogAction, @@ -23,9 +24,6 @@ import { SCThings, } from '@openstapps/core'; import {Logger} from '@openstapps/logger'; -// we only have the @types package because some things type definitions are still missing from the official -// @elastic/elasticsearch package -import {SearchResponse} from 'elasticsearch'; import cron from 'node-cron'; import {MailQueue} from '../../notification/mail-queue'; @@ -131,12 +129,11 @@ export async function setUp( cron.schedule(trigger.executionTime, async () => { // execute watch (search->condition->action) - const result: ApiResponse> = await esClient.search( - watcher.query as RequestParams.Search, - ); + const result = await esClient.search(watcher.query as SearchRequest); // check conditions - const total = result.body.hits.total; + const total = + typeof result.hits.total === 'number' ? result.hits.total : result.hits.total?.value ?? -1; for (const condition of watcher.conditions) { if (conditionFails(condition, total)) { diff --git a/src/storage/elasticsearch/query.ts b/src/storage/elasticsearch/query.ts deleted file mode 100644 index ef174e64..00000000 --- a/src/storage/elasticsearch/query.ts +++ /dev/null @@ -1,501 +0,0 @@ -/* - * Copyright (C) 2019-2021 StApps - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { - SCBackendConfigurationSearchBoostingContext, - SCBackendConfigurationSearchBoostingType, - SCConfigFile, - SCSearchBooleanFilter, - SCSearchContext, - SCSearchFilter, - SCSearchQuery, - SCSearchSort, - SCSportCoursePriceGroup, - SCThingsField, -} from '@openstapps/core'; -import { - ElasticsearchConfig, - ESBooleanFilter, - ESBooleanFilterArguments, - ESDateRange, - ESDateRangeFilter, - ESFunctionScoreQuery, - ESFunctionScoreQueryFunction, - ESGenericRange, - ESGenericSort, - ESGeoBoundingBoxFilter, - ESGeoDistanceFilter, - ESGeoDistanceFilterArguments, - ESGeoDistanceSort, - ESGeoDistanceSortArguments, - ESGeoShapeFilter, - ESNumericRangeFilter, - ESRangeFilter, - ESTermFilter, - ESTypeFilter, - ScriptSort, -} from './types/elasticsearch'; - -/** - * Escapes any reserved character that would otherwise not be accepted by Elasticsearch - * - * Elasticsearch as the following reserved characters: - * + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ / - * It is possible to use all, with the exception of < and >, of them by escaping them with a \ - * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html - * - * @param string_ the string to escape the characters from - */ -function escapeESReservedCharacters(string_: string): string { - return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&'); -} - -/** - * Builds a boolean filter. Returns an elasticsearch boolean filter - * - * @param booleanFilter a search boolean filter for the retrieval of the data - * @returns elasticsearch boolean arguments object - */ -export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments { - const result: ESBooleanFilterArguments = { - minimum_should_match: 0, - must: [], - must_not: [], - should: [], - }; - - if (booleanFilter.arguments.operation === 'and') { - result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter)); - } - - if (booleanFilter.arguments.operation === 'or') { - result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter)); - result.minimum_should_match = 1; - } - - if (booleanFilter.arguments.operation === 'not') { - result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter)); - } - - return result; -} - -/** - * Converts Array of Filters to elasticsearch query-syntax - * - * @param filter A search filter for the retrieval of the data - */ -export function buildFilter( - filter: SCSearchFilter, -): - | ESTermFilter - | ESGeoDistanceFilter - | ESBooleanFilter - | ESGeoShapeFilter - | ESBooleanFilter - | ESRangeFilter { - switch (filter.type) { - case 'value': - return Array.isArray(filter.arguments.value) - ? { - terms: { - [`${filter.arguments.field}.raw`]: filter.arguments.value, - }, - } - : { - term: { - [`${filter.arguments.field}.raw`]: filter.arguments.value, - }, - }; - case 'availability': - const scope = filter.arguments.scope?.charAt(0) ?? 's'; - const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`; - - return { - range: { - [filter.arguments.field]: { - gte: `${time}/${scope}`, - lt: `${time}+1${scope}/${scope}`, - relation: 'intersects', - }, - }, - }; - case 'distance': - const geoObject: ESGeoDistanceFilterArguments = { - distance: `${filter.arguments.distance}m`, - [`${filter.arguments.field}.point.coordinates`]: { - lat: filter.arguments.position[1], - lon: filter.arguments.position[0], - }, - }; - - return { - geo_distance: geoObject, - }; - case 'boolean': - return { - bool: buildBooleanFilter(filter), - }; - case 'numeric range': - const numericRangeObject: ESGenericRange = { - relation: filter.arguments.relation, - }; - if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') { - numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit; - } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') { - numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit; - } - if (filter.arguments.bounds.upperBound?.mode === 'exclusive') { - numericRangeObject.lt = filter.arguments.bounds.upperBound.limit; - } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') { - numericRangeObject.lte = filter.arguments.bounds.upperBound.limit; - } - - const numericRangeFilter: ESNumericRangeFilter = {range: {}}; - numericRangeFilter.range[filter.arguments.field] = numericRangeObject; - - return numericRangeFilter; - case 'date range': - const dateRangeObject: ESDateRange = { - format: filter.arguments.format, - time_zone: filter.arguments.timeZone, - relation: filter.arguments.relation, - }; - if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') { - dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit; - } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') { - dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit; - } - if (filter.arguments.bounds.upperBound?.mode === 'exclusive') { - dateRangeObject.lt = filter.arguments.bounds.upperBound.limit; - } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') { - dateRangeObject.lte = filter.arguments.bounds.upperBound.limit; - } - - const dateRangeFilter: ESDateRangeFilter = {range: {}}; - dateRangeFilter.range[filter.arguments.field] = dateRangeObject; - - return dateRangeFilter; - case 'geo': - // TODO: on ES upgrade, use just geo_shape filters - const geoShapeFilter: ESGeoShapeFilter = { - geo_shape: { - /** - * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3 - */ - // @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type. - ignore_unmapped: true, - [`${filter.arguments.field}.polygon`]: { - shape: filter.arguments.shape, - relation: filter.arguments.spatialRelation, - }, - }, - }; - - if ( - (typeof filter.arguments.spatialRelation === 'undefined' || - filter.arguments.spatialRelation === 'intersects') && - filter.arguments.shape.type === 'envelope' - ) { - return { - bool: { - minimum_should_match: 1, - should: [ - geoShapeFilter, - { - geo_bounding_box: { - /** - * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3 - */ - ignore_unmapped: true, - [`${filter.arguments.field}.point.coordinates`]: { - top_left: filter.arguments.shape.coordinates[0], - bottom_right: filter.arguments.shape.coordinates[1], - }, - }, - }, - ], - }, - }; - } - - return geoShapeFilter; - } -} - -/** - * Builds scoring functions from boosting config - * - * @param boostings Backend boosting configuration for contexts and types - * @param context The context of the app from where the search was initiated - */ -function buildFunctions( - boostings: SCBackendConfigurationSearchBoostingContext, - context: SCSearchContext | undefined, -): ESFunctionScoreQueryFunction[] { - // default context - let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes( - boostings['default' as SCSearchContext], - ); - - if (typeof context !== 'undefined' && context !== 'default') { - // specific context provided, extend default context with additional boosts - functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])]; - } - - return functions; -} - -/** - * Creates boost functions for all type boost configurations - * - * @param boostingTypes Array of type boosting configurations - */ -function buildFunctionsForBoostingTypes( - boostingTypes: SCBackendConfigurationSearchBoostingType[], -): ESFunctionScoreQueryFunction[] { - const functions: ESFunctionScoreQueryFunction[] = []; - - for (const boostingForOneSCType of boostingTypes) { - const typeFilter: ESTypeFilter = { - type: { - value: boostingForOneSCType.type, - }, - }; - - functions.push({ - filter: typeFilter, - weight: boostingForOneSCType.factor, - }); - - if (typeof boostingForOneSCType.fields !== 'undefined') { - const fields = boostingForOneSCType.fields; - - for (const fieldName in boostingForOneSCType.fields) { - if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) { - const boostingForOneField = fields[fieldName]; - - for (const value in boostingForOneField) { - if (boostingForOneField.hasOwnProperty(value)) { - const factor = boostingForOneField[value]; - - // build term filter - const termFilter: ESTermFilter = { - term: {}, - }; - termFilter.term[`${fieldName}.raw`] = value; - - functions.push({ - filter: { - bool: { - must: [typeFilter, termFilter], - should: [], - }, - }, - weight: factor, - }); - } - } - } - } - } - } - - return functions; -} - -/** - * Builds body for Elasticsearch requests - * - * @param parameters Parameters for querying the backend - * @param defaultConfig Default configuration of the backend - * @param elasticsearchConfig Elasticsearch configuration - * @returns ElasticsearchQuery (body of a search-request) - */ -export function buildQuery( - parameters: SCSearchQuery, - defaultConfig: SCConfigFile, - elasticsearchConfig: ElasticsearchConfig, -): ESFunctionScoreQuery { - // if config provides an minMatch parameter we use query_string instead of match query - let query; - if (typeof elasticsearchConfig.query === 'undefined') { - query = { - query_string: { - analyzer: 'search_german', - default_field: 'name', - minimum_should_match: '90%', - query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query), - }, - }; - } else if (elasticsearchConfig.query.queryType === 'query_string') { - query = { - query_string: { - analyzer: 'search_german', - default_field: 'name', - minimum_should_match: elasticsearchConfig.query.minMatch, - query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query), - }, - }; - } else if (elasticsearchConfig.query.queryType === 'dis_max') { - if (parameters.query !== '*') { - query = { - dis_max: { - boost: 1.2, - queries: [ - { - match: { - name: { - boost: elasticsearchConfig.query.matchBoosting, - cutoff_frequency: elasticsearchConfig.query.cutoffFrequency, - fuzziness: elasticsearchConfig.query.fuzziness, - query: typeof parameters.query !== 'string' ? '*' : parameters.query, - }, - }, - }, - { - query_string: { - analyzer: 'search_german', - default_field: 'name', - minimum_should_match: elasticsearchConfig.query.minMatch, - query: - typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query), - }, - }, - ], - tie_breaker: elasticsearchConfig.query.tieBreaker, - }, - }; - } - } else { - throw new Error( - 'Unsupported query type. Check your config file and reconfigure your elasticsearch query', - ); - } - - const functionScoreQuery: ESFunctionScoreQuery = { - function_score: { - functions: buildFunctions(defaultConfig.internal.boostings, parameters.context), - query: { - bool: { - minimum_should_match: 0, // if we have no should, nothing can match - must: [], - should: [], - }, - }, - score_mode: 'multiply', - }, - }; - - const mustMatch = functionScoreQuery.function_score.query.bool.must; - - if (Array.isArray(mustMatch)) { - if (typeof query !== 'undefined') { - mustMatch.push(query); - } - - if (typeof parameters.filter !== 'undefined') { - mustMatch.push(buildFilter(parameters.filter)); - } - } - - return functionScoreQuery; -} - -/** - * converts query to - * - * @param sorts Sorting rules to apply to the data that is being queried - * @returns an array of sort queries - */ -export function buildSort(sorts: SCSearchSort[]): Array { - return sorts.map(sort => { - switch (sort.type) { - case 'generic': - const esGenericSort: ESGenericSort = {}; - esGenericSort[sort.arguments.field] = sort.order; - - return esGenericSort; - case 'ducet': - const esDucetSort: ESGenericSort = {}; - esDucetSort[`${sort.arguments.field}.sort`] = sort.order; - - return esDucetSort; - case 'distance': - const arguments_: ESGeoDistanceSortArguments = { - mode: 'avg', - order: sort.order, - unit: 'm', - }; - - arguments_[`${sort.arguments.field}.point.coordinates`] = { - lat: sort.arguments.position[1], - lon: sort.arguments.position[0], - }; - - return { - _geo_distance: arguments_, - }; - case 'price': - return { - _script: { - order: sort.order, - script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field), - type: 'number' as const, - }, - }; - } - }); -} - -/** - * Provides a script for sorting search results by prices - * - * @param universityRole User group which consumes university services - * @param field Field in which wanted offers with prices are located - */ -export function buildPriceSortScript( - universityRole: keyof SCSportCoursePriceGroup, - field: SCThingsField, -): string { - return ` - // initialize the sort value with the maximum - double price = Double.MAX_VALUE; - - // if we have any offers - if (params._source.containsKey('${field}')) { - // iterate through all offers - for (offer in params._source.${field}) { - // if this offer contains a role specific price - if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) { - // if the role specific price is smaller than the cheapest we found - if (offer.prices.${universityRole} < price) { - // set the role specific price as cheapest for now - price = offer.prices.${universityRole}; - } - } else { // we have no role specific price for our role in this offer - // if the default price of this offer is lower than the cheapest we found - if (offer.price < price) { - // set this price as the cheapest - price = offer.price; - } - } - } - } - - // return cheapest price for our role - return price; - `; -} diff --git a/src/storage/elasticsearch/query/boost/boost-functions.ts b/src/storage/elasticsearch/query/boost/boost-functions.ts new file mode 100644 index 00000000..eeb711ba --- /dev/null +++ b/src/storage/elasticsearch/query/boost/boost-functions.ts @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types'; +import {SCBackendConfigurationSearchBoostingType} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Creates boost functions for all type boost configurations + * + * @param boostingTypes Array of type boosting configurations + */ +export function buildFunctionsForBoostingTypes( + boostingTypes: SCBackendConfigurationSearchBoostingType[], +): QueryDslFunctionScoreContainer[] { + const functions: QueryDslFunctionScoreContainer[] = []; + + for (const boostingForOneSCType of boostingTypes) { + const typeFilter: QueryDslSpecificQueryContainer<'term'> = { + term: { + type: boostingForOneSCType.type, + }, + }; + + functions.push({ + filter: typeFilter, + weight: boostingForOneSCType.factor, + }); + + if (typeof boostingForOneSCType.fields !== 'undefined') { + const fields = boostingForOneSCType.fields; + + for (const fieldName in boostingForOneSCType.fields) { + if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) { + const boostingForOneField = fields[fieldName]; + + for (const value in boostingForOneField) { + if (boostingForOneField.hasOwnProperty(value)) { + const factor = boostingForOneField[value]; + + // build term filter + const termFilter: QueryDslSpecificQueryContainer<'term'> = { + term: {}, + }; + termFilter.term[`${fieldName}.raw`] = value; + + functions.push({ + filter: { + bool: { + must: [typeFilter, termFilter], + should: [], + }, + }, + weight: factor, + }); + } + } + } + } + } + } + + return functions; +} diff --git a/src/storage/elasticsearch/query/boost/scoring-functions.ts b/src/storage/elasticsearch/query/boost/scoring-functions.ts new file mode 100644 index 00000000..58330e03 --- /dev/null +++ b/src/storage/elasticsearch/query/boost/scoring-functions.ts @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types'; +import {SCBackendConfigurationSearchBoostingContext, SCSearchContext} from '@openstapps/core'; +import {buildFunctionsForBoostingTypes} from './boost-functions'; + +/** + * Builds scoring functions from boosting config + * + * @param boostings Backend boosting configuration for contexts and types + * @param context The context of the app from where the search was initiated + */ +export function buildScoringFunctions( + boostings: SCBackendConfigurationSearchBoostingContext, + context: SCSearchContext | undefined, +): QueryDslFunctionScoreContainer[] { + // default context + let functions = buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]); + + if (typeof context !== 'undefined' && context !== 'default') { + // specific context provided, extend default context with additional boosts + functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])]; + } + + return functions; +} diff --git a/src/storage/elasticsearch/query/filter.ts b/src/storage/elasticsearch/query/filter.ts new file mode 100644 index 00000000..70b7b8d7 --- /dev/null +++ b/src/storage/elasticsearch/query/filter.ts @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types'; +import {SCSearchFilter} from '@openstapps/core'; +import {buildBooleanFilter} from './filters/boolean'; +import {buildAvailabilityFilter} from './filters/availability'; +import {buildDateRangeFilter} from './filters/date-range'; +import {buildDistanceFilter} from './filters/distance'; +import {buildGeoFilter} from './filters/geo'; +import {buildNumericRangeFilter} from './filters/numeric-range'; +import {buildValueFilter} from './filters/value'; + +/** + * Converts Array of Filters to elasticsearch query-syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildFilter(filter: SCSearchFilter): QueryDslQueryContainer { + switch (filter.type) { + case 'value': + return buildValueFilter(filter); + case 'availability': + return buildAvailabilityFilter(filter); + case 'distance': + return buildDistanceFilter(filter); + case 'boolean': + return buildBooleanFilter(filter); + case 'numeric range': + return buildNumericRangeFilter(filter); + case 'date range': + return buildDateRangeFilter(filter); + case 'geo': + return buildGeoFilter(filter); + } +} diff --git a/src/storage/elasticsearch/query/filters/availability.ts b/src/storage/elasticsearch/query/filters/availability.ts new file mode 100644 index 00000000..5db3b2b9 --- /dev/null +++ b/src/storage/elasticsearch/query/filters/availability.ts @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SCSearchAvailabilityFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Converts an availability filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildAvailabilityFilter( + filter: SCSearchAvailabilityFilter, +): QueryDslSpecificQueryContainer<'range'> { + const scope = filter.arguments.scope?.charAt(0) ?? 's'; + const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`; + + return { + range: { + [filter.arguments.field]: { + gte: `${time}/${scope}`, + lt: `${time}+1${scope}/${scope}`, + relation: 'intersects', + }, + }, + }; +} diff --git a/src/storage/elasticsearch/query/filters/boolean.ts b/src/storage/elasticsearch/query/filters/boolean.ts new file mode 100644 index 00000000..2a9a9f15 --- /dev/null +++ b/src/storage/elasticsearch/query/filters/boolean.ts @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types'; +import {SCSearchBooleanFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; +import {buildFilter} from '../filter'; + +/** + * Converts a boolean filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildBooleanFilter(filter: SCSearchBooleanFilter): QueryDslSpecificQueryContainer<'bool'> { + const result: QueryDslBoolQuery = { + minimum_should_match: 0, + must: [], + must_not: [], + should: [], + }; + + if (filter.arguments.operation === 'and') { + result.must = filter.arguments.filters.map(it => buildFilter(it)); + } + + if (filter.arguments.operation === 'or') { + result.should = filter.arguments.filters.map(it => buildFilter(it)); + result.minimum_should_match = 1; + } + + if (filter.arguments.operation === 'not') { + result.must_not = filter.arguments.filters.map(it => buildFilter(it)); + } + + return { + bool: result, + }; +} diff --git a/src/storage/elasticsearch/query/filters/date-range.ts b/src/storage/elasticsearch/query/filters/date-range.ts new file mode 100644 index 00000000..1ce0757e --- /dev/null +++ b/src/storage/elasticsearch/query/filters/date-range.ts @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types'; +import {SCSearchDateRangeFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Converts a date range filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildDateRangeFilter( + filter: SCSearchDateRangeFilter, +): QueryDslSpecificQueryContainer<'range'> { + const dateRangeObject: QueryDslDateRangeQuery = { + format: filter.arguments.format, + time_zone: filter.arguments.timeZone, + relation: filter.arguments.relation, + }; + if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') { + dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit; + } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') { + dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit; + } + if (filter.arguments.bounds.upperBound?.mode === 'exclusive') { + dateRangeObject.lt = filter.arguments.bounds.upperBound.limit; + } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') { + dateRangeObject.lte = filter.arguments.bounds.upperBound.limit; + } + + return { + range: { + [filter.arguments.field]: dateRangeObject, + }, + }; +} diff --git a/src/storage/elasticsearch/query/filters/distance.ts b/src/storage/elasticsearch/query/filters/distance.ts new file mode 100644 index 00000000..b2ceb71b --- /dev/null +++ b/src/storage/elasticsearch/query/filters/distance.ts @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types'; +import {SCSearchDistanceFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Converts a distance filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildDistanceFilter( + filter: SCSearchDistanceFilter, +): QueryDslSpecificQueryContainer<'geo_distance'> { + const geoObject: QueryDslGeoDistanceQuery = { + distance: `${filter.arguments.distance}m`, + [`${filter.arguments.field}.point.coordinates`]: { + lat: filter.arguments.position[1], + lon: filter.arguments.position[0], + }, + }; + + return { + geo_distance: geoObject, + }; +} diff --git a/src/storage/elasticsearch/query/filters/geo.ts b/src/storage/elasticsearch/query/filters/geo.ts new file mode 100644 index 00000000..9196d220 --- /dev/null +++ b/src/storage/elasticsearch/query/filters/geo.ts @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SCGeoFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Converts a geo filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> { + return { + geo_shape: { + ignore_unmapped: true, + [`${filter.arguments.field}.polygon`]: { + shape: filter.arguments.shape, + relation: filter.arguments.spatialRelation, + }, + }, + }; +} diff --git a/src/storage/elasticsearch/query/filters/numeric-range.ts b/src/storage/elasticsearch/query/filters/numeric-range.ts new file mode 100644 index 00000000..812cb1be --- /dev/null +++ b/src/storage/elasticsearch/query/filters/numeric-range.ts @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types'; +import {SCSearchNumericRangeFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Converts a numeric range filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildNumericRangeFilter( + filter: SCSearchNumericRangeFilter, +): QueryDslSpecificQueryContainer<'range'> { + const numericRangeObject: QueryDslNumberRangeQuery = { + relation: filter.arguments.relation, + }; + if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') { + numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit; + } else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') { + numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit; + } + if (filter.arguments.bounds.upperBound?.mode === 'exclusive') { + numericRangeObject.lt = filter.arguments.bounds.upperBound.limit; + } else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') { + numericRangeObject.lte = filter.arguments.bounds.upperBound.limit; + } + + return { + range: { + [filter.arguments.field]: numericRangeObject, + }, + }; +} diff --git a/src/storage/elasticsearch/query/filters/value.ts b/src/storage/elasticsearch/query/filters/value.ts new file mode 100644 index 00000000..60288ecc --- /dev/null +++ b/src/storage/elasticsearch/query/filters/value.ts @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SCSearchValueFilter} from '@openstapps/core'; +import {QueryDslSpecificQueryContainer} from '../../types/util'; + +/** + * Converts a value filter to elasticsearch syntax + * + * @param filter A search filter for the retrieval of the data + */ +export function buildValueFilter( + filter: SCSearchValueFilter, +): QueryDslSpecificQueryContainer<'term'> | QueryDslSpecificQueryContainer<'terms'> { + return Array.isArray(filter.arguments.value) + ? { + terms: { + [`${filter.arguments.field}.raw`]: filter.arguments.value, + }, + } + : { + term: { + [`${filter.arguments.field}.raw`]: filter.arguments.value, + }, + }; +} diff --git a/src/storage/elasticsearch/query/query.ts b/src/storage/elasticsearch/query/query.ts new file mode 100644 index 00000000..3f158b89 --- /dev/null +++ b/src/storage/elasticsearch/query/query.ts @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types'; +import {SCConfigFile, SCSearchQuery} from '@openstapps/core'; +import {ElasticsearchConfig} from '../types/elasticsearch-config'; +import {buildFilter} from './filter'; +import {buildScoringFunctions} from './boost/scoring-functions'; + +/** + * Builds body for Elasticsearch requests + * + * @param parameters Parameters for querying the backend + * @param defaultConfig Default configuration of the backend + * @param elasticsearchConfig Elasticsearch configuration + * @returns ElasticsearchQuery (body of a search-request) + */ +export function buildQuery( + parameters: SCSearchQuery, + defaultConfig: SCConfigFile, + elasticsearchConfig: ElasticsearchConfig, +): QueryDslQueryContainer { + // if config provides an minMatch parameter we use query_string instead of match query + let query; + if (typeof elasticsearchConfig.query === 'undefined') { + query = { + query_string: { + analyzer: 'search_german', + default_field: 'name', + minimum_should_match: '90%', + query: typeof parameters.query !== 'string' ? '*' : parameters.query, + }, + }; + } else if (elasticsearchConfig.query.queryType === 'query_string') { + query = { + query_string: { + analyzer: 'search_german', + default_field: 'name', + minimum_should_match: elasticsearchConfig.query.minMatch, + query: typeof parameters.query !== 'string' ? '*' : parameters.query, + }, + }; + } else if (elasticsearchConfig.query.queryType === 'dis_max') { + if (typeof parameters.query === 'string' && parameters.query !== '*') { + query = { + dis_max: { + boost: 1.2, + queries: [ + { + match: { + name: { + boost: elasticsearchConfig.query.matchBoosting, + fuzziness: elasticsearchConfig.query.fuzziness, + query: parameters.query, + }, + }, + }, + { + query_string: { + default_field: 'name', + minimum_should_match: elasticsearchConfig.query.minMatch, + query: parameters.query, + }, + }, + ], + tie_breaker: elasticsearchConfig.query.tieBreaker, + }, + }; + } + } else { + throw new Error( + 'Unsupported query type. Check your config file and reconfigure your elasticsearch query', + ); + } + + const functionScoreQuery: QueryDslQueryContainer = { + function_score: { + functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context), + query: { + bool: { + minimum_should_match: 0, // if we have no should, nothing can match + must: [], + should: [], + }, + }, + score_mode: 'multiply', + }, + }; + + const mustMatch = functionScoreQuery.function_score?.query?.bool?.must; + + if (Array.isArray(mustMatch)) { + if (typeof query !== 'undefined') { + mustMatch.push(query); + } + + if (typeof parameters.filter !== 'undefined') { + mustMatch.push(buildFilter(parameters.filter)); + } + } + + return functionScoreQuery; +} diff --git a/src/storage/elasticsearch/query/sort.ts b/src/storage/elasticsearch/query/sort.ts new file mode 100644 index 00000000..20733504 --- /dev/null +++ b/src/storage/elasticsearch/query/sort.ts @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {Sort} from '@elastic/elasticsearch/lib/api/types'; +import {SCSearchSort} from '@openstapps/core'; +import {buildDistanceSort} from './sort/distance'; +import {buildDucetSort} from './sort/ducet'; +import {buildGenericSort} from './sort/generic'; +import {buildPriceSort} from './sort/price'; + +/** + * converts query to + * + * @param sorts Sorting rules to apply to the data that is being queried + * @returns an array of sort queries + */ +export function buildSort(sorts: SCSearchSort[]): Sort { + return sorts.map(sort => { + switch (sort.type) { + case 'generic': + return buildGenericSort(sort); + case 'ducet': + return buildDucetSort(sort); + case 'distance': + return buildDistanceSort(sort); + case 'price': + return buildPriceSort(sort); + } + }); +} diff --git a/src/storage/elasticsearch/query/sort/distance.ts b/src/storage/elasticsearch/query/sort/distance.ts new file mode 100644 index 00000000..3319f859 --- /dev/null +++ b/src/storage/elasticsearch/query/sort/distance.ts @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SortOptions} from '@elastic/elasticsearch/lib/api/types'; +import {SCDistanceSort} from '@openstapps/core'; + +/** + * Converts a distance sort to elasticsearch syntax + * + * @param sort A sorting definition + */ +export function buildDistanceSort(sort: SCDistanceSort): SortOptions { + return { + _geo_distance: { + mode: 'avg', + order: sort.order, + unit: 'm', + [`${sort.arguments.field}.point.coordinates`]: { + lat: sort.arguments.position[1], + lon: sort.arguments.position[0], + }, + }, + }; +} diff --git a/src/storage/elasticsearch/query/sort/ducet.ts b/src/storage/elasticsearch/query/sort/ducet.ts new file mode 100644 index 00000000..b0936d88 --- /dev/null +++ b/src/storage/elasticsearch/query/sort/ducet.ts @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SortOptions} from '@elastic/elasticsearch/lib/api/types'; +import {SCDucetSort} from '@openstapps/core'; + +/** + * Converts a ducet sort to elasticsearch syntax + * + * @param sort A sorting definition + */ +export function buildDucetSort(sort: SCDucetSort): SortOptions { + return { + [`${sort.arguments.field}.sort`]: sort.order, + }; +} diff --git a/src/storage/elasticsearch/query/sort/generic.ts b/src/storage/elasticsearch/query/sort/generic.ts new file mode 100644 index 00000000..38188c7c --- /dev/null +++ b/src/storage/elasticsearch/query/sort/generic.ts @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SortOptions} from '@elastic/elasticsearch/lib/api/types'; +import {SCGenericSort} from '@openstapps/core'; + +/** + * Converts a generic sort to elasticsearch syntax + * + * @param sort A sorting definition + */ +export function buildGenericSort(sort: SCGenericSort): SortOptions { + return { + [sort.arguments.field]: sort.order, + }; +} diff --git a/src/storage/elasticsearch/query/sort/price.ts b/src/storage/elasticsearch/query/sort/price.ts new file mode 100644 index 00000000..6bd319e4 --- /dev/null +++ b/src/storage/elasticsearch/query/sort/price.ts @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {SortOptions} from '@elastic/elasticsearch/lib/api/types'; +import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core'; + +/** + * Converts a price sort to elasticsearch syntax + * + * @param sort A sorting definition + */ +export function buildPriceSort(sort: SCPriceSort): SortOptions { + return { + _script: { + order: sort.order, + script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field), + type: 'number' as const, + }, + }; +} + +/** + * Provides a script for sorting search results by prices + * + * @param universityRole User group which consumes university services + * @param field Field in which wanted offers with prices are located + */ +export function buildPriceSortScript( + universityRole: keyof SCSportCoursePriceGroup, + field: SCThingsField, +): string { + return ` + // initialize the sort value with the maximum + double price = Double.MAX_VALUE; + + // if we have any offers + if (params._source.containsKey('${field}')) { + // iterate through all offers + for (offer in params._source.${field}) { + // if this offer contains a role specific price + if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) { + // if the role specific price is smaller than the cheapest we found + if (offer.prices.${universityRole} < price) { + // set the role specific price as cheapest for now + price = offer.prices.${universityRole}; + } + } else { // we have no role specific price for our role in this offer + // if the default price of this offer is lower than the cheapest we found + if (offer.price < price) { + // set this price as the cheapest + price = offer.price; + } + } + } + } + + // return cheapest price for our role + return price; + `; +} diff --git a/src/storage/elasticsearch/templating.ts b/src/storage/elasticsearch/templating.ts index 4301f6ff..afb5ad9b 100644 --- a/src/storage/elasticsearch/templating.ts +++ b/src/storage/elasticsearch/templating.ts @@ -29,17 +29,6 @@ export const aggregations = JSON.parse( readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'), ) as AggregationSchema; -/** - * Re-applies all interfaces for every type - * - * @param client An elasticsearch client to use - */ -export async function refreshAllTemplates(client: Client) { - for (const type of Object.values(SCThingType)) { - await putTemplate(client, type as SCThingType); - } -} - /** * Prepares all indices * diff --git a/src/storage/elasticsearch/types/elasticsearch-config.ts b/src/storage/elasticsearch/types/elasticsearch-config.ts new file mode 100644 index 00000000..91231cc4 --- /dev/null +++ b/src/storage/elasticsearch/types/elasticsearch-config.ts @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +/** + * A configuration for using the Dis Max Query + * + * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further + * explanation of what the parameters mean + */ +export interface ElasticsearchQueryDisMaxConfig { + /** + * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear + */ + cutoffFrequency: number; + + /** + * The maximum allowed Levenshtein Edit Distance (or number of edits) + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness + */ + fuzziness: number | string; + + /** + * Increase the importance (relevance score) of a field + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html + */ + matchBoosting: number; + + /** + * Minimal number (or percentage) of words that should match in a query + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html + */ + minMatch: string; + + /** + * Type of the query - in this case 'dis_max' which is a union of its subqueries + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html + */ + queryType: 'dis_max'; + + /** + * Changes behavior of default calculation of the score when multiple results match + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker + */ + tieBreaker: number; +} + +/** + * A configuration for using Query String Query + * + * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further + * explanation of what the parameters mean + */ +export interface ElasticsearchQueryQueryStringConfig { + /** + * Minimal number (or percentage) of words that should match in a query + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html + */ + minMatch: string; + + /** + * Type of the query - in this case 'query_string' which uses a query parser in order to parse content + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html + */ + queryType: 'query_string'; +} + +/** + * An config file for the elasticsearch database interface + * + * The config file extends the SCConfig file by further defining how the database property + */ +export interface ElasticsearchConfigFile { + /** + * Configuration that is not visible to clients + */ + internal: { + /** + * Database configuration + */ + database: ElasticsearchConfig; + }; +} + +/** + * An elasticsearch configuration + */ +export interface ElasticsearchConfig { + /** + * Name of the database + */ + name: 'elasticsearch'; + + /** + * Configuration for using queries + */ + query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig; + + /** + * Version of the used elasticsearch + */ + version: string; +} diff --git a/src/storage/elasticsearch/types/elasticsearch.ts b/src/storage/elasticsearch/types/elasticsearch.ts deleted file mode 100644 index 6804a4f9..00000000 --- a/src/storage/elasticsearch/types/elasticsearch.ts +++ /dev/null @@ -1,605 +0,0 @@ -/* - * Copyright (C) 2019-2021 StApps - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import {SCThing, SCThingType} from '@openstapps/core'; -// we only have the @types package because some things type definitions are still missing from the official -import {NameList} from 'elasticsearch'; -import {Polygon, Position} from 'geojson'; - -/** - * An elasticsearch aggregation bucket - */ -interface Bucket { - /** - * Number of documents in the aggregation bucket - */ - doc_count: number; - - /** - * Text representing the documents in the bucket - */ - key: string; -} - -/** - * An elasticsearch aggregation response - */ -export interface AggregationResponse { - /** - * The individual aggregations - */ - [field: string]: BucketAggregation | NestedAggregation; -} - -/** - * An elasticsearch bucket aggregation - */ -export interface BucketAggregation { - /** - * Buckets in an aggregation - */ - buckets: Bucket[]; - - /** - * Number of documents in an aggregation - */ - doc_count?: number; -} - -/** - * An aggregation that contains more aggregations nested inside - */ -export interface NestedAggregation { - /** - * Number of documents in an aggregation - */ - doc_count: number; - - /** - * Any nested responses - */ - [name: string]: BucketAggregation | number; -} - -/** - * A configuration for using the Dis Max Query - * - * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further - * explanation of what the parameters mean - */ -export interface ElasticsearchQueryDisMaxConfig { - /** - * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear - */ - cutoffFrequency: number; - - /** - * The maximum allowed Levenshtein Edit Distance (or number of edits) - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness - */ - fuzziness: number | string; - - /** - * Increase the importance (relevance score) of a field - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html - */ - matchBoosting: number; - - /** - * Minimal number (or percentage) of words that should match in a query - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html - */ - minMatch: string; - - /** - * Type of the query - in this case 'dis_max' which is a union of its subqueries - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html - */ - queryType: 'dis_max'; - - /** - * Changes behavior of default calculation of the score when multiple results match - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker - */ - tieBreaker: number; -} - -/** - * A configuration for using Query String Query - * - * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further - * explanation of what the parameters mean - */ -export interface ElasticsearchQueryQueryStringConfig { - /** - * Minimal number (or percentage) of words that should match in a query - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html - */ - minMatch: string; - - /** - * Type of the query - in this case 'query_string' which uses a query parser in order to parse content - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html - */ - queryType: 'query_string'; -} - -/** - * A hit in an elasticsearch search result - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html - */ -export interface ElasticsearchObject { - /** - * Unique identifier of a document (object) - */ - _id: string; - - /** - * The index to which the document belongs - */ - _index: string; - - /** - * Relevancy of the document to a query - */ - _score: number; - - /** - * The original JSON representing the body of the document - */ - _source: T; - - /** - * The document's mapping type - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html - */ - _type: string; - - /** - * Version of the document - */ - _version?: number; - - /** - * Used to index the same field in different ways for different purposes - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html - */ - fields?: NameList; - - /** - * Used to highlight search results on one or more fields - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - highlight?: any; - - /** - * Used in when nested/children documents match the query - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - inner_hits?: any; - - /** - * Queries that matched for documents in results - */ - matched_queries?: string[]; - - /** - * Sorting definition - */ - sort?: string[]; -} - -/** - * An config file for the elasticsearch database interface - * - * The config file extends the SCConfig file by further defining how the database property - */ -export interface ElasticsearchConfigFile { - /** - * Configuration that is not visible to clients - */ - internal: { - /** - * Database configuration - */ - database: ElasticsearchConfig; - }; -} - -/** - * An elasticsearch configuration - */ -export interface ElasticsearchConfig { - /** - * Name of the database - */ - name: 'elasticsearch'; - - /** - * Configuration for using queries - */ - query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig; - - /** - * Version of the used elasticsearch - */ - version: string; -} - -/** - * An elasticsearch term filter - */ -export type ESTermFilter = - | { - /** - * Definition of a term to match - */ - term: { - [fieldName: string]: string; - }; - } - | { - /** - * Definition of terms to match (or) - */ - terms: { - [fieldName: string]: string[]; - }; - }; - -export interface ESGenericRange { - /** - * Greater than field - */ - gt?: T; - - /** - * Greater or equal than field - */ - gte?: T; - - /** - * Less than field - */ - lt?: T; - - /** - * Less or equal than field - */ - lte?: T; - - /** - * Relation of the range to a range field - * - * Intersects: Both ranges intersect - * Contains: Search range contains field range - * Within: Field range contains search range - */ - relation?: 'intersects' | 'within' | 'contains'; -} - -interface ESGenericRangeFilter> { - /** - * Range filter definition - */ - range: { - [fieldName: string]: T; - }; -} - -export interface ESDateRange extends ESGenericRange { - /** - * Optional date format override - */ - format?: string; - - /** - * Optional timezone specifier - */ - time_zone?: string; -} - -export type ESNumericRangeFilter = ESGenericRangeFilter>; -export type ESDateRangeFilter = ESGenericRangeFilter; -export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter; - -/** - * An elasticsearch type filter - */ -export interface ESTypeFilter { - /** - * Type filter definition - */ - type: { - /** - * Type name (SCThingType) to filter with - */ - value: SCThingType; - }; -} - -/** - * Filter arguments for an elasticsearch geo distance filter - */ -export interface ESGeoDistanceFilterArguments { - /** - * The radius of the circle centred on the specified location - */ - distance: string; - - [fieldName: string]: - | { - /** - * Latitude - */ - lat: number; - - /** - * Longitude - */ - lon: number; - } - | string; -} - -/** - * An elasticsearch geo distance filter - */ -export interface ESGeoDistanceFilter { - /** - * @see ESGeoDistanceFilterArguments - */ - geo_distance: ESGeoDistanceFilterArguments; -} - -/** - * A rectangular geo shape, representing the top-left and bottom-right corners - * - * This is an extension of the Geojson type - * http://geojson.org/geojson-spec.html - */ -export interface ESEnvelope { - /** - * The top-left and bottom-right corners of the bounding box - */ - coordinates: [Position, Position]; - - /** - * The type of the geometry - */ - type: 'envelope'; -} - -/** - * An Elasticsearch geo bounding box filter - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html - */ -export interface ESGeoBoundingBoxFilter { - /** - * An Elasticsearch geo bounding box filter - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html - */ - geo_bounding_box: { - [fieldName: string]: { - /** - * Geo Shape - */ - bottom_right: Position; - - /** - * Geo Shape - */ - top_left: Position; - }; - }; -} - -/** - * An Elasticsearch geo shape filter - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html - */ -export interface ESGeoShapeFilter { - geo_shape: { - [fieldName: string]: { - /** - * Relation of the two shapes - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_spatial_relations - */ - relation?: 'intersects' | 'disjoint' | 'within' | 'contains'; - - /** - * Geo Shape - */ - shape: Polygon | ESEnvelope; - }; - }; -} - -/** - * Filter arguments for an elasticsearch boolean filter - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html - */ -export interface ESBooleanFilterArguments { - /** - * Minimal number (or percentage) of words that should match in a query - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html - */ - minimum_should_match?: number; - - /** - * The clause (query) must appear in matching documents and will contribute to the score. - */ - must?: T[]; - - /** - * The clause (query) must not appear in the matching documents. - */ - must_not?: T[]; - - /** - * The clause (query) should appear in the matching document. - */ - should?: T[]; -} - -/** - * An elasticsearch boolean filter - */ -export interface ESBooleanFilter { - /** - * @see ESBooleanFilterArguments - */ - bool: ESBooleanFilterArguments; -} - -/** - * An elasticsearch function score query - * - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html - */ -export interface ESFunctionScoreQuery { - /** - * Function score definition - */ - function_score: { - /** - * Functions that compute score for query results (documents) - * - * @see ESFunctionScoreQueryFunction - */ - functions: ESFunctionScoreQueryFunction[]; - - /** - * @see ESBooleanFilter - */ - query: ESBooleanFilter; - - /** - * Specifies how the computed scores are combined - */ - score_mode: 'multiply'; - }; -} - -/** - * An function for an elasticsearch functions score query - */ -export interface ESFunctionScoreQueryFunction { - /** - * Function is applied only if a document matches the given filtering query - */ - filter: ESTermFilter | ESTypeFilter | ESBooleanFilter; - - /** - * Weight (importance) of the filter - */ - weight: number; -} - -/** - * An elasticsearch generic sort - */ -export interface ESGenericSort { - [field: string]: string; -} - -/** - * Sort arguments for an elasticsearch geo distance sort - */ -export interface ESGeoDistanceSortArguments { - /** - * What value to pick for sorting - */ - mode: 'avg' | 'max' | 'median' | 'min'; - - /** - * Order - */ - order: 'asc' | 'desc'; - - /** - * Value unit - */ - unit: 'm'; - - [field: string]: - | { - /** - * Latitude - */ - lat: number; - - /** - * Longitude - */ - lon: number; - } - | string; -} - -/** - * An elasticsearch geo distance sort - */ -export interface ESGeoDistanceSort { - /** - * @see ESGeoDistanceFilterArguments - */ - _geo_distance: ESGeoDistanceSortArguments; -} - -/** - * An elasticsearch script sort - */ -export interface ScriptSort { - /** - * A script - */ - _script: { - /** - * Order - */ - order: 'asc' | 'desc'; - - /** - * The custom script used for sorting - */ - script: string; - - /** - * What type is being sorted - */ - type: 'number' | 'string'; - }; -} diff --git a/src/storage/elasticsearch/types/guards.ts b/src/storage/elasticsearch/types/guards.ts deleted file mode 100644 index c5bf60bc..00000000 --- a/src/storage/elasticsearch/types/guards.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2019-2021 StApps - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { - ESAggMatchAllFilter, - ESAggTypeFilter, - ESNestedAggregation, - ESTermsFilter, -} from '@openstapps/es-mapping-generator/src/types/aggregation'; -import {BucketAggregation, NestedAggregation} from './elasticsearch'; - -/** - * Checks if the type is a BucketAggregation - * - * @param agg the type to check - */ -export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation { - return typeof agg !== 'number'; -} - -/** - * Checks if the type is a NestedAggregation - * - * @param agg the type to check - */ -export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation { - return typeof (agg as BucketAggregation).buckets === 'undefined'; -} - -/** - * Checks if the parameter is of type ESTermsFilter - * - * @param agg the value to check - */ -export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter { - return typeof (agg as ESTermsFilter).terms !== 'undefined'; -} - -/** - * Checks if the parameter is of type ESTermsFilter - * - * @param agg the value to check - */ -export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation { - return typeof (agg as ESNestedAggregation).aggs !== 'undefined'; -} - -/** - * Checks if the parameter is of type - * - * @param filter the filter to narrow the type of - */ -export function isESAggMatchAllFilter( - filter: ESAggTypeFilter | ESAggMatchAllFilter, -): filter is ESAggMatchAllFilter { - return filter.hasOwnProperty('match_all'); -} diff --git a/src/storage/elasticsearch/types/util.ts b/src/storage/elasticsearch/types/util.ts new file mode 100644 index 00000000..15bf449b --- /dev/null +++ b/src/storage/elasticsearch/types/util.ts @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types'; + +export type QueryDslSpecificQueryContainer = Required< + Pick +>; diff --git a/src/storage/elasticsearch/util/alias.ts b/src/storage/elasticsearch/util/alias.ts new file mode 100644 index 00000000..7cef01f8 --- /dev/null +++ b/src/storage/elasticsearch/util/alias.ts @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +import {Logger} from '@openstapps/logger'; + +/** + * Checks for invalid character in alias names and removes them + * + * @param alias The alias name + * @param uid The UID of the current bulk (for debugging purposes) + */ +export function removeInvalidAliasChars(alias: string, uid: string | undefined): string { + let formattedAlias = alias; + + // spaces are included in some types, replace them with underscores + if (formattedAlias.includes(' ')) { + formattedAlias = formattedAlias.trim(); + formattedAlias = formattedAlias.split(' ').join('_'); + } + // List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html + for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) { + if (formattedAlias.includes(value)) { + formattedAlias = formattedAlias.replace(value, ''); + Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks + having the same alias despite having different types, as invalid characters are removed automatically. + New alias name is "${formattedAlias}."`); + } + } + for (const value of ['-', '_', '+']) { + if (formattedAlias.charAt(0) === value) { + formattedAlias = formattedAlias.slice(1); + Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same + alias despite having different types, as invalid characters are removed automatically. + New alias name is "${formattedAlias}."`); + } + } + if (formattedAlias === '.' || formattedAlias === '..') { + Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using + another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`); + + return 'alias_placeholder'; + } + if (formattedAlias.includes(':')) { + Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future + Elasticsearch versions!`); + } + + return formattedAlias; +} diff --git a/src/storage/elasticsearch/util/index.ts b/src/storage/elasticsearch/util/index.ts new file mode 100644 index 00000000..24ba5999 --- /dev/null +++ b/src/storage/elasticsearch/util/index.ts @@ -0,0 +1,63 @@ +import {SCBulkResponse, SCThingType, SCUuid} from '@openstapps/core'; + +/** + * Length of the index UID used for generation of its name + */ +export const INDEX_UID_LENGTH = 8; + +/** + * A string which matches all indices + */ +export const ALL_INDICES_QUERY = 'stapps_*_*_*'; + +/** + * Matches index names such as stapps___ + */ +export const VALID_INDEX_REGEX = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/; + +export interface ParsedIndexName { + type: SCThingType; + source: string; + randomSuffix: string; +} + +/** + * + */ +export function parseIndexName(index: string): ParsedIndexName { + const match = VALID_INDEX_REGEX.exec(index); + if (!match) { + throw new SyntaxError(`Invalid index name ${index}!`); + } + + return { + type: match[1] as SCThingType, + source: match[2], + randomSuffix: match[3], + }; +} + +/** + * Gets the index name in elasticsearch for one SCThingType + * + * @param type SCThingType of data in the index + * @param source source of data in the index + * @param bulk bulk process which created this index + */ +export function getThingIndexName(type: SCThingType, source: string, bulk: SCBulkResponse) { + let out = type.toLowerCase(); + while (out.includes(' ')) { + out = out.replace(' ', '_'); + } + + return `stapps_${out}_${source}_${getIndexUID(bulk.uid)}`; +} + +/** + * Provides the index UID (for its name) from the bulk UID + * + * @param uid Bulk UID + */ +export function getIndexUID(uid: SCUuid) { + return uid.slice(0, Math.max(0, INDEX_UID_LENGTH)); +} diff --git a/src/storage/elasticsearch/util/no-undefined.ts b/src/storage/elasticsearch/util/no-undefined.ts new file mode 100644 index 00000000..118bab84 --- /dev/null +++ b/src/storage/elasticsearch/util/no-undefined.ts @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +/** + * Type guard for filter functions + */ +export function noUndefined(item: T | undefined): item is T { + return typeof item !== 'undefined'; +} diff --git a/src/storage/elasticsearch/util/retry.ts b/src/storage/elasticsearch/util/retry.ts new file mode 100644 index 00000000..acf17e08 --- /dev/null +++ b/src/storage/elasticsearch/util/retry.ts @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2022 StApps + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +export interface RetryOptions { + maxRetries: number; + retryInterval: number; + doAction: () => Promise; + onFailedAttempt: (attempt: number, error: unknown, options: RetryOptions) => void; + onFail: (options: RetryOptions) => never; +} + +/** + * Retries a throwing function at a set interval, until a maximum amount of attempts + */ +export async function retryCatch(options: RetryOptions): Promise { + for (let attempt = 0; attempt < options.maxRetries; attempt++) { + try { + return await options.doAction(); + } catch (error) { + options.onFailedAttempt(attempt, error, options); + await new Promise(resolve => setTimeout(resolve, options.retryInterval)); + } + } + + options.onFail(options); +} diff --git a/test/common.ts b/test/common.ts index ecf8a5fb..98fc11c8 100644 --- a/test/common.ts +++ b/test/common.ts @@ -16,6 +16,7 @@ import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCThingType, SCUuid} from '@openstapps/core'; import {Express} from 'express'; import moment from 'moment'; +import {getIndexUID} from '../src/storage/elasticsearch/util'; import {configureApp} from '../src/app'; import express from 'express'; import http from 'http'; @@ -24,7 +25,6 @@ import {MailQueue} from '../src/notification/mail-queue'; import {Bulk, BulkStorage} from '../src/storage/bulk-storage'; import getPort from 'get-port'; import {Database} from '../src/storage/database'; -import {Elasticsearch} from '../src/storage/elasticsearch/elasticsearch'; import {v4} from 'uuid'; /** @@ -147,5 +147,4 @@ export const getTransport = (verified: boolean) => { }; }; -export const getIndex = (uid?: string) => - `stapps_footype_foosource_${uid ?? Elasticsearch.getIndexUID(v4())}`; +export const getIndex = (uid?: string) => `stapps_footype_foosource_${uid ?? getIndexUID(v4())}`; diff --git a/test/storage/elasticsearch/aggregations.spec.ts b/test/storage/elasticsearch/aggregations.spec.ts index 48584cd6..ed727dca 100644 --- a/test/storage/elasticsearch/aggregations.spec.ts +++ b/test/storage/elasticsearch/aggregations.spec.ts @@ -13,13 +13,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import {AggregateName, AggregationsMultiTermsBucket} from '@elastic/elasticsearch/lib/api/types'; import {SCFacet, SCThingType} from '@openstapps/core'; import {expect} from 'chai'; import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations'; -import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch'; describe('Aggregations', function () { - const aggregations: AggregationResponse = { + const aggregations: Record> = { 'catalog': { 'doc_count': 4, 'superCatalogs.categories': { @@ -76,14 +76,6 @@ describe('Aggregations', function () { buckets: [], }, }, - 'fooType': { - buckets: [ - { - doc_count: 321, - key: 'foo', - }, - ], - }, '@all': { doc_count: 17, type: { @@ -102,33 +94,6 @@ describe('Aggregations', function () { }; const expectedFacets: SCFacet[] = [ - { - buckets: [ - { - count: 13, - key: 'person', - }, - { - count: 4, - key: 'catalog', - }, - ], - field: 'type', - }, - { - buckets: [ - { - count: 8, - key: 'foobar', - }, - { - count: 2, - key: 'bar', - }, - ], - field: 'categories', - onlyOnType: SCThingType.AcademicEvent, - }, { buckets: [ { @@ -153,7 +118,33 @@ describe('Aggregations', function () { field: 'categories', onlyOnType: SCThingType.Catalog, }, - // no fooType as it doesn't appear in the aggregation schema + { + buckets: [ + { + count: 8, + key: 'foobar', + }, + { + count: 2, + key: 'bar', + }, + ], + field: 'categories', + onlyOnType: SCThingType.AcademicEvent, + }, + { + buckets: [ + { + count: 13, + key: 'person', + }, + { + count: 4, + key: 'catalog', + }, + ], + field: 'type', + }, ]; it('should parse the aggregations providing the appropriate facets', function () { diff --git a/test/storage/elasticsearch/common.spec.ts b/test/storage/elasticsearch/common.spec.ts deleted file mode 100644 index 45629424..00000000 --- a/test/storage/elasticsearch/common.spec.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (C) 2020 StApps - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { - ESAggMatchAllFilter, - ESAggTypeFilter, - ESNestedAggregation, - ESTermsFilter, -} from '@openstapps/es-mapping-generator/src/types/aggregation'; -import {expect} from 'chai'; -import { - isNestedAggregation, - isBucketAggregation, - isESTermsFilter, - isESAggMatchAllFilter, - isESNestedAggregation, -} from '../../../lib/storage/elasticsearch/types/guards'; -import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch'; - -describe('Common', function () { - const bucketAggregation: BucketAggregation = {buckets: []}; - const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}}; - const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}}; - - describe('isBucketAggregation', function () { - it('should be false for a number', function () { - expect(isBucketAggregation(123)).to.be.false; - }); - - it('should be true for a bucket aggregation', function () { - expect(isBucketAggregation(bucketAggregation)).to.be.true; - }); - }); - - describe('isNestedAggregation', function () { - it('should be false for a bucket aggregation', function () { - expect(isNestedAggregation(bucketAggregation)).to.be.false; - }); - - it('should be true for a nested aggregation', function () { - const nestedAggregation: NestedAggregation = {doc_count: 123}; - - expect(isNestedAggregation(nestedAggregation)).to.be.true; - }); - }); - - describe('isESTermsFilter', function () { - it('should be false for an elasticsearch nested aggregation', function () { - expect(isESTermsFilter(esNestedAggregation)).to.be.false; - }); - - it('should be true for an elasticsearch terms filter', function () { - expect(isESTermsFilter(esTermsFilter)).to.be.true; - }); - }); - - describe('isESNestedAggregation', function () { - it('should be false for an elasticsearch terms filter', function () { - expect(isESNestedAggregation(esTermsFilter)).to.be.false; - }); - - it('should be true for an elasticsearch nested aggregation', function () { - expect(isESNestedAggregation(esNestedAggregation)).to.be.true; - }); - }); - - describe('isESAggMatchAllFilter', function () { - it('should be false for an elasticsearch aggregation type filter', function () { - const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}}; - expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false; - }); - - it('should be true for an elasticsearch aggregation match all filter', function () { - const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}}; - expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true; - }); - }); -}); diff --git a/test/storage/elasticsearch/elasticsearch.spec.ts b/test/storage/elasticsearch/elasticsearch.spec.ts index 6df70637..1b736c22 100644 --- a/test/storage/elasticsearch/elasticsearch.spec.ts +++ b/test/storage/elasticsearch/elasticsearch.spec.ts @@ -14,7 +14,14 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import {ApiResponse, Client} from '@elastic/elasticsearch'; +import {Client, Diagnostic} from '@elastic/elasticsearch'; +import Indices from '@elastic/elasticsearch/lib/api/api/indices'; +import { + CreateResponse, + SearchHit, + SearchResponse, + SortCombinations, +} from '@elastic/elasticsearch/lib/api/types'; import { SCBook, SCBulkResponse, @@ -30,22 +37,32 @@ import {Logger} from '@openstapps/logger'; import {SMTP} from '@openstapps/logger/lib/smtp'; import {expect, use} from 'chai'; import chaiAsPromised from 'chai-as-promised'; -import {SearchResponse} from 'elasticsearch'; import mockedEnv from 'mocked-env'; -import sinon from 'sinon'; +import {ALL_INDICES_QUERY, parseIndexName} from '../../../src/storage/elasticsearch/util'; +import * as queryModule from '../../../src/storage/elasticsearch/query/query'; +import * as sortModule from '../../../src/storage/elasticsearch/query/sort'; +import sinon, {SinonStub} from 'sinon'; +import {getIndexUID, getThingIndexName, INDEX_UID_LENGTH} from '../../../src/storage/elasticsearch/util'; +import * as utilModule from '../../../src/storage/elasticsearch/util'; +import {removeInvalidAliasChars} from '../../../src/storage/elasticsearch/util/alias'; import {configFile} from '../../../src/common'; import {MailQueue} from '../../../src/notification/mail-queue'; import {aggregations} from '../../../src/storage/elasticsearch/templating'; -import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch'; import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch'; import * as Monitoring from '../../../src/storage/elasticsearch/monitoring'; -import * as query from '../../../src/storage/elasticsearch/query'; import * as templating from '../../../src/storage/elasticsearch/templating'; import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common'; import fs from 'fs'; use(chaiAsPromised); +/** + * + */ +function searchResponse(...hits: SearchHit[]): SearchResponse { + return {hits: {hits}, took: 0, timed_out: false, _shards: {total: 1, failed: 0, successful: 1}}; +} + describe('Elasticsearch', function () { // increase timeout for the suite this.timeout(DEFAULT_TEST_TIMEOUT); @@ -83,6 +100,14 @@ describe('Elasticsearch', function () { }); }); + describe('getAliasMap', function () { + it('should fail after retries', async function () { + const es = new Elasticsearch(configFile); + sandbox.stub(es.client.indices, 'getAlias').throws(); + await expect(es.init({maxRetries: 1, retryInterval: 10})).to.be.rejected; + }); + }); + describe('getIndex (including getIndexUID)', function () { const type = 'foo bar type'; const source = 'foo_source'; @@ -95,59 +120,63 @@ describe('Elasticsearch', function () { }; it('should provide index UID from the provided UID', function () { - const indexUID = Elasticsearch.getIndexUID(bulk.uid); + const indexUID = getIndexUID(bulk.uid); - expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH); + expect(indexUID.length).to.be.equal(INDEX_UID_LENGTH); // test starting and ending character expect(indexUID[0]).to.be.equal(bulk.uid[0]); - expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]); + expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[INDEX_UID_LENGTH - 1]); }); it('should provide index name from the provided data', function () { - expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal( - `stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`, + expect(getThingIndexName(type as SCThingType, source, bulk)).to.be.equal( + `stapps_${type.split(' ').join('_')}_${source}_${getIndexUID(bulk.uid)}`, ); }); + + it('should reject invalid index names', function () { + expect(() => parseIndexName(':)')).to.throw(SyntaxError); + }); }); describe('removeAliasChars', function () { it('should remove spaces from both ends', function () { - expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias'); + expect(removeInvalidAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias'); }); it('should replace inner spaces with underscores', function () { - expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias'); + expect(removeInvalidAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias'); }); it('should remove invalid characters', function () { - expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/* ', 'bulk-uid')).to.be.equal('foobaralias'); + expect(removeInvalidAliasChars('f,o#o\\b|ar/* ', 'bulk-uid')).to.be.equal('foobaralias'); }); it('should remove invalid starting characters', function () { - expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); - expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); - expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); + expect(removeInvalidAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); + expect(removeInvalidAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); + expect(removeInvalidAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias'); }); it('should replace with a placeholder in case of invalid alias', function () { - expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder'); - expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder'); + expect(removeInvalidAliasChars('.', 'bulk-uid')).to.contain('placeholder'); + expect(removeInvalidAliasChars('..', 'bulk-uid')).to.contain('placeholder'); }); it('should work with common cases', function () { expect( - Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'), + removeInvalidAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'), ).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890'); - expect( - Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'), - ).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG'); + expect(removeInvalidAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid')).to.be.equal( + 'THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', + ); }); it('should warn in case of characters that are invalid in future elasticsearch versions', function () { const sandbox = sinon.createSandbox(); const loggerWarnStub = sandbox.stub(Logger, 'warn'); - expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias'); + expect(removeInvalidAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias'); expect(loggerWarnStub.called).to.be.true; }); }); @@ -182,7 +211,7 @@ describe('Elasticsearch', function () { it('should log an error in case of there is one when getting response from the elasticsearch client', async function () { const error = new Error('Foo Error'); const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo'); - sandbox.stub(Client.prototype, 'on').yields(error); + sandbox.stub(Diagnostic.prototype, 'on').yields(error); new Elasticsearch(configFile); @@ -192,7 +221,7 @@ describe('Elasticsearch', function () { it('should log the result in the debug mode when getting response from the elasticsearch client', async function () { const fakeResponse = {foo: 'bar'}; const loggerLogStub = sandbox.stub(Logger, 'log'); - sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse); + sandbox.stub(Diagnostic.prototype, 'on').yields(null, fakeResponse); new Elasticsearch(configFile); expect(loggerLogStub.calledWith(fakeResponse)).to.be.false; @@ -254,26 +283,24 @@ describe('Elasticsearch', function () { describe('Operations with bundle/index', async function () { const sandbox = sinon.createSandbox(); let es: Elasticsearch; + let createStub: SinonStub; + let deleteStub: SinonStub; + let refreshStub: SinonStub; + let updateAliasesStub: SinonStub; + let existsStub: SinonStub; const oldIndex = 'stapps_footype_foosource_oldindex'; beforeEach(function () { + sandbox + .stub(Indices.prototype, 'getAlias') + .resolves({[oldIndex]: {aliases: {[SCThingType.Book]: {}}}} as any); + sandbox.stub(Indices.prototype, 'putTemplate').resolves({} as any); + createStub = sandbox.stub(Indices.prototype, 'create').resolves({} as any); + deleteStub = sandbox.stub(Indices.prototype, 'delete').resolves({} as any); + existsStub = sandbox.stub(Indices.prototype, 'exists').resolves({} as any); + refreshStub = sandbox.stub(Indices.prototype, 'refresh').resolves({} as any); + updateAliasesStub = sandbox.stub(Indices.prototype, 'updateAliases').resolves({} as any); es = new Elasticsearch(configFile); - es.client.indices = { - // @ts-expect-error not assignable - getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}), - // @ts-expect-error not assignable - putTemplate: () => Promise.resolve({}), - // @ts-expect-error not assignable - create: () => Promise.resolve({}), - // @ts-expect-error not assignable - delete: () => Promise.resolve({}), - // @ts-expect-error not assignable - exists: () => Promise.resolve({}), - // @ts-expect-error not assignable - refresh: () => Promise.resolve({}), - // @ts-expect-error not assignable - updateAliases: () => Promise.resolve({}), - }; }); afterEach(function () { @@ -286,8 +313,8 @@ describe('Elasticsearch', function () { }); it('should reject (throw an error) if the index name is not valid', async function () { - sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`); sandbox.createStubInstance(Client, {}); + sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex}`); await es.init(); return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index'); @@ -295,9 +322,8 @@ describe('Elasticsearch', function () { it('should create a new index', async function () { const index = getIndex(); - sandbox.stub(Elasticsearch, 'getIndex').returns(index); + sandbox.stub(utilModule, 'getThingIndexName').returns(index); const putTemplateStub = sandbox.stub(templating, 'putTemplate'); - const createStub = sandbox.stub(es.client.indices, 'create'); await es.init(); await es.bulkCreated(bulk); @@ -313,21 +339,19 @@ describe('Elasticsearch', function () { sandbox.restore(); }); it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () { - sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex()); - const clientDeleteStub = sandbox.stub(es.client.indices, 'delete'); + sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex()); await es.bulkExpired({...bulk, state: 'in progress'}); - expect(clientDeleteStub.called).to.be.true; + expect(deleteStub.called).to.be.true; }); it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () { - sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex()); - const clientDeleteStub = sandbox.stub(es.client.indices, 'delete'); + sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex()); await es.bulkExpired({...bulk, state: 'done'}); - expect(clientDeleteStub.called).to.be.false; + expect(deleteStub.called).to.be.false; }); }); @@ -337,13 +361,23 @@ describe('Elasticsearch', function () { }); it('should reject if the index name is not valid', async function () { - sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`); + sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex()}`); sandbox.createStubInstance(Client, {}); await es.init(); return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index'); }); + it("should create templates if index doesn't exist", async function () { + await es.init(); + existsStub.resolves(false); + const putTemplateSpy = sandbox.spy(templating, 'putTemplate'); + await es.bulkUpdated(bulk); + + expect(createStub.called).to.be.true; + expect(putTemplateSpy.called).to.be.true; + }); + it('should create a new index', async function () { const index = getIndex(); const expectedRefreshActions = [ @@ -354,15 +388,12 @@ describe('Elasticsearch', function () { remove: {index: oldIndex, alias: SCThingType.Book}, }, ]; - sandbox.stub(Elasticsearch, 'getIndex').returns(index); + sandbox.stub(utilModule, 'getThingIndexName').returns(index); sandbox.stub(es, 'aliasMap').value({ [SCThingType.Book]: { [bulk.source]: oldIndex, }, }); - const refreshStub = sandbox.stub(es.client.indices, 'refresh'); - const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases'); - const deleteStub = sandbox.stub(es.client.indices, 'delete'); sandbox.stub(templating, 'putTemplate'); await es.init(); @@ -371,9 +402,7 @@ describe('Elasticsearch', function () { expect(refreshStub.calledWith({index})).to.be.true; expect( updateAliasesStub.calledWith({ - body: { - actions: expectedRefreshActions, - }, + actions: expectedRefreshActions, }), ).to.be.true; expect(deleteStub.called).to.be.true; @@ -394,20 +423,19 @@ describe('Elasticsearch', function () { }); it('should reject if object is not found', async function () { - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}}); + sandbox.stub(es.client, 'search').resolves(searchResponse()); return expect(es.get('123')).to.rejectedWith('found'); }); it('should provide the thing if object is found', async function () { - const foundObject: ElasticsearchObject = { + const foundObject: SearchHit = { _id: '', _index: '', _score: 0, - _type: '', _source: message as SCMessage, }; - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}}); + sandbox.stub(es.client, 'search').resolves(searchResponse(foundObject)); return expect(await es.get('123')).to.be.eql(message); }); @@ -428,56 +456,54 @@ describe('Elasticsearch', function () { it('should not post if the object already exists in an index which will not be rolled over', async function () { const index = getIndex(); const oldIndex = index.replace('foosource', 'barsource'); - const object: ElasticsearchObject = { + const object: SearchHit = { _id: '', _index: oldIndex, _score: 0, - _type: '', _source: message as SCMessage, }; - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}}); - sandbox.stub(Elasticsearch, 'getIndex').returns(index); + sandbox.stub(es.client, 'search').resolves(searchResponse(object)); + sandbox.stub(utilModule, 'getThingIndexName').returns(index); - return expect(es.post(object._source, bulk)).to.rejectedWith('exist'); + return expect(es.post(object._source!, bulk)).to.rejectedWith('exist'); }); it('should not reject if the object already exists but in an index which will be rolled over', async function () { - const object: ElasticsearchObject = { + const object: SearchHit = { _id: '', _index: getIndex(), _score: 0, - _type: '', _source: message as SCMessage, }; - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}}); + sandbox.stub(es.client, 'search').resolves(searchResponse(object)); // return index name with different generated UID (see getIndex method) - sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex()); + sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex()); - return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist'); + return expect(es.post(object._source!, bulk)).to.not.rejectedWith('exist'); }); it('should reject if there is an object creation error on the elasticsearch side', async function () { - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}}); - sandbox.stub(es.client, 'create').resolves({body: {created: false}}); + sandbox.stub(es.client, 'search').resolves(searchResponse()); + sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse); return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation'); }); it('should create a new object', async function () { let caughtParameter: any; - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}}); + sandbox.stub(es.client, 'search').resolves(searchResponse()); // @ts-expect-error call const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => { caughtParameter = parameter; - return Promise.resolve({body: {created: true}}); + return Promise.resolve({result: 'created'}); }); await es.post(message as SCMessage, bulk); expect(createStub.called).to.be.true; - expect(caughtParameter.body).to.be.eql({ + expect(caughtParameter.document).to.be.eql({ ...message, - creation_date: caughtParameter.body.creation_date, + creation_date: caughtParameter.document.creation_date, }); }); }); @@ -493,29 +519,27 @@ describe('Elasticsearch', function () { sandbox.restore(); }); it('should reject to put if the object does not already exist', async function () { - const object: ElasticsearchObject = { + const object: SearchHit = { _id: '', _index: getIndex(), _score: 0, - _type: '', _source: message as SCMessage, }; - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}}); + sandbox.stub(es.client, 'search').resolves(searchResponse()); - return expect(es.put(object._source)).to.rejectedWith('exist'); + return expect(es.put(object._source!)).to.rejectedWith('exist'); }); // noinspection JSUnusedLocalSymbols it('should update the object if it already exists', async function () { let caughtParameter: any; - const object: ElasticsearchObject = { + const object: SearchHit = { _id: '', _index: getIndex(), _score: 0, - _type: '', _source: message as SCMessage, }; - sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}}); + sandbox.stub(es.client, 'search').resolves(searchResponse(object)); // @ts-expect-error unused // eslint-disable-next-line @typescript-eslint/no-unused-vars const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => { @@ -523,7 +547,7 @@ describe('Elasticsearch', function () { return Promise.resolve({body: {created: true}}); }); - await es.put(object._source); + await es.put(object._source!); expect(caughtParameter.body.doc).to.be.eql(object._source); }); @@ -532,18 +556,16 @@ describe('Elasticsearch', function () { describe('search', async function () { let es: Elasticsearch; const sandbox = sinon.createSandbox(); - const objectMessage: ElasticsearchObject = { + const objectMessage: SearchHit = { _id: '123', _index: getIndex(), _score: 0, - _type: '', _source: message as SCMessage, }; - const objectBook: ElasticsearchObject = { + const objectBook: SearchHit = { _id: '321', _index: getIndex(), _score: 0, - _type: '', _source: book as SCBook, }; const fakeEsAggregations = { @@ -565,26 +587,16 @@ describe('Elasticsearch', function () { }, }, }; - const fakeSearchResponse: Partial>> = { - body: { - took: 12, - timed_out: false, - // @ts-expect-error not assignable - _shards: {}, - // @ts-expect-error not assignable - hits: { - hits: [objectMessage, objectBook], - total: 123, - }, - aggregations: fakeEsAggregations, + const fakeSearchResponse: SearchResponse = { + took: 12, + timed_out: false, + // @ts-expect-error not assignable + _shards: {}, + hits: { + hits: [objectMessage, objectBook], + total: 123, }, - headers: {}, - // @ts-expect-error not assignable - meta: {}, - // @ts-expect-error not assignable - statusCode: {}, - // @ts-expect-error not assignable - warnings: {}, + aggregations: fakeEsAggregations, }; let searchStub: sinon.SinonStub; before(function () { @@ -625,9 +637,9 @@ describe('Elasticsearch', function () { const {pagination} = await es.search({from}); expect(pagination).to.be.eql({ - count: fakeSearchResponse.body!.hits.hits.length, + count: fakeSearchResponse.hits.hits.length, offset: from, - total: fakeSearchResponse.body!.hits.total, + total: fakeSearchResponse.hits.total, }); }); @@ -659,22 +671,20 @@ describe('Elasticsearch', function () { }, }, }; - const fakeResponse = {foo: 'bar'}; + const fakeResponse = {foo: 'bar'} as SortCombinations; const fakeBuildSortResponse = [fakeResponse]; // @ts-expect-error not assignable - sandbox.stub(query, 'buildQuery').returns(fakeResponse); - sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse); + sandbox.stub(queryModule, 'buildQuery').returns(fakeResponse); + sandbox.stub(sortModule, 'buildSort').returns(fakeBuildSortResponse); await es.search(parameters); sandbox.assert.calledWithMatch(searchStub, { - body: { - aggs: aggregations, - query: fakeResponse, - sort: fakeBuildSortResponse, - }, + aggs: aggregations, + query: fakeResponse, + sort: fakeBuildSortResponse, from: parameters.from, - index: Elasticsearch.getListOfAllIndices(), + index: ALL_INDICES_QUERY, size: parameters.size, }); }); diff --git a/test/storage/elasticsearch/monitoring.spec.ts b/test/storage/elasticsearch/monitoring.spec.ts index 52ccc35f..135f03e9 100644 --- a/test/storage/elasticsearch/monitoring.spec.ts +++ b/test/storage/elasticsearch/monitoring.spec.ts @@ -14,7 +14,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import {ApiResponse, Client} from '@elastic/elasticsearch'; +import {Client} from '@elastic/elasticsearch'; +import {SearchResponse} from '@elastic/elasticsearch/lib/api/types'; import { SCMonitoringConfiguration, SCMonitoringLogAction, @@ -23,7 +24,6 @@ import { SCThings, } from '@openstapps/core'; import {Logger} from '@openstapps/logger'; -import {SearchResponse} from 'elasticsearch'; import {MailQueue} from '../../../src/notification/mail-queue'; import {setUp} from '../../../src/storage/elasticsearch/monitoring'; @@ -111,16 +111,14 @@ describe('Monitoring', async function () { }); it('should log errors where conditions failed', async function () { - const fakeSearchResponse: Partial>> = { - body: { - took: 12, - timed_out: false, - // @ts-expect-error not assignable - _shards: {}, - // @ts-expect-error not assignable - hits: { - total: 123, - }, + const fakeSearchResponse: SearchResponse = { + took: 12, + timed_out: false, + // @ts-expect-error not assignable + _shards: {}, + // @ts-expect-error not assignable + hits: { + total: 123, }, }; const fakeClient = new Client({node: 'http://foohost:9200'}); diff --git a/test/storage/elasticsearch/query.spec.ts b/test/storage/elasticsearch/query.spec.ts index 7b4a108f..13fc8cd5 100644 --- a/test/storage/elasticsearch/query.spec.ts +++ b/test/storage/elasticsearch/query.spec.ts @@ -25,25 +25,14 @@ import { SCThingType, } from '@openstapps/core'; import {expect} from 'chai'; -import { - ESDateRangeFilter, - ESRangeFilter, - ESNumericRangeFilter, - ElasticsearchConfig, - ESBooleanFilter, - ESGenericSort, - ESGeoDistanceFilter, - ESGeoDistanceSort, - ESTermFilter, - ScriptSort, -} from '../../../src/storage/elasticsearch/types/elasticsearch'; +import {buildFilter} from '../../../src/storage/elasticsearch/query/filter'; +import {buildBooleanFilter} from '../../../src/storage/elasticsearch/query/filters/boolean'; +import {buildQuery} from '../../../src/storage/elasticsearch/query/query'; +import {buildSort} from '../../../src/storage/elasticsearch/query/sort'; +import {ElasticsearchConfig} from '../../../src/storage/elasticsearch/types/elasticsearch-config'; +import {QueryDslSpecificQueryContainer} from '../../../src/storage/elasticsearch/types/util'; import {configFile} from '../../../src/common'; -import { - buildBooleanFilter, - buildFilter, - buildQuery, - buildSort, -} from '../../../src/storage/elasticsearch/query'; +import {SortCombinations} from '@elastic/elasticsearch/lib/api/types'; describe('Query', function () { describe('buildBooleanFilter', function () { @@ -74,7 +63,7 @@ describe('Query', function () { or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}}, not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}}, }; - const expectedEsFilters: Array = [ + const expectedEsFilters: Array> = [ { term: { 'type.raw': 'catalog', @@ -88,20 +77,20 @@ describe('Query', function () { ]; it('should create appropriate elasticsearch "and" filter argument', function () { - const {must} = buildBooleanFilter(booleanFilters.and); + const {must} = buildBooleanFilter(booleanFilters.and).bool; expect(must).to.be.eql(expectedEsFilters); }); it('should create appropriate elasticsearch "or" filter argument', function () { - const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or); + const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or).bool; expect(should).to.be.eql(expectedEsFilters); expect(minimum_should_match).to.be.equal(1); }); it('should create appropriate elasticsearch "not" filter argument', function () { - const {must_not} = buildBooleanFilter(booleanFilters.not); + const {must_not} = buildBooleanFilter(booleanFilters.not).bool; expect(must_not).to.be.eql(expectedEsFilters); }); @@ -196,6 +185,10 @@ describe('Query', function () { expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type'); }); + + it('should accept other search contexts', function () { + expect(buildQuery({context: 'place', ...parameters}, config, esConfig)).to.be.an('object'); + }); }); describe('buildFilter', function () { @@ -267,7 +260,7 @@ describe('Query', function () { it('should build value filter', function () { const filter = buildFilter(searchFilters.value); - const expectedFilter: ESTermFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'term'> = { term: { 'type.raw': SCThingType.Dish, }, @@ -279,7 +272,7 @@ describe('Query', function () { it('should build numeric range filters', function () { for (const upperMode of ['inclusive', 'exclusive', null]) { for (const lowerMode of ['inclusive', 'exclusive', null]) { - const expectedFilter: ESNumericRangeFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'range'> = { range: { price: { relation: undefined, @@ -304,7 +297,7 @@ describe('Query', function () { mode: bound as 'inclusive' | 'exclusive', limit: out, }; - expectedFilter.range.price[ + expectedFilter.range.price![ `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}` ] = out; } @@ -312,7 +305,7 @@ describe('Query', function () { setBound('upperBound', upperMode); setBound('lowerBound', lowerMode); - const filter = buildFilter(rawFilter) as ESNumericRangeFilter; + const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'term'>; expect(filter).to.deep.equal(expectedFilter); for (const bound of ['g', 'l']) { // @ts-expect-error implicit any @@ -330,7 +323,7 @@ describe('Query', function () { it('should build date range filters', function () { for (const upperMode of ['inclusive', 'exclusive', null]) { for (const lowerMode of ['inclusive', 'exclusive', null]) { - const expectedFilter: ESDateRangeFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'range'> = { range: { price: { format: 'thisIsADummyFormat', @@ -359,7 +352,7 @@ describe('Query', function () { mode: bound as 'inclusive' | 'exclusive', limit: out, }; - expectedFilter.range.price[ + expectedFilter.range.price![ `${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}` ] = out; } @@ -367,7 +360,7 @@ describe('Query', function () { setBound('upperBound', upperMode); setBound('lowerBound', lowerMode); - const filter = buildFilter(rawFilter) as ESNumericRangeFilter; + const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'range'>; expect(filter).to.deep.equal(expectedFilter); for (const bound of ['g', 'l']) { // @ts-expect-error implicit any @@ -394,7 +387,7 @@ describe('Query', function () { }, }); - const expectedFilter: ESRangeFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'range'> = { range: { 'offers.availabilityRange': { gte: `test||/${scope}`, @@ -415,7 +408,7 @@ describe('Query', function () { }, }); - const expectedFilter: ESRangeFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'range'> = { range: { 'offers.availabilityRange': { gte: 'test||/s', @@ -436,7 +429,7 @@ describe('Query', function () { }, }); - const expectedFilter: ESRangeFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'range'> = { range: { 'offers.availabilityRange': { gte: `test||/d`, @@ -456,7 +449,7 @@ describe('Query', function () { }, }); - const expectedFilter: ESRangeFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'range'> = { range: { 'offers.availabilityRange': { gte: `now/d`, @@ -470,7 +463,7 @@ describe('Query', function () { it('should build distance filter', function () { const filter = buildFilter(searchFilters.distance); - const expectedFilter: ESGeoDistanceFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'geo_distance'> = { geo_distance: { 'distance': '1000m', 'geo.point.coordinates': { @@ -486,34 +479,18 @@ describe('Query', function () { it('should build geo filter for shapes and points', function () { const filter = buildFilter(searchFilters.geoPoint); const expectedFilter = { - bool: { - minimum_should_match: 1, - should: [ - { - geo_shape: { - 'geo.polygon': { - relation: undefined, - shape: { - type: 'envelope', - coordinates: [ - [50.123, 8.123], - [50.123, 8.123], - ], - }, - }, - 'ignore_unmapped': true, - }, + geo_shape: { + 'geo.polygon': { + relation: undefined, + shape: { + type: 'envelope', + coordinates: [ + [50.123, 8.123], + [50.123, 8.123], + ], }, - { - geo_bounding_box: { - 'geo.point.coordinates': { - bottom_right: [50.123, 8.123], - top_left: [50.123, 8.123], - }, - 'ignore_unmapped': true, - }, - }, - ], + }, + 'ignore_unmapped': true, }, }; @@ -543,7 +520,7 @@ describe('Query', function () { it('should build boolean filter', function () { const filter = buildFilter(searchFilters.boolean); - const expectedFilter: ESBooleanFilter = { + const expectedFilter: QueryDslSpecificQueryContainer<'bool'> = { bool: { minimum_should_match: 0, must: [ @@ -604,8 +581,8 @@ describe('Query', function () { }, }, ]; - let sorts: Array = []; - const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = { + let sorts: SortCombinations[] = []; + const expectedSorts: {[key: string]: SortCombinations} = { ducet: { 'name.sort': 'desc', }, @@ -632,7 +609,7 @@ describe('Query', function () { }, }; before(function () { - sorts = buildSort(searchSCSearchSort); + sorts = buildSort(searchSCSearchSort) as SortCombinations[]; }); it('should build ducet sort', function () { @@ -649,10 +626,10 @@ describe('Query', function () { it('should build price sort', function () { const priceSortNoScript = { - ...sorts[3], + ...(sorts[3] as any), _script: { - ...(sorts[3] as ScriptSort)._script, - script: (expectedSorts.price as ScriptSort)._script.script, + ...(sorts[3] as any)._script, + script: (expectedSorts.price as any)._script.script, }, }; expect(priceSortNoScript).to.be.eql(expectedSorts.price); diff --git a/tsconfig.json b/tsconfig.json index 4dbcef9c..c4b40d1f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,9 @@ "extends": "./node_modules/@openstapps/configuration/tsconfig.json", "compilerOptions": { "resolveJsonModule": true, - "useUnknownInCatchVariables": false + "skipLibCheck": true, + "useUnknownInCatchVariables": false, + "lib": ["ES2020"] }, "exclude": [ "./config/",