mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2025-12-27 08:36:14 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
f4b2d747a3 | ||
|
|
4ebe44a5a7 | ||
|
|
3471591a7d | ||
|
d16ae93a7a | ||
|
|
de71d68051 | ||
|
c9b83b5d71 | ||
|
|
515a6eeea5 | ||
|
|
3e490aeeb9 |
@@ -46,7 +46,7 @@ integration:
|
||||
services:
|
||||
- docker:dind
|
||||
script:
|
||||
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
- docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --abort-on-container-exit --exit-code-from apicli
|
||||
tags:
|
||||
- gitlab-org-docker
|
||||
@@ -110,7 +110,7 @@ ci:
|
||||
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
|
||||
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
|
||||
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
|
||||
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
|
||||
- docker push $IMAGETAG_BASE
|
||||
except:
|
||||
@@ -124,22 +124,22 @@ ci:
|
||||
.publish_version_template: &publish_version_template
|
||||
script:
|
||||
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
|
||||
- export VERSION=$(echo -n "$CI_BUILD_REF_NAME" | cut -c 2-)
|
||||
- export VERSION=$(echo -n "$CI_COMMIT_REF_NAME" | cut -c 2-)
|
||||
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE
|
||||
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
|
||||
- export IMAGETAG_VERSION=$IMAGETAG_BASE:$VERSION
|
||||
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
|
||||
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_VERSION -t $IMAGETAG_CORE_VERSION .
|
||||
- docker push $IMAGETAG_BASE
|
||||
|
||||
.publish_branch_template: &publish_branch_template
|
||||
script:
|
||||
- export CORE_VERSION=$(openstapps-projectmanagement get-used-version @openstapps/core)
|
||||
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_BUILD_REF_NAME
|
||||
- export IMAGETAG_BASE=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_NAME
|
||||
- export IMAGETAG_CORE_VERSION=$IMAGETAG_BASE:core-$CORE_VERSION
|
||||
- export IMAGETAG_LATEST=$IMAGETAG_BASE:latest
|
||||
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
- docker build -t $IMAGETAG_LATEST -t $IMAGETAG_CORE_VERSION .
|
||||
- docker push $IMAGETAG_BASE
|
||||
|
||||
|
||||
16
CHANGELOG.md
16
CHANGELOG.md
@@ -1,3 +1,19 @@
|
||||
# [1.0.0](https://gitlab.com/openstapps/backend/compare/v0.6.0...v1.0.0) (2023-05-08)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* openapi docs generation ([4ebe44a](https://gitlab.com/openstapps/backend/commit/4ebe44a5a7a1b7bfd0aa5b84d47d4056d3068ffe))
|
||||
* rename deprecated Gitlab CI variables ([3471591](https://gitlab.com/openstapps/backend/commit/3471591a7d458df70447c8dac91f96f3c83e763c))
|
||||
* semster boosting ([515a6ee](https://gitlab.com/openstapps/backend/commit/515a6eeea56305a37510d99b9f84a6b118b66f8a))
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* update to of elasticsearch 8.4 ([c9b83b5](https://gitlab.com/openstapps/backend/commit/c9b83b5d71610f82bd1d99e837e29ad445758aea))
|
||||
|
||||
|
||||
|
||||
# [0.6.0](https://gitlab.com/openstapps/backend/compare/v0.5.0...v0.6.0) (2023-01-30)
|
||||
|
||||
|
||||
|
||||
@@ -18,9 +18,10 @@ you with everything you need to run this backend.
|
||||
|
||||
# Local usage for development purposes
|
||||
## Requirements
|
||||
* Elasticsearch (5.6)
|
||||
* Elasticsearch (8.4)
|
||||
- [ICU analysis plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html)
|
||||
- OR Docker
|
||||
* Node.js (~14) / NPM
|
||||
* Docker
|
||||
|
||||
### Startup Behaviour
|
||||
|
||||
@@ -34,7 +35,7 @@ with the backend. To save you some work we provide a
|
||||
[docker image](https://gitlab.com/openstapps/database) which
|
||||
only needs to be executed to work with the backend.
|
||||
|
||||
Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:master`
|
||||
Run `docker run -d -p 9200:9200 registry.gitlab.com/openstapps/database:latest`
|
||||
|
||||
Elasticsearch should be running at port 9200 now. If you have problems with
|
||||
getting elasticsearch to work, have a look in the
|
||||
|
||||
@@ -17,536 +17,341 @@ const markdownSources: {
|
||||
} = {en: {}, de: {}};
|
||||
|
||||
markdownSources.de!.privacyPolicy = `
|
||||
Diese Datenschutzerklärung dient zur Erfüllung der nach Artikel 13 EU DSGVO geforderten Informationspflicht bei Erhebung von Daten zum Zeitpunkt der Erhebung bei betroffenen Personen.
|
||||
# Datenschutzerklärung
|
||||
|
||||
# **Name und Anschrift des Verantwortlichen**
|
||||
Johann Wolfgang Goethe-Universität Frankfurt am Main<br />
|
||||
## Kontaktdaten des Verantwortlichen
|
||||
|
||||
Verantwortlich im Sinne der Datenschutz-Grundverordnung und weiterer Vorschriften zum Datenschutz ist die:
|
||||
|
||||
Johann Wolfgang Goethe-Universität Frankfurt am Main vertreten durch ihren Präsidenten<br />
|
||||
Theodor-W.-Adorno-Platz 1<br />
|
||||
60323 Frankfurt am Main
|
||||
|
||||
Postanschrift:<br />
|
||||
Goethe-Universität Frankfurt am Main<br />
|
||||
60629 Frankfurt<br />
|
||||
60629 Frankfurt
|
||||
|
||||
Telefon: +49-69-798-0 | Fax: +49-69-798-18383<br />
|
||||
Internet: http://www.uni-frankfurt.de<br />
|
||||
Website: http://www.uni-frankfurt.de
|
||||
|
||||
Bei Anfragen oder Beschwerden zum Datenschutz können Sie sich mit den Datenschutzbeauftragten der Goethe-Universität in Verbindung setzen.
|
||||
## Kontaktdaten der Datenschutzbeauftragten an der Goethe-Universität
|
||||
|
||||
# **Kontaktdaten der Datenschutzbeauftragten**
|
||||
Johann Wolfgang Goethe-Universität Frankfurt am Main<br />
|
||||
Die behördlichen Datenschutzbeauftragten<br />
|
||||
Theodor-W.-Adorno-Platz 1<br />
|
||||
60323 Frankfurt am Main<br />
|
||||
Sie erreichen die behördlichen Datenschutzbeauftragten der Johann Wolfgang Goethe-Universität Frankfurt am Main unter:<br />
|
||||
Mail: <dsb@uni-frankfurt.de><br />
|
||||
Website: http://www.uni-frankfurt.de/47859992/datenschutzbeauftragte
|
||||
|
||||
Internet: http://www.uni-frankfurt.de/47859992/datenschutzbeauftragte
|
||||
## Informationen zur Verarbeitung personenbezogener Daten
|
||||
|
||||
# **Rechte und Beschwerdemöglichkeiten**
|
||||
Sie haben das Recht sich bei datenschutzrechtlichen Problemen bei der zuständigen Fachaufsichtsbehörde zu beschweren.<br />
|
||||
### <u>1. Umfang der Verarbeitung personenbezogener Daten</u>
|
||||
|
||||
Kontaktadresse der Fachaufsichtsbehörde der Goethe-Universität Frankfurt am Main:<br />
|
||||
Personenbezogene Daten sind gemäß Artikel 4 DSGVO alle Informationen, die sich auf eine identifizierte oder identifizierbare natürliche Person beziehen.
|
||||
|
||||
Der Hessische Datenschutzbeauftragte<br />
|
||||
Wir verarbeiten personenbezogene Daten von Ihnen als Nutzer:innen der Goethe-Uni-App, soweit dies zur Bereitstellung einer **funktionsfähigen Applikation** technisch erforderlich ist.
|
||||
|
||||
Weiterhin kann eine Datenverarbeitung auf Ihrer freiwilligen Einwilligung basieren, wenn Sie **spezifische Funktionen** nutzen möchten.
|
||||
|
||||
Wir unterscheiden daher nachfolgend zwischen
|
||||
|
||||
- Zugriffsdaten bei der Nutzung der App: Inhalt der Anfragen, IP-Adressen, Datum/Uhrzeit der Anfrage, Angefragte URL, Fehlermeldungen, Browser-Kennung, HTTP-Header
|
||||
|
||||
- Standortbestimmung und Navigation: freiwillige Standortangaben
|
||||
|
||||
- Nutzer:inneneinstellungen: freiwillige Angabe von a) Sprachpräferenzen (derzeit: deutsch/englisch), b) Status (z. B. Gast/Student) oder c) spezifischen Suchanfragen und Suchergebnissen (Notifications)
|
||||
|
||||
- Kalenderfunktion: freiwillige Nutzung der Kalenderfunktion (optional mit freiwilliger Nutzung einer Synchronisationsfunktion: Opt-in) oder der integrierten Stundenplanfunktion, hierbei werden folgende Daten auf dem Endgerät verarbeitet und gespeichert: Termine und Veranstaltungen
|
||||
|
||||
- Feedbackfunktion und Kontaktaufnahme: freiwillige Nutzung mit der Angabe von Kontaktdaten und ggf. freiwilliger Übermittlung von Protokolldaten
|
||||
|
||||
- Campus Dienste: freiwillige Nutzung mit Verarbeitung von Notenansicht, Matrikelnummer, E-Mailadresse, Name
|
||||
|
||||
- Funktionen der Bibliothek: freiwillige Nutzung mit Verarbeitung von Bibliothekskontodaten, wie z.B. Ausweisnummer mit Name, E-Mailadresse, postalischer Adresse, Nutzungsberechtigung, Bestelldaten, Gebühren, Vormerkung, Ausleihdaten. Die vollständigen Angaben zur Verarbeitung finden Sie in der Datenschutzerklärung der Bibliothek:<br />
|
||||
https://www.ub.uni-frankfurt.de/benutzung/datenschutz.html
|
||||
|
||||
Die App verlinkt an einigen Stellen auf die Website der Goethe-Universität sowie auf andere, externe Websites, die in einem In-App-Browser dargestellt werden. Wir bitten Sie bei Aufruf dieser Websites, die dort geltenden gesonderte Datenschutzhinweise und Erklärungen zu beachten.
|
||||
|
||||
### <u>2. Zweck(e) der Datenverarbeitung</u>
|
||||
|
||||
**Zugriff auf Standortdaten**
|
||||
|
||||
Für die Navigation benötigt die Goethe-Uni-App Zugriff auf den Standort des verwendeten Endgerätes (Location Based Services). Bei einer Anfrage erhebt die App den aktuellen Standort über GPS, Funkzellendaten und WLAN-Datenbanken, um Ihnen als Nutzer:in Informationen zu Ihrer unmittelbaren Umgebung geben zu können. Der Zugriff auf die Standortdaten erfolgt nur, wenn Sie den Zugriff auf die Standortdaten erlauben. Daten zu Ihrem Standort werden ausschließlich für die Bearbeitung von standortbezogenen Anfragen genutzt und um Ihren Standort auf der Karte anzuzeigen.
|
||||
|
||||
**Zugriff auf Zugriffsdaten**
|
||||
|
||||
Die Speicherung und Verarbeitung von Protokolldateien erfolgt, um die Funktionsfähigkeit der Goethe Uni-App für Sie sicherzustellen. Zudem benötigen wir die die Daten aus Gründen der Sicherheit unserer informationstechnischen Systeme. Eine anderweitige Auswertung oder Weitergabe findet in diesem Zusammenhang nicht statt.
|
||||
|
||||
**Zugriff auf Spracheinstellungen**
|
||||
|
||||
Der Zugriff auf die Spracheinstellung erfolgt um Ihnen die Oberfläche der App in der von Ihnen gewünschten Sprache anzuzeigen.
|
||||
|
||||
**Zugriff auf die Einstellung der Statusgruppe**
|
||||
|
||||
Der Zugriff auf die Einstellung der Statusgruppe erfolgt um Ihnen in der App die für Ihre Gruppe zutreffenden Informationen anzuzeigen, z.B. Mensapreise
|
||||
|
||||
**Zugriff auf personenbezogene Daten bei der Nutzung der Feedbackfunktion**
|
||||
|
||||
Die Verarbeitung der personenbezogenen Daten aus der Feedbackfunktion dient uns zur Kontaktaufnahme und Fehlerbehebung.
|
||||
|
||||
**Zugriff auf personenbezogene Daten bei der Kalendersynchronisation**
|
||||
|
||||
Der Zugriff auf die Termindaten erfolgt um sie bei aktivierter Kalenderfunktion in den Gerätekalender zu schreiben.
|
||||
|
||||
**Zugriff auf Daten der Campus Dienste**
|
||||
|
||||
Der Zugriff auf das Campus Management Systems erfolgt ausschließlich um persönliche Daten der Studierendenverwaltung in der App anzuzeigen (z.B. Prüfungsnoten).
|
||||
|
||||
**Zugriff auf bibliotheksspezifische personenbezogene Daten**
|
||||
|
||||
Der Zugriff auf die Daten (z.B. Ausweisnummer, Name, Postanschrift) erfolgt zur Durchführung von Bestell- und Ausleihverfahren von Büchern und sonstigen Materialien der Universitätsbibliothek. Die vollständigen Angaben zu den Verarbeitungszwecken finden Sie in der Datenschutzerklärung der Bibliothek: https://www.ub.uni-frankfurt.de/benutzung/datenschutz.html
|
||||
|
||||
### <u>3. Rechtsgrundlage(n) für die Datenverarbeitung</u>
|
||||
|
||||
Die Nutzung der Nutzungs-/Zugriffsdaten („Protokolldateien") basiert auf Artikel 6 Absatz 1 lit. f) DSGVO.
|
||||
|
||||
Für alle spezifischen Funktionen, bei denen die Datenverarbeitung auf Ihrer freiwilligen Einwilligung als Nutzer:innen basiert, werden explizit Einwilligungen bzw. aktive Zustimmungsakte („Opt-In") eingeholt. Die Bereitstellung personenbezogener Daten zu Ihrer Person gegenüber der Goethe-Universität erfolgen dabei auf freiwilliger Basis. Die Rechtsgrundlage ist in diesen Fällen jeweils Artikel 6 Absatz 1 lit. a) DSGVO. Sie können Ihre jeweilige Einwilligung jederzeit einzeln widerrufen bzw. Ihre Einstellungen ändern.
|
||||
|
||||
### <u>4. Datenlöschung und Speicherdauer</u>
|
||||
|
||||
Die in den Protokolldateien der App erfassten Daten werden sieben Tage nach dem Ende des Zugriffs automatisch gelöscht oder anonymisiert.
|
||||
|
||||
Die Löschfristen bzw. Speicherdauer der in den Bibliotheksystemen erfassten Daten finden Sie in der Datenschutzerklärung der Bibliothek: https://www.ub.uni-frankfurt.de/benutzung/datenschutz.html
|
||||
|
||||
Für alle anderen Funktionen und Dienste gilt: Die Löschung erfolgt hier je nach Vorgabe des genutzten Dienstes. Die personenbezogenen Daten der betroffenen Person werden gelöscht oder gesperrt, sobald der Zweck der Speicherung entfällt.
|
||||
|
||||
### <u>5. Datenweitergabe/Datenübermittlung</u>
|
||||
|
||||
Ihre personenbezogenen Daten werden von uns nicht an Dritte weitergegeben.
|
||||
|
||||
Von Betreiberseite wird durch technische und organisatorische Maßnahmen sichergestellt, dass Dritte keinen Zugriff auf die verarbeiteten Daten, wie z. B. Nutzungsdaten, erhalten. Ein Auftragsverarbeitungsverhältnis nach Art. 28 DSGVO besteht nicht, da ausschließlich eigene Server verwendet werden.
|
||||
|
||||
### <u>6. Automatisierte Entscheidungsfindung</u>
|
||||
|
||||
Eine automatisierte Entscheidungsfindung einschließlich Profiling erfolgt nicht.
|
||||
|
||||
## Rechte der betroffenen Person
|
||||
|
||||
Werden personenbezogene Daten von Ihnen verarbeitet, sind Sie Betroffener im Sinne der DSGVO. Die Geltendmachung Ihrer Betroffenenrechte ist kostenfrei. Sie können sich dafür selbstverständlich an uns wenden. Es stehen Ihnen folgende Betroffenenrechte gegenüber der Goethe-Universität zu:
|
||||
|
||||
### <u>1. Auskunftsrecht</u>
|
||||
|
||||
Sie können von uns als verantwortlicher Stelle eine Bestätigung darüber verlangen, ob und welche Ihrer personenbezogenen Daten von uns verarbeitet werden. Sie haben das Recht, von uns Kopien Ihrer personenbezogenen Daten zu verlangen. Bitte beachten Sie die Ausnahmen, die sich durch spezifische Vorschriften ergeben können.
|
||||
|
||||
### <u>2. Recht auf Berichtigung</u>
|
||||
|
||||
Sie haben das Recht von uns die Berichtigung und/oder Vervollständigung zu verlangen, sofern die verarbeiteten personenbezogenen Daten, die Sie betreffen, nicht (mehr) richtig oder nicht (mehr) vollständig sind.
|
||||
|
||||
### <u>3. Recht auf Einschränkung der Verarbeitung</u>
|
||||
|
||||
Unter bestimmten Voraussetzungen können Sie die Einschränkung der Verarbeitung der Sie betreffenden personenbezogenen Daten verlangen, d. h. dass dann Ihre personenbezogenen Daten zwar nicht gelöscht, aber gekennzeichnet werden, so dass eine weitere Verarbeitung eingeschränkt ist.
|
||||
|
||||
### <u>4. Recht auf Löschung</u>
|
||||
|
||||
Sie können unter bestimmten Voraussetzungen von uns verlangen, dass die Sie betreffenden personenbezogenen Daten unverzüglich gelöscht werden. Dies ist insbesondere der Fall, wenn die personenbezogenen Daten zu dem Zweck, zu dem sie ursprünglich erhoben oder verarbeitet wurden, nicht mehr erforderlich sind.
|
||||
|
||||
### <u>5. Recht auf Unterrichtung</u>
|
||||
|
||||
Haben Sie das Recht auf Berichtigung, Löschung oder Einschränkung der Verarbeitung uns gegenüber geltend gemacht, sind wir verpflichtet, allen Empfänger/innen, denen die Sie betreffenden personenbezogenen Daten offengelegt wurden, diese Berichtigung oder Löschung der Daten oder Einschränkung der Verarbeitung mitzuteilen, es sei denn, dies erweist sich als unmöglich oder ist mit einem unverhältnismäßigen Aufwand verbunden. Sie sind berechtigt, über diese Empfänger unterrichtet zu werden.
|
||||
|
||||
### <u>6. Recht auf Datenübertragbarkeit</u>
|
||||
|
||||
Sie haben unter bestimmten Voraussetzungen das Recht von uns zu verlangen, dass Ihre personenbezogenen Daten von uns direkt an einen anderen Verantwortlichen oder an eine andere Organisation übermittelt werden. Alternativ haben Sie unter bestimmten Voraussetzungen das Recht von uns zu verlangen, dass wir Ihnen selbst die Daten in einem maschinenlesbaren Format bereitstellen.
|
||||
|
||||
### <u>7. Widerspruchsrecht</u>
|
||||
|
||||
Wenn wir Ihre personenbezogenen Daten verarbeiten, weil die Verarbeitung im öffentlichen Interesse, Teil unserer öffentlichen Aufgaben ist bzw. wenn wir Ihre Daten auf Basis eines berechtigten Interesses verarbeiten, haben Sie aus Gründen, die sich aus Ihrer besonderen Situation ergeben, das Recht, jederzeit der Verarbeitung der Sie betreffenden Daten zu widersprechen.
|
||||
|
||||
### <u>8. Recht auf Widerruf der datenschutzrechtlichen Einwilligungserklärung</u>
|
||||
|
||||
Wenn wir Ihre personenbezogenen Daten verarbeiten, weil Sie uns Ihre Einwilligung gegeben haben, haben Sie jederzeit das Recht, Ihre Einwilligungserklärung zu widerrufen.
|
||||
|
||||
### <u>9. Recht auf Beschwerde bei einer Aufsichtsbehörde</u>
|
||||
|
||||
Sie haben ferner das Recht auf Beschwerde bei einer Aufsichtsbehörde. Die zuständige Aufsichtsbehörde wird Ihre Beschwerde prüfen.
|
||||
|
||||
## **Kontaktdaten der Aufsichtsbehörde im Bereich Datenschutz**
|
||||
|
||||
Wenn Sie der Ansicht sind, dass eine Verarbeitung der Sie betreffenden personenbezogenen Daten gegen Datenschutzvorschriften verstößt, wenn Sie eine allgemeine Anfrage haben oder wenn Sie sich bei einer zuständigen Fachaufsichtsbehörde beschweren wollen, können Sie sich an den Hessischen Beauftragten für Datenschutz und Informationsfreiheit (HBDI) wenden.
|
||||
|
||||
**Der Hessische Beauftragte für Datenschutz und Informationsfreiheit ist auf unterschiedlichen Wegen erreichbar:**
|
||||
|
||||
<u>**Der Hessische Beauftragte für Datenschutz und Informationsfreiheit**</u><br />
|
||||
Postfach 3163<br />
|
||||
65021 Wiesbaden<br /><br />
|
||||
E-Mail an HDSB (Link zum Kontaktformular des Hessischen Datenschutzbeauftragten:
|
||||
https://datenschutz.hessen.de/über-uns/kontakt)<br />
|
||||
65021 Wiesbaden
|
||||
|
||||
Telefon: +49 611 1408 - 0<br />
|
||||
Telefax: +49 611 1408 – 611
|
||||
Telefon: +49 611 1408 -- 0
|
||||
|
||||
Sie haben gegenüber der Goethe-Universität folgende Rechte hinsichtlich Ihrer gespeicherten personenbezogenen Daten:
|
||||
* Recht auf Auskunft,
|
||||
* Recht auf Berichtigung oder Löschung,
|
||||
* Recht auf Einschränkung der Verarbeitung,
|
||||
* Recht auf Widerruf Ihrer Einwilligung,
|
||||
* Recht auf Widerspruch gegen die Verarbeitung,
|
||||
* Recht auf Datenübertragbarkeit, in einer gängigen, strukturierten und maschinenlesbaren Form
|
||||
(ab dem 25. Mai 2018).
|
||||
|
||||
Zur Geltungsmachung dieser Rechte wenden Sie sich an <a href="mailto:app@rz.uni-frankfurt.de">app@rz.uni-frankfurt.de</a>.<br />
|
||||
|
||||
# **Art der gespeicherten Daten, Zweck und Rechtgrundlagen, Löschungsfristen**
|
||||
|
||||
**Umgang mit personenbezogenen Daten**<br />
|
||||
|
||||
Personenbezogene Daten sind Informationen, mit deren Hilfe eine natürliche Person bestimmbar ist, also Angaben, durch die Personen identifizierbar sind. Dazu gehören insbesondere Namen, E-Mail-Adressen, Matrikelnummern oder Telefonnummern. Aber auch Daten über Vorlieben, Hobbies, Mitgliedschaften oder auch Informationen über Webseiten, die aufgesucht wurden, zählen zu personenbezogenen Daten.<br />
|
||||
|
||||
Personenbezogene Daten werden von uns nur dann erhoben, genutzt und weitergegeben, wenn dies gesetzlich erlaubt ist oder die Nutzer in die Datenerhebung einwilligt haben.<br />
|
||||
|
||||
Die Nutzung personenbezogener Daten der Studierenden zum Zwecke des Studiums basieren weitestgehend auf dem geltenden Hessischen Hochschulgesetz in Verbindung mit der geltenden Immatrikulationsverordnung des Landes Hessen und beziehen sich somit auf EU DSGVO Artikel 6 Absatz 1 c).<br />
|
||||
|
||||
Die Daten der Beschäftigten der Goethe-Universität zum Zwecke der Personalverwaltung, der Lehr-, Forschungs- und Prüfungstätigkeiten werden auf Basis des Hessischen Hochschulgesetzes, der Immatrikulationsverordnung des Landes Hessen, TV-GU, beamtenrechtliche und personalrechtliche Regelungen erhoben und verarbeitet.<br />
|
||||
|
||||
**Zugriffsdaten/Server-Logdateien**<br />
|
||||
|
||||
Beim Zugriff auf die Seiten dieses Webservers werden im Allgemeinen folgende Daten in den Server-Logfiles gespeichert
|
||||
1. IP-Adresse
|
||||
2. Datum und Uhrzeit
|
||||
3. Typ des Client Browsers
|
||||
4. URL der aufgerufenen Seite
|
||||
5. Gegebenenfalls die Fehlermeldung zum aufgetretenen Fehler
|
||||
6. Gegebenenfalls der anfragende Provider
|
||||
|
||||
Diese Daten dienen ausschließlich zum Zwecke der Kontrolle der Funktionalität, der Sicherheit und Fehlerbehebung. Diese Nutzung basiert auf EU DSGVO Artikel 6 Absatz 1 f). Alle Logdateien werden automatisiert nach spätestens 7 Tagen gelöscht oder anonymisiert.
|
||||
|
||||
**Kontaktaufnahme**<br />
|
||||
|
||||
Zur Kontaktaufnahme mit Mitgliedern der Goethe-Universität (zum Beispiel per Kontaktformular oder E-Mail) werden Ihre Angaben zwecks Bearbeitung der Anfrage sowie für den Fall, dass Anschlussfragen entstehen, gespeichert. Nach Bearbeitung Ihres Anliegens bzw. nach Erfüllung der Rechtspflicht oder des genutzten Dienstes werden die Daten gelöscht, es sei denn, die Aufbewahrung der Daten ist zur Umsetzung berechtigter Interessen der Goethe Universität oder auf Grund einer gesetzlichen Vorschrift (z.B. Gesetz, Rechtsverordnung, Satzung der Goethe Universität etc.) erforderlich.
|
||||
|
||||
**Einbindung von Diensten Dritter**<br />
|
||||
|
||||
Innerhalb einiger Seiten dieses Onlineangebotes werden Inhalte Dritter, (wie z.B. Videos von YouTube, Kartenmaterial von Google-Maps, RSS-Feeds, Grafiken, etc.) von anderen Webseiten eingebunden. Dies setzt immer voraus, dass die Anbieter dieser Inhalte (nachfolgend bezeichnet als "Dritt-Anbieter") Ihre IP-Adresse wahrnehmen. Denn ohne die IP-Adresse könnten die Dritt-Anbieter die Inhalte nicht an Ihren Browser senden. Die IP-Adresse ist damit für die Darstellung dieser Inhalte erforderlich. Wir bemühen uns nur solche Inhalte zu verwenden, deren jeweilige Anbieter die IP-Adresse lediglich zur Auslieferung der Inhalte verwenden. Jedoch haben wir auf eine weitere Verwendung Ihre Daten keinen Einfluss (z.B. falls die Dritt-Anbieter die IP-Adresse für statistische Zwecke speichern).
|
||||
|
||||
# **Artikel 13 EU DSGVO**
|
||||
## **Informationspflicht bei Erhebung von personenbezogenen Daten bei der betroffenen Person**
|
||||
|
||||
1. Werden personenbezogene Daten bei der betroffenen Person erhoben, so teilt der
|
||||
Verantwortliche der betroffenen Person zum Zeitpunkt der Erhebung dieser Daten Folgendes mit:
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>den Namen und die Kontaktdaten des Verantwortlichen sowie gegebenenfalls seines
|
||||
Vertreters;</li>
|
||||
<li>gegebenenfalls die Kontaktdaten des Datenschutzbeauftragten;
|
||||
<li>die Zwecke, für die die personenbezogenen Daten verarbeitet werden sollen, sowie die Rechtsgrundlage für die Verarbeitung;</li>
|
||||
<li>wenn die Verarbeitung auf Artikel 6 Absatz 1 Buchstabe f beruht, die berechtigten</li>
|
||||
Interessen, die von dem Verantwortlichen oder einem Dritten verfolgt werden;
|
||||
<li>gegebenenfalls die Empfänger oder Kategorien von Empfängern der
|
||||
personenbezogenen Daten und</li>
|
||||
<li>gegebenenfalls die Absicht des Verantwortlichen, die personenbezogenen Daten an
|
||||
ein Drittland oder eine internationale Organisation zu übermitteln, sowie das
|
||||
Vorhandensein oder das Fehlen eines Angemessenheitsbeschlusses der Kommission
|
||||
oder im Falle von Übermittlungen gemäß Artikel 46 oder Artikel 47 oder Artikel 49
|
||||
Absatz 1 Unterabsatz 2 einen Verweis auf die geeigneten oder angemessenen
|
||||
Garantien und die Möglichkeit, wie eine Kopie von ihnen zu erhalten ist, oder wo sie verfügbar sind.</li>
|
||||
</ol>
|
||||
2. Zusätzlich zu den Informationen gemäß Absatz 1 stellt der Verantwortliche der betroffenen
|
||||
Person zum Zeitpunkt der Erhebung dieser Daten folgende weitere Informationen zur
|
||||
Verfügung, die notwendig sind, um eine faire und transparente Verarbeitung zu
|
||||
gewährleisten:
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>die Dauer, für die die personenbezogenen Daten gespeichert werden oder, falls dies nicht möglich ist, die Kriterien für die Festlegung dieser Dauer;</li>
|
||||
<li>das Bestehen eines Rechts auf Auskunft seitens des Verantwortlichen über die betreffenden personenbezogenen Daten sowie auf Berichtigung oder Löschung oder auf Einschränkung der Verarbeitung oder eines Widerspruchsrechts gegen die Verarbeitung sowie des Rechts auf Datenübertragbarkeit;</li>
|
||||
<li>wenn die Verarbeitung auf Artikel 6 Absatz 1 Buchstabe a oder Artikel 9 Absatz 2 Buchstabe a beruht, das Bestehen eines Rechts, die Einwilligung jederzeit zu widerrufen, ohne dass die Rechtmäßigkeit der aufgrund der Einwilligung bis zum Widerruf erfolgten Verarbeitung berührt wird;</li>
|
||||
<li>das Bestehen eines Beschwerderechts bei einer Aufsichtsbehörde;</li>
|
||||
<li>ob die Bereitstellung der personenbezogenen Daten gesetzlich oder vertraglich vorgeschrieben oder für einen Vertragsabschluss erforderlich ist, ob die betroffene Person verpflichtet ist, die personenbezogenen Daten bereitzustellen, und welche mögliche Folgen die Nichtbereitstellung hätte und</li>
|
||||
<li>das Bestehen einer automatisierten Entscheidungsfindung einschließlich Profiling gemäß Artikel 22 Absätze 1 und 4 und – zumindest in diesen Fällen – aussagekräftige Informationen über die involvierte Logik sowie die Tragweite und die angestrebten Auswirkungen einer derartigen Verarbeitung für die betroffene Person.</li>
|
||||
</ol>
|
||||
3. Beabsichtigt der Verantwortliche, die personenbezogenen Daten für einen anderen Zweck weiterzuverarbeiten als den, für den die personenbezogenen Daten erhoben wurden, so stellt er der betroffenen Person vor dieser Weiterverarbeitung Informationen über diesen anderen Zweck und alle anderen maßgeblichen Informationen gemäß Absatz 2 zur Verfügung.
|
||||
4. Die Absätze 1, 2 und 3 finden keine Anwendung, wenn und soweit die betroffene Person bereits über die Informationen verfügt.
|
||||
|
||||
# **Art. 6**
|
||||
## **DSGVO Rechtmäßigkeit der Verarbeitung**
|
||||
1. Die Verarbeitung ist nur rechtmäßig, wenn mindestens eine der nachstehenden Bedingungen erfüllt ist:
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>Die betroffene Person hat ihre Einwilligung zu der Verarbeitung der sie betreffenden personenbezogenen Daten für einen oder mehrere bestimmte Zwecke gegeben;</li>
|
||||
<li>die Verarbeitung ist für die Erfüllung eines Vertrags, dessen Vertragspartei die betroffene Person ist, oder zur Durchführung vorvertraglicher Maßnahmen erforderlich, die auf Anfrage der betroffenen Person erfolgen;</li>
|
||||
<li>die Verarbeitung ist zur Erfüllung einer rechtlichen Verpflichtung erforderlich, der der Verantwortliche unterliegt;</li>
|
||||
<li>die Verarbeitung ist erforderlich, um lebenswichtige Interessen der betroffenen
|
||||
Person oder einer anderen natürlichen Person zu schützen;</li>
|
||||
<li>die Verarbeitung ist für die Wahrnehmung einer Aufgabe erforderlich, die im
|
||||
öffentlichen Interesse liegt oder in Ausübung öffentlicher Gewalt erfolgt, die dem Verantwortlichen übertragen wurde;</li>
|
||||
<li>die Verarbeitung ist zur Wahrung der berechtigten Interessen des Verantwortlichen oder eines Dritten erforderlich, sofern nicht die Interessen oder Grundrechte und Grundfreiheiten der betroffenen Person, die den Schutz personenbezogener Daten erfordern, überwiegen, insbesondere dann, wenn es sich bei der betroffenen Person um ein Kind handelt.</li>
|
||||
</ol>
|
||||
Unterabsatz 1 Buchstabe f gilt nicht für die von Behörden in Erfüllung ihrer
|
||||
Aufgaben vorgenommene Verarbeitung.
|
||||
|
||||
2. Die Mitgliedstaaten können spezifischere Bestimmungen zur Anpassung der Anwendung der Vorschriften dieser Verordnung in Bezug auf die Verarbeitung zur Erfüllung von Absatz 1 Buchstaben c und e beibehalten oder einführen, indem sie spezifische Anforderungen für die Verarbeitung sowie sonstige Maßnahmen präziser bestimmen, um eine rechtmäßig und nach Treu und Glauben erfolgende Verarbeitung zu gewährleisten, einschließlich für andere besondere Verarbeitungssituationen gemäß Kapitel IX.
|
||||
3. Die Rechtsgrundlage für die Verarbeitungen gemäß Absatz 1 Buchstaben c und e wird
|
||||
festgelegt durch
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>Unionsrecht oder</li>
|
||||
<li>das Recht der Mitgliedstaaten, dem der Verantwortliche unterliegt.</li>
|
||||
</ol>
|
||||
Der Zweck der Verarbeitung muss in dieser Rechtsgrundlage festgelegt oder hinsichtlich der
|
||||
Verarbeitung gemäß Absatz 1 Buchstabe e für die Erfüllung einer Aufgabe erforderlich sein, die im öffentlichen Interesse liegt oder in Ausübung öffentlicher Gewalt erfolgt, die dem Verantwortlichen übertragen wurde. 3Diese Rechtsgrundlage kann spezifische Bestimmungen zur Anpassung der Anwendung der Vorschriften dieser Verordnung enthalten, unter anderem Bestimmungen darüber, welche allgemeinen Bedingungen für die Regelung der Rechtmäßigkeit der Verarbeitung durch den Verantwortlichen gelten, welche Arten von Daten verarbeitet werden, welche Personen betroffen sind, an welche Einrichtungen und für welche Zwecke die personenbezogenen Daten offengelegt werden dürfen, welcher Zweckbindung sie unterliegen, wie lange sie gespeichert werden dürfen und welche Verarbeitungsvorgänge und ‐verfahren angewandt werden dürfen, einschließlich Maßnahmen zur Gewährleistung einer rechtmäßig und nach Treu und Glauben erfolgenden Verarbeitung, wie solche für sonstige besondere Verarbeitungssituationen gemäß Kapitel IX. 4Das Unionsrecht oder das Recht der Mitgliedstaaten müssen ein im öffentlichen Interesse liegendes Ziel verfolgen und in einem angemessenen Verhältnis zu dem verfolgten legitimen Zweck stehen.
|
||||
4. Beruht die Verarbeitung zu einem anderen Zweck als zu demjenigen, zu dem die personenbezogenen Daten erhoben wurden, nicht auf der Einwilligung der betroffenen Person oder auf einer Rechtsvorschrift der Union oder der Mitgliedstaaten, die in einer demokratischen Gesellschaft eine notwendige und verhältnismäßige Maßnahme zum Schutz der in Artikel 23 Absatz 1 genannten Ziele darstellt, so berücksichtigt der Verantwortliche – um festzustellen, ob die Verarbeitung zu einem anderen Zweck mit demjenigen, zu dem die personenbezogenen Daten ursprünglich erhoben wurden, vereinbar ist – unter anderem
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>jede Verbindung zwischen den Zwecken, für die die personenbezogenen Daten
|
||||
erhoben wurden, und den Zwecken der beabsichtigten Weiterverarbeitung,</li>
|
||||
<li>den Zusammenhang, in dem die personenbezogenen Daten erhoben wurden,
|
||||
insbesondere hinsichtlich des Verhältnisses zwischen den betroffenen Personen und
|
||||
dem Verantwortlichen,</li>
|
||||
<li>die Art der personenbezogenen Daten, insbesondere ob besondere Kategorien
|
||||
personenbezogener Daten gemäß Artikel 9 verarbeitet werden oder ob
|
||||
personenbezogene Daten über strafrechtliche Verurteilungen und Straftaten gemäß
|
||||
Artikel 10 verarbeitet werden,</li>
|
||||
<li>die möglichen Folgen der beabsichtigten Weiterverarbeitung für die betroffenen
|
||||
Personen,</li>
|
||||
<li>das Vorhandensein geeigneter Garantien, wozu Verschlüsselung oder
|
||||
Pseudonymisierung gehören kann.</li>
|
||||
</ol>
|
||||
`;
|
||||
|
||||
markdownSources.de!.termsAndConditions = `
|
||||
<p>Stand: 04. November 2015</p>
|
||||
<p>Es gilt der jeweilige Datenschutz der Hochschule im jeweiligen Bundesland. Darüber hinaus gelten die folgenden Vereinbarungen. Mit der Installation erklären Sie sich bereit die folgenden Bedingungen zu akzeptieren. <br>
|
||||
</p>
|
||||
<ol>
|
||||
<li>Geltungsbereich und verwendete Bezeichnungen
|
||||
<ol>
|
||||
<li> Der Entwicklungsverbund, nachfolgend mit <strong>App-Anbieter</strong> bezeichnet, vermittelt Informationen und Leistungen über ihre mobile App, nachfolgend mit <strong>Open StApps</strong> bezeichnet. </li>
|
||||
<li> Diensteanbieter der entsprechenden Hochschulauswahl ist die jeweilige Hochschule mit jeweils hochschulspezifischen und nichthochschulspezifischen Angeboten selbst. Nachfolgend werden folgende Unterscheidungen getroffen:
|
||||
<ul>
|
||||
<li>Hochschule, nachfolgend als <strong>Datenanbieter</strong> bezeichnet, stellt hochschulspezifische Angebote bereit.</li>
|
||||
<li>Bibliotheken, Cafés, Copyshops, Mensen und weitere nichthochschulspezifische Einrichtungen, nachfolgend als <strong>Drittdatenanbieter</strong> bezeichnet, stellt nichthochschulspezifische Angebote bereit.</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li> Vertragspartner mit dem App-Anbieter werden nachfolgend mit <strong>Studierende</strong> bezeichnet. </li>
|
||||
<li> Die vom App-Anbieter angebotenen Informationen und Leistungen werden folgend als <strong>Dienstangebot</strong> bezeichnet. </li>
|
||||
<li>Die Gesamtausführung einer Funktionalität bis zum Erreichen eines Endzustands, wie bspw. Abschluss einer Prüfungsanmeldung, Ausleihe eines Mediums, wird als <strong>Prozessabwicklung</strong> bezeichnet.</li>
|
||||
<li> Die nachstehenden Allgemeinen Geschäftsbedingungen gelten nur dem App-Anbieter. Die angezeigten Dienstangebote haben darauf keinen Einfluss. </li>
|
||||
<li> Die Erbringung eines Dienstangebots erfolgt zu den Allgemeinen Geschäftsbedingungen des jeweiligen Daten- oder Drittdatenanbieters. Sofern dem App-Anbieter diese Allgemeinen Geschäftsbedingungen vorliegen, können diese hier eingesehen werden. </li>
|
||||
<li> Es gilt das jeweilige Impressum des Daten- oder Drittdatenanbieters. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Leistungen von Open StApps
|
||||
<ol>
|
||||
<li>Open StApps tritt im Rahmen seiner Tätigkeit ausschließlich als App-Anbieter von Dienstangeboten auf. Nach Ausübung von Prozessabwicklungen innerhalb des App-Anbieters, können weitere administrative Tätigkeiten vom App-Anbieter hinzutreten. </li>
|
||||
<li>Der App-Anbieter bietet selbst keine eigenen Dienstangebote an, außer sie dienen der Kennzeichnung und Zuordnung dieser. Somit gilt bspw. die Prüfungsanmeldung nicht zu den Vertragspflichten des App-Anbieters. Eine erfolgreiche Prozessabwicklung eines Dienstangebots kommt ausschließlich zwischen den Studierenden und dem jeweiligen Daten- oder Drittdatenanbieter zustande. Der App-Anbieter haftet daher auch nicht für die Verfügbarkeit, Durchführung, Qualität, Begleitumstände oder etwaige Störungen oder Änderungen der Dienstangebote. Derartige Sachverhalte liegen grundsätzlich im alleinigen Verantwortungsbereich der Daten- oder Drittdatenanbieter. </li>
|
||||
<li> Der App-Anbieter haftet entsprechend auch nicht für die Richtigkeit, Vollständigkeit oder Aktualität der Dienstangebote, die während der Prozessabwicklung zu den jeweiligen Dienstangeboten dargestellt werden. Die Vermittlung der Dienstangebote erfolgt auf Basis der von den Daten- oder Drittdatenanbieter bereitgestellten und den Studierenden übermittelten Dienstangeboten. Es erfolgt dabei seitens des App-Anbieters keine vorherige, begleitende, allgemeine oder individuelle Korrektheits- oder Verfügbarkeitsprüfung. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li> Vereinbarung
|
||||
<ol>
|
||||
<li>Die von Studierenden abgeschlossenen Prozessabwicklungen sind verbindlich.</li>
|
||||
<li>Die von Studierenden abgeschlossenen Prozessabwicklungen werden über den App-Anbieter vermittelt.</li>
|
||||
<li>Sofern die Prozessabwicklung des Studierenden erfolgreich vermittelt werden konnte, stellt der App-Anbieter den Erfolg oder Misserfolg der Prozessabwicklung dar. Erst mit dem Erfolg einer Prozessabwicklung gilt Punkt III.1.</li>
|
||||
<li> Der Studierende hat im Interesse einer ordnungsgemäßen Prozessabwicklung folgende Verpflichtungen:
|
||||
<ul>
|
||||
<li>Die Verfügbarkeit einer stetigen Internetverbindung während einer Prozessabwicklung</li>
|
||||
<li>Die ordnungsgemäße Benutzung der Open StApps</li>
|
||||
<li>Eine innerhalb der Open StApps erfolgreiche Authentifizierung und Authorisierung falls eine Prozessabwicklung diese benötigt</li>
|
||||
<li>Die ordnungsgemäße Eingabe und Prüfung der zur Prozessabwicklung erforderlichen Daten</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li> Nach erfolgter Prozessabwicklung ist die Korrektheit der Daten beim Daten- oder Drittdatenanbieter sicherzustellen. </li>
|
||||
<li> Der Studierende hat regelmäßig die zur Prozessabwicklung geforderten Daten auf Änderungen beim Daten- oder Drittdatenanbieter zu kontrollieren. </li>
|
||||
<li> Der App-Anbieter weist darauf hin, dass die Vermittlung von Prozessabwicklungen nicht unter die gesetzlichen Vorschriften für Fernabsatzverträge fällt. Der Studierende hat insofern kein besonderes Widerrufs- und Rückgaberecht gegenüber dem App-Anbieter. </li>
|
||||
<li>Der Abschluss einer Prozessabwicklung steht unter dem Vorbehalt einer erfolgreichen Vermittlung des App-Anbieters. In Ausnahmefällen sind Daten- oder Drittdatenanbieter nicht befugt oder berechtigt Prozessabwicklungen erfolgreich abzuschließen. Dies ist für den App-Anbieter unter Umständen im Vorhinein nicht ersichtlich, sodass der App-Anbieter dafür nicht haftet.</li>
|
||||
<li>Wird eine erfolgreiche Prozessabwicklung von einer schuldhaften Verletzung des Studierenden ausgeübt, ist der App-Anbieter berechtigt, den Daten- oder Drittdatenanbieter darüber zu informieren. Über das weitere Verfahren entscheidet der Daten- oder Drittdatenanbieter.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Änderungen von abgeschlossenen Prozessabwicklungen
|
||||
<ol>
|
||||
<li>Im Falle einer abgeschlossenen Prozessabwicklung, die zu Ungunsten des Studierenden ausfällt, ist unverzüglich der Daten- oder Drittdatenanbieter zu informieren. Der Daten- oder Drittdatenanbieter entscheidet selbstständig über das weitere Vorgehen.</li>
|
||||
<li>Im Fall einer nicht verschuldeten Prozessabwicklung auf Basis des Dienstangebots, erfolgt keine Gewährleistung von Seiten des App-Anbieters. Die weitere Bearbeitung erfolgt dann durch den Daten- oder Drittdatenanbieter.</li>
|
||||
<li>Sofern Änderungen an Daten der abgeschlossenen Prozessabwicklungen getätigt werden müssen und nicht über den App-Anbieter ausgeführt werden können, ist der Daten- oder Drittdatenanbieter über einen entsprechenden Änderungswunsch zu informieren.</li>
|
||||
<li>Aus Gründen der Beweissicherung werden Prozessabwicklungen personen- und gerätebezogen gespeichert, falls Prozessabwicklungen einen Personenbezug erfordern. Die Speicherung erfolgt nach den üblichen Gesetzen, mindestens aber für 90 Tage. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Datenschutz und Nutzung der Open StApps
|
||||
<ol>
|
||||
<li>Die Open StApps wird als App über die Vertriebsanbieter "Google Play Store" oder in "Apple iTunes" bereitgestellt. Der Vertriebspartner stellt eigene Nutzungs- und Datenschutzerklärungen bereit. Open StApps hat keinerlei Einfluss auf deren Nutzungsbedingungen und Datenschutzerklärungen.</li>
|
||||
<li>Personenbezogene und sonstige Daten des Studierenden werden vom App-Anbieter nur erhoben, gespeichert und an Daten- oder Drittdatenanbieter weitergeleitet, sofern dies für eine abgeschlossene Prozessabwicklung mit Personenbezug erforderlich ist.</li>
|
||||
<li> Sofern der Studierende Kontaktinformationen angibt, kann eine Kontaktaufnahme von Seiten des App-Anbieters erfolgen. </li>
|
||||
<li>Der App-Anbieter gewährt den Nutzenden ein kostenfreies, zeitlich unbefristetes, nichtkommerzielles Recht zu Nutzung (<strong>Lizenz</strong>) der Open StApps ein.</li>
|
||||
<li>Die Lizenz berechtigt den Nutzenden die Nutzung der Open StApps im Rahmen eines normalen Gebrauchs. Dies umfasst die Installation, Ausführung das Laden der Open StApps in den Arbeitsspeicher und seinen Ablauf. Andere Nutzungsarten sind ausgeschlossen.</li>
|
||||
<li>Die Abänderung, Übersetzung, Bearbeitung, Umgestaltungen, Zurückentwickelung (sog. Reverse Engineering) und Vervielfältigung, auch teilweise oder vorübergehend, der Open StApps und des Programmcodes sind unzulässig und stellen ein Verstoß von § 39 Abs. 2 UrhG dar. Im Übrigen bleiben §§ 69d, 69e UrhG unberührt.</li>
|
||||
<li>Der App-Anbieter ist Inhaber sämtlicher gewerblicher Schutz-, Nutzungs- und Urheberrechte an der Open StApps sowie zugehöriger Dokumentationen. Hinweise auf Urheberrechte oder auf sonstige gewerbliche Schutzrechte, die sich in der Open StApps befinden, dürfen weder verändert, beseitigt noch sonst unkenntlich gemacht werden.<br>
|
||||
</li>
|
||||
<li>Dienstangebote der Daten- oder Drittdatenanbieter dürfen nicht verändert, kopiert, verbreitet, übertragen, veröffentlicht, lizenziert, abgetreten oder verkauft und/oder keine davon abgeleiteten Werke erstellt werden.</li>
|
||||
<li>Über die Einhaltung der Rechte, wie bspw. dem Urheberrecht und Personenrecht, ist der Daten- oder Drittdatenanbieter bei seinen Dienstangeboten zuständig. Die für die ordnungsgemäße Funktion der Open StApps notwendigen Rechte sind vorhanden und/oder wurden rechtlich eingeräumt.</li>
|
||||
<li>Links, die außerhalb der Open StApps führen, werden den Studierenden nur als Hinweise zur Verfügung gestellt. Der App-Anbieter hat auf die Inhalte, die über den Link abrufbar sind keinen Einfluss und bedeutet gleichsam keine Billigung dieser Inhalte, noch stehen diese Inhalte in Verbindung zwischen Open StApps und den Betreibern. </li>
|
||||
<li>Für alle Inhalte innerhalb der Open StApps gilt das Copyright © 2015 für "Open StApps - Das Studierenden-App Development Kit" und deren Daten- oder Drittdatenanbieter. Alle Rechte sind vorbehalten. </li>
|
||||
<li>Alle Funktionen innerhalb der Open StApps stellen ein urheberrechtlich geschütztes Werk der Open StApps und deren Verbundmitglieder dar. Die Nutzung der Open StApps durch die Studierenden unterliegen den Bedingungen der Endbenutzer-Lizenzvereinbarung (EULA). Die Reproduktion oder Weiterverbreitung der Funktionen innerhalb der Open StApps sowie der Open StApps selbst ist verboten. Ferner darf die Software der Open StApps weder durch reverse engineering zurückverfolgt, dekompiliert oder disassembliert werden. Die Open StApps darf weder direkt noch indirekt in Länder exportiert und/oder weiterexportiert werden, die den Exportbeschränkungen von Deutschland oder deren Schengenstaaten unterliegen.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Nutzung von Standortdaten
|
||||
<ol>
|
||||
<li>Die App ermöglicht das Aktivieren des Ortungsdienstes über die Einstellungen. Der Ortungsdienst ermöglicht den Zugriff auf Ihre Standortdaten um detaillierte Entfernungsdaten darzustellen.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Gewährleistung
|
||||
<ol>
|
||||
<li>Der App-Anbieter gewährleistet, gemäß den Vorschriften der § 434 ff BGB, dass Open StApps mit größter gebotener Sorgfalt und Fachkenntnis erstellt worden ist. Ein völliger Ausschluss von Softwarefehlern innerhalb der Open StApps ist zum derzeitigen Kenntnisstand nicht möglich.</li>
|
||||
<li>Der App-Anbieter der Open StApps korrigiert Fehler, die eine bestimmungsmäßige Nutzung der Open StApps erheblich beeinträchtigen. Die Fehlerbehebung erfolgt nach jeweiliger Fehlereinstufung durch den App-Anbieter zu einem ihm festgelegten Zeitpunkt. Zur Fehlerbeseitigung ist die nutzende Person verpflichtet die fehlerbereinigte Version über die vorhandene fehlerbehaftete Version zu installieren und auszuführen. Eine Mitwirkung der nutzenden Person zur Fehlerbehebung kann erforderlich sein. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Haftung
|
||||
<ol>
|
||||
<li>Die vertraglichen Pflichten von Open StApps umfassen ausschließlich die ordnungsgemäße Vermittlung von Dienstangeboten zwischen Studierenden und Daten- oder Drittdatenanbieter unter der Berücksichtung der in dieser AGB aufgeführten Vereinbarungen.</li>
|
||||
<li>Der App-Anbieter haftet für Schäden, die er vorsätzlich oder grob fahrlässig verursacht hat.</li>
|
||||
<li>Der App-Anbieter haftet nicht für die Wiederbeschaffung von Daten.</li>
|
||||
<li>Die nutzende Person ist zur regelmäßigen Sicherung seiner Daten verpflichtet.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Schlussbestimmungen
|
||||
<ol>
|
||||
<li>Sollte eine Bestimmung dieses Vertrages rechtsunwirksam sein oder werden, so ist dies ohne Einfluss auf die Gültigkeit der übrigen Vertragsbestimmungen und des Vertrages selbst. Die unwirksame Bestimmung wird durch eine solche wirksame Bestimmung ersetzt, die ihr wirtschaftlich nahezu entspricht. Dasselbe gilt für Vertragslücken oder nicht ausreichende vertragliche Regelungen. </li>
|
||||
<li>Die Allgemeinen Geschäftsbedingungen der Open StApps und der Daten- oder Drittdatenanbieter können jederzeit ohne gesonderte Vorankündigung geändert werden. Bereits abgeschlossene bzw. laufende Verträge sind von Änderungen nicht rückwirkend betroffen. Open StApps stellt stets die eigene aktuelle und gültige Version ihrer Allgemeinen Geschäftsbedingungen bereit. </li>
|
||||
<li>Es gilt das Recht der Bundesrepublik Deutschland.</li>
|
||||
<li>Es gilt Berlin als Gerichtsstand.</li>
|
||||
</ol>
|
||||
</li>
|
||||
</ol>
|
||||
<p>Fragen oder Anmerkungen zu dieser AGB richten Sie bitte an: <a href="mailto:app@rz.uni-frankfurt.de">app@rz.uni-frankfurt.de</a></p>
|
||||
Für allgemeine Anfragen können Sie ein Kontaktformular nutzen:<br />
|
||||
<https://datenschutz.hessen.de/kontakt><br />
|
||||
<br />
|
||||
Für Beschwerden steht Ihnen zudem ein Beschwerdeformular zur Verfügung:<br />
|
||||
<https://datenschutz.hessen.de/service/beschwerde-uebermitteln>
|
||||
`;
|
||||
|
||||
markdownSources.en!.privacyPolicy = `
|
||||
This data protection declaration serves to fulfill the information obligation required under Article 13 EU DSGVO when data is collected at the time of collection from data subjects.
|
||||
# Privacy policy
|
||||
|
||||
# **Name and address of the responsible person**
|
||||
Johann Wolfgang Goethe University Frankfurt am Main<br />
|
||||
## Contact details of the person responsible
|
||||
|
||||
Responsible in the sense of the General Data Protection Regulation and further regulations on data protection is the:
|
||||
|
||||
Johann Wolfgang Goethe-Universität Frankfurt am Main represented by its president<br />
|
||||
Theodor-W.-Adorno-Platz 1<br />
|
||||
60323 Frankfurt am Main
|
||||
|
||||
Postal address:<br />
|
||||
Goethe University Frankfurt am Main<br />
|
||||
60629 Frankfurt<br />
|
||||
Postanschrift:<br />
|
||||
Goethe-Universität Frankfurt am Main<br />
|
||||
60629 Frankfurt
|
||||
|
||||
Phone: +49-69-798-0 | Fax: +49-69-798-18383<br />
|
||||
Internet: http://www.uni-frankfurt.de<br />
|
||||
Website: http://www.uni-frankfurt.de
|
||||
|
||||
If you have any questions or complaints about data protection, you can contact the data protection officer at Goethe University.
|
||||
## Contact details of the data protection officer at Goethe University
|
||||
|
||||
# **Contact details of the data protection officer**
|
||||
Johann Wolfgang Goethe University Frankfurt am Main<br />
|
||||
The official data protection officers<br />
|
||||
Theodor-W.-Adorno-Platz 1<br />
|
||||
60323 Frankfurt am Main<br />
|
||||
You can reach the data protection officers at Johann Wolfgang Goethe University Frankfurt am Main at:<br />
|
||||
Mail: <dsb@uni-frankfurt.de><br />
|
||||
Website: http://www.uni-frankfurt.de/47859992/datenschutzbeauftragte
|
||||
|
||||
Internet: http://www.uni-frankfurt.de/47859992/datenschutzbeauftragte
|
||||
## Information on the processing of personal data
|
||||
|
||||
# **Rights and Complaints**
|
||||
You have the right to complain to the competent supervisory authority in the event of data protection problems.<br />
|
||||
### <u>1. Scope of the processing of personal data</u>.
|
||||
|
||||
Contact address of the technical supervisory authority of the Goethe University Frankfurt am Main:<br />
|
||||
According to Article 4 DSGVO, personal data is any information relating to an identified or identifiable natural person.
|
||||
|
||||
The Hessian Data Protection Officer<br />
|
||||
We process personal data of you as a user inside of the Goethe University App to the extent that this is technically necessary for the provision of a **functional application**.
|
||||
|
||||
Furthermore, data processing may be based on your voluntary consent if you wish to use **specific functions**.
|
||||
|
||||
We therefore distinguish below between
|
||||
|
||||
- Access data when using the app: content of requests, IP addresses, date/time of request, requested URL, error codes, browser identifier, HTTP header.
|
||||
|
||||
- Location and navigation: voluntary location information
|
||||
|
||||
- User settings: voluntary specification of a) language preferences (currently: German/English), b) status (e.g. guest/student) or c) specific search queries and search results (notifications)
|
||||
|
||||
- Calendar function: voluntary use of the calendar function (optional with voluntary use of a sync function: opt-in) or the integrated timetable function. The following data is processed and stored on the users device: appointments and events
|
||||
|
||||
- Feedback function and contacting: voluntary use with the provision of contact data and, if applicable, voluntary transmission of log data
|
||||
|
||||
- Campus services: voluntary use with processing of grade view, matriculation number, email address, name
|
||||
|
||||
- Services of the library: voluntary use with processing of library account data, such as ID number with name, e-mail address, postal address, right of use, order data, fees, reservation, loan data. Full details of processing can be found in the library's privacy policy:<br />
|
||||
https://www.ub.uni-frankfurt.de/benutzung/datenschutz.html
|
||||
|
||||
In some places, the app links to the Goethe University website and to other external websites that are displayed in an in-app browser. When you visit these websites, we ask you to pay attention to the separate data protection notices and declarations that apply there.
|
||||
|
||||
### <u>2. Purpose(s) of data processing</u>
|
||||
|
||||
**Access to location data**.
|
||||
|
||||
For navigation, the Goethe University app requires access to the location of the end device used (location-based services). When a request is made, the app collects the current location via GPS, radio cell data and WLAN databases in order to be able to give you as a user:in information about your immediate surroundings. The location data is only accessed if you allow access to the location data. Data about your location is only used to process location-related requests and to display your location on the map.
|
||||
|
||||
**Access to access data**.
|
||||
|
||||
Log files are stored and processed to ensure that the Goethe Uni app functions properly for you. In addition, we need the data for reasons of security of our information technology systems. No other evaluation or disclosure takes place in this context.
|
||||
|
||||
**Access to language settings**.
|
||||
|
||||
Access to the language setting is made in order to display the interface of the app in the language of your choice.
|
||||
|
||||
**Access to the status group setting**.
|
||||
|
||||
Access to the status group setting is provided to show you the information in the app that applies to your group, e.g. canteen prices.
|
||||
|
||||
**Access to personal data when using the feedback function**.
|
||||
|
||||
We use the processing of personal data from the feedback function to contact you and troubleshoot problems.
|
||||
|
||||
**Access to personal data when synchronizing calendars**.
|
||||
|
||||
Appointment data is accessed in order to write it to the device calendar when the calendar function is enabled.
|
||||
|
||||
**Access to Campus Services data**.
|
||||
|
||||
Access to the Campus Management System is solely for the purpose of displaying personal student management data in the app (e.g., exam grades).
|
||||
|
||||
**Access to library-specific personal data**.
|
||||
|
||||
Access to data (e.g., ID number, name, mailing address) is for the purpose of carrying out ordering and borrowing procedures for books and other materials from the University Library. Full details of the purposes of processing can be found in the Library's Privacy Policy: https://www.ub.uni-frankfurt.de/benutzung/datenschutz.html
|
||||
|
||||
### <u>3. Rechtsgrundlage(n) für die Datenverarbeitung</u>
|
||||
|
||||
The use of usage/access data ("log files") is based on Article 6(1)(f) DSGVO.
|
||||
|
||||
For all specific functions where data processing is based on your voluntary consent as a user:in, explicit consent or active acts of consent ("opt-in") are obtained. The provision of personal data about you to Goethe University is done on a voluntary basis. The legal basis in each of these cases is Article 6 (1) a) DSGVO. You can individually revoke your respective consent or change your settings at any time.
|
||||
|
||||
### <u>4. Data deletion and storage duration</u>
|
||||
|
||||
The data collected in the log files of the app are automatically deleted or anonymized seven days after the end of the access.
|
||||
|
||||
The deletion periods or storage duration of the data collected in the library systems can be found in the library's privacy policy: https://www.ub.uni-frankfurt.de/benutzung/datenschutz.html
|
||||
|
||||
For all other functions and services, the following applies: deletion takes place here depending on the specifications of the service used. The personal data of the data subject will be deleted or blocked as soon as the purpose of the storage no longer applies.
|
||||
|
||||
### <u>5. Data disclosure/data transfer</u>
|
||||
|
||||
We will not pass on your personal data to third parties.
|
||||
|
||||
On the part of the operator, technical and organizational measures are taken to ensure that third parties do not gain access to the processed data, such as usage data. An order processing relationship according to Art. 28 DSGVO does not exist, as only our own servers are used.
|
||||
|
||||
### <u>6. Automated decision-making</u>
|
||||
|
||||
Automated decision-making, including profiling, does not take place.
|
||||
|
||||
## Rights of the data subject
|
||||
|
||||
If personal data is processed by you, you are a data subject within the meaning of the GDPR. The assertion of your data subject rights is free of charge. You can, of course, contact us for this purpose. You are entitled to the following data subject rights vis-à-vis Goethe University:
|
||||
|
||||
### <u>1. Right of access</u>
|
||||
|
||||
You can request confirmation from us as the controller as to whether and which of your personal data is being processed by us. You have the right to request copies of your personal data from us. Please note the exceptions that may arise due to specific regulations.
|
||||
|
||||
### <u>2. Right of rectification</u>
|
||||
|
||||
You have the right to request us to rectify and/or complete, if the processed personal data concerning you is not (anymore) accurate or not (anymore) complete.
|
||||
|
||||
### <u>3. Right to restriction of processing</u>
|
||||
|
||||
Under certain conditions, you can request the restriction of the processing of personal data concerning you, i.e. that your personal data is then not deleted, but marked so that further processing is restricted.
|
||||
|
||||
### <u>4. Right to erasure</u>
|
||||
|
||||
Under certain conditions, you can demand that we delete the personal data concerning you without delay. This is particularly the case if the personal data is no longer necessary for the purpose for which it was originally collected or processed.
|
||||
|
||||
### <u>5. Right to information</u>
|
||||
|
||||
If you have asserted the right to rectification, erasure or restriction of processing against us, we are obliged to inform all recipients to whom the personal data concerning you have been disclosed of this rectification or erasure of the data or restriction of processing, unless this proves impossible or involves a disproportionate effort. You are entitled to be informed about these recipients.
|
||||
|
||||
### <u>6. Right to data portability</u>
|
||||
|
||||
Under certain conditions, you have the right to request that we transfer your personal data directly to another controller or organization. Alternatively, under certain conditions, you have the right to request that we ourselves provide you with the data in a machine-readable format.
|
||||
|
||||
### <u>7. Right to object</u>
|
||||
|
||||
If we process your personal data because the processing is in the public interest, part of our public duties, or if we process your data on the basis of a legitimate interest, you have the right to object at any time to the processing of data relating to you for reasons arising from your particular situation.
|
||||
|
||||
### <u>8. Right to revoke the declaration of consent under data protection law</u>
|
||||
|
||||
If we process your personal data because you have given us your consent, you have the right to revoke your declaration of consent at any time.
|
||||
|
||||
### <u>9. Right to lodge a complaint with a supervisory authority</u>
|
||||
|
||||
You also have the right to lodge a complaint with a supervisory authority. The competent supervisory authority will examine your complaint.
|
||||
|
||||
## **Contact details of the supervisory authority in the area of data protection**
|
||||
|
||||
If you believe that the processing of your personal data violates data protection regulations, if you have a general inquiry or if you want to complain to a competent supervisory authority, you can contact the Hessian Commissioner for Data Protection and Freedom of Information (HBDI).
|
||||
|
||||
**The Hessian Commissioner for Data Protection and Freedom of Information can be reached in different ways:**
|
||||
|
||||
<u>**The Hessian Commissioner for Data Protection and Freedom of Information**</u><br />
|
||||
PO Box 3163<br />
|
||||
65021 Wiesbaden<br /><br />
|
||||
E-mail to HDSB (link to the contact form of the Hessian data protection officer:
|
||||
https://datenschutz.hessen.de/über-uns/kontakt)<br />
|
||||
65021 Wiesbaden
|
||||
|
||||
Telephone: +49 611 1408 - 0<br />
|
||||
Fax: +49 611 1408 – 611
|
||||
Telephone: +49 611 1408 -- 0
|
||||
|
||||
You have the following rights vis-à-vis the Goethe University with regard to your stored personal data:
|
||||
* right to information,
|
||||
* right to rectification or erasure,
|
||||
* right to restriction of processing,
|
||||
* right to withdraw your consent,
|
||||
* Right to object to processing,
|
||||
* Right to data portability, in a commonly used, structured and machine-readable form
|
||||
(from May 25, 2018).
|
||||
|
||||
To assert these rights, please contact <a href="mailto:app@rz.uni-frankfurt.de">app@rz.uni-frankfurt.de</a>.<br />
|
||||
|
||||
# **Type of data stored, purpose and legal basis, deletion periods**
|
||||
|
||||
**Handling of personal data**<br />
|
||||
|
||||
Personal data is information that can be used to identify a natural person, i.e. information that can be used to identify individuals. This includes in particular names, e-mail addresses, matriculation numbers or telephone numbers. But data about preferences, hobbies, memberships or information about websites that have been visited also count as personal data.<br />
|
||||
|
||||
Personal data is only collected, used and passed on by us if this is permitted by law or if the user has consented to the data collection.<br />
|
||||
|
||||
The use of students' personal data for the purpose of studying is largely based on the applicable Hessian Higher Education Act in conjunction with the applicable enrollment ordinance of the State of Hesse and thus refers to EU DSGVO Article 6 Paragraph 1 c).<br />
|
||||
|
||||
The data of the employees of the Goethe University for the purpose of personnel administration, teaching, research and examination activities are collected and processed on the basis of the Hessian Higher Education Act, the enrollment ordinance of the State of Hesse, TV-GU, civil service and personnel law regulations.<br />
|
||||
|
||||
**Access Data/Server Log Files**<br />
|
||||
|
||||
When accessing the pages of this web server, the following data is generally stored in the server log files
|
||||
1. IP address
|
||||
2. Date and time
|
||||
3. Type of client browser
|
||||
4. URL of the accessed page
|
||||
5. If applicable, the error message for the error that has occurred
|
||||
6. If applicable, the requesting provider
|
||||
|
||||
This data is used solely for the purpose of checking functionality, security and troubleshooting. This use is based on EU GDPR Article 6 Paragraph 1 f). All log files are automatically deleted or made anonymous after 7 days at the latest.
|
||||
|
||||
**Contact**<br />
|
||||
|
||||
To contact members of the Goethe University (e.g. via contact form or e-mail), your details will be stored for the purpose of processing the request and in the event that follow-up questions arise. After your request has been processed or after the legal obligation has been fulfilled or the service used has been fulfilled, the data will be deleted, unless the storage of the data is necessary for the implementation of legitimate interests of Goethe University or on the basis of a statutory provision (e.g. law, statutory ordinance, statutes of the Goethe university etc.) required.
|
||||
|
||||
**Inclusion of third party services**<br />
|
||||
|
||||
Content from third parties (e.g. videos from YouTube, map material from Google Maps, RSS feeds, graphics, etc.) from other websites is integrated within some pages of this online offer. This always presupposes that the providers of this content (hereinafter referred to as "third-party providers") perceive your IP address. Because without the IP address, the third-party providers could not send the content to your browser. The IP address is therefore required for the display of this content. We endeavor to only use content whose respective providers only use the IP address to deliver the content. However, we have no influence on the further use of your data (e.g. if the third-party providers save the IP address for statistical purposes).
|
||||
|
||||
# **Article 13 EU GDPR**
|
||||
## **Duty to provide information when collecting personal data from the data subject**
|
||||
|
||||
1. If personal data is collected from the data subject, the
|
||||
responsible for the data subject at the time this data was collected:
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>The name and contact details of the person responsible and, if applicable, his
|
||||
representative;</li>
|
||||
<li>if applicable, the contact details of the data protection officer;
|
||||
<li>the purposes for which the personal data are to be processed and the legal basis for the processing;</li>
|
||||
<li>if the processing is based on Article 6 paragraph 1 letter f, the legitimate</li>
|
||||
Interests pursued by the controller or a third party;
|
||||
<li>if applicable, the recipients or categories of recipients of the
|
||||
personal data and</li>
|
||||
<li>if applicable, the intention of the controller to provide the personal data
|
||||
to a third country or an international organization, as well as that
|
||||
Presence or absence of an adequacy decision by the Commission
|
||||
or in the case of transfers pursuant to Article 46 or Article 47 or Article 49
|
||||
Paragraph 1 subparagraph 2 a reference to the appropriate or reasonable
|
||||
Warranties and how to obtain a copy of them, or where they are available.</li>
|
||||
</ol>
|
||||
2. In addition to the information referred to in paragraph 1, the person responsible provides the data subject
|
||||
person at the time this data was collected:
|
||||
Disposal necessary to ensure fair and transparent processing
|
||||
guarantee:
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>the period for which the personal data will be stored or, if this is not possible, the criteria used to determine that period;</li>
|
||||
<li>The existence of a right to information on the part of the person responsible about the personal data concerned and to correction or deletion or restriction of processing or a right to object to processing and the right to data portability;</li>
|
||||
<li>if the processing is based on Article 6 paragraph 1 letter a or Article 9 paragraph 2 letter a, the existence of a right to withdraw consent at any time without affecting the lawfulness of the processing carried out on the basis of the consent up to the point of withdrawal;< /li>
|
||||
<li>The existence of a right of appeal to a supervisory authority;</li>
|
||||
<li>whether the provision of the personal data is required by law or contract or is necessary for the conclusion of a contract, whether the data subject is obliged to provide the personal data and what the possible consequences of non-provision would be and</li>
|
||||
<li>The existence of automated decision-making including profiling in accordance with Article 22 Paragraphs 1 and 4 and - at least in these cases - meaningful information about the logic involved and the scope and intended effects of such processing for the data subject.</li>
|
||||
</ol>
|
||||
3. If the controller intends to further process the personal data for a purpose other than that for which the personal data was collected, he shall provide the data subject with information about this other purpose and any other relevant information pursuant to paragraph 2 prior to such further processing.
|
||||
4. Paragraphs 1, 2 and 3 do not apply if and to the extent that the data subject already has the information.
|
||||
|
||||
# **Art. 6**
|
||||
## **GDPR lawfulness of processing**
|
||||
1. Processing is lawful only if at least one of the following conditions is met:
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>The data subject has given their consent to the processing of their personal data for one or more specific purposes;</li>
|
||||
<li>the processing is necessary for the performance of a contract to which the data subject is party or in order to take steps at the request of the data subject prior to entering into a contract;</li>
|
||||
<li>the processing is necessary for compliance with a legal obligation to which the controller is subject;</li>
|
||||
<li>The processing is necessary to protect the vital interests of the data subject
|
||||
person or another natural person;</li>
|
||||
<li>the processing is necessary for the performance of a task that is
|
||||
is in the public interest or in the exercise of official authority that has been transferred to the person responsible;</li>
|
||||
<li>The processing is necessary to safeguard the legitimate interests of the person responsible or a third party, unless the interests or fundamental rights and freedoms of the data subject that require the protection of personal data prevail, in particular if the data subject is a child is acting.</li>
|
||||
</ol>
|
||||
Point (f) of the first subparagraph shall not apply to public authorities in the performance of their
|
||||
processing performed on tasks.
|
||||
|
||||
2. Member States may maintain or introduce more specific provisions adapting the application of the rules of this Regulation in relation to processing to comply with points (c) and (e) of paragraph 1 by specifying specific requirements for processing and other measures to ensure a lawful and to ensure fair processing, including for other special processing situations in accordance with Chapter IX.
|
||||
3. The legal basis for the processing pursuant to paragraph 1 letters c and e is
|
||||
set by
|
||||
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>Union law or</li>
|
||||
<li>The law of the Member States to which the controller is subject.</li>
|
||||
</ol>
|
||||
The purpose of the processing must be specified in this legal basis or in relation to the
|
||||
Processing pursuant to paragraph 1 letter e is necessary for the performance of a task that is in the public interest or in the exercise of official authority that has been assigned to the person responsible. 3This legal basis may contain specific provisions adjusting the application of the provisions of this Regulation, including provisions on which general conditions apply to regulate the lawfulness of processing by the controller, what types of data are processed, which subjects are concerned, to which entities and for what purposes the personal data may be disclosed, the purpose limitations, how long they may be stored and what processing operations and procedures may be used, including measures to ensure lawful and fair processing, such as those for others special processing situations according to Chapter IX. 4Union law or the law of the Member States must pursue an objective in the public interest and be proportionate to the legitimate aim pursued.
|
||||
4. If the processing for a purpose other than the one for which the personal data was collected is not based on the consent of the data subject or on a law of the Union or of the Member States which, in a democratic society, provides a necessary and proportionate measure of protection of the objectives referred to in Article 23(1), the controller shall, in order to determine whether the processing for another purpose is compatible with the one for which the personal data were originally collected, take into account, among other things
|
||||
<ol style="list-style: lower-alpha">
|
||||
<li>any connection between the purposes for which the personal data
|
||||
were collected and the purposes of the intended further processing,</li>
|
||||
<li>the context in which the personal data was collected,
|
||||
in particular with regard to the relationship between the persons concerned and
|
||||
the person responsible,</li>
|
||||
<li>the type of personal data, in particular whether special categories
|
||||
personal data are processed in accordance with Article 9 or whether
|
||||
personal data on criminal convictions and offenses pursuant to
|
||||
Article 10 are processed,</li>
|
||||
<li>The possible consequences of the intended further processing for those affected
|
||||
people,</li>
|
||||
<li>the existence of appropriate safeguards, such as encryption or
|
||||
Pseudonymization may include.</li>
|
||||
</ol>
|
||||
`;
|
||||
|
||||
markdownSources.en!.termsAndConditions = `
|
||||
<p>As of November 04, 2015</p>
|
||||
<p>The respective data protection of the university in the respective federal state applies. In addition, the following agreements apply. By installing you agree to accept the following terms and conditions. <br>
|
||||
</p>
|
||||
<ol>
|
||||
<li>Scope and terms used
|
||||
<ol>
|
||||
<li> The development association, hereinafter referred to as <strong>app provider</strong>, provides information and services via its mobile app, hereinafter referred to as <strong>Open StApps</strong>. </li>
|
||||
<li> The service provider of the corresponding university selection is the respective university itself with university-specific and non-university-specific offers. The following distinctions are made:
|
||||
<ul>
|
||||
<li>University, hereinafter referred to as <strong>data provider</strong>, provides university-specific offers.</li>
|
||||
<li>Libraries, cafés, copy shops, canteens and other non-university-specific facilities, hereinafter referred to as <strong>third-party data providers</strong>, provide non-university-specific offers.</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li> Contractual partners with the app provider are hereinafter referred to as <strong>students</strong>. </li>
|
||||
<li> The information and services offered by the app provider are hereinafter referred to as <strong>service offer</strong>. </li>
|
||||
<li>The overall execution of a functionality until it reaches a final state, such as completing an examination registration or borrowing a medium, is referred to as <strong>process handling</strong>.</li>
|
||||
<li> The following general terms and conditions apply only to the app provider. The displayed service offers have no influence on this. </li>
|
||||
<li> The provision of a service is subject to the general terms and conditions of the respective data or third-party data provider. If the app provider has these General Terms and Conditions, they can be viewed here. </li>
|
||||
<li> The respective imprint of the data or third-party data provider applies. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Services provided by Open StApps
|
||||
<ol>
|
||||
<li>Open StApps acts exclusively as an app provider of services within the scope of its activities. After carrying out process handling within the app provider, further administrative activities can be added by the app provider. </li>
|
||||
<li>The app provider does not offer its own services, unless they serve to identify and assign them. Thus, for example, the exam registration does not apply to the contractual obligations of the app provider. A successful process handling of a service comes about exclusively between the students and the respective data or third-party data provider. The app provider is therefore not liable for the availability, implementation, quality, accompanying circumstances or any disruptions or changes to the service offerings. Such matters are fundamentally the sole responsibility of the data or third-party data provider. </li>
|
||||
<li> Accordingly, the app provider is not liable for the correctness, completeness or topicality of the service offers that are presented during the processing of the respective service offers. The mediation of the service offers is based on the service offers provided by the data or third-party data providers and transmitted to the students. The app provider does not carry out any prior, accompanying, general or individual checks for correctness or availability. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li> agreement
|
||||
<ol>
|
||||
<li>The processes completed by students are binding.</li>
|
||||
<li>The processes completed by students are mediated by the app provider.</li>
|
||||
<li>If the process handling of the student could be successfully conveyed, the app provider shows the success or failure of the process handling. Point III.1 only applies if the process handling is successful.</li>
|
||||
<li> The student has the following obligations in the interest of proper processing:
|
||||
<ul>
|
||||
<li>The availability of a constant internet connection during a process execution</li>
|
||||
<li>Proper use of Open StApps</li>
|
||||
<li>Successful authentication and authorization within the Open StApps if a process requires this</li>
|
||||
<li>The proper entry and verification of the data required for process execution</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li> After the process has been completed, the correctness of the data from the data or third-party data provider must be ensured. </li>
|
||||
<li> The student must regularly check the data required for process handling for changes at the data or third-party data provider. </li>
|
||||
<li> The app provider points out that the mediation of process handling does not fall under the statutory provisions for distance contracts. In this respect, the student has no special right of revocation or return to the app provider. </li>
|
||||
<li>The completion of a process is subject to the successful mediation of the app provider. In exceptional cases, data or third-party data providers are not authorized or authorized to successfully complete processes. This may not be apparent to the app provider in advance, so the app provider is not liable for it.</li>
|
||||
<li>If the process is successfully completed by a culpable violation on the part of the student, the app provider is entitled to inform the data or third-party data provider about this. The data or third-party data provider decides on the further procedure.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Changes to completed process executions
|
||||
<ol>
|
||||
<li>In the event of a completed process that is to the detriment of the student, the data or third-party data provider must be informed immediately. The data or third-party data provider decides independently how to proceed.</li>
|
||||
<li>In the case of a non-culpable process handling based on the service offer, there is no guarantee on the part of the app provider. Further processing is then carried out by the data or third-party data provider.</li>
|
||||
<li>If changes have to be made to the data of the completed process executions and cannot be carried out via the app provider, the data or third-party data provider must be informed of a corresponding change request.</li>
|
||||
<li>For reasons of preserving evidence, processes are stored in relation to persons and devices if processes require personal reference. The storage takes place according to the usual laws, but at least for 90 days. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Privacy and use of the Open StApps
|
||||
<ol>
|
||||
<li>The Open StApps is distributed as an app via the distribution provider "Google Play Store" or in "Apple iTunes" provided. The sales partner provides its own usage and data protection declarations. Open StApps has no influence on their terms of use and privacy policies.</li>
|
||||
<li>Personal and other data of the student are only collected, stored and forwarded to data or third-party data providers by the app provider if this is necessary for a completed process with personal reference.</li>
|
||||
<li> If the student provides contact information, the app provider can contact you. </li>
|
||||
<li>The app provider grants the user a free, unlimited, non-commercial right to use (<strong>license</strong>) the Open StApps.</li>
|
||||
<li>The license entitles the user to use the Open StApps as part of normal use. This includes the installation, execution, loading of the Open StApps into memory and its expiration. Other types of use are excluded.</li>
|
||||
<li>The modification, translation, processing, redesign, reverse engineering (so-called reverse engineering) and duplication, even partial or temporary, of the Open StApps and the program code are not permitted and constitute a violation of § 39 Para. 2 UrhG. The rest remain §§ 69d, 69e UrhG unaffected.</li>
|
||||
<li>The app provider is the owner of all industrial property rights, usage rights and copyrights to the Open StApps and related documentation. References to copyrights or other industrial property rights that are in the Open StApps may not be changed, removed or otherwise made unrecognizable.<br>
|
||||
</li>
|
||||
<li>Service offerings from the data or third party data providers may not be modified, copied, distributed, transmitted, published, licensed, assigned or sold and/or no derivative works created therefrom.</li>
|
||||
<li>The data or third-party data provider is responsible for compliance with the rights, such as copyright and personal rights, in his service offerings. The rights necessary for the proper functioning of the Open StApps exist and/or have been legally granted.</li>
|
||||
<li>Links that lead outside of the Open StApps are only provided to the students as information. The app provider has no influence on the content that can be accessed via the link and does not signify any approval of this content, nor is this content connected between Open StApps and the operators. </li>
|
||||
<li>All content within Open StApps is copyright © 2015 for "Open StApps - The Student App Development Kit" and their data or third party data providers. All rights reserved. </li>
|
||||
<li>All functions within the Open StApps are the copyrighted work of the Open StApps and their network members. The use of the Open StApps by students is subject to the terms of the End User License Agreement (EULA). The reproduction or redistribution of the functions within the Open StApps and the Open StApps themselves is prohibited. Furthermore, the Open StApps software may not be reverse engineered, decompiled or disassembled. The Open StApps may not be directly or indirectly exported and/or re-exported to countries that are subject to the export restrictions of Germany or their Schengen states.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Use of location data
|
||||
<ol>
|
||||
<li>The app allows activating the location service via the settings. The location service allows access to your location data to display detailed distance data.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Warranty
|
||||
<ol>
|
||||
<li>The app provider guarantees, in accordance with the provisions of § 434 ff BGB, that Open StApps has been created with the utmost care and expertise. A complete exclusion of software errors within the Open StApps is not possible at the current state of knowledge.</li>
|
||||
<li>The app provider of the Open StApps corrects errors that significantly impair the intended use of the Open StApps. Troubleshooting is carried out according to the respective error classification by the app provider at a time specified by the app provider. To eliminate errors, the user is obliged to install and run the error-corrected version over the existing error-affected version. The user's involvement in troubleshooting may be required. </li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Liability
|
||||
<ol>
|
||||
<li>The contractual obligations of Open StApps exclusively include the proper mediation of service offers between students and data or third-party data providers, taking into account the agreements listed in these terms and conditions.</li>
|
||||
<li>The app provider is liable for damage caused intentionally or through gross negligence.</li>
|
||||
<li>The app provider is not liable for recovering data.</li>
|
||||
<li>The user is obliged to regularly back up their data.</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>Final Provisions
|
||||
<ol>
|
||||
<li>Should a provision of this contract be or become legally ineffective, this has no effect on the validity of the remaining contractual provisions and the contract itself. The ineffective provision will be replaced by an effective provision that is economically almost identical to it. The same applies to gaps in the contract or insufficient contractual provisions. </li>
|
||||
<li>The general terms and conditions of the Open StApps and the data or third-party data providers can be changed at any time without separate prior notice. Contracts that have already been concluded or are in progress are not retrospectively affected by changes. Open StApps always provides its own current and valid version of its general terms and conditions. </li>
|
||||
<li>The law of the Federal Republic of Germany applies.</li>
|
||||
<li>Berlin is the place of jurisdiction.</li>
|
||||
</ol>
|
||||
</li>
|
||||
</ol>
|
||||
<p>If you have any questions or comments about these terms and conditions, please contact: <a href="mailto:app@rz.uni-frankfurt.de">app@rz.uni-frankfurt.de</a></p>
|
||||
For general inquiries you can use a contact form:<br />
|
||||
<https://datenschutz.hessen.de/kontakt><br />
|
||||
<br />
|
||||
A complaint form is also available for complaints:<br />
|
||||
<https://datenschutz.hessen.de/service/beschwerde-uebermitteln>
|
||||
`;
|
||||
|
||||
/**
|
||||
@@ -562,10 +367,12 @@ const config: RecursivePartial<SCConfigFile> = {
|
||||
},
|
||||
endpoints: {
|
||||
authorization: 'https://cas.rz.uni-frankfurt.de/cas/oauth2.0/authorize',
|
||||
endSession: 'https://cas.rz.uni-frankfurt.de/cas/logout',
|
||||
mapping: {
|
||||
id: '$.id',
|
||||
email: '$.attributes.mailPrimaryAddress',
|
||||
familyName: '$.attributes.sn',
|
||||
givenName: '$.attributes.givenName',
|
||||
name: '$.attributes.givenName',
|
||||
role: '$.attributes.eduPersonPrimaryAffiliation',
|
||||
studentId: '$.attributes.employeeNumber',
|
||||
@@ -574,6 +381,25 @@ const config: RecursivePartial<SCConfigFile> = {
|
||||
userinfo: 'https://cas.rz.uni-frankfurt.de/cas/oauth2.0/profile',
|
||||
},
|
||||
},
|
||||
paia: {
|
||||
client: {
|
||||
clientId: '',
|
||||
scopes: '',
|
||||
url: 'https://hds.hebis.de/Shibboleth.sso/UBFFM?target=https://hds.hebis.de/ubffm/paia_login_stub.php',
|
||||
},
|
||||
endpoints: {
|
||||
authorization:
|
||||
'https://hds.hebis.de/Shibboleth.sso/UBFFM?target=https://hds.hebis.de/ubffm/paia_login_stub.php',
|
||||
endSession: 'https://ubffm.hds.hebis.de/Shibboleth.sso/Logout',
|
||||
mapping: {
|
||||
id: '$.email',
|
||||
name: '$.name',
|
||||
role: '$.type',
|
||||
},
|
||||
token: 'https://hds.hebis.de/paia/auth/login',
|
||||
userinfo: 'https://hds.hebis.de/paia/core',
|
||||
},
|
||||
},
|
||||
},
|
||||
app: {
|
||||
features: {
|
||||
@@ -588,6 +414,10 @@ const config: RecursivePartial<SCConfigFile> = {
|
||||
hebisProxy: {
|
||||
url: 'https://proxy.ub.uni-frankfurt.de/login?qurl=',
|
||||
},
|
||||
paia: {
|
||||
authProvider: 'paia',
|
||||
url: 'https://hds.hebis.de/paia/core',
|
||||
},
|
||||
},
|
||||
},
|
||||
aboutPages: {
|
||||
@@ -713,17 +543,6 @@ const config: RecursivePartial<SCConfigFile> = {
|
||||
},
|
||||
type: SCAboutPageContentType.ROUTER_LINK,
|
||||
},
|
||||
{
|
||||
icon: 'text_snippet',
|
||||
title: 'Allgemeine Geschäftsbedingungen',
|
||||
link: 'terms',
|
||||
translations: {
|
||||
en: {
|
||||
title: 'Terms and conditions',
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.ROUTER_LINK,
|
||||
},
|
||||
{
|
||||
icon: 'copyright',
|
||||
title: 'Bibliotheken und Lizenzen',
|
||||
@@ -746,36 +565,17 @@ const config: RecursivePartial<SCConfigFile> = {
|
||||
title: 'Impressum',
|
||||
content: [
|
||||
{
|
||||
title: 'Beteiligte Universitäten',
|
||||
card: true,
|
||||
content: {
|
||||
value: `
|
||||
|
||||
[Johann Wolfgang Goethe-Universität Frankfurt am Main](https://uni-frankfurt.de)<br>
|
||||
[Universität Kassel](https://www.uni-kassel.de)<br>
|
||||
[Philipps-Universität Marburg](https://www.uni-marburg.de)<br>
|
||||
[Technische Hochschule Mittelhessen](https://www.thm.de)<br>
|
||||
weitere Hochschulen und Mitarbeitende
|
||||
value: `
|
||||
[Impressum der Johann Wolfgang Goethe-Universität Frankfurt am Main](https://www.uni-frankfurt.de/impressum)
|
||||
`,
|
||||
translations: {
|
||||
en: {
|
||||
value: `
|
||||
[Goethe University Frankfurt](https://uni-frankfurt.de)<br>
|
||||
[University of Kassel](https://www.uni-kassel.de)<br>
|
||||
[University of Marburg](https://www.uni-marburg.de)<br>
|
||||
[University of Applied Sciences Mittelhessen](https://www.thm.de)<br>
|
||||
further universities and developers
|
||||
`,
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.MARKDOWN,
|
||||
},
|
||||
translations: {
|
||||
en: {
|
||||
title: 'Collaborating Universities',
|
||||
value: `
|
||||
[Imprint of the Goethe University Frankfurt](https://www.uni-frankfurt.de/impressum)
|
||||
`,
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.SECTION,
|
||||
type: SCAboutPageContentType.MARKDOWN,
|
||||
},
|
||||
],
|
||||
translations: {
|
||||
@@ -803,25 +603,6 @@ const config: RecursivePartial<SCConfigFile> = {
|
||||
},
|
||||
},
|
||||
},
|
||||
'about/terms': {
|
||||
title: 'AGB',
|
||||
content: [
|
||||
{
|
||||
value: markdownSources.de!.termsAndConditions,
|
||||
translations: {
|
||||
en: {
|
||||
value: markdownSources.en!.termsAndConditions,
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.MARKDOWN,
|
||||
},
|
||||
],
|
||||
translations: {
|
||||
en: {
|
||||
title: 'Terms and conditions',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
// tslint:disable:no-default-export
|
||||
// tslint:disable:no-magic-numbers
|
||||
import {
|
||||
SCAboutPageContentType,
|
||||
SCConfigFile,
|
||||
@@ -13,21 +11,31 @@ import {readFileSync} from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* Evaluates if a number is within the given range
|
||||
* Generates a range of numbers that represent consecutive calendric months
|
||||
*
|
||||
* @param number_ The number that should be checked
|
||||
* @param range Array of two numbers representing a range (inclusive interval)
|
||||
* @param startMonth The month to start with (inclusive)
|
||||
* @param endMonth The month to end with (inclusive)
|
||||
*/
|
||||
export function inRangeInclusive(number_: number, range: number[]): boolean {
|
||||
return number_ >= range[0] && number_ <= range[1];
|
||||
export function yearSlice(startMonth: number, endMonth: number) {
|
||||
let months = [...Array.from({length: 13}).keys()].slice(1);
|
||||
months = [...months, ...months];
|
||||
if (!months.includes(startMonth) || !months.includes(endMonth)) {
|
||||
throw new Error(`Given months not part of a year! Check ${startMonth} or ${endMonth}!`);
|
||||
}
|
||||
|
||||
const startIndex = months.indexOf(startMonth);
|
||||
const endIndex =
|
||||
months.indexOf(endMonth) <= startIndex ? months.lastIndexOf(endMonth) : months.indexOf(endMonth);
|
||||
|
||||
return months.slice(startIndex, endIndex + 1);
|
||||
}
|
||||
|
||||
const sommerRange = [4, 1];
|
||||
const winterRange = [10, 1];
|
||||
const sommerRange = yearSlice(3, 8);
|
||||
const winterRange = yearSlice(9, 2);
|
||||
const month = new Date().getMonth();
|
||||
const year = new Date().getFullYear();
|
||||
const winterYearOffset = month < winterRange[0] ? -1 : 0;
|
||||
const sommerYear = year + (month <= winterRange[1] ? -1 : 0);
|
||||
const sommerYear = year + (month <= winterRange[winterRange.length] ? -1 : 0);
|
||||
const winterYear = `${year + winterYearOffset}/${(year + 1 + winterYearOffset).toString().slice(-2)}`;
|
||||
|
||||
const wsAcronymShort = `WS ${winterYear}`;
|
||||
@@ -111,7 +119,7 @@ const languageSetting: SCLanguageSetting = {
|
||||
* IDE to read the TSDoc documentation.
|
||||
*/
|
||||
|
||||
const config: Partial<SCConfigFile> = {
|
||||
const config: SCConfigFile = {
|
||||
app: {
|
||||
aboutPages: {
|
||||
'about': {
|
||||
@@ -242,17 +250,6 @@ const config: Partial<SCConfigFile> = {
|
||||
},
|
||||
type: SCAboutPageContentType.ROUTER_LINK,
|
||||
},
|
||||
{
|
||||
icon: 'text_snippet',
|
||||
title: 'Allgemeine Geschäftsbedingungen',
|
||||
link: 'terms',
|
||||
translations: {
|
||||
en: {
|
||||
title: 'Terms and conditions',
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.ROUTER_LINK,
|
||||
},
|
||||
{
|
||||
icon: 'copyright',
|
||||
title: 'Bibliotheken und Lizenzen',
|
||||
@@ -279,17 +276,21 @@ const config: Partial<SCConfigFile> = {
|
||||
card: true,
|
||||
content: {
|
||||
value: `
|
||||
[Nimrasi Universität Null Island]()
|
||||
|
||||
[Königliche Hochschule Lummerland]()
|
||||
`,
|
||||
[Johann Wolfgang Goethe-Universität Frankfurt am Main](https://uni-frankfurt.de)<br>
|
||||
[Philipps-Universität Marburg](https://www.uni-marburg.de)<br>
|
||||
[Technische Hochschule Mittelhessen](https://www.thm.de)<br>
|
||||
[Universität Kassel](https://www.uni-kassel.de)<br>
|
||||
weitere Hochschulen und Mitarbeitende
|
||||
`,
|
||||
translations: {
|
||||
en: {
|
||||
value: `
|
||||
[Nimrasi University of Null Island]()
|
||||
|
||||
[Royal Institute of Make-Believe]()
|
||||
`,
|
||||
[Goethe University Frankfurt](https://uni-frankfurt.de)<br>
|
||||
[University of Marburg](https://www.uni-marburg.de)<br>
|
||||
[University of Applied Sciences Mittelhessen](https://www.thm.de)<br>
|
||||
[University of Kassel](https://www.uni-kassel.de)<br>
|
||||
further universities and developers
|
||||
`,
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.MARKDOWN,
|
||||
@@ -327,25 +328,6 @@ const config: Partial<SCConfigFile> = {
|
||||
},
|
||||
},
|
||||
},
|
||||
'about/terms': {
|
||||
title: 'Allgemeine Geschäftsbedingungen',
|
||||
content: [
|
||||
{
|
||||
value: 'Hier wären die AGB',
|
||||
translations: {
|
||||
en: {
|
||||
value: 'This would be the terms & conditions',
|
||||
},
|
||||
},
|
||||
type: SCAboutPageContentType.MARKDOWN,
|
||||
},
|
||||
],
|
||||
translations: {
|
||||
en: {
|
||||
title: 'Terms and conditions',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
campusPolygon: {
|
||||
coordinates: [
|
||||
@@ -359,14 +341,7 @@ const config: Partial<SCConfigFile> = {
|
||||
],
|
||||
type: 'Polygon',
|
||||
},
|
||||
features: {
|
||||
extern: {
|
||||
paia: {
|
||||
authProvider: 'paia',
|
||||
url: 'https://hds.hebis.de/paia/core',
|
||||
},
|
||||
},
|
||||
},
|
||||
features: {},
|
||||
menus: [
|
||||
{
|
||||
icon: 'home',
|
||||
@@ -538,10 +513,10 @@ const config: Partial<SCConfigFile> = {
|
||||
title: 'about',
|
||||
translations: {
|
||||
de: {
|
||||
title: 'Über Open StApps',
|
||||
title: 'Über die App',
|
||||
},
|
||||
en: {
|
||||
title: 'About Open StApps',
|
||||
title: 'About the App',
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -562,26 +537,7 @@ const config: Partial<SCConfigFile> = {
|
||||
privacyPolicyUrl: 'https://mobile.server.uni-frankfurt.de/_static/privacy.md',
|
||||
settings: [userGroupSetting, languageSetting],
|
||||
},
|
||||
auth: {
|
||||
paia: {
|
||||
client: {
|
||||
clientId: '',
|
||||
scopes: '',
|
||||
url: 'https://hds.hebis.de/Shibboleth.sso/UBFFM?target=https://hds.hebis.de/ubffm/paia_login_stub.php',
|
||||
},
|
||||
endpoints: {
|
||||
authorization:
|
||||
'https://hds.hebis.de/Shibboleth.sso/UBFFM?target=https://hds.hebis.de/ubffm/paia_login_stub.php',
|
||||
mapping: {
|
||||
id: '$.email',
|
||||
name: '$.name',
|
||||
role: '$.type',
|
||||
},
|
||||
token: 'https://hds.hebis.de/paia/auth/login',
|
||||
userinfo: 'https://hds.hebis.de/paia/core',
|
||||
},
|
||||
},
|
||||
},
|
||||
auth: {},
|
||||
backend: {
|
||||
SCVersion: JSON.parse(readFileSync(path.resolve('.', '.', 'package.json'), 'utf8').toString())
|
||||
.dependencies['@openstapps/core'],
|
||||
@@ -695,10 +651,10 @@ const config: Partial<SCConfigFile> = {
|
||||
factor: 1,
|
||||
fields: {
|
||||
'academicTerms.acronym': {
|
||||
[ssAcronymShort]: inRangeInclusive(month, sommerRange) ? 1.1 : 1.05,
|
||||
[wsAcronymShort]: inRangeInclusive(month, winterRange) ? 1.1 : 1.05,
|
||||
[ssAcronymLong]: inRangeInclusive(month, sommerRange) ? 1.1 : 1.05,
|
||||
[wsAcronymLong]: inRangeInclusive(month, winterRange) ? 1.1 : 1.05,
|
||||
[ssAcronymShort]: sommerRange.includes(month) ? 1.1 : 1.05,
|
||||
[wsAcronymShort]: winterRange.includes(month) ? 1.1 : 1.05,
|
||||
[ssAcronymLong]: sommerRange.includes(month) ? 1.1 : 1.05,
|
||||
[wsAcronymLong]: winterRange.includes(month) ? 1.1 : 1.05,
|
||||
},
|
||||
},
|
||||
type: SCThingType.AcademicEvent,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// tslint:disable:no-default-export
|
||||
// tslint:disable:no-magic-numbers
|
||||
import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch';
|
||||
import {ElasticsearchConfigFile} from '../src/storage/elasticsearch/types/elasticsearch-config';
|
||||
|
||||
/**
|
||||
* This is the default configuration for elasticsearch (a database)
|
||||
@@ -19,13 +19,13 @@ const config: ElasticsearchConfigFile = {
|
||||
internal: {
|
||||
database: {
|
||||
name: 'elasticsearch',
|
||||
version: '5.6',
|
||||
version: '8.4',
|
||||
query: {
|
||||
minMatch: '75%',
|
||||
queryType: 'dis_max',
|
||||
matchBoosting: 1.3,
|
||||
fuzziness: 'AUTO',
|
||||
cutoffFrequency: 0.0,
|
||||
cutoffFrequency: 0,
|
||||
tieBreaker: 0,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
elasticsearch:
|
||||
ports:
|
||||
- "9200:9200"
|
||||
image: "registry.gitlab.com/openstapps/database:master"
|
||||
image: "registry.gitlab.com/openstapps/database:latest"
|
||||
|
||||
apicli:
|
||||
image: "registry.gitlab.com/openstapps/api/cli:latest"
|
||||
|
||||
5177
package-lock.json
generated
5177
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
32
package.json
32
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@openstapps/backend",
|
||||
"version": "0.6.0",
|
||||
"version": "1.0.0",
|
||||
"description": "A reference implementation for a StApps backend",
|
||||
"license": "AGPL-3.0-only",
|
||||
"author": "André Bierlein <andre.mt.bierlein@gmail.com>",
|
||||
@@ -19,7 +19,7 @@
|
||||
"changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0 && git add CHANGELOG.md",
|
||||
"check-configuration": "openstapps-configuration",
|
||||
"compile": "rimraf lib && tsc && prepend lib/cli.js '#!/usr/bin/env node\n'",
|
||||
"documentation": "typedoc --includeVersion --out docs --readme README.md --listInvalidSymbolLinks --entryPointStrategy expand src && openstapps-core-tools openapi ./node_modules/@openstapps/core/lib ./docs/openapi && redoc-cli bundle docs/openapi/openapi.json -o docs/openapi/index.html",
|
||||
"documentation": "typedoc --includeVersion --out docs --readme README.md --listInvalidSymbolLinks --entryPointStrategy expand src && openstapps-core-tools openapi ./node_modules/@openstapps/core/lib ./docs/openapi && openapi build-docs docs/openapi/openapi.json -o docs/openapi/index.html",
|
||||
"version": "npm run changelog",
|
||||
"prepublishOnly": "npm ci && npm run build",
|
||||
"preversion": "npm run prepublishOnly",
|
||||
@@ -27,17 +27,18 @@
|
||||
"start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js",
|
||||
"start-debug": "STAPPS_LOG_LEVEL=31 NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true node ./lib/cli.js --require ts-node/register",
|
||||
"test": "npm run test-unit && npm run test-integration",
|
||||
"test-unit": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
|
||||
"test-unit": "cross-env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true STAPPS_LOG_LEVEL=0 nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
|
||||
"test-integration": "docker-compose -f integration-test.yml pull && docker-compose -f integration-test.yml up --build --abort-on-container-exit --exit-code-from apicli",
|
||||
"lint": "eslint -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/",
|
||||
"lint:fix": "eslint --fix -c .eslintrc.json --ignore-path .eslintignore --ext .ts src/ test/"
|
||||
},
|
||||
"dependencies": {
|
||||
"@elastic/elasticsearch": "5.6.22",
|
||||
"@openstapps/core": "0.74.0",
|
||||
"@elastic/elasticsearch": "8.4.0",
|
||||
"@openstapps/core": "1.0.1",
|
||||
"@openstapps/core-tools": "0.34.0",
|
||||
"@openstapps/logger": "1.1.1",
|
||||
"@types/node": "14.18.36",
|
||||
"@redocly/cli": "1.0.0-beta.125",
|
||||
"@types/node": "14.18.43",
|
||||
"config": "3.3.9",
|
||||
"cors": "2.8.5",
|
||||
"express": "4.18.2",
|
||||
@@ -46,26 +47,25 @@
|
||||
"got": "11.8.6",
|
||||
"moment": "2.29.4",
|
||||
"morgan": "1.10.0",
|
||||
"nock": "13.3.0",
|
||||
"nock": "13.3.1",
|
||||
"node-cache": "5.1.2",
|
||||
"node-cron": "3.0.2",
|
||||
"nodemailer": "6.9.1",
|
||||
"prom-client": "14.1.1",
|
||||
"prom-client": "14.2.0",
|
||||
"promise-queue": "2.2.5",
|
||||
"ts-node": "10.9.1",
|
||||
"uuid": "8.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openstapps/configuration": "0.34.0",
|
||||
"@openstapps/es-mapping-generator": "0.4.0",
|
||||
"@openstapps/es-mapping-generator": "0.6.0",
|
||||
"@openstapps/eslint-config": "1.1.0",
|
||||
"@testdeck/mocha": "0.3.3",
|
||||
"@types/chai": "4.3.4",
|
||||
"@types/chai-as-promised": "7.1.5",
|
||||
"@types/config": "3.3.0",
|
||||
"@types/cors": "2.8.13",
|
||||
"@types/elasticsearch": "5.0.40",
|
||||
"@types/express": "4.17.16",
|
||||
"@types/express": "4.17.17",
|
||||
"@types/geojson": "1.0.6",
|
||||
"@types/mocha": "10.0.1",
|
||||
"@types/morgan": "1.9.4",
|
||||
@@ -80,9 +80,10 @@
|
||||
"chai": "4.3.7",
|
||||
"chai-as-promised": "7.1.1",
|
||||
"conventional-changelog-cli": "2.2.2",
|
||||
"eslint": "8.33.0",
|
||||
"eslint-config-prettier": "8.6.0",
|
||||
"eslint-plugin-jsdoc": "39.7.4",
|
||||
"cross-env": "7.0.3",
|
||||
"eslint": "8.39.0",
|
||||
"eslint-config-prettier": "8.8.0",
|
||||
"eslint-plugin-jsdoc": "39.9.1",
|
||||
"eslint-plugin-prettier": "4.2.1",
|
||||
"eslint-plugin-unicorn": "43.0.2",
|
||||
"get-port": "5.1.1",
|
||||
@@ -90,8 +91,7 @@
|
||||
"mocked-env": "1.3.5",
|
||||
"nyc": "15.1.0",
|
||||
"prepend-file-cli": "1.0.6",
|
||||
"prettier": "2.8.3",
|
||||
"redoc-cli": "0.13.20",
|
||||
"prettier": "2.8.8",
|
||||
"rimraf": "3.0.2",
|
||||
"sinon": "14.0.2",
|
||||
"sinon-express-mock": "2.2.1",
|
||||
|
||||
@@ -64,6 +64,7 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
|
||||
}),
|
||||
);
|
||||
|
||||
/* istanbul ignore if */
|
||||
if (process.env.PROMETHEUS_MIDDLEWARE === 'true') {
|
||||
app.use(getPrometheusMiddleware());
|
||||
}
|
||||
@@ -142,7 +143,10 @@ export async function configureApp(app: Express, databases: {[name: string]: Dat
|
||||
});
|
||||
|
||||
// validate config file
|
||||
await validator.addSchemas(path.join('node_modules', '@openstapps', 'core', 'lib', 'schema'));
|
||||
await validator.addSchemas(
|
||||
// eslint-disable-next-line unicorn/prefer-module
|
||||
path.join(path.dirname(require.resolve('@openstapps/core/package.json')), 'lib', 'schema'),
|
||||
);
|
||||
|
||||
// validate the config file
|
||||
const configValidation = validator.validate(configFile, 'SCConfigFile');
|
||||
|
||||
@@ -13,73 +13,45 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCFacet, SCThingType} from '@openstapps/core';
|
||||
import {aggregations} from './templating';
|
||||
import {AggregationResponse} from './types/elasticsearch';
|
||||
import {
|
||||
isBucketAggregation,
|
||||
isESAggMatchAllFilter,
|
||||
isESNestedAggregation,
|
||||
isESTermsFilter,
|
||||
isNestedAggregation,
|
||||
} from './types/guards';
|
||||
AggregateName,
|
||||
AggregationsAggregate,
|
||||
AggregationsFiltersAggregate,
|
||||
AggregationsMultiTermsBucket,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCFacet, SCThingType} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* Parses elasticsearch aggregations (response from es) to facets for the app
|
||||
*
|
||||
* @param aggregationResponse - aggregations response from elasticsearch
|
||||
*/
|
||||
export function parseAggregations(aggregationResponse: AggregationResponse): SCFacet[] {
|
||||
export function parseAggregations(
|
||||
aggregationResponse: Record<AggregateName, AggregationsAggregate>,
|
||||
): SCFacet[] {
|
||||
const facets: SCFacet[] = [];
|
||||
|
||||
// get all names of the types an aggregation is on
|
||||
for (const typeName in aggregations) {
|
||||
if (aggregations.hasOwnProperty(typeName) && aggregationResponse.hasOwnProperty(typeName)) {
|
||||
// the type object from the schema
|
||||
const type = aggregations[typeName];
|
||||
// the "real" type object from the response
|
||||
const realType = aggregationResponse[typeName];
|
||||
for (const aggregateName in aggregationResponse) {
|
||||
const aggregation = aggregationResponse[aggregateName] as AggregationsMultiTermsBucket;
|
||||
const type = aggregateName === '@all' ? {} : {onlyOnType: aggregateName as SCThingType};
|
||||
|
||||
// both conditions must apply, else we have an error somewhere
|
||||
if (isESNestedAggregation(type) && isNestedAggregation(realType)) {
|
||||
for (const fieldName in type.aggs) {
|
||||
if (type.aggs.hasOwnProperty(fieldName) && realType.hasOwnProperty(fieldName)) {
|
||||
// the field object from the schema
|
||||
const field = type.aggs[fieldName];
|
||||
// the "real" field object from the response
|
||||
const realField = realType[fieldName];
|
||||
for (const field in aggregation) {
|
||||
const fieldAggregate = aggregation[field] as AggregationsFiltersAggregate;
|
||||
if (typeof fieldAggregate !== 'object') continue;
|
||||
|
||||
// this should always be true in theory...
|
||||
if (isESTermsFilter(field) && isBucketAggregation(realField) && realField.buckets.length > 0) {
|
||||
const facet: SCFacet = {
|
||||
buckets: realField.buckets.map(bucket => {
|
||||
return {
|
||||
count: bucket.doc_count,
|
||||
key: bucket.key,
|
||||
};
|
||||
}),
|
||||
field: fieldName,
|
||||
};
|
||||
// if it's not for all types then create the appropriate field and set the type name
|
||||
if (!isESAggMatchAllFilter(type.filter)) {
|
||||
facet.onlyOnType = type.filter.type.value as SCThingType;
|
||||
}
|
||||
facets.push(facet);
|
||||
}
|
||||
}
|
||||
}
|
||||
// the last part here means that it is a bucket aggregation
|
||||
} else if (isESTermsFilter(type) && !isNestedAggregation(realType) && realType.buckets.length > 0) {
|
||||
facets.push({
|
||||
buckets: realType.buckets.map(bucket => {
|
||||
return {
|
||||
count: bucket.doc_count,
|
||||
key: bucket.key,
|
||||
};
|
||||
}),
|
||||
field: typeName,
|
||||
});
|
||||
}
|
||||
const buckets = Object.values(fieldAggregate.buckets).map(bucket => {
|
||||
return {
|
||||
count: bucket.doc_count,
|
||||
key: bucket.key as string,
|
||||
};
|
||||
});
|
||||
if (buckets.length === 0) continue;
|
||||
|
||||
facets.push({
|
||||
buckets,
|
||||
field,
|
||||
...type,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2019 StApps
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
@@ -13,58 +13,47 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {ApiResponse, Client, events, RequestParams} from '@elastic/elasticsearch';
|
||||
import {Client, events} from '@elastic/elasticsearch';
|
||||
import {
|
||||
SCBulkResponse,
|
||||
SCConfigFile,
|
||||
SCFacet,
|
||||
SCSearchQuery,
|
||||
SCSearchResponse,
|
||||
SCThings,
|
||||
SCThingType,
|
||||
SCUuid,
|
||||
} from '@openstapps/core';
|
||||
AggregateName,
|
||||
AggregationsMultiTermsBucket,
|
||||
IndicesGetAliasResponse,
|
||||
IndicesUpdateAliasesAction,
|
||||
SearchHit,
|
||||
SearchResponse,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
|
||||
import {Logger} from '@openstapps/logger';
|
||||
// we only have the @types package because some things type definitions are still missing from the official
|
||||
// @elastic/elasticsearch package
|
||||
import {IndicesUpdateAliasesParamsAction, SearchResponse} from 'elasticsearch';
|
||||
import moment from 'moment';
|
||||
import {MailQueue} from '../../notification/mail-queue';
|
||||
import {Bulk} from '../bulk-storage';
|
||||
import {Database} from '../database';
|
||||
import {parseAggregations} from './aggregations';
|
||||
import * as Monitoring from './monitoring';
|
||||
import {buildQuery, buildSort} from './query';
|
||||
import {buildQuery} from './query/query';
|
||||
import {buildSort} from './query/sort';
|
||||
import {aggregations, putTemplate} from './templating';
|
||||
import {
|
||||
AggregationResponse,
|
||||
ElasticsearchConfig,
|
||||
ElasticsearchObject,
|
||||
ElasticsearchQueryDisMaxConfig,
|
||||
ElasticsearchQueryQueryStringConfig,
|
||||
} from './types/elasticsearch';
|
||||
|
||||
/**
|
||||
* Matches index names such as stapps_<type>_<source>_<random suffix>
|
||||
*/
|
||||
const indexRegex = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
|
||||
} from './types/elasticsearch-config';
|
||||
import {ALL_INDICES_QUERY, getThingIndexName, parseIndexName, VALID_INDEX_REGEX} from './util';
|
||||
import {removeInvalidAliasChars} from './util/alias';
|
||||
import {noUndefined} from './util/no-undefined';
|
||||
import {retryCatch, RetryOptions} from './util/retry';
|
||||
|
||||
/**
|
||||
* A database interface for elasticsearch
|
||||
*/
|
||||
export class Elasticsearch implements Database {
|
||||
/**
|
||||
* Length of the index UID used for generation of its name
|
||||
*/
|
||||
static readonly INDEX_UID_LENGTH = 8;
|
||||
|
||||
/**
|
||||
* Holds a map of all elasticsearch indices that are available to search
|
||||
*/
|
||||
aliasMap: {
|
||||
// each scType has a alias which can contain multiple sources
|
||||
// each scType has an alias which can contain multiple sources
|
||||
[scType: string]: {
|
||||
// each source is assigned a index name in elasticsearch
|
||||
// each source is assigned an index name in elasticsearch
|
||||
[source: string]: string;
|
||||
};
|
||||
};
|
||||
@@ -97,89 +86,11 @@ export class Elasticsearch implements Database {
|
||||
return 'http://localhost:9200';
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the index name in elasticsearch for one SCThingType
|
||||
*
|
||||
* @param type SCThingType of data in the index
|
||||
* @param source source of data in the index
|
||||
* @param bulk bulk process which created this index
|
||||
*/
|
||||
static getIndex(type: SCThingType, source: string, bulk: SCBulkResponse) {
|
||||
let out = type.toLowerCase();
|
||||
while (out.includes(' ')) {
|
||||
out = out.replace(' ', '_');
|
||||
}
|
||||
|
||||
return `stapps_${out}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides the index UID (for its name) from the bulk UID
|
||||
*
|
||||
* @param uid Bulk UID
|
||||
*/
|
||||
static getIndexUID(uid: SCUuid) {
|
||||
return uid.slice(0, Math.max(0, Elasticsearch.INDEX_UID_LENGTH));
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a string which matches all indices
|
||||
*/
|
||||
static getListOfAllIndices(): string {
|
||||
// map each SC type in upper camel case
|
||||
return 'stapps_*_*_*';
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks for invalid character in alias names and removes them
|
||||
*
|
||||
* @param alias The alias name
|
||||
* @param uid The UID of the current bulk (for debugging purposes)
|
||||
*/
|
||||
static removeAliasChars(alias: string, uid: string | undefined): string {
|
||||
let formattedAlias = alias;
|
||||
|
||||
// spaces are included in some types, replace them with underscores
|
||||
if (formattedAlias.includes(' ')) {
|
||||
formattedAlias = formattedAlias.trim();
|
||||
formattedAlias = formattedAlias.split(' ').join('_');
|
||||
}
|
||||
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
|
||||
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
|
||||
if (formattedAlias.includes(value)) {
|
||||
formattedAlias = formattedAlias.replace(value, '');
|
||||
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
|
||||
having the same alias despite having different types, as invalid characters are removed automatically.
|
||||
New alias name is "${formattedAlias}."`);
|
||||
}
|
||||
}
|
||||
for (const value of ['-', '_', '+']) {
|
||||
if (formattedAlias.charAt(0) === value) {
|
||||
formattedAlias = formattedAlias.slice(1);
|
||||
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
|
||||
alias despite having different types, as invalid characters are removed automatically.
|
||||
New alias name is "${formattedAlias}."`);
|
||||
}
|
||||
}
|
||||
if (formattedAlias === '.' || formattedAlias === '..') {
|
||||
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
|
||||
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
|
||||
|
||||
return 'alias_placeholder';
|
||||
}
|
||||
if (formattedAlias.includes(':')) {
|
||||
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
|
||||
Elasticsearch versions!`);
|
||||
}
|
||||
|
||||
return formattedAlias;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new interface for elasticsearch
|
||||
*
|
||||
* @param config an assembled config file
|
||||
* @param mailQueue a mailqueue for monitoring
|
||||
* @param mailQueue a mail queue for monitoring
|
||||
*/
|
||||
constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) {
|
||||
if (
|
||||
@@ -192,7 +103,7 @@ export class Elasticsearch implements Database {
|
||||
this.client = new Client({
|
||||
node: Elasticsearch.getElasticsearchUrl(),
|
||||
});
|
||||
this.client.on(events.REQUEST, async (error: Error | null, result: ApiResponse<unknown>) => {
|
||||
this.client.diagnostic.on(events.REQUEST, async (error: Error | null, result: unknown) => {
|
||||
if (error !== null) {
|
||||
await Logger.error(error);
|
||||
}
|
||||
@@ -210,73 +121,40 @@ export class Elasticsearch implements Database {
|
||||
/**
|
||||
* Gets a map which contains each alias and all indices that are associated with each alias
|
||||
*/
|
||||
private async getAliasMap() {
|
||||
// delay after which alias map will be fetched again
|
||||
const RETRY_INTERVAL = 5000;
|
||||
// maximum number of retries
|
||||
const RETRY_COUNT = 3;
|
||||
// create a list of old indices that are not in use
|
||||
const oldIndicesToDelete: string[] = [];
|
||||
|
||||
let aliases:
|
||||
| {
|
||||
[index: string]: {
|
||||
/**
|
||||
* Aliases of an index
|
||||
*/
|
||||
aliases: {
|
||||
[K in SCThingType]: unknown;
|
||||
};
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
|
||||
for (const retry of [...Array.from({length: RETRY_COUNT})].map((_, i) => i + 1)) {
|
||||
if (typeof aliases !== 'undefined') {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const aliasResponse = await this.client.indices.getAlias({});
|
||||
aliases = aliasResponse.body;
|
||||
} catch (error) {
|
||||
private async getAliasMap(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}) {
|
||||
const aliasResponse = await retryCatch({
|
||||
maxRetries: 10,
|
||||
retryInterval: 2000,
|
||||
doAction: () => this.client.indices.getAlias(),
|
||||
onFailedAttempt: (attempt, error, {maxRetries, retryInterval}) => {
|
||||
Logger.warn('Failed getting alias map:', error);
|
||||
Logger.warn(`Retrying in ${RETRY_INTERVAL} milliseconds. (${retry} of ${RETRY_COUNT})`);
|
||||
await new Promise(resolve => setTimeout(resolve, RETRY_INTERVAL));
|
||||
}
|
||||
}
|
||||
Logger.warn(`Retrying in ${retryInterval} milliseconds. (${attempt} of ${maxRetries})`);
|
||||
},
|
||||
onFail: ({maxRetries}) => {
|
||||
throw new TypeError(`Failed to retrieve alias map after ${maxRetries} attempts!`);
|
||||
},
|
||||
...retryOptions,
|
||||
});
|
||||
|
||||
if (typeof aliases === 'undefined') {
|
||||
throw new TypeError(`Failed to retrieve alias map after ${RETRY_COUNT} attempts!`);
|
||||
}
|
||||
const aliases = Object.entries(aliasResponse)
|
||||
.filter(([index]) => !index.startsWith('.'))
|
||||
.map(([index, alias]) => ({
|
||||
index,
|
||||
alias,
|
||||
...parseIndexName(index),
|
||||
}));
|
||||
|
||||
for (const index in aliases) {
|
||||
if (aliases.hasOwnProperty(index)) {
|
||||
const matches = indexRegex.exec(index);
|
||||
if (matches !== null) {
|
||||
const type = matches[1];
|
||||
const source = matches[2];
|
||||
|
||||
// check if there is an alias for the current index
|
||||
// check that alias equals type
|
||||
const hasAlias = type in aliases[index].aliases;
|
||||
if (hasAlias) {
|
||||
if (typeof this.aliasMap[type] === 'undefined') {
|
||||
this.aliasMap[type] = {};
|
||||
}
|
||||
this.aliasMap[type][source] = index;
|
||||
} else {
|
||||
oldIndicesToDelete.push(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const {type, index, source} of aliases.filter(({type, alias}) => type in alias.aliases)) {
|
||||
this.aliasMap[type] = this.aliasMap[type] || {};
|
||||
this.aliasMap[type][source] = index;
|
||||
}
|
||||
|
||||
this.ready = true;
|
||||
|
||||
// delete old indices that are not used in any alias
|
||||
if (oldIndicesToDelete.length > 0) {
|
||||
const unusedIndices = aliases.filter(({type, alias}) => !(type in alias.aliases)).map(({index}) => index);
|
||||
if (unusedIndices.length > 0) {
|
||||
await this.client.indices.delete({
|
||||
index: oldIndicesToDelete,
|
||||
index: unusedIndices,
|
||||
});
|
||||
Logger.warn(`Deleted old indices: oldIndicesToDelete`);
|
||||
}
|
||||
@@ -291,8 +169,8 @@ export class Elasticsearch implements Database {
|
||||
* @param uid an UID to use for the search
|
||||
* @returns an elasticsearch object containing the thing
|
||||
*/
|
||||
private async getObject(uid: SCUuid): Promise<ElasticsearchObject<SCThings> | undefined> {
|
||||
const searchResponse: ApiResponse<SearchResponse<SCThings>> = await this.client.search({
|
||||
private async getObject(uid: SCUuid): Promise<SearchHit<SCThings> | undefined> {
|
||||
const searchResponse = await this.client.search<SCThings>({
|
||||
body: {
|
||||
query: {
|
||||
term: {
|
||||
@@ -303,43 +181,44 @@ export class Elasticsearch implements Database {
|
||||
},
|
||||
},
|
||||
from: 0,
|
||||
index: Elasticsearch.getListOfAllIndices(),
|
||||
index: ALL_INDICES_QUERY,
|
||||
size: 1,
|
||||
});
|
||||
|
||||
// return data from response
|
||||
return searchResponse.body.hits.hits[0];
|
||||
return searchResponse.hits.hits[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Should be called, when a new bulk was created. Creates a new index and applies a the mapping to the index
|
||||
*
|
||||
* @param bulk the bulk process that was created
|
||||
*/
|
||||
public async bulkCreated(bulk: Bulk): Promise<void> {
|
||||
// if our es instance is not ready yet, we cannot serve this request
|
||||
private async prepareBulkWrite(bulk: Bulk): Promise<{index: string; alias: string}> {
|
||||
if (!this.ready) {
|
||||
throw new Error('No connection to elasticsearch established yet.');
|
||||
}
|
||||
|
||||
// index name for elasticsearch
|
||||
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
|
||||
|
||||
// there already is an index with this type and source. We will index the new one and switch the alias to it
|
||||
// the old one is deleted
|
||||
const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
|
||||
const index = getThingIndexName(bulk.type, bulk.source, bulk);
|
||||
const alias = removeInvalidAliasChars(bulk.type, bulk.uid);
|
||||
|
||||
if (typeof this.aliasMap[alias] === 'undefined') {
|
||||
this.aliasMap[alias] = {};
|
||||
}
|
||||
|
||||
if (!indexRegex.test(index)) {
|
||||
if (!VALID_INDEX_REGEX.test(index)) {
|
||||
throw new Error(
|
||||
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
|
||||
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
|
||||
);
|
||||
}
|
||||
|
||||
return {index, alias};
|
||||
}
|
||||
|
||||
/**
|
||||
* Should be called, when a new bulk was created. Creates a new index and applies the mapping to the index
|
||||
*
|
||||
* @param bulk the bulk process that was created
|
||||
*/
|
||||
public async bulkCreated(bulk: Bulk): Promise<void> {
|
||||
const {index} = await this.prepareBulkWrite(bulk);
|
||||
|
||||
// re-apply the index template before each new bulk operation
|
||||
await putTemplate(this.client, bulk.type);
|
||||
await this.client.indices.create({
|
||||
@@ -355,8 +234,7 @@ export class Elasticsearch implements Database {
|
||||
* @param bulk the bulk process that is expired
|
||||
*/
|
||||
public async bulkExpired(bulk: Bulk): Promise<void> {
|
||||
// index name for elasticsearch
|
||||
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
|
||||
const index: string = getThingIndexName(bulk.type, bulk.source, bulk);
|
||||
|
||||
Logger.info('Bulk expired. Deleting index', index);
|
||||
|
||||
@@ -375,31 +253,11 @@ export class Elasticsearch implements Database {
|
||||
* @param bulk the new bulk process that should replace the old one with same type and source
|
||||
*/
|
||||
public async bulkUpdated(bulk: Bulk): Promise<void> {
|
||||
// if our es instance is not ready yet, we cannot serve this request
|
||||
if (!this.ready) {
|
||||
throw new Error('No connection to elasticsearch established yet.');
|
||||
}
|
||||
const {index, alias} = await this.prepareBulkWrite(bulk);
|
||||
|
||||
// index name for elasticsearch
|
||||
const index: string = Elasticsearch.getIndex(bulk.type, bulk.source, bulk);
|
||||
|
||||
// alias for the indices
|
||||
const alias = Elasticsearch.removeAliasChars(bulk.type, bulk.uid);
|
||||
|
||||
if (typeof this.aliasMap[alias] === 'undefined') {
|
||||
this.aliasMap[alias] = {};
|
||||
}
|
||||
|
||||
if (!indexRegex.test(index)) {
|
||||
throw new Error(
|
||||
`Index names can only consist of lowercase letters from a-z, "-", "_" and integer numbers.
|
||||
Make sure to set the bulk "source" and "type" to names consisting of the characters above.`,
|
||||
);
|
||||
}
|
||||
|
||||
// create the new index if it does not exists
|
||||
// create the new index if it does not exist
|
||||
// eslint-disable-next-line unicorn/no-await-expression-member
|
||||
if (!(await this.client.indices.exists({index})).body) {
|
||||
if (!(await this.client.indices.exists({index}))) {
|
||||
// re-apply the index template before each new bulk operation
|
||||
await putTemplate(this.client, bulk.type);
|
||||
await this.client.indices.create({
|
||||
@@ -412,7 +270,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
// add our new index to the alias
|
||||
// this was type safe with @types/elasticsearch, the new package however provides no type definitions
|
||||
const actions: IndicesUpdateAliasesParamsAction[] = [
|
||||
const actions: IndicesUpdateAliasesAction[] = [
|
||||
{
|
||||
add: {index: index, alias: alias},
|
||||
},
|
||||
@@ -427,16 +285,10 @@ export class Elasticsearch implements Database {
|
||||
}
|
||||
|
||||
// refresh the index (fsync changes)
|
||||
await this.client.indices.refresh({
|
||||
index: index,
|
||||
});
|
||||
await this.client.indices.refresh({index});
|
||||
|
||||
// execute our alias actions
|
||||
await this.client.indices.updateAliases({
|
||||
body: {
|
||||
actions,
|
||||
},
|
||||
});
|
||||
await this.client.indices.updateAliases({actions});
|
||||
|
||||
// swap the index in our aliasMap
|
||||
this.aliasMap[alias][bulk.source] = index;
|
||||
@@ -457,7 +309,7 @@ export class Elasticsearch implements Database {
|
||||
public async get(uid: SCUuid): Promise<SCThings> {
|
||||
const object = await this.getObject(uid);
|
||||
|
||||
if (typeof object === 'undefined') {
|
||||
if (typeof object?._source === 'undefined') {
|
||||
throw new TypeError('Item not found.');
|
||||
}
|
||||
|
||||
@@ -467,7 +319,7 @@ export class Elasticsearch implements Database {
|
||||
/**
|
||||
* Initialize the elasticsearch database (call all needed methods)
|
||||
*/
|
||||
public async init(): Promise<void> {
|
||||
public async init(retryOptions: Partial<RetryOptions<IndicesGetAliasResponse>> = {}): Promise<void> {
|
||||
const monitoringConfiguration = this.config.internal.monitoring;
|
||||
|
||||
if (typeof monitoringConfiguration !== 'undefined') {
|
||||
@@ -480,7 +332,7 @@ export class Elasticsearch implements Database {
|
||||
await Monitoring.setUp(monitoringConfiguration, this.client, this.mailQueue);
|
||||
}
|
||||
|
||||
return this.getAliasMap();
|
||||
return this.getAliasMap(retryOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -490,7 +342,7 @@ export class Elasticsearch implements Database {
|
||||
* @param bulk the bulk process which item belongs to
|
||||
*/
|
||||
public async post(object: SCThings, bulk: Bulk): Promise<void> {
|
||||
const object_: SCThings & {creation_date: string} = {
|
||||
const thing: SCThings & {creation_date: string} = {
|
||||
...object,
|
||||
creation_date: moment().format(),
|
||||
};
|
||||
@@ -499,7 +351,7 @@ export class Elasticsearch implements Database {
|
||||
|
||||
// check that the item will get replaced if the index is rolled over (index with the same name excluding ending uid)
|
||||
if (typeof item !== 'undefined') {
|
||||
const indexOfNew = Elasticsearch.getIndex(object_.type, bulk.source, bulk);
|
||||
const indexOfNew = getThingIndexName(thing.type, bulk.source, bulk);
|
||||
const oldIndex = item._index;
|
||||
|
||||
// new item doesn't replace the old one
|
||||
@@ -509,22 +361,23 @@ export class Elasticsearch implements Database {
|
||||
) {
|
||||
throw new Error(
|
||||
// eslint-disable-next-line unicorn/no-null
|
||||
`Object "${object_.uid}" already exists. Object was: ${JSON.stringify(object_, null, 2)}`,
|
||||
`Object "${thing.uid}" already exists. Object was: ${JSON.stringify(thing, null, 2)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// regular bulk update (item gets replaced when bulk is updated)
|
||||
const searchResponse = await this.client.create({
|
||||
body: object_,
|
||||
id: object_.uid,
|
||||
index: Elasticsearch.getIndex(object_.type, bulk.source, bulk),
|
||||
const searchResponse = await this.client.create<SCThings>({
|
||||
document: thing,
|
||||
id: thing.uid,
|
||||
index: getThingIndexName(thing.type, bulk.source, bulk),
|
||||
timeout: '90s',
|
||||
type: object_.type,
|
||||
});
|
||||
|
||||
if (!searchResponse.body.created) {
|
||||
throw new Error(`Object creation Error: Instance was: ${JSON.stringify(object_)}`);
|
||||
if (searchResponse.result !== 'created') {
|
||||
throw new Error(
|
||||
`Object creation Error (${searchResponse.result}: Instance was: ${JSON.stringify(thing)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -543,7 +396,6 @@ export class Elasticsearch implements Database {
|
||||
},
|
||||
id: object.uid,
|
||||
index: item._index,
|
||||
type: object.type.toLowerCase(),
|
||||
});
|
||||
|
||||
return;
|
||||
@@ -562,65 +414,46 @@ export class Elasticsearch implements Database {
|
||||
throw new TypeError('Database is undefined. You have to configure the query build');
|
||||
}
|
||||
|
||||
// create elasticsearch configuration out of data from database configuration
|
||||
const esConfig: ElasticsearchConfig = {
|
||||
name: this.config.internal.database.name as 'elasticsearch',
|
||||
version: this.config.internal.database.version as string,
|
||||
};
|
||||
|
||||
if (typeof this.config.internal.database.query !== 'undefined') {
|
||||
esConfig.query = this.config.internal.database.query as
|
||||
query: this.config.internal.database.query as
|
||||
| ElasticsearchQueryDisMaxConfig
|
||||
| ElasticsearchQueryQueryStringConfig;
|
||||
}
|
||||
| ElasticsearchQueryQueryStringConfig
|
||||
| undefined,
|
||||
};
|
||||
|
||||
const searchRequest: RequestParams.Search = {
|
||||
body: {
|
||||
aggs: aggregations,
|
||||
query: buildQuery(parameters, this.config, esConfig),
|
||||
},
|
||||
const query = {
|
||||
aggs: aggregations,
|
||||
query: buildQuery(parameters, this.config, esConfig),
|
||||
from: parameters.from,
|
||||
index: Elasticsearch.getListOfAllIndices(),
|
||||
index: ALL_INDICES_QUERY,
|
||||
size: parameters.size,
|
||||
sort: typeof parameters.sort !== 'undefined' ? buildSort(parameters.sort) : undefined,
|
||||
};
|
||||
|
||||
if (typeof parameters.sort !== 'undefined') {
|
||||
searchRequest.body.sort = buildSort(parameters.sort);
|
||||
}
|
||||
|
||||
// perform the search against elasticsearch
|
||||
const response: ApiResponse<SearchResponse<SCThings>> = await this.client.search(searchRequest);
|
||||
|
||||
// gather pagination information
|
||||
const pagination = {
|
||||
count: response.body.hits.hits.length,
|
||||
offset: typeof parameters.from === 'number' ? parameters.from : 0,
|
||||
total: response.body.hits.total,
|
||||
};
|
||||
|
||||
// gather statistics about this search
|
||||
const stats = {
|
||||
time: response.body.took,
|
||||
};
|
||||
|
||||
// we only directly return the _source documents
|
||||
// elasticsearch provides much more information, the user shouldn't see
|
||||
const data = response.body.hits.hits.map(hit => {
|
||||
return hit._source; // SCThing
|
||||
});
|
||||
|
||||
let facets: SCFacet[] = [];
|
||||
|
||||
// read the aggregations from elasticsearch and parse them to facets by our configuration
|
||||
if (typeof response.body.aggregations !== 'undefined') {
|
||||
facets = parseAggregations(response.body.aggregations as AggregationResponse);
|
||||
}
|
||||
const response: SearchResponse<SCThings> = await this.client.search(query);
|
||||
|
||||
return {
|
||||
data,
|
||||
facets,
|
||||
pagination,
|
||||
stats,
|
||||
data: response.hits.hits
|
||||
.map(hit => {
|
||||
// we only directly return the _source documents
|
||||
// elasticsearch provides much more information, the user shouldn't see
|
||||
return hit._source;
|
||||
})
|
||||
.filter(noUndefined),
|
||||
facets:
|
||||
typeof response.aggregations !== 'undefined'
|
||||
? parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>)
|
||||
: [],
|
||||
pagination: {
|
||||
count: response.hits.hits.length,
|
||||
offset: typeof parameters.from === 'number' ? parameters.from : 0,
|
||||
total:
|
||||
typeof response.hits.total === 'number' ? response.hits.total : response.hits.total?.value ?? 0,
|
||||
},
|
||||
stats: {
|
||||
time: response.took,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {ApiResponse, Client, RequestParams} from '@elastic/elasticsearch';
|
||||
import {Client} from '@elastic/elasticsearch';
|
||||
import {SearchRequest} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
SCMonitoringConfiguration,
|
||||
SCMonitoringLogAction,
|
||||
@@ -23,9 +24,6 @@ import {
|
||||
SCThings,
|
||||
} from '@openstapps/core';
|
||||
import {Logger} from '@openstapps/logger';
|
||||
// we only have the @types package because some things type definitions are still missing from the official
|
||||
// @elastic/elasticsearch package
|
||||
import {SearchResponse} from 'elasticsearch';
|
||||
import cron from 'node-cron';
|
||||
import {MailQueue} from '../../notification/mail-queue';
|
||||
|
||||
@@ -131,12 +129,11 @@ export async function setUp(
|
||||
|
||||
cron.schedule(trigger.executionTime, async () => {
|
||||
// execute watch (search->condition->action)
|
||||
const result: ApiResponse<SearchResponse<SCThings>> = await esClient.search(
|
||||
watcher.query as RequestParams.Search,
|
||||
);
|
||||
const result = await esClient.search<SCThings>(watcher.query as SearchRequest);
|
||||
|
||||
// check conditions
|
||||
const total = result.body.hits.total;
|
||||
const total =
|
||||
typeof result.hits.total === 'number' ? result.hits.total : result.hits.total?.value ?? -1;
|
||||
|
||||
for (const condition of watcher.conditions) {
|
||||
if (conditionFails(condition, total)) {
|
||||
|
||||
@@ -1,501 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
SCBackendConfigurationSearchBoostingContext,
|
||||
SCBackendConfigurationSearchBoostingType,
|
||||
SCConfigFile,
|
||||
SCSearchBooleanFilter,
|
||||
SCSearchContext,
|
||||
SCSearchFilter,
|
||||
SCSearchQuery,
|
||||
SCSearchSort,
|
||||
SCSportCoursePriceGroup,
|
||||
SCThingsField,
|
||||
} from '@openstapps/core';
|
||||
import {
|
||||
ElasticsearchConfig,
|
||||
ESBooleanFilter,
|
||||
ESBooleanFilterArguments,
|
||||
ESDateRange,
|
||||
ESDateRangeFilter,
|
||||
ESFunctionScoreQuery,
|
||||
ESFunctionScoreQueryFunction,
|
||||
ESGenericRange,
|
||||
ESGenericSort,
|
||||
ESGeoBoundingBoxFilter,
|
||||
ESGeoDistanceFilter,
|
||||
ESGeoDistanceFilterArguments,
|
||||
ESGeoDistanceSort,
|
||||
ESGeoDistanceSortArguments,
|
||||
ESGeoShapeFilter,
|
||||
ESNumericRangeFilter,
|
||||
ESRangeFilter,
|
||||
ESTermFilter,
|
||||
ESTypeFilter,
|
||||
ScriptSort,
|
||||
} from './types/elasticsearch';
|
||||
|
||||
/**
|
||||
* Escapes any reserved character that would otherwise not be accepted by Elasticsearch
|
||||
*
|
||||
* Elasticsearch as the following reserved characters:
|
||||
* + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
|
||||
* It is possible to use all, with the exception of < and >, of them by escaping them with a \
|
||||
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
|
||||
*
|
||||
* @param string_ the string to escape the characters from
|
||||
*/
|
||||
function escapeESReservedCharacters(string_: string): string {
|
||||
return string_.replace(/[+\-=!(){}\[\]^"~*?:\\/]|(&&)|(\|\|)/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a boolean filter. Returns an elasticsearch boolean filter
|
||||
*
|
||||
* @param booleanFilter a search boolean filter for the retrieval of the data
|
||||
* @returns elasticsearch boolean arguments object
|
||||
*/
|
||||
export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<unknown> {
|
||||
const result: ESBooleanFilterArguments<unknown> = {
|
||||
minimum_should_match: 0,
|
||||
must: [],
|
||||
must_not: [],
|
||||
should: [],
|
||||
};
|
||||
|
||||
if (booleanFilter.arguments.operation === 'and') {
|
||||
result.must = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
|
||||
}
|
||||
|
||||
if (booleanFilter.arguments.operation === 'or') {
|
||||
result.should = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
|
||||
result.minimum_should_match = 1;
|
||||
}
|
||||
|
||||
if (booleanFilter.arguments.operation === 'not') {
|
||||
result.must_not = booleanFilter.arguments.filters.map(filter => buildFilter(filter));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts Array of Filters to elasticsearch query-syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildFilter(
|
||||
filter: SCSearchFilter,
|
||||
):
|
||||
| ESTermFilter
|
||||
| ESGeoDistanceFilter
|
||||
| ESBooleanFilter<ESGeoShapeFilter | ESGeoBoundingBoxFilter>
|
||||
| ESGeoShapeFilter
|
||||
| ESBooleanFilter<unknown>
|
||||
| ESRangeFilter {
|
||||
switch (filter.type) {
|
||||
case 'value':
|
||||
return Array.isArray(filter.arguments.value)
|
||||
? {
|
||||
terms: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
}
|
||||
: {
|
||||
term: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
};
|
||||
case 'availability':
|
||||
const scope = filter.arguments.scope?.charAt(0) ?? 's';
|
||||
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
|
||||
|
||||
return {
|
||||
range: {
|
||||
[filter.arguments.field]: {
|
||||
gte: `${time}/${scope}`,
|
||||
lt: `${time}+1${scope}/${scope}`,
|
||||
relation: 'intersects',
|
||||
},
|
||||
},
|
||||
};
|
||||
case 'distance':
|
||||
const geoObject: ESGeoDistanceFilterArguments = {
|
||||
distance: `${filter.arguments.distance}m`,
|
||||
[`${filter.arguments.field}.point.coordinates`]: {
|
||||
lat: filter.arguments.position[1],
|
||||
lon: filter.arguments.position[0],
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
geo_distance: geoObject,
|
||||
};
|
||||
case 'boolean':
|
||||
return {
|
||||
bool: buildBooleanFilter(filter),
|
||||
};
|
||||
case 'numeric range':
|
||||
const numericRangeObject: ESGenericRange<number> = {
|
||||
relation: filter.arguments.relation,
|
||||
};
|
||||
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
|
||||
numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
|
||||
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
|
||||
numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
|
||||
}
|
||||
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
|
||||
numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
|
||||
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
|
||||
numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
|
||||
}
|
||||
|
||||
const numericRangeFilter: ESNumericRangeFilter = {range: {}};
|
||||
numericRangeFilter.range[filter.arguments.field] = numericRangeObject;
|
||||
|
||||
return numericRangeFilter;
|
||||
case 'date range':
|
||||
const dateRangeObject: ESDateRange = {
|
||||
format: filter.arguments.format,
|
||||
time_zone: filter.arguments.timeZone,
|
||||
relation: filter.arguments.relation,
|
||||
};
|
||||
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
|
||||
dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
|
||||
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
|
||||
dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
|
||||
}
|
||||
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
|
||||
dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
|
||||
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
|
||||
dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
|
||||
}
|
||||
|
||||
const dateRangeFilter: ESDateRangeFilter = {range: {}};
|
||||
dateRangeFilter.range[filter.arguments.field] = dateRangeObject;
|
||||
|
||||
return dateRangeFilter;
|
||||
case 'geo':
|
||||
// TODO: on ES upgrade, use just geo_shape filters
|
||||
const geoShapeFilter: ESGeoShapeFilter = {
|
||||
geo_shape: {
|
||||
/**
|
||||
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
|
||||
*/
|
||||
// @ts-expect-error unfortunately, typescript is stupid and won't allow me to map this to an actual type.
|
||||
ignore_unmapped: true,
|
||||
[`${filter.arguments.field}.polygon`]: {
|
||||
shape: filter.arguments.shape,
|
||||
relation: filter.arguments.spatialRelation,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (
|
||||
(typeof filter.arguments.spatialRelation === 'undefined' ||
|
||||
filter.arguments.spatialRelation === 'intersects') &&
|
||||
filter.arguments.shape.type === 'envelope'
|
||||
) {
|
||||
return {
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
geoShapeFilter,
|
||||
{
|
||||
geo_bounding_box: {
|
||||
/**
|
||||
* https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_ignore_unmapped_3
|
||||
*/
|
||||
ignore_unmapped: true,
|
||||
[`${filter.arguments.field}.point.coordinates`]: {
|
||||
top_left: filter.arguments.shape.coordinates[0],
|
||||
bottom_right: filter.arguments.shape.coordinates[1],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return geoShapeFilter;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds scoring functions from boosting config
|
||||
*
|
||||
* @param boostings Backend boosting configuration for contexts and types
|
||||
* @param context The context of the app from where the search was initiated
|
||||
*/
|
||||
function buildFunctions(
|
||||
boostings: SCBackendConfigurationSearchBoostingContext,
|
||||
context: SCSearchContext | undefined,
|
||||
): ESFunctionScoreQueryFunction[] {
|
||||
// default context
|
||||
let functions: ESFunctionScoreQueryFunction[] = buildFunctionsForBoostingTypes(
|
||||
boostings['default' as SCSearchContext],
|
||||
);
|
||||
|
||||
if (typeof context !== 'undefined' && context !== 'default') {
|
||||
// specific context provided, extend default context with additional boosts
|
||||
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
|
||||
}
|
||||
|
||||
return functions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates boost functions for all type boost configurations
|
||||
*
|
||||
* @param boostingTypes Array of type boosting configurations
|
||||
*/
|
||||
function buildFunctionsForBoostingTypes(
|
||||
boostingTypes: SCBackendConfigurationSearchBoostingType[],
|
||||
): ESFunctionScoreQueryFunction[] {
|
||||
const functions: ESFunctionScoreQueryFunction[] = [];
|
||||
|
||||
for (const boostingForOneSCType of boostingTypes) {
|
||||
const typeFilter: ESTypeFilter = {
|
||||
type: {
|
||||
value: boostingForOneSCType.type,
|
||||
},
|
||||
};
|
||||
|
||||
functions.push({
|
||||
filter: typeFilter,
|
||||
weight: boostingForOneSCType.factor,
|
||||
});
|
||||
|
||||
if (typeof boostingForOneSCType.fields !== 'undefined') {
|
||||
const fields = boostingForOneSCType.fields;
|
||||
|
||||
for (const fieldName in boostingForOneSCType.fields) {
|
||||
if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
|
||||
const boostingForOneField = fields[fieldName];
|
||||
|
||||
for (const value in boostingForOneField) {
|
||||
if (boostingForOneField.hasOwnProperty(value)) {
|
||||
const factor = boostingForOneField[value];
|
||||
|
||||
// build term filter
|
||||
const termFilter: ESTermFilter = {
|
||||
term: {},
|
||||
};
|
||||
termFilter.term[`${fieldName}.raw`] = value;
|
||||
|
||||
functions.push({
|
||||
filter: {
|
||||
bool: {
|
||||
must: [typeFilter, termFilter],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
weight: factor,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return functions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds body for Elasticsearch requests
|
||||
*
|
||||
* @param parameters Parameters for querying the backend
|
||||
* @param defaultConfig Default configuration of the backend
|
||||
* @param elasticsearchConfig Elasticsearch configuration
|
||||
* @returns ElasticsearchQuery (body of a search-request)
|
||||
*/
|
||||
export function buildQuery(
|
||||
parameters: SCSearchQuery,
|
||||
defaultConfig: SCConfigFile,
|
||||
elasticsearchConfig: ElasticsearchConfig,
|
||||
): ESFunctionScoreQuery {
|
||||
// if config provides an minMatch parameter we use query_string instead of match query
|
||||
let query;
|
||||
if (typeof elasticsearchConfig.query === 'undefined') {
|
||||
query = {
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: '90%',
|
||||
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'query_string') {
|
||||
query = {
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query: typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
|
||||
if (parameters.query !== '*') {
|
||||
query = {
|
||||
dis_max: {
|
||||
boost: 1.2,
|
||||
queries: [
|
||||
{
|
||||
match: {
|
||||
name: {
|
||||
boost: elasticsearchConfig.query.matchBoosting,
|
||||
cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
|
||||
fuzziness: elasticsearchConfig.query.fuzziness,
|
||||
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query:
|
||||
typeof parameters.query !== 'string' ? '*' : escapeESReservedCharacters(parameters.query),
|
||||
},
|
||||
},
|
||||
],
|
||||
tie_breaker: elasticsearchConfig.query.tieBreaker,
|
||||
},
|
||||
};
|
||||
}
|
||||
} else {
|
||||
throw new Error(
|
||||
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
|
||||
);
|
||||
}
|
||||
|
||||
const functionScoreQuery: ESFunctionScoreQuery = {
|
||||
function_score: {
|
||||
functions: buildFunctions(defaultConfig.internal.boostings, parameters.context),
|
||||
query: {
|
||||
bool: {
|
||||
minimum_should_match: 0, // if we have no should, nothing can match
|
||||
must: [],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
score_mode: 'multiply',
|
||||
},
|
||||
};
|
||||
|
||||
const mustMatch = functionScoreQuery.function_score.query.bool.must;
|
||||
|
||||
if (Array.isArray(mustMatch)) {
|
||||
if (typeof query !== 'undefined') {
|
||||
mustMatch.push(query);
|
||||
}
|
||||
|
||||
if (typeof parameters.filter !== 'undefined') {
|
||||
mustMatch.push(buildFilter(parameters.filter));
|
||||
}
|
||||
}
|
||||
|
||||
return functionScoreQuery;
|
||||
}
|
||||
|
||||
/**
|
||||
* converts query to
|
||||
*
|
||||
* @param sorts Sorting rules to apply to the data that is being queried
|
||||
* @returns an array of sort queries
|
||||
*/
|
||||
export function buildSort(sorts: SCSearchSort[]): Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> {
|
||||
return sorts.map(sort => {
|
||||
switch (sort.type) {
|
||||
case 'generic':
|
||||
const esGenericSort: ESGenericSort = {};
|
||||
esGenericSort[sort.arguments.field] = sort.order;
|
||||
|
||||
return esGenericSort;
|
||||
case 'ducet':
|
||||
const esDucetSort: ESGenericSort = {};
|
||||
esDucetSort[`${sort.arguments.field}.sort`] = sort.order;
|
||||
|
||||
return esDucetSort;
|
||||
case 'distance':
|
||||
const arguments_: ESGeoDistanceSortArguments = {
|
||||
mode: 'avg',
|
||||
order: sort.order,
|
||||
unit: 'm',
|
||||
};
|
||||
|
||||
arguments_[`${sort.arguments.field}.point.coordinates`] = {
|
||||
lat: sort.arguments.position[1],
|
||||
lon: sort.arguments.position[0],
|
||||
};
|
||||
|
||||
return {
|
||||
_geo_distance: arguments_,
|
||||
};
|
||||
case 'price':
|
||||
return {
|
||||
_script: {
|
||||
order: sort.order,
|
||||
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
|
||||
type: 'number' as const,
|
||||
},
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a script for sorting search results by prices
|
||||
*
|
||||
* @param universityRole User group which consumes university services
|
||||
* @param field Field in which wanted offers with prices are located
|
||||
*/
|
||||
export function buildPriceSortScript(
|
||||
universityRole: keyof SCSportCoursePriceGroup,
|
||||
field: SCThingsField,
|
||||
): string {
|
||||
return `
|
||||
// initialize the sort value with the maximum
|
||||
double price = Double.MAX_VALUE;
|
||||
|
||||
// if we have any offers
|
||||
if (params._source.containsKey('${field}')) {
|
||||
// iterate through all offers
|
||||
for (offer in params._source.${field}) {
|
||||
// if this offer contains a role specific price
|
||||
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
|
||||
// if the role specific price is smaller than the cheapest we found
|
||||
if (offer.prices.${universityRole} < price) {
|
||||
// set the role specific price as cheapest for now
|
||||
price = offer.prices.${universityRole};
|
||||
}
|
||||
} else { // we have no role specific price for our role in this offer
|
||||
// if the default price of this offer is lower than the cheapest we found
|
||||
if (offer.price < price) {
|
||||
// set this price as the cheapest
|
||||
price = offer.price;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// return cheapest price for our role
|
||||
return price;
|
||||
`;
|
||||
}
|
||||
75
src/storage/elasticsearch/query/boost/boost-functions.ts
Normal file
75
src/storage/elasticsearch/query/boost/boost-functions.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCBackendConfigurationSearchBoostingType} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Creates boost functions for all type boost configurations
|
||||
*
|
||||
* @param boostingTypes Array of type boosting configurations
|
||||
*/
|
||||
export function buildFunctionsForBoostingTypes(
|
||||
boostingTypes: SCBackendConfigurationSearchBoostingType[],
|
||||
): QueryDslFunctionScoreContainer[] {
|
||||
const functions: QueryDslFunctionScoreContainer[] = [];
|
||||
|
||||
for (const boostingForOneSCType of boostingTypes) {
|
||||
const typeFilter: QueryDslSpecificQueryContainer<'term'> = {
|
||||
term: {
|
||||
type: boostingForOneSCType.type,
|
||||
},
|
||||
};
|
||||
|
||||
functions.push({
|
||||
filter: typeFilter,
|
||||
weight: boostingForOneSCType.factor,
|
||||
});
|
||||
|
||||
if (typeof boostingForOneSCType.fields !== 'undefined') {
|
||||
const fields = boostingForOneSCType.fields;
|
||||
|
||||
for (const fieldName in boostingForOneSCType.fields) {
|
||||
if (boostingForOneSCType.fields.hasOwnProperty(fieldName)) {
|
||||
const boostingForOneField = fields[fieldName];
|
||||
|
||||
for (const value in boostingForOneField) {
|
||||
if (boostingForOneField.hasOwnProperty(value)) {
|
||||
const factor = boostingForOneField[value];
|
||||
|
||||
// build term filter
|
||||
const termFilter: QueryDslSpecificQueryContainer<'term'> = {
|
||||
term: {},
|
||||
};
|
||||
termFilter.term[`${fieldName}.raw`] = value;
|
||||
|
||||
functions.push({
|
||||
filter: {
|
||||
bool: {
|
||||
must: [typeFilter, termFilter],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
weight: factor,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return functions;
|
||||
}
|
||||
38
src/storage/elasticsearch/query/boost/scoring-functions.ts
Normal file
38
src/storage/elasticsearch/query/boost/scoring-functions.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCBackendConfigurationSearchBoostingContext, SCSearchContext} from '@openstapps/core';
|
||||
import {buildFunctionsForBoostingTypes} from './boost-functions';
|
||||
|
||||
/**
|
||||
* Builds scoring functions from boosting config
|
||||
*
|
||||
* @param boostings Backend boosting configuration for contexts and types
|
||||
* @param context The context of the app from where the search was initiated
|
||||
*/
|
||||
export function buildScoringFunctions(
|
||||
boostings: SCBackendConfigurationSearchBoostingContext,
|
||||
context: SCSearchContext | undefined,
|
||||
): QueryDslFunctionScoreContainer[] {
|
||||
// default context
|
||||
let functions = buildFunctionsForBoostingTypes(boostings['default' as SCSearchContext]);
|
||||
|
||||
if (typeof context !== 'undefined' && context !== 'default') {
|
||||
// specific context provided, extend default context with additional boosts
|
||||
functions = [...functions, ...buildFunctionsForBoostingTypes(boostings[context])];
|
||||
}
|
||||
|
||||
return functions;
|
||||
}
|
||||
47
src/storage/elasticsearch/query/filter.ts
Normal file
47
src/storage/elasticsearch/query/filter.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCSearchFilter} from '@openstapps/core';
|
||||
import {buildBooleanFilter} from './filters/boolean';
|
||||
import {buildAvailabilityFilter} from './filters/availability';
|
||||
import {buildDateRangeFilter} from './filters/date-range';
|
||||
import {buildDistanceFilter} from './filters/distance';
|
||||
import {buildGeoFilter} from './filters/geo';
|
||||
import {buildNumericRangeFilter} from './filters/numeric-range';
|
||||
import {buildValueFilter} from './filters/value';
|
||||
|
||||
/**
|
||||
* Converts Array of Filters to elasticsearch query-syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildFilter(filter: SCSearchFilter): QueryDslQueryContainer {
|
||||
switch (filter.type) {
|
||||
case 'value':
|
||||
return buildValueFilter(filter);
|
||||
case 'availability':
|
||||
return buildAvailabilityFilter(filter);
|
||||
case 'distance':
|
||||
return buildDistanceFilter(filter);
|
||||
case 'boolean':
|
||||
return buildBooleanFilter(filter);
|
||||
case 'numeric range':
|
||||
return buildNumericRangeFilter(filter);
|
||||
case 'date range':
|
||||
return buildDateRangeFilter(filter);
|
||||
case 'geo':
|
||||
return buildGeoFilter(filter);
|
||||
}
|
||||
}
|
||||
38
src/storage/elasticsearch/query/filters/availability.ts
Normal file
38
src/storage/elasticsearch/query/filters/availability.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCSearchAvailabilityFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Converts an availability filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildAvailabilityFilter(
|
||||
filter: SCSearchAvailabilityFilter,
|
||||
): QueryDslSpecificQueryContainer<'range'> {
|
||||
const scope = filter.arguments.scope?.charAt(0) ?? 's';
|
||||
const time = typeof filter.arguments.time === 'undefined' ? 'now' : `${filter.arguments.time}||`;
|
||||
|
||||
return {
|
||||
range: {
|
||||
[filter.arguments.field]: {
|
||||
gte: `${time}/${scope}`,
|
||||
lt: `${time}+1${scope}/${scope}`,
|
||||
relation: 'intersects',
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
49
src/storage/elasticsearch/query/filters/boolean.ts
Normal file
49
src/storage/elasticsearch/query/filters/boolean.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCSearchBooleanFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
import {buildFilter} from '../filter';
|
||||
|
||||
/**
|
||||
* Converts a boolean filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildBooleanFilter(filter: SCSearchBooleanFilter): QueryDslSpecificQueryContainer<'bool'> {
|
||||
const result: QueryDslBoolQuery = {
|
||||
minimum_should_match: 0,
|
||||
must: [],
|
||||
must_not: [],
|
||||
should: [],
|
||||
};
|
||||
|
||||
if (filter.arguments.operation === 'and') {
|
||||
result.must = filter.arguments.filters.map(it => buildFilter(it));
|
||||
}
|
||||
|
||||
if (filter.arguments.operation === 'or') {
|
||||
result.should = filter.arguments.filters.map(it => buildFilter(it));
|
||||
result.minimum_should_match = 1;
|
||||
}
|
||||
|
||||
if (filter.arguments.operation === 'not') {
|
||||
result.must_not = filter.arguments.filters.map(it => buildFilter(it));
|
||||
}
|
||||
|
||||
return {
|
||||
bool: result,
|
||||
};
|
||||
}
|
||||
48
src/storage/elasticsearch/query/filters/date-range.ts
Normal file
48
src/storage/elasticsearch/query/filters/date-range.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCSearchDateRangeFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Converts a date range filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildDateRangeFilter(
|
||||
filter: SCSearchDateRangeFilter,
|
||||
): QueryDslSpecificQueryContainer<'range'> {
|
||||
const dateRangeObject: QueryDslDateRangeQuery = {
|
||||
format: filter.arguments.format,
|
||||
time_zone: filter.arguments.timeZone,
|
||||
relation: filter.arguments.relation,
|
||||
};
|
||||
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
|
||||
dateRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
|
||||
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
|
||||
dateRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
|
||||
}
|
||||
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
|
||||
dateRangeObject.lt = filter.arguments.bounds.upperBound.limit;
|
||||
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
|
||||
dateRangeObject.lte = filter.arguments.bounds.upperBound.limit;
|
||||
}
|
||||
|
||||
return {
|
||||
range: {
|
||||
[filter.arguments.field]: dateRangeObject,
|
||||
},
|
||||
};
|
||||
}
|
||||
38
src/storage/elasticsearch/query/filters/distance.ts
Normal file
38
src/storage/elasticsearch/query/filters/distance.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCSearchDistanceFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Converts a distance filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildDistanceFilter(
|
||||
filter: SCSearchDistanceFilter,
|
||||
): QueryDslSpecificQueryContainer<'geo_distance'> {
|
||||
const geoObject: QueryDslGeoDistanceQuery = {
|
||||
distance: `${filter.arguments.distance}m`,
|
||||
[`${filter.arguments.field}.point.coordinates`]: {
|
||||
lat: filter.arguments.position[1],
|
||||
lon: filter.arguments.position[0],
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
geo_distance: geoObject,
|
||||
};
|
||||
}
|
||||
33
src/storage/elasticsearch/query/filters/geo.ts
Normal file
33
src/storage/elasticsearch/query/filters/geo.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCGeoFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Converts a geo filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> {
|
||||
return {
|
||||
geo_shape: {
|
||||
ignore_unmapped: true,
|
||||
[`${filter.arguments.field}.polygon`]: {
|
||||
shape: filter.arguments.shape,
|
||||
relation: filter.arguments.spatialRelation,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
47
src/storage/elasticsearch/query/filters/numeric-range.ts
Normal file
47
src/storage/elasticsearch/query/filters/numeric-range.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCSearchNumericRangeFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Converts a numeric range filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildNumericRangeFilter(
|
||||
filter: SCSearchNumericRangeFilter,
|
||||
): QueryDslSpecificQueryContainer<'range'> {
|
||||
const numericRangeObject: QueryDslNumberRangeQuery = {
|
||||
relation: filter.arguments.relation,
|
||||
};
|
||||
if (filter.arguments.bounds.lowerBound?.mode === 'exclusive') {
|
||||
numericRangeObject.gt = filter.arguments.bounds.lowerBound.limit;
|
||||
} else if (filter.arguments.bounds.lowerBound?.mode === 'inclusive') {
|
||||
numericRangeObject.gte = filter.arguments.bounds.lowerBound.limit;
|
||||
}
|
||||
if (filter.arguments.bounds.upperBound?.mode === 'exclusive') {
|
||||
numericRangeObject.lt = filter.arguments.bounds.upperBound.limit;
|
||||
} else if (filter.arguments.bounds.upperBound?.mode === 'inclusive') {
|
||||
numericRangeObject.lte = filter.arguments.bounds.upperBound.limit;
|
||||
}
|
||||
|
||||
return {
|
||||
range: {
|
||||
[filter.arguments.field]: numericRangeObject,
|
||||
},
|
||||
};
|
||||
}
|
||||
37
src/storage/elasticsearch/query/filters/value.ts
Normal file
37
src/storage/elasticsearch/query/filters/value.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCSearchValueFilter} from '@openstapps/core';
|
||||
import {QueryDslSpecificQueryContainer} from '../../types/util';
|
||||
|
||||
/**
|
||||
* Converts a value filter to elasticsearch syntax
|
||||
*
|
||||
* @param filter A search filter for the retrieval of the data
|
||||
*/
|
||||
export function buildValueFilter(
|
||||
filter: SCSearchValueFilter,
|
||||
): QueryDslSpecificQueryContainer<'term'> | QueryDslSpecificQueryContainer<'terms'> {
|
||||
return Array.isArray(filter.arguments.value)
|
||||
? {
|
||||
terms: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
}
|
||||
: {
|
||||
term: {
|
||||
[`${filter.arguments.field}.raw`]: filter.arguments.value,
|
||||
},
|
||||
};
|
||||
}
|
||||
114
src/storage/elasticsearch/query/query.ts
Normal file
114
src/storage/elasticsearch/query/query.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCConfigFile, SCSearchQuery} from '@openstapps/core';
|
||||
import {ElasticsearchConfig} from '../types/elasticsearch-config';
|
||||
import {buildFilter} from './filter';
|
||||
import {buildScoringFunctions} from './boost/scoring-functions';
|
||||
|
||||
/**
|
||||
* Builds body for Elasticsearch requests
|
||||
*
|
||||
* @param parameters Parameters for querying the backend
|
||||
* @param defaultConfig Default configuration of the backend
|
||||
* @param elasticsearchConfig Elasticsearch configuration
|
||||
* @returns ElasticsearchQuery (body of a search-request)
|
||||
*/
|
||||
export function buildQuery(
|
||||
parameters: SCSearchQuery,
|
||||
defaultConfig: SCConfigFile,
|
||||
elasticsearchConfig: ElasticsearchConfig,
|
||||
): QueryDslQueryContainer {
|
||||
// if config provides an minMatch parameter we use query_string instead of match query
|
||||
let query;
|
||||
if (typeof elasticsearchConfig.query === 'undefined') {
|
||||
query = {
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: '90%',
|
||||
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'query_string') {
|
||||
query = {
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query: typeof parameters.query !== 'string' ? '*' : parameters.query,
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
|
||||
if (typeof parameters.query === 'string' && parameters.query !== '*') {
|
||||
query = {
|
||||
dis_max: {
|
||||
boost: 1.2,
|
||||
queries: [
|
||||
{
|
||||
match: {
|
||||
name: {
|
||||
boost: elasticsearchConfig.query.matchBoosting,
|
||||
fuzziness: elasticsearchConfig.query.fuzziness,
|
||||
query: parameters.query,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
query_string: {
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query: parameters.query,
|
||||
},
|
||||
},
|
||||
],
|
||||
tie_breaker: elasticsearchConfig.query.tieBreaker,
|
||||
},
|
||||
};
|
||||
}
|
||||
} else {
|
||||
throw new Error(
|
||||
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
|
||||
);
|
||||
}
|
||||
|
||||
const functionScoreQuery: QueryDslQueryContainer = {
|
||||
function_score: {
|
||||
functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
|
||||
query: {
|
||||
bool: {
|
||||
minimum_should_match: 0, // if we have no should, nothing can match
|
||||
must: [],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
score_mode: 'multiply',
|
||||
},
|
||||
};
|
||||
|
||||
const mustMatch = functionScoreQuery.function_score?.query?.bool?.must;
|
||||
|
||||
if (Array.isArray(mustMatch)) {
|
||||
if (typeof query !== 'undefined') {
|
||||
mustMatch.push(query);
|
||||
}
|
||||
|
||||
if (typeof parameters.filter !== 'undefined') {
|
||||
mustMatch.push(buildFilter(parameters.filter));
|
||||
}
|
||||
}
|
||||
|
||||
return functionScoreQuery;
|
||||
}
|
||||
41
src/storage/elasticsearch/query/sort.ts
Normal file
41
src/storage/elasticsearch/query/sort.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {Sort} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCSearchSort} from '@openstapps/core';
|
||||
import {buildDistanceSort} from './sort/distance';
|
||||
import {buildDucetSort} from './sort/ducet';
|
||||
import {buildGenericSort} from './sort/generic';
|
||||
import {buildPriceSort} from './sort/price';
|
||||
|
||||
/**
|
||||
* converts query to
|
||||
*
|
||||
* @param sorts Sorting rules to apply to the data that is being queried
|
||||
* @returns an array of sort queries
|
||||
*/
|
||||
export function buildSort(sorts: SCSearchSort[]): Sort {
|
||||
return sorts.map(sort => {
|
||||
switch (sort.type) {
|
||||
case 'generic':
|
||||
return buildGenericSort(sort);
|
||||
case 'ducet':
|
||||
return buildDucetSort(sort);
|
||||
case 'distance':
|
||||
return buildDistanceSort(sort);
|
||||
case 'price':
|
||||
return buildPriceSort(sort);
|
||||
}
|
||||
});
|
||||
}
|
||||
35
src/storage/elasticsearch/query/sort/distance.ts
Normal file
35
src/storage/elasticsearch/query/sort/distance.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCDistanceSort} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* Converts a distance sort to elasticsearch syntax
|
||||
*
|
||||
* @param sort A sorting definition
|
||||
*/
|
||||
export function buildDistanceSort(sort: SCDistanceSort): SortOptions {
|
||||
return {
|
||||
_geo_distance: {
|
||||
mode: 'avg',
|
||||
order: sort.order,
|
||||
unit: 'm',
|
||||
[`${sort.arguments.field}.point.coordinates`]: {
|
||||
lat: sort.arguments.position[1],
|
||||
lon: sort.arguments.position[0],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
27
src/storage/elasticsearch/query/sort/ducet.ts
Normal file
27
src/storage/elasticsearch/query/sort/ducet.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCDucetSort} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* Converts a ducet sort to elasticsearch syntax
|
||||
*
|
||||
* @param sort A sorting definition
|
||||
*/
|
||||
export function buildDucetSort(sort: SCDucetSort): SortOptions {
|
||||
return {
|
||||
[`${sort.arguments.field}.sort`]: sort.order,
|
||||
};
|
||||
}
|
||||
27
src/storage/elasticsearch/query/sort/generic.ts
Normal file
27
src/storage/elasticsearch/query/sort/generic.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCGenericSort} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* Converts a generic sort to elasticsearch syntax
|
||||
*
|
||||
* @param sort A sorting definition
|
||||
*/
|
||||
export function buildGenericSort(sort: SCGenericSort): SortOptions {
|
||||
return {
|
||||
[sort.arguments.field]: sort.order,
|
||||
};
|
||||
}
|
||||
71
src/storage/elasticsearch/query/sort/price.ts
Normal file
71
src/storage/elasticsearch/query/sort/price.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* Converts a price sort to elasticsearch syntax
|
||||
*
|
||||
* @param sort A sorting definition
|
||||
*/
|
||||
export function buildPriceSort(sort: SCPriceSort): SortOptions {
|
||||
return {
|
||||
_script: {
|
||||
order: sort.order,
|
||||
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
|
||||
type: 'number' as const,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a script for sorting search results by prices
|
||||
*
|
||||
* @param universityRole User group which consumes university services
|
||||
* @param field Field in which wanted offers with prices are located
|
||||
*/
|
||||
export function buildPriceSortScript(
|
||||
universityRole: keyof SCSportCoursePriceGroup,
|
||||
field: SCThingsField,
|
||||
): string {
|
||||
return `
|
||||
// initialize the sort value with the maximum
|
||||
double price = Double.MAX_VALUE;
|
||||
|
||||
// if we have any offers
|
||||
if (params._source.containsKey('${field}')) {
|
||||
// iterate through all offers
|
||||
for (offer in params._source.${field}) {
|
||||
// if this offer contains a role specific price
|
||||
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
|
||||
// if the role specific price is smaller than the cheapest we found
|
||||
if (offer.prices.${universityRole} < price) {
|
||||
// set the role specific price as cheapest for now
|
||||
price = offer.prices.${universityRole};
|
||||
}
|
||||
} else { // we have no role specific price for our role in this offer
|
||||
// if the default price of this offer is lower than the cheapest we found
|
||||
if (offer.price < price) {
|
||||
// set this price as the cheapest
|
||||
price = offer.price;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// return cheapest price for our role
|
||||
return price;
|
||||
`;
|
||||
}
|
||||
@@ -29,17 +29,6 @@ export const aggregations = JSON.parse(
|
||||
readFileSync(path.resolve(mappingsPath, 'aggregations.json'), 'utf8'),
|
||||
) as AggregationSchema;
|
||||
|
||||
/**
|
||||
* Re-applies all interfaces for every type
|
||||
*
|
||||
* @param client An elasticsearch client to use
|
||||
*/
|
||||
export async function refreshAllTemplates(client: Client) {
|
||||
for (const type of Object.values(SCThingType)) {
|
||||
await putTemplate(client, type as SCThingType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares all indices
|
||||
*
|
||||
|
||||
121
src/storage/elasticsearch/types/elasticsearch-config.ts
Normal file
121
src/storage/elasticsearch/types/elasticsearch-config.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A configuration for using the Dis Max Query
|
||||
*
|
||||
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
|
||||
* explanation of what the parameters mean
|
||||
*/
|
||||
export interface ElasticsearchQueryDisMaxConfig {
|
||||
/**
|
||||
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
|
||||
*/
|
||||
cutoffFrequency: number;
|
||||
|
||||
/**
|
||||
* The maximum allowed Levenshtein Edit Distance (or number of edits)
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
|
||||
*/
|
||||
fuzziness: number | string;
|
||||
|
||||
/**
|
||||
* Increase the importance (relevance score) of a field
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
|
||||
*/
|
||||
matchBoosting: number;
|
||||
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minMatch: string;
|
||||
|
||||
/**
|
||||
* Type of the query - in this case 'dis_max' which is a union of its subqueries
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
|
||||
*/
|
||||
queryType: 'dis_max';
|
||||
|
||||
/**
|
||||
* Changes behavior of default calculation of the score when multiple results match
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
|
||||
*/
|
||||
tieBreaker: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A configuration for using Query String Query
|
||||
*
|
||||
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
|
||||
* explanation of what the parameters mean
|
||||
*/
|
||||
export interface ElasticsearchQueryQueryStringConfig {
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minMatch: string;
|
||||
|
||||
/**
|
||||
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
|
||||
*/
|
||||
queryType: 'query_string';
|
||||
}
|
||||
|
||||
/**
|
||||
* An config file for the elasticsearch database interface
|
||||
*
|
||||
* The config file extends the SCConfig file by further defining how the database property
|
||||
*/
|
||||
export interface ElasticsearchConfigFile {
|
||||
/**
|
||||
* Configuration that is not visible to clients
|
||||
*/
|
||||
internal: {
|
||||
/**
|
||||
* Database configuration
|
||||
*/
|
||||
database: ElasticsearchConfig;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch configuration
|
||||
*/
|
||||
export interface ElasticsearchConfig {
|
||||
/**
|
||||
* Name of the database
|
||||
*/
|
||||
name: 'elasticsearch';
|
||||
|
||||
/**
|
||||
* Configuration for using queries
|
||||
*/
|
||||
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
|
||||
|
||||
/**
|
||||
* Version of the used elasticsearch
|
||||
*/
|
||||
version: string;
|
||||
}
|
||||
@@ -1,605 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCThing, SCThingType} from '@openstapps/core';
|
||||
// we only have the @types package because some things type definitions are still missing from the official
|
||||
import {NameList} from 'elasticsearch';
|
||||
import {Polygon, Position} from 'geojson';
|
||||
|
||||
/**
|
||||
* An elasticsearch aggregation bucket
|
||||
*/
|
||||
interface Bucket {
|
||||
/**
|
||||
* Number of documents in the aggregation bucket
|
||||
*/
|
||||
doc_count: number;
|
||||
|
||||
/**
|
||||
* Text representing the documents in the bucket
|
||||
*/
|
||||
key: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch aggregation response
|
||||
*/
|
||||
export interface AggregationResponse {
|
||||
/**
|
||||
* The individual aggregations
|
||||
*/
|
||||
[field: string]: BucketAggregation | NestedAggregation;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch bucket aggregation
|
||||
*/
|
||||
export interface BucketAggregation {
|
||||
/**
|
||||
* Buckets in an aggregation
|
||||
*/
|
||||
buckets: Bucket[];
|
||||
|
||||
/**
|
||||
* Number of documents in an aggregation
|
||||
*/
|
||||
doc_count?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* An aggregation that contains more aggregations nested inside
|
||||
*/
|
||||
export interface NestedAggregation {
|
||||
/**
|
||||
* Number of documents in an aggregation
|
||||
*/
|
||||
doc_count: number;
|
||||
|
||||
/**
|
||||
* Any nested responses
|
||||
*/
|
||||
[name: string]: BucketAggregation | number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A configuration for using the Dis Max Query
|
||||
*
|
||||
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
|
||||
* explanation of what the parameters mean
|
||||
*/
|
||||
export interface ElasticsearchQueryDisMaxConfig {
|
||||
/**
|
||||
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
|
||||
*/
|
||||
cutoffFrequency: number;
|
||||
|
||||
/**
|
||||
* The maximum allowed Levenshtein Edit Distance (or number of edits)
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
|
||||
*/
|
||||
fuzziness: number | string;
|
||||
|
||||
/**
|
||||
* Increase the importance (relevance score) of a field
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
|
||||
*/
|
||||
matchBoosting: number;
|
||||
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minMatch: string;
|
||||
|
||||
/**
|
||||
* Type of the query - in this case 'dis_max' which is a union of its subqueries
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
|
||||
*/
|
||||
queryType: 'dis_max';
|
||||
|
||||
/**
|
||||
* Changes behavior of default calculation of the score when multiple results match
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
|
||||
*/
|
||||
tieBreaker: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A configuration for using Query String Query
|
||||
*
|
||||
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
|
||||
* explanation of what the parameters mean
|
||||
*/
|
||||
export interface ElasticsearchQueryQueryStringConfig {
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minMatch: string;
|
||||
|
||||
/**
|
||||
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
|
||||
*/
|
||||
queryType: 'query_string';
|
||||
}
|
||||
|
||||
/**
|
||||
* A hit in an elasticsearch search result
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
|
||||
*/
|
||||
export interface ElasticsearchObject<T extends SCThing> {
|
||||
/**
|
||||
* Unique identifier of a document (object)
|
||||
*/
|
||||
_id: string;
|
||||
|
||||
/**
|
||||
* The index to which the document belongs
|
||||
*/
|
||||
_index: string;
|
||||
|
||||
/**
|
||||
* Relevancy of the document to a query
|
||||
*/
|
||||
_score: number;
|
||||
|
||||
/**
|
||||
* The original JSON representing the body of the document
|
||||
*/
|
||||
_source: T;
|
||||
|
||||
/**
|
||||
* The document's mapping type
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
|
||||
*/
|
||||
_type: string;
|
||||
|
||||
/**
|
||||
* Version of the document
|
||||
*/
|
||||
_version?: number;
|
||||
|
||||
/**
|
||||
* Used to index the same field in different ways for different purposes
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
|
||||
*/
|
||||
fields?: NameList;
|
||||
|
||||
/**
|
||||
* Used to highlight search results on one or more fields
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
highlight?: any;
|
||||
|
||||
/**
|
||||
* Used in when nested/children documents match the query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
inner_hits?: any;
|
||||
|
||||
/**
|
||||
* Queries that matched for documents in results
|
||||
*/
|
||||
matched_queries?: string[];
|
||||
|
||||
/**
|
||||
* Sorting definition
|
||||
*/
|
||||
sort?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* An config file for the elasticsearch database interface
|
||||
*
|
||||
* The config file extends the SCConfig file by further defining how the database property
|
||||
*/
|
||||
export interface ElasticsearchConfigFile {
|
||||
/**
|
||||
* Configuration that is not visible to clients
|
||||
*/
|
||||
internal: {
|
||||
/**
|
||||
* Database configuration
|
||||
*/
|
||||
database: ElasticsearchConfig;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch configuration
|
||||
*/
|
||||
export interface ElasticsearchConfig {
|
||||
/**
|
||||
* Name of the database
|
||||
*/
|
||||
name: 'elasticsearch';
|
||||
|
||||
/**
|
||||
* Configuration for using queries
|
||||
*/
|
||||
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
|
||||
|
||||
/**
|
||||
* Version of the used elasticsearch
|
||||
*/
|
||||
version: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch term filter
|
||||
*/
|
||||
export type ESTermFilter =
|
||||
| {
|
||||
/**
|
||||
* Definition of a term to match
|
||||
*/
|
||||
term: {
|
||||
[fieldName: string]: string;
|
||||
};
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* Definition of terms to match (or)
|
||||
*/
|
||||
terms: {
|
||||
[fieldName: string]: string[];
|
||||
};
|
||||
};
|
||||
|
||||
export interface ESGenericRange<T> {
|
||||
/**
|
||||
* Greater than field
|
||||
*/
|
||||
gt?: T;
|
||||
|
||||
/**
|
||||
* Greater or equal than field
|
||||
*/
|
||||
gte?: T;
|
||||
|
||||
/**
|
||||
* Less than field
|
||||
*/
|
||||
lt?: T;
|
||||
|
||||
/**
|
||||
* Less or equal than field
|
||||
*/
|
||||
lte?: T;
|
||||
|
||||
/**
|
||||
* Relation of the range to a range field
|
||||
*
|
||||
* Intersects: Both ranges intersect
|
||||
* Contains: Search range contains field range
|
||||
* Within: Field range contains search range
|
||||
*/
|
||||
relation?: 'intersects' | 'within' | 'contains';
|
||||
}
|
||||
|
||||
interface ESGenericRangeFilter<G, T extends ESGenericRange<G>> {
|
||||
/**
|
||||
* Range filter definition
|
||||
*/
|
||||
range: {
|
||||
[fieldName: string]: T;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ESDateRange extends ESGenericRange<string> {
|
||||
/**
|
||||
* Optional date format override
|
||||
*/
|
||||
format?: string;
|
||||
|
||||
/**
|
||||
* Optional timezone specifier
|
||||
*/
|
||||
time_zone?: string;
|
||||
}
|
||||
|
||||
export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<number>>;
|
||||
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
|
||||
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;
|
||||
|
||||
/**
|
||||
* An elasticsearch type filter
|
||||
*/
|
||||
export interface ESTypeFilter {
|
||||
/**
|
||||
* Type filter definition
|
||||
*/
|
||||
type: {
|
||||
/**
|
||||
* Type name (SCThingType) to filter with
|
||||
*/
|
||||
value: SCThingType;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter arguments for an elasticsearch geo distance filter
|
||||
*/
|
||||
export interface ESGeoDistanceFilterArguments {
|
||||
/**
|
||||
* The radius of the circle centred on the specified location
|
||||
*/
|
||||
distance: string;
|
||||
|
||||
[fieldName: string]:
|
||||
| {
|
||||
/**
|
||||
* Latitude
|
||||
*/
|
||||
lat: number;
|
||||
|
||||
/**
|
||||
* Longitude
|
||||
*/
|
||||
lon: number;
|
||||
}
|
||||
| string;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch geo distance filter
|
||||
*/
|
||||
export interface ESGeoDistanceFilter {
|
||||
/**
|
||||
* @see ESGeoDistanceFilterArguments
|
||||
*/
|
||||
geo_distance: ESGeoDistanceFilterArguments;
|
||||
}
|
||||
|
||||
/**
|
||||
* A rectangular geo shape, representing the top-left and bottom-right corners
|
||||
*
|
||||
* This is an extension of the Geojson type
|
||||
* http://geojson.org/geojson-spec.html
|
||||
*/
|
||||
export interface ESEnvelope {
|
||||
/**
|
||||
* The top-left and bottom-right corners of the bounding box
|
||||
*/
|
||||
coordinates: [Position, Position];
|
||||
|
||||
/**
|
||||
* The type of the geometry
|
||||
*/
|
||||
type: 'envelope';
|
||||
}
|
||||
|
||||
/**
|
||||
* An Elasticsearch geo bounding box filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
|
||||
*/
|
||||
export interface ESGeoBoundingBoxFilter {
|
||||
/**
|
||||
* An Elasticsearch geo bounding box filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-bounding-box-query.html
|
||||
*/
|
||||
geo_bounding_box: {
|
||||
[fieldName: string]: {
|
||||
/**
|
||||
* Geo Shape
|
||||
*/
|
||||
bottom_right: Position;
|
||||
|
||||
/**
|
||||
* Geo Shape
|
||||
*/
|
||||
top_left: Position;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* An Elasticsearch geo shape filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html
|
||||
*/
|
||||
export interface ESGeoShapeFilter {
|
||||
geo_shape: {
|
||||
[fieldName: string]: {
|
||||
/**
|
||||
* Relation of the two shapes
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-geo-shape-query.html#_spatial_relations
|
||||
*/
|
||||
relation?: 'intersects' | 'disjoint' | 'within' | 'contains';
|
||||
|
||||
/**
|
||||
* Geo Shape
|
||||
*/
|
||||
shape: Polygon | ESEnvelope;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter arguments for an elasticsearch boolean filter
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
|
||||
*/
|
||||
export interface ESBooleanFilterArguments<T> {
|
||||
/**
|
||||
* Minimal number (or percentage) of words that should match in a query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
||||
*/
|
||||
minimum_should_match?: number;
|
||||
|
||||
/**
|
||||
* The clause (query) must appear in matching documents and will contribute to the score.
|
||||
*/
|
||||
must?: T[];
|
||||
|
||||
/**
|
||||
* The clause (query) must not appear in the matching documents.
|
||||
*/
|
||||
must_not?: T[];
|
||||
|
||||
/**
|
||||
* The clause (query) should appear in the matching document.
|
||||
*/
|
||||
should?: T[];
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch boolean filter
|
||||
*/
|
||||
export interface ESBooleanFilter<T> {
|
||||
/**
|
||||
* @see ESBooleanFilterArguments
|
||||
*/
|
||||
bool: ESBooleanFilterArguments<T>;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch function score query
|
||||
*
|
||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
|
||||
*/
|
||||
export interface ESFunctionScoreQuery {
|
||||
/**
|
||||
* Function score definition
|
||||
*/
|
||||
function_score: {
|
||||
/**
|
||||
* Functions that compute score for query results (documents)
|
||||
*
|
||||
* @see ESFunctionScoreQueryFunction
|
||||
*/
|
||||
functions: ESFunctionScoreQueryFunction[];
|
||||
|
||||
/**
|
||||
* @see ESBooleanFilter
|
||||
*/
|
||||
query: ESBooleanFilter<unknown>;
|
||||
|
||||
/**
|
||||
* Specifies how the computed scores are combined
|
||||
*/
|
||||
score_mode: 'multiply';
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* An function for an elasticsearch functions score query
|
||||
*/
|
||||
export interface ESFunctionScoreQueryFunction {
|
||||
/**
|
||||
* Function is applied only if a document matches the given filtering query
|
||||
*/
|
||||
filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;
|
||||
|
||||
/**
|
||||
* Weight (importance) of the filter
|
||||
*/
|
||||
weight: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch generic sort
|
||||
*/
|
||||
export interface ESGenericSort {
|
||||
[field: string]: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort arguments for an elasticsearch geo distance sort
|
||||
*/
|
||||
export interface ESGeoDistanceSortArguments {
|
||||
/**
|
||||
* What value to pick for sorting
|
||||
*/
|
||||
mode: 'avg' | 'max' | 'median' | 'min';
|
||||
|
||||
/**
|
||||
* Order
|
||||
*/
|
||||
order: 'asc' | 'desc';
|
||||
|
||||
/**
|
||||
* Value unit
|
||||
*/
|
||||
unit: 'm';
|
||||
|
||||
[field: string]:
|
||||
| {
|
||||
/**
|
||||
* Latitude
|
||||
*/
|
||||
lat: number;
|
||||
|
||||
/**
|
||||
* Longitude
|
||||
*/
|
||||
lon: number;
|
||||
}
|
||||
| string;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch geo distance sort
|
||||
*/
|
||||
export interface ESGeoDistanceSort {
|
||||
/**
|
||||
* @see ESGeoDistanceFilterArguments
|
||||
*/
|
||||
_geo_distance: ESGeoDistanceSortArguments;
|
||||
}
|
||||
|
||||
/**
|
||||
* An elasticsearch script sort
|
||||
*/
|
||||
export interface ScriptSort {
|
||||
/**
|
||||
* A script
|
||||
*/
|
||||
_script: {
|
||||
/**
|
||||
* Order
|
||||
*/
|
||||
order: 'asc' | 'desc';
|
||||
|
||||
/**
|
||||
* The custom script used for sorting
|
||||
*/
|
||||
script: string;
|
||||
|
||||
/**
|
||||
* What type is being sorted
|
||||
*/
|
||||
type: 'number' | 'string';
|
||||
};
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2021 StApps
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
ESAggMatchAllFilter,
|
||||
ESAggTypeFilter,
|
||||
ESNestedAggregation,
|
||||
ESTermsFilter,
|
||||
} from '@openstapps/es-mapping-generator/src/types/aggregation';
|
||||
import {BucketAggregation, NestedAggregation} from './elasticsearch';
|
||||
|
||||
/**
|
||||
* Checks if the type is a BucketAggregation
|
||||
*
|
||||
* @param agg the type to check
|
||||
*/
|
||||
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
|
||||
return typeof agg !== 'number';
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the type is a NestedAggregation
|
||||
*
|
||||
* @param agg the type to check
|
||||
*/
|
||||
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
|
||||
return typeof (agg as BucketAggregation).buckets === 'undefined';
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the parameter is of type ESTermsFilter
|
||||
*
|
||||
* @param agg the value to check
|
||||
*/
|
||||
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
|
||||
return typeof (agg as ESTermsFilter).terms !== 'undefined';
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the parameter is of type ESTermsFilter
|
||||
*
|
||||
* @param agg the value to check
|
||||
*/
|
||||
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
|
||||
return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the parameter is of type
|
||||
*
|
||||
* @param filter the filter to narrow the type of
|
||||
*/
|
||||
export function isESAggMatchAllFilter(
|
||||
filter: ESAggTypeFilter | ESAggMatchAllFilter,
|
||||
): filter is ESAggMatchAllFilter {
|
||||
return filter.hasOwnProperty('match_all');
|
||||
}
|
||||
20
src/storage/elasticsearch/types/util.ts
Normal file
20
src/storage/elasticsearch/types/util.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export type QueryDslSpecificQueryContainer<T extends keyof QueryDslQueryContainer> = Required<
|
||||
Pick<QueryDslQueryContainer, T>
|
||||
>;
|
||||
60
src/storage/elasticsearch/util/alias.ts
Normal file
60
src/storage/elasticsearch/util/alias.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {Logger} from '@openstapps/logger';
|
||||
|
||||
/**
|
||||
* Checks for invalid character in alias names and removes them
|
||||
*
|
||||
* @param alias The alias name
|
||||
* @param uid The UID of the current bulk (for debugging purposes)
|
||||
*/
|
||||
export function removeInvalidAliasChars(alias: string, uid: string | undefined): string {
|
||||
let formattedAlias = alias;
|
||||
|
||||
// spaces are included in some types, replace them with underscores
|
||||
if (formattedAlias.includes(' ')) {
|
||||
formattedAlias = formattedAlias.trim();
|
||||
formattedAlias = formattedAlias.split(' ').join('_');
|
||||
}
|
||||
// List of invalid characters: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/indices-create-index.html
|
||||
for (const value of ['\\', '/', '*', '?', '"', '<', '>', '|', ',', '#']) {
|
||||
if (formattedAlias.includes(value)) {
|
||||
formattedAlias = formattedAlias.replace(value, '');
|
||||
Logger.warn(`Type of the bulk ${uid} contains an invalid character '${value}'. This can lead to two bulks
|
||||
having the same alias despite having different types, as invalid characters are removed automatically.
|
||||
New alias name is "${formattedAlias}."`);
|
||||
}
|
||||
}
|
||||
for (const value of ['-', '_', '+']) {
|
||||
if (formattedAlias.charAt(0) === value) {
|
||||
formattedAlias = formattedAlias.slice(1);
|
||||
Logger.warn(`Type of the bulk ${uid} begins with '${value}'. This can lead to two bulks having the same
|
||||
alias despite having different types, as invalid characters are removed automatically.
|
||||
New alias name is "${formattedAlias}."`);
|
||||
}
|
||||
}
|
||||
if (formattedAlias === '.' || formattedAlias === '..') {
|
||||
Logger.warn(`Type of the bulk ${uid} is ${formattedAlias}. This is an invalid name, please consider using
|
||||
another one, as it will be replaced with 'alias_placeholder', which can lead to strange errors.`);
|
||||
|
||||
return 'alias_placeholder';
|
||||
}
|
||||
if (formattedAlias.includes(':')) {
|
||||
Logger.warn(`Type of the bulk ${uid} contains a ':'. This isn't an issue now, but will be in future
|
||||
Elasticsearch versions!`);
|
||||
}
|
||||
|
||||
return formattedAlias;
|
||||
}
|
||||
63
src/storage/elasticsearch/util/index.ts
Normal file
63
src/storage/elasticsearch/util/index.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import {SCBulkResponse, SCThingType, SCUuid} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* Length of the index UID used for generation of its name
|
||||
*/
|
||||
export const INDEX_UID_LENGTH = 8;
|
||||
|
||||
/**
|
||||
* A string which matches all indices
|
||||
*/
|
||||
export const ALL_INDICES_QUERY = 'stapps_*_*_*';
|
||||
|
||||
/**
|
||||
* Matches index names such as stapps_<type>_<source>_<random suffix>
|
||||
*/
|
||||
export const VALID_INDEX_REGEX = /^stapps_([A-z0-9_]+)_([a-z0-9-_]+)_([-a-z0-9^_]+)$/;
|
||||
|
||||
export interface ParsedIndexName {
|
||||
type: SCThingType;
|
||||
source: string;
|
||||
randomSuffix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
export function parseIndexName(index: string): ParsedIndexName {
|
||||
const match = VALID_INDEX_REGEX.exec(index);
|
||||
if (!match) {
|
||||
throw new SyntaxError(`Invalid index name ${index}!`);
|
||||
}
|
||||
|
||||
return {
|
||||
type: match[1] as SCThingType,
|
||||
source: match[2],
|
||||
randomSuffix: match[3],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the index name in elasticsearch for one SCThingType
|
||||
*
|
||||
* @param type SCThingType of data in the index
|
||||
* @param source source of data in the index
|
||||
* @param bulk bulk process which created this index
|
||||
*/
|
||||
export function getThingIndexName(type: SCThingType, source: string, bulk: SCBulkResponse) {
|
||||
let out = type.toLowerCase();
|
||||
while (out.includes(' ')) {
|
||||
out = out.replace(' ', '_');
|
||||
}
|
||||
|
||||
return `stapps_${out}_${source}_${getIndexUID(bulk.uid)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides the index UID (for its name) from the bulk UID
|
||||
*
|
||||
* @param uid Bulk UID
|
||||
*/
|
||||
export function getIndexUID(uid: SCUuid) {
|
||||
return uid.slice(0, Math.max(0, INDEX_UID_LENGTH));
|
||||
}
|
||||
21
src/storage/elasticsearch/util/no-undefined.ts
Normal file
21
src/storage/elasticsearch/util/no-undefined.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Type guard for filter functions
|
||||
*/
|
||||
export function noUndefined<T>(item: T | undefined): item is T {
|
||||
return typeof item !== 'undefined';
|
||||
}
|
||||
38
src/storage/elasticsearch/util/retry.ts
Normal file
38
src/storage/elasticsearch/util/retry.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2022 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
export interface RetryOptions<T> {
|
||||
maxRetries: number;
|
||||
retryInterval: number;
|
||||
doAction: () => Promise<T>;
|
||||
onFailedAttempt: (attempt: number, error: unknown, options: RetryOptions<T>) => void;
|
||||
onFail: (options: RetryOptions<T>) => never;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retries a throwing function at a set interval, until a maximum amount of attempts
|
||||
*/
|
||||
export async function retryCatch<T>(options: RetryOptions<T>): Promise<T> {
|
||||
for (let attempt = 0; attempt < options.maxRetries; attempt++) {
|
||||
try {
|
||||
return await options.doAction();
|
||||
} catch (error) {
|
||||
options.onFailedAttempt(attempt, error, options);
|
||||
await new Promise(resolve => setTimeout(resolve, options.retryInterval));
|
||||
}
|
||||
}
|
||||
|
||||
options.onFail(options);
|
||||
}
|
||||
@@ -13,21 +13,21 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {inRangeInclusive} from '../config/default';
|
||||
import {yearSlice} from '../config/default';
|
||||
import {expect} from 'chai';
|
||||
|
||||
describe('Common', function () {
|
||||
describe('inRangeInclusive', function () {
|
||||
it('should provide true if the given number is in the range', function () {
|
||||
expect(inRangeInclusive(1, [1, 3])).to.be.true;
|
||||
expect(inRangeInclusive(2, [1, 3])).to.be.true;
|
||||
expect(inRangeInclusive(1.1, [1, 3])).to.be.true;
|
||||
expect(inRangeInclusive(3, [1, 3])).to.be.true;
|
||||
describe('yearSlice', function () {
|
||||
it('should provide correct ascending month number ranges', function () {
|
||||
expect(yearSlice(1, 12)).to.eql([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]);
|
||||
});
|
||||
|
||||
it('should provide false if the given number is not in the range', function () {
|
||||
expect(inRangeInclusive(3.1, [1, 3])).to.be.false;
|
||||
expect(inRangeInclusive(0, [1, 3])).to.be.false;
|
||||
it('should provide correct month number ranges for year rollovers', function () {
|
||||
expect(yearSlice(12, 1)).to.eql([12, 1]);
|
||||
});
|
||||
|
||||
it('should provide correct month number ranges for a whole year', function () {
|
||||
expect(yearSlice(12, 12)).to.eql([12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCThingType, SCUuid} from '@openstapps/core';
|
||||
import {Express} from 'express';
|
||||
import moment from 'moment';
|
||||
import {getIndexUID} from '../src/storage/elasticsearch/util';
|
||||
import {configureApp} from '../src/app';
|
||||
import express from 'express';
|
||||
import http from 'http';
|
||||
@@ -24,7 +25,6 @@ import {MailQueue} from '../src/notification/mail-queue';
|
||||
import {Bulk, BulkStorage} from '../src/storage/bulk-storage';
|
||||
import getPort from 'get-port';
|
||||
import {Database} from '../src/storage/database';
|
||||
import {Elasticsearch} from '../src/storage/elasticsearch/elasticsearch';
|
||||
import {v4} from 'uuid';
|
||||
|
||||
/**
|
||||
@@ -147,5 +147,4 @@ export const getTransport = (verified: boolean) => {
|
||||
};
|
||||
};
|
||||
|
||||
export const getIndex = (uid?: string) =>
|
||||
`stapps_footype_foosource_${uid ?? Elasticsearch.getIndexUID(v4())}`;
|
||||
export const getIndex = (uid?: string) => `stapps_footype_foosource_${uid ?? getIndexUID(v4())}`;
|
||||
|
||||
@@ -13,13 +13,13 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {AggregateName, AggregationsMultiTermsBucket} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {SCFacet, SCThingType} from '@openstapps/core';
|
||||
import {expect} from 'chai';
|
||||
import {parseAggregations} from '../../../src/storage/elasticsearch/aggregations';
|
||||
import {AggregationResponse} from '../../../src/storage/elasticsearch/types/elasticsearch';
|
||||
|
||||
describe('Aggregations', function () {
|
||||
const aggregations: AggregationResponse = {
|
||||
const aggregations: Record<AggregateName, Partial<AggregationsMultiTermsBucket>> = {
|
||||
'catalog': {
|
||||
'doc_count': 4,
|
||||
'superCatalogs.categories': {
|
||||
@@ -76,14 +76,6 @@ describe('Aggregations', function () {
|
||||
buckets: [],
|
||||
},
|
||||
},
|
||||
'fooType': {
|
||||
buckets: [
|
||||
{
|
||||
doc_count: 321,
|
||||
key: 'foo',
|
||||
},
|
||||
],
|
||||
},
|
||||
'@all': {
|
||||
doc_count: 17,
|
||||
type: {
|
||||
@@ -102,33 +94,6 @@ describe('Aggregations', function () {
|
||||
};
|
||||
|
||||
const expectedFacets: SCFacet[] = [
|
||||
{
|
||||
buckets: [
|
||||
{
|
||||
count: 13,
|
||||
key: 'person',
|
||||
},
|
||||
{
|
||||
count: 4,
|
||||
key: 'catalog',
|
||||
},
|
||||
],
|
||||
field: 'type',
|
||||
},
|
||||
{
|
||||
buckets: [
|
||||
{
|
||||
count: 8,
|
||||
key: 'foobar',
|
||||
},
|
||||
{
|
||||
count: 2,
|
||||
key: 'bar',
|
||||
},
|
||||
],
|
||||
field: 'categories',
|
||||
onlyOnType: SCThingType.AcademicEvent,
|
||||
},
|
||||
{
|
||||
buckets: [
|
||||
{
|
||||
@@ -153,7 +118,33 @@ describe('Aggregations', function () {
|
||||
field: 'categories',
|
||||
onlyOnType: SCThingType.Catalog,
|
||||
},
|
||||
// no fooType as it doesn't appear in the aggregation schema
|
||||
{
|
||||
buckets: [
|
||||
{
|
||||
count: 8,
|
||||
key: 'foobar',
|
||||
},
|
||||
{
|
||||
count: 2,
|
||||
key: 'bar',
|
||||
},
|
||||
],
|
||||
field: 'categories',
|
||||
onlyOnType: SCThingType.AcademicEvent,
|
||||
},
|
||||
{
|
||||
buckets: [
|
||||
{
|
||||
count: 13,
|
||||
key: 'person',
|
||||
},
|
||||
{
|
||||
count: 4,
|
||||
key: 'catalog',
|
||||
},
|
||||
],
|
||||
field: 'type',
|
||||
},
|
||||
];
|
||||
|
||||
it('should parse the aggregations providing the appropriate facets', function () {
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 StApps
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
ESAggMatchAllFilter,
|
||||
ESAggTypeFilter,
|
||||
ESNestedAggregation,
|
||||
ESTermsFilter,
|
||||
} from '@openstapps/es-mapping-generator/src/types/aggregation';
|
||||
import {expect} from 'chai';
|
||||
import {
|
||||
isNestedAggregation,
|
||||
isBucketAggregation,
|
||||
isESTermsFilter,
|
||||
isESAggMatchAllFilter,
|
||||
isESNestedAggregation,
|
||||
} from '../../../lib/storage/elasticsearch/types/guards';
|
||||
import {BucketAggregation, NestedAggregation} from '../../../src/storage/elasticsearch/types/elasticsearch';
|
||||
|
||||
describe('Common', function () {
|
||||
const bucketAggregation: BucketAggregation = {buckets: []};
|
||||
const esNestedAggregation: ESNestedAggregation = {aggs: {}, filter: {match_all: true}};
|
||||
const esTermsFilter: ESTermsFilter = {terms: {field: 'foo'}};
|
||||
|
||||
describe('isBucketAggregation', function () {
|
||||
it('should be false for a number', function () {
|
||||
expect(isBucketAggregation(123)).to.be.false;
|
||||
});
|
||||
|
||||
it('should be true for a bucket aggregation', function () {
|
||||
expect(isBucketAggregation(bucketAggregation)).to.be.true;
|
||||
});
|
||||
});
|
||||
|
||||
describe('isNestedAggregation', function () {
|
||||
it('should be false for a bucket aggregation', function () {
|
||||
expect(isNestedAggregation(bucketAggregation)).to.be.false;
|
||||
});
|
||||
|
||||
it('should be true for a nested aggregation', function () {
|
||||
const nestedAggregation: NestedAggregation = {doc_count: 123};
|
||||
|
||||
expect(isNestedAggregation(nestedAggregation)).to.be.true;
|
||||
});
|
||||
});
|
||||
|
||||
describe('isESTermsFilter', function () {
|
||||
it('should be false for an elasticsearch nested aggregation', function () {
|
||||
expect(isESTermsFilter(esNestedAggregation)).to.be.false;
|
||||
});
|
||||
|
||||
it('should be true for an elasticsearch terms filter', function () {
|
||||
expect(isESTermsFilter(esTermsFilter)).to.be.true;
|
||||
});
|
||||
});
|
||||
|
||||
describe('isESNestedAggregation', function () {
|
||||
it('should be false for an elasticsearch terms filter', function () {
|
||||
expect(isESNestedAggregation(esTermsFilter)).to.be.false;
|
||||
});
|
||||
|
||||
it('should be true for an elasticsearch nested aggregation', function () {
|
||||
expect(isESNestedAggregation(esNestedAggregation)).to.be.true;
|
||||
});
|
||||
});
|
||||
|
||||
describe('isESAggMatchAllFilter', function () {
|
||||
it('should be false for an elasticsearch aggregation type filter', function () {
|
||||
const aggregationTypeFilter: ESAggTypeFilter = {type: {value: 'foo'}};
|
||||
expect(isESAggMatchAllFilter(aggregationTypeFilter)).to.be.false;
|
||||
});
|
||||
|
||||
it('should be true for an elasticsearch aggregation match all filter', function () {
|
||||
const esAggMatchAllFilter: ESAggMatchAllFilter = {match_all: {}};
|
||||
expect(isESAggMatchAllFilter(esAggMatchAllFilter)).to.be.true;
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,14 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {ApiResponse, Client} from '@elastic/elasticsearch';
|
||||
import {Client, Diagnostic} from '@elastic/elasticsearch';
|
||||
import Indices from '@elastic/elasticsearch/lib/api/api/indices';
|
||||
import {
|
||||
CreateResponse,
|
||||
SearchHit,
|
||||
SearchResponse,
|
||||
SortCombinations,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
SCBook,
|
||||
SCBulkResponse,
|
||||
@@ -30,22 +37,32 @@ import {Logger} from '@openstapps/logger';
|
||||
import {SMTP} from '@openstapps/logger/lib/smtp';
|
||||
import {expect, use} from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import {SearchResponse} from 'elasticsearch';
|
||||
import mockedEnv from 'mocked-env';
|
||||
import sinon from 'sinon';
|
||||
import {ALL_INDICES_QUERY, parseIndexName} from '../../../src/storage/elasticsearch/util';
|
||||
import * as queryModule from '../../../src/storage/elasticsearch/query/query';
|
||||
import * as sortModule from '../../../src/storage/elasticsearch/query/sort';
|
||||
import sinon, {SinonStub} from 'sinon';
|
||||
import {getIndexUID, getThingIndexName, INDEX_UID_LENGTH} from '../../../src/storage/elasticsearch/util';
|
||||
import * as utilModule from '../../../src/storage/elasticsearch/util';
|
||||
import {removeInvalidAliasChars} from '../../../src/storage/elasticsearch/util/alias';
|
||||
import {configFile} from '../../../src/common';
|
||||
import {MailQueue} from '../../../src/notification/mail-queue';
|
||||
import {aggregations} from '../../../src/storage/elasticsearch/templating';
|
||||
import {ElasticsearchObject} from '../../../src/storage/elasticsearch/types/elasticsearch';
|
||||
import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch';
|
||||
import * as Monitoring from '../../../src/storage/elasticsearch/monitoring';
|
||||
import * as query from '../../../src/storage/elasticsearch/query';
|
||||
import * as templating from '../../../src/storage/elasticsearch/templating';
|
||||
import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common';
|
||||
import fs from 'fs';
|
||||
|
||||
use(chaiAsPromised);
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
function searchResponse<T>(...hits: SearchHit<T>[]): SearchResponse<T> {
|
||||
return {hits: {hits}, took: 0, timed_out: false, _shards: {total: 1, failed: 0, successful: 1}};
|
||||
}
|
||||
|
||||
describe('Elasticsearch', function () {
|
||||
// increase timeout for the suite
|
||||
this.timeout(DEFAULT_TEST_TIMEOUT);
|
||||
@@ -83,6 +100,14 @@ describe('Elasticsearch', function () {
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAliasMap', function () {
|
||||
it('should fail after retries', async function () {
|
||||
const es = new Elasticsearch(configFile);
|
||||
sandbox.stub(es.client.indices, 'getAlias').throws();
|
||||
await expect(es.init({maxRetries: 1, retryInterval: 10})).to.be.rejected;
|
||||
});
|
||||
});
|
||||
|
||||
describe('getIndex (including getIndexUID)', function () {
|
||||
const type = 'foo bar type';
|
||||
const source = 'foo_source';
|
||||
@@ -95,59 +120,63 @@ describe('Elasticsearch', function () {
|
||||
};
|
||||
|
||||
it('should provide index UID from the provided UID', function () {
|
||||
const indexUID = Elasticsearch.getIndexUID(bulk.uid);
|
||||
const indexUID = getIndexUID(bulk.uid);
|
||||
|
||||
expect(indexUID.length).to.be.equal(Elasticsearch.INDEX_UID_LENGTH);
|
||||
expect(indexUID.length).to.be.equal(INDEX_UID_LENGTH);
|
||||
// test starting and ending character
|
||||
expect(indexUID[0]).to.be.equal(bulk.uid[0]);
|
||||
expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[Elasticsearch.INDEX_UID_LENGTH - 1]);
|
||||
expect(indexUID[indexUID.length - 1]).to.be.equal(bulk.uid[INDEX_UID_LENGTH - 1]);
|
||||
});
|
||||
|
||||
it('should provide index name from the provided data', function () {
|
||||
expect(Elasticsearch.getIndex(type as SCThingType, source, bulk)).to.be.equal(
|
||||
`stapps_${type.split(' ').join('_')}_${source}_${Elasticsearch.getIndexUID(bulk.uid)}`,
|
||||
expect(getThingIndexName(type as SCThingType, source, bulk)).to.be.equal(
|
||||
`stapps_${type.split(' ').join('_')}_${source}_${getIndexUID(bulk.uid)}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('should reject invalid index names', function () {
|
||||
expect(() => parseIndexName(':)')).to.throw(SyntaxError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeAliasChars', function () {
|
||||
it('should remove spaces from both ends', function () {
|
||||
expect(Elasticsearch.removeAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(removeInvalidAliasChars(' foobaralias ', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
});
|
||||
|
||||
it('should replace inner spaces with underscores', function () {
|
||||
expect(Elasticsearch.removeAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
|
||||
expect(removeInvalidAliasChars('foo bar alias', 'bulk-uid')).to.be.equal('foo_bar_alias');
|
||||
});
|
||||
|
||||
it('should remove invalid characters', function () {
|
||||
expect(Elasticsearch.removeAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(removeInvalidAliasChars('f,o#o\\b|ar/<?alias>* ', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
});
|
||||
|
||||
it('should remove invalid starting characters', function () {
|
||||
expect(Elasticsearch.removeAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(Elasticsearch.removeAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(Elasticsearch.removeAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(removeInvalidAliasChars('-foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(removeInvalidAliasChars('_foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
expect(removeInvalidAliasChars('+foobaralias', 'bulk-uid')).to.be.equal('foobaralias');
|
||||
});
|
||||
|
||||
it('should replace with a placeholder in case of invalid alias', function () {
|
||||
expect(Elasticsearch.removeAliasChars('.', 'bulk-uid')).to.contain('placeholder');
|
||||
expect(Elasticsearch.removeAliasChars('..', 'bulk-uid')).to.contain('placeholder');
|
||||
expect(removeInvalidAliasChars('.', 'bulk-uid')).to.contain('placeholder');
|
||||
expect(removeInvalidAliasChars('..', 'bulk-uid')).to.contain('placeholder');
|
||||
});
|
||||
|
||||
it('should work with common cases', function () {
|
||||
expect(
|
||||
Elasticsearch.removeAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
|
||||
removeInvalidAliasChars('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890', 'bulk-uid'),
|
||||
).to.be.equal('the-quick-brown-fox-jumps-over-the-lazy-dog-1234567890');
|
||||
expect(
|
||||
Elasticsearch.removeAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid'),
|
||||
).to.be.equal('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG');
|
||||
expect(removeInvalidAliasChars('THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG', 'bulk-uid')).to.be.equal(
|
||||
'THE_QUICK_BROWN_FOX_JUMPS_OVER_THE_LAZY_DOG',
|
||||
);
|
||||
});
|
||||
|
||||
it('should warn in case of characters that are invalid in future elasticsearch versions', function () {
|
||||
const sandbox = sinon.createSandbox();
|
||||
const loggerWarnStub = sandbox.stub(Logger, 'warn');
|
||||
|
||||
expect(Elasticsearch.removeAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
|
||||
expect(removeInvalidAliasChars('foo:bar:alias', 'bulk-uid')).to.contain('foo:bar:alias');
|
||||
expect(loggerWarnStub.called).to.be.true;
|
||||
});
|
||||
});
|
||||
@@ -182,7 +211,7 @@ describe('Elasticsearch', function () {
|
||||
it('should log an error in case of there is one when getting response from the elasticsearch client', async function () {
|
||||
const error = new Error('Foo Error');
|
||||
const loggerErrorStub = sandbox.stub(Logger, 'error').resolves('foo');
|
||||
sandbox.stub(Client.prototype, 'on').yields(error);
|
||||
sandbox.stub(Diagnostic.prototype, 'on').yields(error);
|
||||
|
||||
new Elasticsearch(configFile);
|
||||
|
||||
@@ -192,7 +221,7 @@ describe('Elasticsearch', function () {
|
||||
it('should log the result in the debug mode when getting response from the elasticsearch client', async function () {
|
||||
const fakeResponse = {foo: 'bar'};
|
||||
const loggerLogStub = sandbox.stub(Logger, 'log');
|
||||
sandbox.stub(Client.prototype, 'on').yields(null, fakeResponse);
|
||||
sandbox.stub(Diagnostic.prototype, 'on').yields(null, fakeResponse);
|
||||
|
||||
new Elasticsearch(configFile);
|
||||
expect(loggerLogStub.calledWith(fakeResponse)).to.be.false;
|
||||
@@ -254,26 +283,24 @@ describe('Elasticsearch', function () {
|
||||
describe('Operations with bundle/index', async function () {
|
||||
const sandbox = sinon.createSandbox();
|
||||
let es: Elasticsearch;
|
||||
let createStub: SinonStub;
|
||||
let deleteStub: SinonStub;
|
||||
let refreshStub: SinonStub;
|
||||
let updateAliasesStub: SinonStub;
|
||||
let existsStub: SinonStub;
|
||||
const oldIndex = 'stapps_footype_foosource_oldindex';
|
||||
|
||||
beforeEach(function () {
|
||||
sandbox
|
||||
.stub(Indices.prototype, 'getAlias')
|
||||
.resolves({[oldIndex]: {aliases: {[SCThingType.Book]: {}}}} as any);
|
||||
sandbox.stub(Indices.prototype, 'putTemplate').resolves({} as any);
|
||||
createStub = sandbox.stub(Indices.prototype, 'create').resolves({} as any);
|
||||
deleteStub = sandbox.stub(Indices.prototype, 'delete').resolves({} as any);
|
||||
existsStub = sandbox.stub(Indices.prototype, 'exists').resolves({} as any);
|
||||
refreshStub = sandbox.stub(Indices.prototype, 'refresh').resolves({} as any);
|
||||
updateAliasesStub = sandbox.stub(Indices.prototype, 'updateAliases').resolves({} as any);
|
||||
es = new Elasticsearch(configFile);
|
||||
es.client.indices = {
|
||||
// @ts-expect-error not assignable
|
||||
getAlias: () => Promise.resolve({body: [{[oldIndex]: {aliases: {[SCThingType.Book]: {}}}}]}),
|
||||
// @ts-expect-error not assignable
|
||||
putTemplate: () => Promise.resolve({}),
|
||||
// @ts-expect-error not assignable
|
||||
create: () => Promise.resolve({}),
|
||||
// @ts-expect-error not assignable
|
||||
delete: () => Promise.resolve({}),
|
||||
// @ts-expect-error not assignable
|
||||
exists: () => Promise.resolve({}),
|
||||
// @ts-expect-error not assignable
|
||||
refresh: () => Promise.resolve({}),
|
||||
// @ts-expect-error not assignable
|
||||
updateAliases: () => Promise.resolve({}),
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
@@ -286,8 +313,8 @@ describe('Elasticsearch', function () {
|
||||
});
|
||||
|
||||
it('should reject (throw an error) if the index name is not valid', async function () {
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex}`);
|
||||
sandbox.createStubInstance(Client, {});
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex}`);
|
||||
await es.init();
|
||||
|
||||
return expect(es.bulkCreated(bulk)).to.be.rejectedWith('Index');
|
||||
@@ -295,9 +322,8 @@ describe('Elasticsearch', function () {
|
||||
|
||||
it('should create a new index', async function () {
|
||||
const index = getIndex();
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
|
||||
const putTemplateStub = sandbox.stub(templating, 'putTemplate');
|
||||
const createStub = sandbox.stub(es.client.indices, 'create');
|
||||
await es.init();
|
||||
|
||||
await es.bulkCreated(bulk);
|
||||
@@ -313,21 +339,19 @@ describe('Elasticsearch', function () {
|
||||
sandbox.restore();
|
||||
});
|
||||
it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
|
||||
const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
|
||||
|
||||
await es.bulkExpired({...bulk, state: 'in progress'});
|
||||
|
||||
expect(clientDeleteStub.called).to.be.true;
|
||||
expect(deleteStub.called).to.be.true;
|
||||
});
|
||||
|
||||
it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
|
||||
const clientDeleteStub = sandbox.stub(es.client.indices, 'delete');
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
|
||||
|
||||
await es.bulkExpired({...bulk, state: 'done'});
|
||||
|
||||
expect(clientDeleteStub.called).to.be.false;
|
||||
expect(deleteStub.called).to.be.false;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -337,13 +361,23 @@ describe('Elasticsearch', function () {
|
||||
});
|
||||
|
||||
it('should reject if the index name is not valid', async function () {
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(`invalid_${getIndex()}`);
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(`invalid_${getIndex()}`);
|
||||
sandbox.createStubInstance(Client, {});
|
||||
await es.init();
|
||||
|
||||
return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
|
||||
});
|
||||
|
||||
it("should create templates if index doesn't exist", async function () {
|
||||
await es.init();
|
||||
existsStub.resolves(false);
|
||||
const putTemplateSpy = sandbox.spy(templating, 'putTemplate');
|
||||
await es.bulkUpdated(bulk);
|
||||
|
||||
expect(createStub.called).to.be.true;
|
||||
expect(putTemplateSpy.called).to.be.true;
|
||||
});
|
||||
|
||||
it('should create a new index', async function () {
|
||||
const index = getIndex();
|
||||
const expectedRefreshActions = [
|
||||
@@ -354,15 +388,12 @@ describe('Elasticsearch', function () {
|
||||
remove: {index: oldIndex, alias: SCThingType.Book},
|
||||
},
|
||||
];
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
|
||||
sandbox.stub(es, 'aliasMap').value({
|
||||
[SCThingType.Book]: {
|
||||
[bulk.source]: oldIndex,
|
||||
},
|
||||
});
|
||||
const refreshStub = sandbox.stub(es.client.indices, 'refresh');
|
||||
const updateAliasesStub = sandbox.stub(es.client.indices, 'updateAliases');
|
||||
const deleteStub = sandbox.stub(es.client.indices, 'delete');
|
||||
sandbox.stub(templating, 'putTemplate');
|
||||
await es.init();
|
||||
|
||||
@@ -371,9 +402,7 @@ describe('Elasticsearch', function () {
|
||||
expect(refreshStub.calledWith({index})).to.be.true;
|
||||
expect(
|
||||
updateAliasesStub.calledWith({
|
||||
body: {
|
||||
actions: expectedRefreshActions,
|
||||
},
|
||||
actions: expectedRefreshActions,
|
||||
}),
|
||||
).to.be.true;
|
||||
expect(deleteStub.called).to.be.true;
|
||||
@@ -394,20 +423,19 @@ describe('Elasticsearch', function () {
|
||||
});
|
||||
|
||||
it('should reject if object is not found', async function () {
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse());
|
||||
|
||||
return expect(es.get('123')).to.rejectedWith('found');
|
||||
});
|
||||
|
||||
it('should provide the thing if object is found', async function () {
|
||||
const foundObject: ElasticsearchObject<SCMessage> = {
|
||||
const foundObject: SearchHit<SCMessage> = {
|
||||
_id: '',
|
||||
_index: '',
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: message as SCMessage,
|
||||
};
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [foundObject]}}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse(foundObject));
|
||||
|
||||
return expect(await es.get('123')).to.be.eql(message);
|
||||
});
|
||||
@@ -428,56 +456,54 @@ describe('Elasticsearch', function () {
|
||||
it('should not post if the object already exists in an index which will not be rolled over', async function () {
|
||||
const index = getIndex();
|
||||
const oldIndex = index.replace('foosource', 'barsource');
|
||||
const object: ElasticsearchObject<SCMessage> = {
|
||||
const object: SearchHit<SCMessage> = {
|
||||
_id: '',
|
||||
_index: oldIndex,
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: message as SCMessage,
|
||||
};
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(index);
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(index);
|
||||
|
||||
return expect(es.post(object._source, bulk)).to.rejectedWith('exist');
|
||||
return expect(es.post(object._source!, bulk)).to.rejectedWith('exist');
|
||||
});
|
||||
|
||||
it('should not reject if the object already exists but in an index which will be rolled over', async function () {
|
||||
const object: ElasticsearchObject<SCMessage> = {
|
||||
const object: SearchHit<SCMessage> = {
|
||||
_id: '',
|
||||
_index: getIndex(),
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: message as SCMessage,
|
||||
};
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
|
||||
// return index name with different generated UID (see getIndex method)
|
||||
sandbox.stub(Elasticsearch, 'getIndex').returns(getIndex());
|
||||
sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
|
||||
|
||||
return expect(es.post(object._source, bulk)).to.not.rejectedWith('exist');
|
||||
return expect(es.post(object._source!, bulk)).to.not.rejectedWith('exist');
|
||||
});
|
||||
|
||||
it('should reject if there is an object creation error on the elasticsearch side', async function () {
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
|
||||
sandbox.stub(es.client, 'create').resolves({body: {created: false}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse());
|
||||
sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse);
|
||||
|
||||
return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
|
||||
});
|
||||
|
||||
it('should create a new object', async function () {
|
||||
let caughtParameter: any;
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse());
|
||||
// @ts-expect-error call
|
||||
const createStub = sandbox.stub(es.client, 'create').callsFake(parameter => {
|
||||
caughtParameter = parameter;
|
||||
return Promise.resolve({body: {created: true}});
|
||||
return Promise.resolve({result: 'created'});
|
||||
});
|
||||
|
||||
await es.post(message as SCMessage, bulk);
|
||||
|
||||
expect(createStub.called).to.be.true;
|
||||
expect(caughtParameter.body).to.be.eql({
|
||||
expect(caughtParameter.document).to.be.eql({
|
||||
...message,
|
||||
creation_date: caughtParameter.body.creation_date,
|
||||
creation_date: caughtParameter.document.creation_date,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -493,29 +519,27 @@ describe('Elasticsearch', function () {
|
||||
sandbox.restore();
|
||||
});
|
||||
it('should reject to put if the object does not already exist', async function () {
|
||||
const object: ElasticsearchObject<SCMessage> = {
|
||||
const object: SearchHit<SCMessage> = {
|
||||
_id: '',
|
||||
_index: getIndex(),
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: message as SCMessage,
|
||||
};
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: []}}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse());
|
||||
|
||||
return expect(es.put(object._source)).to.rejectedWith('exist');
|
||||
return expect(es.put(object._source!)).to.rejectedWith('exist');
|
||||
});
|
||||
|
||||
// noinspection JSUnusedLocalSymbols
|
||||
it('should update the object if it already exists', async function () {
|
||||
let caughtParameter: any;
|
||||
const object: ElasticsearchObject<SCMessage> = {
|
||||
const object: SearchHit<SCMessage> = {
|
||||
_id: '',
|
||||
_index: getIndex(),
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: message as SCMessage,
|
||||
};
|
||||
sandbox.stub(es.client, 'search').resolves({body: {hits: {hits: [object]}}});
|
||||
sandbox.stub(es.client, 'search').resolves(searchResponse(object));
|
||||
// @ts-expect-error unused
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const stubUpdate = sandbox.stub(es.client, 'update').callsFake(parameters => {
|
||||
@@ -523,7 +547,7 @@ describe('Elasticsearch', function () {
|
||||
return Promise.resolve({body: {created: true}});
|
||||
});
|
||||
|
||||
await es.put(object._source);
|
||||
await es.put(object._source!);
|
||||
|
||||
expect(caughtParameter.body.doc).to.be.eql(object._source);
|
||||
});
|
||||
@@ -532,18 +556,16 @@ describe('Elasticsearch', function () {
|
||||
describe('search', async function () {
|
||||
let es: Elasticsearch;
|
||||
const sandbox = sinon.createSandbox();
|
||||
const objectMessage: ElasticsearchObject<SCMessage> = {
|
||||
const objectMessage: SearchHit<SCMessage> = {
|
||||
_id: '123',
|
||||
_index: getIndex(),
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: message as SCMessage,
|
||||
};
|
||||
const objectBook: ElasticsearchObject<SCBook> = {
|
||||
const objectBook: SearchHit<SCBook> = {
|
||||
_id: '321',
|
||||
_index: getIndex(),
|
||||
_score: 0,
|
||||
_type: '',
|
||||
_source: book as SCBook,
|
||||
};
|
||||
const fakeEsAggregations = {
|
||||
@@ -565,26 +587,16 @@ describe('Elasticsearch', function () {
|
||||
},
|
||||
},
|
||||
};
|
||||
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
|
||||
body: {
|
||||
took: 12,
|
||||
timed_out: false,
|
||||
// @ts-expect-error not assignable
|
||||
_shards: {},
|
||||
// @ts-expect-error not assignable
|
||||
hits: {
|
||||
hits: [objectMessage, objectBook],
|
||||
total: 123,
|
||||
},
|
||||
aggregations: fakeEsAggregations,
|
||||
const fakeSearchResponse: SearchResponse<SCThings> = {
|
||||
took: 12,
|
||||
timed_out: false,
|
||||
// @ts-expect-error not assignable
|
||||
_shards: {},
|
||||
hits: {
|
||||
hits: [objectMessage, objectBook],
|
||||
total: 123,
|
||||
},
|
||||
headers: {},
|
||||
// @ts-expect-error not assignable
|
||||
meta: {},
|
||||
// @ts-expect-error not assignable
|
||||
statusCode: {},
|
||||
// @ts-expect-error not assignable
|
||||
warnings: {},
|
||||
aggregations: fakeEsAggregations,
|
||||
};
|
||||
let searchStub: sinon.SinonStub;
|
||||
before(function () {
|
||||
@@ -625,9 +637,9 @@ describe('Elasticsearch', function () {
|
||||
const {pagination} = await es.search({from});
|
||||
|
||||
expect(pagination).to.be.eql({
|
||||
count: fakeSearchResponse.body!.hits.hits.length,
|
||||
count: fakeSearchResponse.hits.hits.length,
|
||||
offset: from,
|
||||
total: fakeSearchResponse.body!.hits.total,
|
||||
total: fakeSearchResponse.hits.total,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -659,22 +671,20 @@ describe('Elasticsearch', function () {
|
||||
},
|
||||
},
|
||||
};
|
||||
const fakeResponse = {foo: 'bar'};
|
||||
const fakeResponse = {foo: 'bar'} as SortCombinations;
|
||||
const fakeBuildSortResponse = [fakeResponse];
|
||||
// @ts-expect-error not assignable
|
||||
sandbox.stub(query, 'buildQuery').returns(fakeResponse);
|
||||
sandbox.stub(query, 'buildSort').returns(fakeBuildSortResponse);
|
||||
sandbox.stub(queryModule, 'buildQuery').returns(fakeResponse);
|
||||
sandbox.stub(sortModule, 'buildSort').returns(fakeBuildSortResponse);
|
||||
|
||||
await es.search(parameters);
|
||||
|
||||
sandbox.assert.calledWithMatch(searchStub, {
|
||||
body: {
|
||||
aggs: aggregations,
|
||||
query: fakeResponse,
|
||||
sort: fakeBuildSortResponse,
|
||||
},
|
||||
aggs: aggregations,
|
||||
query: fakeResponse,
|
||||
sort: fakeBuildSortResponse,
|
||||
from: parameters.from,
|
||||
index: Elasticsearch.getListOfAllIndices(),
|
||||
index: ALL_INDICES_QUERY,
|
||||
size: parameters.size,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {ApiResponse, Client} from '@elastic/elasticsearch';
|
||||
import {Client} from '@elastic/elasticsearch';
|
||||
import {SearchResponse} from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
SCMonitoringConfiguration,
|
||||
SCMonitoringLogAction,
|
||||
@@ -23,7 +24,6 @@ import {
|
||||
SCThings,
|
||||
} from '@openstapps/core';
|
||||
import {Logger} from '@openstapps/logger';
|
||||
import {SearchResponse} from 'elasticsearch';
|
||||
import {MailQueue} from '../../../src/notification/mail-queue';
|
||||
import {setUp} from '../../../src/storage/elasticsearch/monitoring';
|
||||
|
||||
@@ -111,16 +111,14 @@ describe('Monitoring', async function () {
|
||||
});
|
||||
|
||||
it('should log errors where conditions failed', async function () {
|
||||
const fakeSearchResponse: Partial<ApiResponse<SearchResponse<SCThings>>> = {
|
||||
body: {
|
||||
took: 12,
|
||||
timed_out: false,
|
||||
// @ts-expect-error not assignable
|
||||
_shards: {},
|
||||
// @ts-expect-error not assignable
|
||||
hits: {
|
||||
total: 123,
|
||||
},
|
||||
const fakeSearchResponse: SearchResponse<SCThings> = {
|
||||
took: 12,
|
||||
timed_out: false,
|
||||
// @ts-expect-error not assignable
|
||||
_shards: {},
|
||||
// @ts-expect-error not assignable
|
||||
hits: {
|
||||
total: 123,
|
||||
},
|
||||
};
|
||||
const fakeClient = new Client({node: 'http://foohost:9200'});
|
||||
|
||||
@@ -25,25 +25,14 @@ import {
|
||||
SCThingType,
|
||||
} from '@openstapps/core';
|
||||
import {expect} from 'chai';
|
||||
import {
|
||||
ESDateRangeFilter,
|
||||
ESRangeFilter,
|
||||
ESNumericRangeFilter,
|
||||
ElasticsearchConfig,
|
||||
ESBooleanFilter,
|
||||
ESGenericSort,
|
||||
ESGeoDistanceFilter,
|
||||
ESGeoDistanceSort,
|
||||
ESTermFilter,
|
||||
ScriptSort,
|
||||
} from '../../../src/storage/elasticsearch/types/elasticsearch';
|
||||
import {buildFilter} from '../../../src/storage/elasticsearch/query/filter';
|
||||
import {buildBooleanFilter} from '../../../src/storage/elasticsearch/query/filters/boolean';
|
||||
import {buildQuery} from '../../../src/storage/elasticsearch/query/query';
|
||||
import {buildSort} from '../../../src/storage/elasticsearch/query/sort';
|
||||
import {ElasticsearchConfig} from '../../../src/storage/elasticsearch/types/elasticsearch-config';
|
||||
import {QueryDslSpecificQueryContainer} from '../../../src/storage/elasticsearch/types/util';
|
||||
import {configFile} from '../../../src/common';
|
||||
import {
|
||||
buildBooleanFilter,
|
||||
buildFilter,
|
||||
buildQuery,
|
||||
buildSort,
|
||||
} from '../../../src/storage/elasticsearch/query';
|
||||
import {SortCombinations} from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
describe('Query', function () {
|
||||
describe('buildBooleanFilter', function () {
|
||||
@@ -74,7 +63,7 @@ describe('Query', function () {
|
||||
or: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'or'}},
|
||||
not: {...booleanFilter, arguments: {...booleanFilter.arguments, operation: 'not'}},
|
||||
};
|
||||
const expectedEsFilters: Array<ESTermFilter> = [
|
||||
const expectedEsFilters: Array<QueryDslSpecificQueryContainer<'term'>> = [
|
||||
{
|
||||
term: {
|
||||
'type.raw': 'catalog',
|
||||
@@ -88,20 +77,20 @@ describe('Query', function () {
|
||||
];
|
||||
|
||||
it('should create appropriate elasticsearch "and" filter argument', function () {
|
||||
const {must} = buildBooleanFilter(booleanFilters.and);
|
||||
const {must} = buildBooleanFilter(booleanFilters.and).bool;
|
||||
|
||||
expect(must).to.be.eql(expectedEsFilters);
|
||||
});
|
||||
|
||||
it('should create appropriate elasticsearch "or" filter argument', function () {
|
||||
const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or);
|
||||
const {should, minimum_should_match} = buildBooleanFilter(booleanFilters.or).bool;
|
||||
|
||||
expect(should).to.be.eql(expectedEsFilters);
|
||||
expect(minimum_should_match).to.be.equal(1);
|
||||
});
|
||||
|
||||
it('should create appropriate elasticsearch "not" filter argument', function () {
|
||||
const {must_not} = buildBooleanFilter(booleanFilters.not);
|
||||
const {must_not} = buildBooleanFilter(booleanFilters.not).bool;
|
||||
|
||||
expect(must_not).to.be.eql(expectedEsFilters);
|
||||
});
|
||||
@@ -196,6 +185,10 @@ describe('Query', function () {
|
||||
|
||||
expect(() => buildQuery(parameters, config, esConfig)).to.throw('query type');
|
||||
});
|
||||
|
||||
it('should accept other search contexts', function () {
|
||||
expect(buildQuery({context: 'place', ...parameters}, config, esConfig)).to.be.an('object');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildFilter', function () {
|
||||
@@ -267,7 +260,7 @@ describe('Query', function () {
|
||||
|
||||
it('should build value filter', function () {
|
||||
const filter = buildFilter(searchFilters.value);
|
||||
const expectedFilter: ESTermFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'term'> = {
|
||||
term: {
|
||||
'type.raw': SCThingType.Dish,
|
||||
},
|
||||
@@ -279,7 +272,7 @@ describe('Query', function () {
|
||||
it('should build numeric range filters', function () {
|
||||
for (const upperMode of ['inclusive', 'exclusive', null]) {
|
||||
for (const lowerMode of ['inclusive', 'exclusive', null]) {
|
||||
const expectedFilter: ESNumericRangeFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
|
||||
range: {
|
||||
price: {
|
||||
relation: undefined,
|
||||
@@ -304,7 +297,7 @@ describe('Query', function () {
|
||||
mode: bound as 'inclusive' | 'exclusive',
|
||||
limit: out,
|
||||
};
|
||||
expectedFilter.range.price[
|
||||
expectedFilter.range.price![
|
||||
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
|
||||
] = out;
|
||||
}
|
||||
@@ -312,7 +305,7 @@ describe('Query', function () {
|
||||
setBound('upperBound', upperMode);
|
||||
setBound('lowerBound', lowerMode);
|
||||
|
||||
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
|
||||
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'term'>;
|
||||
expect(filter).to.deep.equal(expectedFilter);
|
||||
for (const bound of ['g', 'l']) {
|
||||
// @ts-expect-error implicit any
|
||||
@@ -330,7 +323,7 @@ describe('Query', function () {
|
||||
it('should build date range filters', function () {
|
||||
for (const upperMode of ['inclusive', 'exclusive', null]) {
|
||||
for (const lowerMode of ['inclusive', 'exclusive', null]) {
|
||||
const expectedFilter: ESDateRangeFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
|
||||
range: {
|
||||
price: {
|
||||
format: 'thisIsADummyFormat',
|
||||
@@ -359,7 +352,7 @@ describe('Query', function () {
|
||||
mode: bound as 'inclusive' | 'exclusive',
|
||||
limit: out,
|
||||
};
|
||||
expectedFilter.range.price[
|
||||
expectedFilter.range.price![
|
||||
`${location === 'lowerBound' ? 'g' : 'l'}${bound === 'inclusive' ? 'te' : 't'}`
|
||||
] = out;
|
||||
}
|
||||
@@ -367,7 +360,7 @@ describe('Query', function () {
|
||||
setBound('upperBound', upperMode);
|
||||
setBound('lowerBound', lowerMode);
|
||||
|
||||
const filter = buildFilter(rawFilter) as ESNumericRangeFilter;
|
||||
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'range'>;
|
||||
expect(filter).to.deep.equal(expectedFilter);
|
||||
for (const bound of ['g', 'l']) {
|
||||
// @ts-expect-error implicit any
|
||||
@@ -394,7 +387,7 @@ describe('Query', function () {
|
||||
},
|
||||
});
|
||||
|
||||
const expectedFilter: ESRangeFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
|
||||
range: {
|
||||
'offers.availabilityRange': {
|
||||
gte: `test||/${scope}`,
|
||||
@@ -415,7 +408,7 @@ describe('Query', function () {
|
||||
},
|
||||
});
|
||||
|
||||
const expectedFilter: ESRangeFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
|
||||
range: {
|
||||
'offers.availabilityRange': {
|
||||
gte: 'test||/s',
|
||||
@@ -436,7 +429,7 @@ describe('Query', function () {
|
||||
},
|
||||
});
|
||||
|
||||
const expectedFilter: ESRangeFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
|
||||
range: {
|
||||
'offers.availabilityRange': {
|
||||
gte: `test||/d`,
|
||||
@@ -456,7 +449,7 @@ describe('Query', function () {
|
||||
},
|
||||
});
|
||||
|
||||
const expectedFilter: ESRangeFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'range'> = {
|
||||
range: {
|
||||
'offers.availabilityRange': {
|
||||
gte: `now/d`,
|
||||
@@ -470,7 +463,7 @@ describe('Query', function () {
|
||||
|
||||
it('should build distance filter', function () {
|
||||
const filter = buildFilter(searchFilters.distance);
|
||||
const expectedFilter: ESGeoDistanceFilter = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'geo_distance'> = {
|
||||
geo_distance: {
|
||||
'distance': '1000m',
|
||||
'geo.point.coordinates': {
|
||||
@@ -486,34 +479,18 @@ describe('Query', function () {
|
||||
it('should build geo filter for shapes and points', function () {
|
||||
const filter = buildFilter(searchFilters.geoPoint);
|
||||
const expectedFilter = {
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: [
|
||||
{
|
||||
geo_shape: {
|
||||
'geo.polygon': {
|
||||
relation: undefined,
|
||||
shape: {
|
||||
type: 'envelope',
|
||||
coordinates: [
|
||||
[50.123, 8.123],
|
||||
[50.123, 8.123],
|
||||
],
|
||||
},
|
||||
},
|
||||
'ignore_unmapped': true,
|
||||
},
|
||||
geo_shape: {
|
||||
'geo.polygon': {
|
||||
relation: undefined,
|
||||
shape: {
|
||||
type: 'envelope',
|
||||
coordinates: [
|
||||
[50.123, 8.123],
|
||||
[50.123, 8.123],
|
||||
],
|
||||
},
|
||||
{
|
||||
geo_bounding_box: {
|
||||
'geo.point.coordinates': {
|
||||
bottom_right: [50.123, 8.123],
|
||||
top_left: [50.123, 8.123],
|
||||
},
|
||||
'ignore_unmapped': true,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
'ignore_unmapped': true,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -543,7 +520,7 @@ describe('Query', function () {
|
||||
|
||||
it('should build boolean filter', function () {
|
||||
const filter = buildFilter(searchFilters.boolean);
|
||||
const expectedFilter: ESBooleanFilter<any> = {
|
||||
const expectedFilter: QueryDslSpecificQueryContainer<'bool'> = {
|
||||
bool: {
|
||||
minimum_should_match: 0,
|
||||
must: [
|
||||
@@ -604,8 +581,8 @@ describe('Query', function () {
|
||||
},
|
||||
},
|
||||
];
|
||||
let sorts: Array<ESGenericSort | ESGeoDistanceSort | ScriptSort> = [];
|
||||
const expectedSorts: {[key: string]: ESGenericSort | ESGeoDistanceSort | ScriptSort} = {
|
||||
let sorts: SortCombinations[] = [];
|
||||
const expectedSorts: {[key: string]: SortCombinations} = {
|
||||
ducet: {
|
||||
'name.sort': 'desc',
|
||||
},
|
||||
@@ -632,7 +609,7 @@ describe('Query', function () {
|
||||
},
|
||||
};
|
||||
before(function () {
|
||||
sorts = buildSort(searchSCSearchSort);
|
||||
sorts = buildSort(searchSCSearchSort) as SortCombinations[];
|
||||
});
|
||||
|
||||
it('should build ducet sort', function () {
|
||||
@@ -649,10 +626,10 @@ describe('Query', function () {
|
||||
|
||||
it('should build price sort', function () {
|
||||
const priceSortNoScript = {
|
||||
...sorts[3],
|
||||
...(sorts[3] as any),
|
||||
_script: {
|
||||
...(sorts[3] as ScriptSort)._script,
|
||||
script: (expectedSorts.price as ScriptSort)._script.script,
|
||||
...(sorts[3] as any)._script,
|
||||
script: (expectedSorts.price as any)._script.script,
|
||||
},
|
||||
};
|
||||
expect(priceSortNoScript).to.be.eql(expectedSorts.price);
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
"extends": "./node_modules/@openstapps/configuration/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"resolveJsonModule": true,
|
||||
"useUnknownInCatchVariables": false
|
||||
"skipLibCheck": true,
|
||||
"useUnknownInCatchVariables": false,
|
||||
"lib": ["ES2020"]
|
||||
},
|
||||
"exclude": [
|
||||
"./config/",
|
||||
|
||||
Reference in New Issue
Block a user