feat: backend index diffing

Closes #215

.changeset/cold-years-listen.md

@@ -0,0 +1,5 @@
+---
+"@openstapps/easy-ast": patch
+---
+
+Fixed docs generation

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -24,6 +24,7 @@ report-junit.xml
 # NixOS flake
 result
 hsperfdata_root
+.direnv/
 
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov

.gitlab-ci.yml

@@ -133,7 +133,7 @@ audit:
   allow_failure: true
   needs: []
   script:
-    - pnpm audit --prod
+    - pnpm audit --prod --audit-level critical
   rules:
     - if: $CI_COMMIT_BRANCH == 'main'
       allow_failure: false

.gitlab/publishing.gitlab-ci.yml

@@ -1,6 +1,11 @@
 .limit_publish_pipelines:
   rules:
-      - if: '($CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "develop") && $CI_COMMIT_MESSAGE =~ /ci: publish release/ && $CI_PIPELINE_SOURCE != "schedule"'
+      - if: '$CI_COMMIT_BRANCH == "develop" && $CI_COMMIT_MESSAGE =~ /ci: publish prerelease/ && $CI_PIPELINE_SOURCE != "schedule"'
+        variables:
+          PUBLISH_TAG: next
+      - if: '$CI_COMMIT_BRANCH == "main" && $CI_COMMIT_MESSAGE =~ /ci: publish release/ && $CI_PIPELINE_SOURCE != "schedule"'
+        variables:
+          PUBLISH_TAG: latest
 
 deploy:
   stage: publish
@@ -24,8 +29,6 @@ publish image:
   image:
     name: gcr.io/kaniko-project/executor:v1.12.1-debug
     entrypoint: [""]
-  variables:
-    PUBLISH_TAG: next
   script:
     - >
        /kaniko/executor
@@ -50,9 +53,6 @@ publish image:
       - IMAGE_NAME: app
         DEPLOY_DIR: frontend/app
   rules:
-    - if: $CI_COMMIT_BRANCH == 'main'
-      variables:
-        PUBLISH_TAG: latest
     - !reference [.limit_publish_pipelines, rules]
 
 publish packages:
@@ -61,16 +61,12 @@ publish packages:
   variables:
     GIT_STRATEGY: clone
     GIT_DEPTH: 0
-    PUBLISH_TAG: next
   script:
     - pnpm install
     - pnpm build
     - pnpm config set '//registry.npmjs.org/:_authToken' "${NPM_AUTH_TOKEN}"
     - pnpm publish -r --publish-branch ${CI_COMMIT_BRANCH} --tag ${PUBLISH_TAG} --no-git-checks # TODO: Git checks...
   rules:
-    - if: $CI_COMMIT_BRANCH == 'main'
-      variables:
-        PUBLISH_TAG: latest
     - !reference [.limit_publish_pipelines, rules]
 
 publish docs:
@@ -84,5 +80,4 @@ publish docs:
     paths:
       - public
   rules:
-    - if: $CI_COMMIT_BRANCH == 'main'
     - !reference [.limit_publish_pipelines, rules]

.prettierignore

@@ -0,0 +1 @@
+pnpm-lock.yaml

.syncpackrc.cjs

@@ -2,7 +2,13 @@
 
 /** @type {import('syncpack').RcFile} */
 const config = {
-  semverRange: '',
+  semverGroups: [
+    {
+      range: '',
+      dependencies: ['**'],
+      packages: ['**'],
+    }
+  ],
   source: ['package.json', '**/package.json'],
   indent: '  ',
   sortFirst: [
@@ -49,7 +55,7 @@ const config = {
     {
       label: 'Packages should use workspace version',
       dependencies: ['@openstapps/**'],
-      dependencyTypes: ['prod', 'dev'],
+      dependencyTypes: ['prod', 'dev', 'peer'],
       packages: ['**'],
       pinVersion: 'workspace:*',
     },

.tool-versions

@@ -0,0 +1,3 @@
+nodejs 18.19.1
+pnpm 8.15.4
+python 3.11.5

CONTRIBUTING.md

@@ -37,13 +37,13 @@ Adding new types requires changes at multiple locations for it to work correctly
 - Add your SCThing and SCThingWithoutReferences to `src/things/your-thing-name.ts` and make them extend `SCThingWithoutReferences` and `SCThing` respectively
 - Add your SCThingMeta to `src/things/your-thing-name.ts` and make it extend `SCThingMeta`
 - Add your SCThingMeta to `SCClasses` in `src/meta.ts`
-- Add your SCThing to `SCThingsWithoutDiff` in `src/meta.ts`
+- Add your SCThing to `SCIndexableThings ` in `src/meta.ts`
 - Add your SCThingWithoutReferences to `SCAssociatedThingWithoutReferences` in `src/meta.ts`
 - Add your SCThing to `SCAssociatedThing` in `src/meta.ts`
 - Add your SCThing to the `SCThingType` enum in `src/things/abstract/thing.ts`
 - Add an example file for your SCThing in `test/resources/YourThingName.json`
 - Add the following lines for your SCThing in `test/type.spec.ts`:
-
+- Make sure your SCThing (but not SCThingWithoutReferences!) includes the `@indexable` and `@validatable` JSDoc annotations, otherwise neither JSON Schemas nor Elasticsearch mappings will be generated
 ```typescript
 /**
  * Types of properties of SCYourThingName

backend/backend/CHANGELOG.md

@@ -1,5 +1,71 @@
 # @openstapps/backend
 
+## 3.3.1
+
+### Patch Changes
+
+- 67ab1fd6: fix for geo.point mapping
+
+## 3.3.0
+
+### Minor Changes
+
+- 688bc5f2: v3.3.0 changes
+
+### Patch Changes
+
+- Updated dependencies [688bc5f2]
+  - @openstapps/core@3.3.0
+  - @openstapps/core-tools@3.3.0
+  - @openstapps/logger@3.0.0
+
+## 3.2.0
+
+### Minor Changes
+
+- 912ae422: Add the ability to filter by existence of a field
+
+### Patch Changes
+
+- 689ac68b: pin alpine version to 3.18 and add healthchecks
+- e8d72683: Backend unit tests break every year
+- Updated dependencies [912ae422]
+  - @openstapps/core@4.0.0
+  - @openstapps/core-tools@3.0.0
+  - @openstapps/logger@3.0.0
+
+## 3.1.2
+
+### Patch Changes
+
+- Fix backend rejecting plugins
+- Fix backend sliently falling back to default configs
+
+## 3.1.1
+
+### Patch Changes
+
+- Fix version history offered by backend
+- Updated dependencies
+  - @openstapps/core@3.1.1
+
+## 3.1.0
+
+### Minor Changes
+
+- 06b8ca10: Add job portal feature
+
+### Patch Changes
+
+- Updated dependencies [06b8ca10]
+  - @openstapps/core@3.1.0
+
+## 3.0.1
+
+### Patch Changes
+
+- Fix missing dependency
+
 ## 3.0.0
 
 ### Major Changes

backend/backend/Dockerfile

@@ -9,4 +9,6 @@ ENV NODE_ENV=production
 WORKDIR /app
 
 EXPOSE 3000
+HEALTHCHECK --interval=10s --timeout=10s --start-period=10s --retries=12 CMD curl -s --fail --request POST --data '{}' --header 'Content-Type: application/json' http://localhost:3000/ >/dev/null || exit 1
+
 ENTRYPOINT ["node", "app.js"]

backend/backend/config/b-tu/elasticsearchrc.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 /**
  * This is the database configuration for the technical university of berlin
  *

backend/backend/config/default/app/index.js

@@ -22,6 +22,7 @@ const app = {
   name: 'Goethe-Uni',
   privacyPolicyUrl: 'https://mobile.server.uni-frankfurt.de/_static/privacy.md',
   settings: [userGroupSetting, languageSetting],
+  versionHistory: [],
 };
 
 export default app;

backend/backend/config/default/app/language-setting.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {SCSettingInputType, SCThingOriginType, SCThingType} from '@openstapps/core';
 
 /** @type {import('@openstapps/core').SCLanguageSetting}  */

backend/backend/config/default/app/menu.js

@@ -1,4 +1,3 @@
-// @ts-check
 /** @type {import('@openstapps/core').SCAppConfigurationMenuCategory[]} */
 const menus = [
   {
@@ -56,6 +55,19 @@ const menus = [
           },
         },
       },
+      {
+        icon: 'work',
+        route: '/jobs',
+        title: 'job postings',
+        translations: {
+          de: {
+            title: 'Jobangebote',
+          },
+          en: {
+            title: 'job postings',
+          },
+        },
+      },
     ],
     title: 'overview',
     route: '/overview',

backend/backend/config/default/app/user-group-setting.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {SCSettingInputType, SCThingOriginType, SCThingType} from '@openstapps/core';
 
 /** @type {import('@openstapps/core').SCUserGroupSetting} */

backend/backend/config/default/backend/aggregations.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {SCThingType} from '@openstapps/core';
 
 /** @type {import('@openstapps/core').SCBackendAggregationConfiguration[]} */

backend/backend/config/default/backend/boostings.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {
   month,
   sommerRange,

backend/backend/config/default/backend/index.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {SCThingType} from '@openstapps/core';
 import aggregations from './aggregations.js';
 import boostings from './boostings.js';
@@ -17,7 +16,7 @@ export const backend = {
   hiddenTypes: [SCThingType.DateSeries, SCThingType.Diff, SCThingType.Floor],
   mappingIgnoredTags: ['minlength', 'pattern', 'see', 'tjs-format'],
   maxMultiSearchRouteQueries: 5,
-  maxRequestBodySize: 512 * 1024,
+  maxRequestBodySize: 2e6,
   name: 'Goethe-Universität Frankfurt am Main',
   namespace: '909a8cbc-8520-456c-b474-ef1525f14209',
   sortableFields: [

backend/backend/config/default/backendrc.js

@@ -1,4 +1,3 @@
-// @ts-check
 import app from './app/index.js';
 import {backend, internal} from './backend/index.js';
 

backend/backend/config/default/elasticsearchrc.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 /**
  * This is the default configuration for elasticsearch (a database)
  *
@@ -11,13 +9,13 @@
  *
  * To get more information about the meaning of specific fields, please use your IDE to read the TSDoc documentation.
  *
- * @type {import('../../src/storage/elasticsearch/types/elasticsearch-config.js')}
+ * @type {import('../../src/storage/elasticsearch/types/elasticsearch-config.js').ElasticsearchConfigFile}
  */
 const config = {
   internal: {
     database: {
       name: 'elasticsearch',
-      version: '5.6',
+      version: '8.4.2',
       query: {
         minMatch: '75%',
         queryType: 'dis_max',

backend/backend/config/default/tools/markdown.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {readFile} from 'fs/promises';
 import {SCAboutPageContentType} from '@openstapps/core';
 

backend/backend/config/default/tools/semester-acronym.js

@@ -1,4 +1,3 @@
-// @ts-check
 /**
  * Generates a range of numbers that represent consecutive calendar months
  *

backend/backend/config/default/tools/version.js

@@ -0,0 +1,37 @@
+import {readFile, readdir} from 'fs/promises';
+import url from 'url';
+import path from 'path';
+
+/**
+ * @example version(1, import.meta.url)
+ * @param options {Omit<import('@openstapps/core').SCAppVersionInfo, 'releaseNotes' | 'translations'>}
+ * @param base {string}
+ * @returns {Promise<import('@openstapps/core').SCAppVersionInfo>}
+ */
+export async function version(options, base) {
+  const de = await readFile(new URL(`${options.version}.de.md`, base), 'utf8');
+  const en = await readFile(new URL(`${options.version}.en.md`, base), 'utf8');
+
+  return {
+    ...options,
+    releaseNotes: de,
+    translations: {
+      en: {
+        releaseNotes: en,
+      },
+    },
+  };
+}
+
+/**
+ * @param base {string} Base path of the file as `import.meta.url`
+ * @returns {Promise<import('@openstapps/core').SCAppVersionInfo[]>}
+ */
+export async function versions(base) {
+  const directory = await readdir(path.dirname(url.fileURLToPath(base)));
+  const versions = [
+    ...new Set(directory.filter(it => it.endsWith('.md')).map(it => it.replace(/\.\w+\.md$/, ''))),
+  ].sort((a, b) => -a.localeCompare(b, undefined, {numeric: true}));
+
+  return Promise.all(versions.map(versionName => version({version: versionName}, base)));
+}

backend/backend/config/f-u/about-pages/about.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {SCAboutPageContentType} from '@openstapps/core';
 import {markdown} from '../../default/tools/markdown.js';
 

backend/backend/config/f-u/about-pages/imprint.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {SCAboutPageContentType} from '@openstapps/core';
 
 /** @type {import('@openstapps/core').SCAboutPage} */

backend/backend/config/f-u/about-pages/index.js

@@ -1,4 +1,3 @@
-// @ts-check
 import about from './about.js';
 import imprint from './imprint.js';
 import privacy from './privacy.js';

backend/backend/config/f-u/about-pages/privacy.js

@@ -1,4 +1,3 @@
-// @ts-check
 import {markdown} from '../../default/tools/markdown.js';
 
 /** @type {import('@openstapps/core').SCAboutPage} */

backend/backend/config/f-u/backendrc.js

@@ -1,7 +1,7 @@
-// @ts-check
 import aboutPages from './about-pages/index.js';
 import defaultApp from '../default/app/index.js';
 import {backend as defaultBackend, internal as defaultInternal} from '../default/backend/index.js';
+import versionHistory from './version-history/index.js';
 
 /**
  * This is the default configuration for the Goethe university of Frankfurt
@@ -76,6 +76,7 @@ const config = {
         } */
       },
     },
+    versionHistory,
     aboutPages,
   },
   backend: defaultBackend,

backend/backend/config/f-u/version-history/2.4.0.de.md

@@ -0,0 +1,52 @@
+# Goethe-Uni App 2.4
+
+Wir freuen uns euch mehr in der Goethe-Uni App
+bieten zu können.
+
+## Navigation zu Gebäuden und Orten
+
+Als eines der Ergebnisse des Ideenwettbewerbs wurde jetzt
+ein Navigationsfeature in die App integriert.
+
+Orte auf der Karte, Mensen, sowie sogar Termine (wenn hinterlegt)
+bieten jetzt direkt die Option eine Verbindung zu finden, gestützt
+durch die Karten App auf deinem Gerät.
+
+## Integration der Jobbörse
+
+Jobs findest du ab sofort auch in der Goethe-Uni App.
+
+Auch das ist ein Ergebnis des Ideenwettbewerbs,
+und wir freuen uns es euch hier präsentieren zu können!
+
+## Der Umweltscore
+
+Der Umweltscore für Gerichte wird nun auch in der App angezeigt.
+
+> Nachhaltigkeit, Umweltschutz, Gesundheit und Klimawandel sind
+> zentrale Begriffe im gesellschaftlichen Miteinander.
+> Unsere Ernährung spielt hierbei eine wichtige Rolle.
+> Das Studierendenwerk Frankfurt am Main zeichnet seine Speisenpläne
+> ab sofort mit einem Umweltscore aus.
+> Anhand dieser Bewertung können Sie direkt ersehen,
+> welchen Einfluss Ihre Essenauswahl auf das Klima hat.
+
+## Weitere Verbesserungen
+
+### Performance
+
+Die Performance der App beim Navigieren wurde stark verbessert und ist datensparender.
+
+### Kalender
+
+Die Kalenderabschnitte haben jetzt neue Namen bekommen:
+
+- Der _Kalender_ zeigt Termine für spezifische Tage
+- Die _Wochenübersicht_ ist ein Stundenplan mit allen Termine, die sich wiederholen (z. B. Vorlesungen)
+- Die _Einzeltermine_ zeigen alle Termine, die sich nicht wiederholen
+  (z. B. Klausuren)
+
+### Meine App
+
+Der "Meine Kurse" Abschnitt wurde überarbeitet, und zeigt jetzt Termine
+für die nächsten Tage und mit mehr Details an.

backend/backend/config/f-u/version-history/2.4.0.en.md

@@ -0,0 +1,49 @@
+# Goethe-Uni App 2.4
+
+The Goethe-Uni App got even better!
+
+## Navigation to buildings and places
+
+As part of the "Ideenwettbewerb," the idea competition,
+we have now integrated a navigation feature into the app.
+
+Orte auf der Karte, Mensen, sowie sogar Termine (wenn hinterlegt)
+bieten jetzt direkt die Option eine Verbindung zu finden, gestützt
+durch die Karten App auf deinem Gerät.
+
+## Integration of the job market
+
+Jobs are now also available in the Goethe-Uni App.
+
+This feature is also a result of the idea competition,
+and we're happy to be able to present it to you here!
+
+## The environment score
+
+The environment score for dishes is now displayed inside the app.
+
+> Sustainability, environment protection, health, and climate change are
+> central topics in how we live today in our society.
+> Our eating habits play an important role in it.
+> The "Studierendenwerk Frankfurt am Main" is marking up its menus
+> from now on with the so-called "Umweltscore," the environment score.
+> Based on this rating, you can see the impact your meal choice would have on our climate.
+
+## Further improvements
+
+### Performance
+
+The performance while navigating around the app has been heavily improved and requires less data to work.
+
+### Calendar
+
+The calendar sections have new names:
+
+- The _calendar_ shows appointments on specific days
+- The _week overview_ is a schedule with all events that repeat (e.g. lectures)
+- The _single events_ show all appointments that don't repeat (e.g. exams)
+
+### My App
+
+The "my courses" section has been revamped,
+and now shows events for the next days and with more detail.

backend/backend/config/f-u/version-history/2.5.0.de.md

@@ -0,0 +1,15 @@
+# Goethe-Uni App 2.5
+
+Die Goethe-Uni App ist noch besser geworden!
+
+## Komplett neue Kartenansicht
+
+Wir haben die Karte überarbeitet, um eine klarere und schnellere Übersicht zu bieten.
+
+## Deutschlandticket mit an Bord
+
+Wenn du das Upgrade des Semesterticket zum Deutschlandticket gemacht hast und eingeloggt bist, findet es sich jetzt auch in der App.
+
+## Bibliotheksdienste sind wieder voll funktionsfähig
+
+Aufgrund einiger Adhoc-Änderungen im Bibliothekssystem haben wir die App so angepasst, dass sie damit umgehen kann.

backend/backend/config/f-u/version-history/2.5.0.en.md

@@ -0,0 +1,15 @@
+# Goethe-Uni App 2.5
+
+The Goethe-Uni App got even better!
+
+## Completely new map view
+
+We overhauled the map to offer you a clearer and faster and overview.
+
+## Deutschlandticket included
+
+If you upgraded your Semesterticket to a Deutschlandticket it will now reside in the App if you are logged in.
+
+## Library services are fully functional again
+
+Due to some adhoc changes in the library system we adjusted the app to handle them properly.

backend/backend/config/f-u/version-history/index.js

@@ -0,0 +1,6 @@
+import {versions} from '../../default/tools/version.js';
+
+/** @type {import('@openstapps/core').SCAppVersionInfo[]} */
+const versionHistory = await versions(import.meta.url);
+
+export default versionHistory;

backend/backend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openstapps/backend",
   "description": "A reference implementation for a StApps backend",
-  "version": "3.0.0",
+  "version": "3.3.1",
   "private": true,
   "type": "module",
   "license": "AGPL-3.0-only",
@@ -59,11 +59,12 @@
     "body-parser": "1.20.2",
     "cors": "2.8.5",
     "cosmiconfig": "8.1.3",
+    "deepmerge": "4.3.1",
     "express": "4.18.2",
     "express-prom-bundle": "6.6.0",
     "express-promise-router": "4.1.1",
     "got": "12.6.0",
-    "moment": "2.29.4",
+    "moment": "2.30.1",
     "morgan": "1.10.0",
     "nock": "13.3.1",
     "node-cache": "5.1.2",
@@ -97,14 +98,17 @@
     "sinon": "15.0.4",
     "sinon-express-mock": "2.2.1",
     "supertest": "6.3.3",
-    "ts-node": "10.9.1",
+    "ts-node": "10.9.2",
     "tsup": "6.7.0",
-    "typescript": "5.1.6"
+    "typescript": "5.4.2"
   },
   "tsup": {
     "entry": [
       "src/cli.ts"
     ],
+    "loader": {
+      ".groovy": "text"
+    },
     "sourcemap": true,
     "clean": true,
     "target": "es2022",

backend/backend/src/config.ts

@@ -2,6 +2,7 @@ import {cosmiconfig, PublicExplorer} from 'cosmiconfig';
 import {SCConfigFile} from '@openstapps/core';
 import path from 'path';
 import deepmerge from 'deepmerge';
+import {Logger} from '@openstapps/logger';
 
 const fallbackNamespace = 'default';
 const configPath = 'config';
@@ -23,31 +24,25 @@ function configLoader(moduleName: string): PublicExplorer {
  * Find and load a config file
  */
 async function findConfig<T>(moduleName: string, namespace = fallbackNamespace): Promise<T> {
+  const config = await configLoader(moduleName).search(path.posix.join('.', configPath, namespace));
+
+  if (config) {
+    Logger.info(`Using ${namespace} config for ${moduleName}`);
+    return config.config;
+  } else {
+    Logger.info(`Using ${fallbackNamespace} config for ${moduleName}`);
     return configLoader(moduleName)
-    .search(path.posix.join('.', configPath, namespace))
-    .then(it => it!.config as T)
-    .catch(() =>
-      configLoader(moduleName)
       .search(path.posix.join('.', configPath, fallbackNamespace))
-        .then(it => it!.config),
+      .then(it => it!.config);
-    );
+  }
 }
 
-/**
- * Loads a config file
- */
-async function loadConfig<T>(moduleName: string): Promise<T> {
 const namespace = process.env.NODE_APP_INSTANCE;
 const database = process.env.NODE_CONFIG_ENV;
 
-  const config = await findConfig<T>(moduleName, namespace);
+export const prometheusConfig = await findConfig<unknown>('prometheus', namespace);
-  if (database) {
-    const databaseConfig = await findConfig<T>(database, namespace);
-    return deepmerge(config, databaseConfig);
-  }
 
-  return config;
+const backendConfigWithoutDatabase = await findConfig<SCConfigFile>('backend', namespace);
-}
+export const backendConfig = database
-
+  ? deepmerge(backendConfigWithoutDatabase, await findConfig<never>(database, namespace))
-export const backendConfig = await loadConfig<SCConfigFile>('backend');
+  : backendConfigWithoutDatabase;
-export const prometheusConfig = await loadConfig<unknown>('prometheus');

backend/backend/src/storage/database.ts

@@ -13,15 +13,34 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
+import {
+  SCConfigFile,
+  SCPlace,
+  SCPlaceWithoutReferences,
+  SCSearchQuery,
+  SCSearchResponse,
+  SCThingWithCategoriesWithoutReferences,
+  SCThings,
+  SCUuid,
+} from '@openstapps/core';
 import {MailQueue} from '../notification/mail-queue.js';
 import {Bulk} from './bulk-storage.js';
+import {FeatureCollection, Point, Polygon} from 'geojson';
 
 /**
  * Creates an instance of a database
  */
 export type DatabaseConstructor = new (config: SCConfigFile, mailQueue?: MailQueue) => Database;
 
+export type SupplementaryGeoJSON = FeatureCollection<Point | Polygon, SupplementaryGeoJSONThing>;
+export type SupplementaryGeoJSONThing = Pick<
+  Extract<SCThings, SCPlace>,
+  Exclude<
+    keyof SCPlaceWithoutReferences | keyof SCThingWithCategoriesWithoutReferences<never, never>,
+    'geo' | 'origin' | 'translations'
+  >
+>;
+
 /**
  * Defines what one database class needs to have defined
  */
@@ -82,4 +101,9 @@ export interface Database {
    * @param params Parameters which form a search query to search the backend data
    */
   search(parameters: SCSearchQuery): Promise<SCSearchResponse>;
+
+  /**
+   * Get geo info for display on a map
+   */
+  geo(): Promise<SupplementaryGeoJSON>;
 }

backend/backend/src/storage/elasticsearch/diff-index.groovy

@@ -0,0 +1,65 @@
+void traverse(def a, def b, ArrayList path, HashMap result) {
+  if (a instanceof Map && b instanceof Map) {
+    for (key in a.keySet()) {
+      path.add(key);
+      traverse(a.get(key), b.get(key), path, result);
+      path.remove(path.size() - 1);
+    }
+  } else if (a instanceof List && b instanceof List) {
+    int la = a.size();
+    int lb = b.size();
+    int max = la > lb ? la : lb;
+    for (int i = 0; i < max; i++) {
+      path.add(i);
+      if (i < la && i < lb) {
+        traverse(a[i], b[i], path, result);
+      } else if (i >= la) {
+        result.added.add(path.toArray());
+      } else {
+        result.removed.add(path.toArray());
+      }
+      path.remove(path.size() - 1);
+    }
+  } else if (a == null && b != null) {
+    result.removed.add(path.toArray());
+  } else if (a != null && b == null) {
+    result.added.add(path.toArray());
+  } else if (!a.equals(b)) {
+    result.changed.add(path.toArray());
+  }
+}
+
+def to;
+def from;
+
+for (state in states) {
+  if (state.index.equals(params.newIndex)) {
+    to = state.doc;
+  } else {
+    from = state.doc;
+  }
+}
+
+HashMap result = [
+  'added': [],
+  'removed': [],
+  'changed': []
+];
+
+traverse(to, from, new ArrayList(), result);
+
+if (to == null && from != null) {
+  result.status = 'removed';
+} else if (to != null && from == null) {
+  result.status = 'added';
+} else if (
+  result.added.size() == 0 &&
+  result.removed.size() == 0 &&
+  result.changed.size() == 0
+) {
+  result.status = 'unchanged';
+} else {
+  result.status = 'changed';
+}
+
+return result;

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -26,7 +26,7 @@ import {Logger} from '@openstapps/logger';
 import moment from 'moment';
 import {MailQueue} from '../../notification/mail-queue.js';
 import {Bulk} from '../bulk-storage.js';
-import {Database} from '../database.js';
+import {Database, SupplementaryGeoJSON, SupplementaryGeoJSONThing} from '../database.js';
 import {parseAggregations} from './aggregations.js';
 import * as Monitoring from './monitoring.js';
 import {buildQuery} from './query/query.js';
@@ -46,6 +46,8 @@ import {
 } from './util/index.js';
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
+import {Feature, Point, Polygon} from 'geojson';
+import indexDiffScript from './diff-index.groovy';
 
 /**
  * A database interface for elasticsearch
@@ -84,7 +86,10 @@ export class Elasticsearch implements Database {
    * @param config an assembled config file
    * @param mailQueue a mail queue for monitoring
    */
-  constructor(private readonly config: SCConfigFile, mailQueue?: MailQueue) {
+  constructor(
+    private readonly config: SCConfigFile,
+    mailQueue?: MailQueue,
+  ) {
     if (config.internal.database === undefined || typeof config.internal.database.version !== 'string') {
       throw new TypeError('Database version is undefined. Check your config file');
     }
@@ -235,6 +240,42 @@ export class Elasticsearch implements Database {
       .then(it => Object.entries(it).map(([name]) => name))
       .catch(() => [] as string[]);
 
+    if (activeIndices.length <= 1) {
+      const result = await this.client.transform.previewTransform({
+        source: {
+          index: [...activeIndices, index],
+          query: {match_all: {}},
+        },
+        dest: {index: 'compare'},
+        pivot: {
+          group_by: {
+            uid: {terms: {field: 'uid.raw'}},
+          },
+          aggregations: {
+            compare: {
+              scripted_metric: {
+                map_script: `
+                  state.index = doc['_index'];
+                  state.doc = params['_source'];`,
+                combine_script: `
+                  state.index = state.index[0];
+                  return state;
+                `,
+                reduce_script: {
+                  source: indexDiffScript,
+                  params: {
+                    newIndex: index,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      console.log(JSON.stringify(result.preview, null, 2))
+    }
+
     await this.client.indices.updateAliases({
       actions: [
         {
@@ -402,4 +443,49 @@ export class Elasticsearch implements Database {
       },
     };
   }
+
+  async geo(): Promise<SupplementaryGeoJSON> {
+    const searchResponse = await this.client.search<Extract<SCThings, {geo: unknown}>>({
+      body: {
+        query: {
+          exists: {
+            field: 'geo',
+          },
+        },
+      },
+      from: 0,
+      allow_no_indices: true,
+      index: ACTIVE_INDICES_ALIAS,
+      size: 1,
+    });
+
+    return {
+      type: 'FeatureCollection',
+      features: searchResponse.hits.hits
+        .map(thing => {
+          return thing._source?.geo
+            ? ({
+                id: Number(thing._source.identifiers?.['OSM']) || undefined,
+                type: 'Feature',
+                geometry: thing._source.geo.polygon ?? thing._source.geo.point,
+                properties: {
+                  name: thing._source.name,
+                  sameAs: thing._source.sameAs,
+                  image: thing._source.image,
+                  alternateNames: thing._source.alternateNames,
+                  description: thing._source.description,
+                  identifiers: thing._source.identifiers,
+                  categories: thing._source.categories,
+                  categorySpecificValues: thing._source.categorySpecificValues,
+                  openingHours: thing._source.openingHours,
+                  address: thing._source.address,
+                  uid: thing._source.uid,
+                  type: thing._source.type,
+                },
+              } satisfies Feature<Polygon | Point, SupplementaryGeoJSONThing>)
+            : undefined;
+        })
+        .filter(noUndefined),
+    };
+  }
 }

backend/backend/src/storage/elasticsearch/query/filters/distance.ts

@@ -25,7 +25,7 @@ export function buildDistanceFilter(
 ): QueryDslSpecificQueryContainer<'geo_distance'> {
   const geoObject: QueryDslGeoDistanceQuery = {
     distance: `${filter.arguments.distance}m`,
-    [`${filter.arguments.field}.point.coordinates`]: {
+    [`${filter.arguments.field}.point`]: {
       lat: filter.arguments.position[1],
       lon: filter.arguments.position[0],
     },

backend/backend/src/storage/elasticsearch/query/filters/geo.ts

@@ -19,8 +19,11 @@ import {QueryDslSpecificQueryContainer} from '../../types/util.js';
  * Converts a geo filter to elasticsearch syntax
  * @param filter A search filter for the retrieval of the data
  */
-export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> {
+export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'bool'> {
   return {
+    bool: {
+      should: [
+        {
           geo_shape: {
             ignore_unmapped: true,
             [`${filter.arguments.field}.polygon`]: {
@@ -28,5 +31,17 @@ export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContai
               relation: filter.arguments.spatialRelation,
             },
           },
+        } satisfies QueryDslSpecificQueryContainer<'geo_shape'>,
+        {
+          geo_shape: {
+            ignore_unmapped: true,
+            [`${filter.arguments.field}.point`]: {
+              shape: filter.arguments.shape,
+              relation: filter.arguments.spatialRelation,
+            },
+          },
+        } satisfies QueryDslSpecificQueryContainer<'geo_shape'>,
+      ],
+    },
   };
 }

backend/backend/src/storage/elasticsearch/query/filters/value.ts

@@ -21,16 +21,31 @@ import {QueryDslSpecificQueryContainer} from '../../types/util.js';
  */
 export function buildValueFilter(
   filter: SCSearchValueFilter,
-): QueryDslSpecificQueryContainer<'term'> | QueryDslSpecificQueryContainer<'terms'> {
+):
-  return Array.isArray(filter.arguments.value)
+  | QueryDslSpecificQueryContainer<'exists'>
-    ? {
+  | QueryDslSpecificQueryContainer<'term'>
-        terms: {
+  | QueryDslSpecificQueryContainer<'terms'> {
-          [`${filter.arguments.field}.raw`]: filter.arguments.value,
+  switch (typeof filter.arguments.value) {
+    case 'undefined': {
+      return {
+        exists: {
+          field: filter.arguments.field,
         },
+      };
     }
-    : {
+    case 'string': {
+      return {
         term: {
           [`${filter.arguments.field}.raw`]: filter.arguments.value,
         },
       };
     }
+    case 'object': {
+      return {
+        terms: {
+          [`${filter.arguments.field}.raw`]: filter.arguments.value,
+        },
+      };
+    }
+  }
+}

backend/backend/src/storage/elasticsearch/query/sort/distance.ts

@@ -25,7 +25,7 @@ export function buildDistanceSort(sort: SCDistanceSort): SortOptions {
       mode: 'avg',
       order: sort.order,
       unit: 'm',
-      [`${sort.arguments.field}.point.coordinates`]: {
+      [`${sort.arguments.field}.point`]: {
         lat: sort.arguments.position[1],
         lon: sort.arguments.position[0],
       },

backend/backend/src/storage/elasticsearch/query/sort/price-sort.groovy

@@ -0,0 +1,26 @@
+// initialize the sort value with the maximum
+double price = Double.MAX_VALUE;
+
+// if we have any offers
+if (params._source.containsKey(params.field)) {
+  // iterate through all offers
+  for (offer in params._source[params.field]) {
+    // if this offer contains a role specific price
+    if (offer.containsKey('prices') && offer.prices.containsKey(params.universityRole)) {
+      // if the role specific price is smaller than the cheapest we found
+      if (offer.prices[params.universityRole] < price) {
+        // set the role specific price as cheapest for now
+        price = offer.prices[params.universityRole];
+      }
+    } else { // we have no role specific price for our role in this offer
+      // if the default price of this offer is lower than the cheapest we found
+      if (offer.price < price) {
+        // set this price as the cheapest
+        price = offer.price;
+      }
+    }
+  }
+}
+
+// return cheapest price for our role
+return price;

backend/backend/src/storage/elasticsearch/query/sort/price.ts

@@ -13,7 +13,8 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
-import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
+import {SCPriceSort} from '@openstapps/core';
+import priceSortScript from './price-sort.groovy';
 
 /**
  * Converts a price sort to elasticsearch syntax
@@ -23,47 +24,11 @@ export function buildPriceSort(sort: SCPriceSort): SortOptions {
   return {
     _script: {
       order: sort.order,
-      script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+      script: {
+        source: priceSortScript,
+        params: sort.arguments,
+      },
       type: 'number' as const,
     },
   };
 }
-
-/**
- * Provides a script for sorting search results by prices
- * @param universityRole User group which consumes university services
- * @param field Field in which wanted offers with prices are located
- */
-export function buildPriceSortScript(
-  universityRole: keyof SCSportCoursePriceGroup,
-  field: SCThingsField,
-): string {
-  return `
-  // initialize the sort value with the maximum
-  double price = Double.MAX_VALUE;
-
-  // if we have any offers
-  if (params._source.containsKey('${field}')) {
-    // iterate through all offers
-    for (offer in params._source.${field}) {
-      // if this offer contains a role specific price
-      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
-        // if the role specific price is smaller than the cheapest we found
-        if (offer.prices.${universityRole} < price) {
-          // set the role specific price as cheapest for now
-          price = offer.prices.${universityRole};
-        }
-      } else { // we have no role specific price for our role in this offer
-        // if the default price of this offer is lower than the cheapest we found
-        if (offer.price < price) {
-          // set this price as the cheapest
-          price = offer.price;
-        }
-      }
-    }
-  }
-
-  // return cheapest price for our role
-  return price;
-  `;
-}

backend/backend/src/types.d.ts

@@ -0,0 +1,6 @@
+declare module '*.groovy' {
+  const content: string;
+  export default content;
+}
+
+export {};

backend/backend/test/common.ts

@@ -22,7 +22,7 @@ import http from 'http';
 import {MailQueue} from '../src/notification/mail-queue.js';
 import {Bulk, BulkStorage} from '../src/storage/bulk-storage.js';
 import getPort from 'get-port';
-import {Database} from '../src/storage/database.js';
+import {Database, SupplementaryGeoJSON} from '../src/storage/database.js';
 import {v4} from 'uuid';
 import {backendConfig} from '../src/config.js';
 import {getIndexUID} from '../src/storage/elasticsearch/util/index.js';
@@ -58,7 +58,6 @@ export async function startApp(): Promise<Express> {
  * An elasticsearch mock
  */
 export class ElasticsearchMock implements Database {
-  // @ts-expect-error never read
   private bulk: Bulk | undefined;
 
   private storageMock = new Map<string, SCThings>();
@@ -67,6 +66,10 @@ export class ElasticsearchMock implements Database {
     // Nothing to do here
   }
 
+  geo(): Promise<SupplementaryGeoJSON> {
+    throw new Error('Method not implemented.');
+  }
+
   bulkCreated(bulk: Bulk): Promise<void> {
     this.bulk = bulk;
     return Promise.resolve(undefined);

backend/backend/test/routes/bulk-route.spec.ts

@@ -14,6 +14,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 import {
+  SCBook,
   SCBulkAddRoute,
   SCBulkDoneRoute,
   SCBulkRequest,
@@ -23,29 +24,30 @@ import {
 import {expect} from 'chai';
 import {bulk, DEFAULT_TEST_TIMEOUT} from '../common.js';
 import {testApp} from '../tests-setup.js';
-import {readFile} from 'fs/promises';
 import {v4} from 'uuid';
+import bookFile from '@openstapps/core/test/resources/indexable/Book.2.json' assert {type: 'json'};
 
-const book = JSON.parse(
+const book = bookFile.instance as SCBook;
-  await readFile('node_modules/@openstapps/core/test/resources/indexable/Book.2.json', 'utf8'),
-).instance;
 
 describe('Bulk routes', async function () {
   // increase timeout for the suite
   this.timeout(DEFAULT_TEST_TIMEOUT);
 
-  const request: SCBulkRequest = {
+  let request: SCBulkRequest;
+  let bulkRoute: SCBulkRoute;
+  let bulkAddRoute: SCBulkAddRoute;
+  let bulkDoneRoute: SCBulkDoneRoute;
+
+  before(function () {
+    request = {
       expiration: bulk.expiration,
       source: bulk.source,
       type: bulk.type,
     };
-  const bulkRoute = new SCBulkRoute();
+    bulkRoute = new SCBulkRoute();
-  const bulkAddRoute = new SCBulkAddRoute();
+    bulkAddRoute = new SCBulkAddRoute();
-  const bulkDoneRoute = new SCBulkDoneRoute();
+    bulkDoneRoute = new SCBulkDoneRoute();
-
+  });
-  // afterEach(async function() {
-  // TODO: Delete saved bulks
-  // });
 
   it('should create bulk', async function () {
     const {status, body, error} = await testApp

backend/backend/test/routes/index-route.spec.ts

@@ -21,7 +21,12 @@ import {expect} from 'chai';
 describe('Index route', async function () {
   // increase timeout for the suite
   this.timeout(DEFAULT_TEST_TIMEOUT);
-  const indexRoute = new SCIndexRoute();
+
+  let indexRoute: SCIndexRoute;
+
+  before(function () {
+    indexRoute = new SCIndexRoute();
+  });
 
   it('should respond with both app and backend configuration', async function () {
     const request: SCIndexRequest = {};

backend/backend/test/routes/plugin-register-route.spec.ts

@@ -30,15 +30,11 @@ import chaiAsPromised from 'chai-as-promised';
 import {DEFAULT_TEST_TIMEOUT} from '../common.js';
 import {testApp} from '../tests-setup.js';
 import {backendConfig} from '../../src/config.js';
-import {readFile} from 'fs/promises';
+import registerRequest from '@openstapps/core/test/resources/PluginRegisterRequest.1.json' assert {type: 'json'};
 
 // for using promises in expectations (to.eventually.be...)
 use(chaiAsPromised);
 
-const registerRequest = JSON.parse(
-  await readFile('node_modules/@openstapps/core/test/resources/PluginRegisterRequest.1.json', 'utf8'),
-);
-
 // cast it because of "TS2322: Type 'string' is not assignable to type '"add"'"
 export const registerAddRequest: SCPluginAdd = registerRequest.instance as SCPluginAdd;
 

backend/backend/test/routes/route.spec.ts

@@ -47,8 +47,8 @@ describe('Create route', async function () {
   const statusCodeSuccess = 222;
   const bodySuccess = {foo: true};
   const sandbox = sinon.createSandbox();
-  const validationError = new SCValidationErrorResponse([]);
+  let validationError: SCValidationErrorResponse;
-  const internalServerError = new SCInternalServerErrorResponse();
+  let internalServerError: SCInternalServerErrorResponse;
 
   beforeEach(function () {
     app = express();
@@ -64,6 +64,9 @@ describe('Create route', async function () {
     handler = (_request, _app) => {
       return Promise.resolve(bodySuccess);
     };
+
+    validationError = new SCValidationErrorResponse([]);
+    internalServerError = new SCInternalServerErrorResponse();
   });
 
   afterEach(function () {

backend/backend/test/routes/search-route.spec.ts

@@ -29,14 +29,21 @@ import {backendConfig} from '../../src/config.js';
 describe('Search route', async function () {
   // increase timeout for the suite
   this.timeout(DEFAULT_TEST_TIMEOUT);
-  const searchRoute = new SCSearchRoute();
+  let searchRoute: SCSearchRoute;
-  const multiSearchRoute = new SCMultiSearchRoute();
+  let multiSearchRoute: SCMultiSearchRoute;
-  const syntaxError = new SCSyntaxErrorResponse('Foo Message');
+  let syntaxError: SCSyntaxErrorResponse;
-  const methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
+  let methodNotAllowedError: SCMethodNotAllowedErrorResponse;
-  const tooManyRequestsError = new SCTooManyRequestsErrorResponse();
+  let tooManyRequestsError: SCTooManyRequestsErrorResponse;
+
+  before(function () {
+    searchRoute = new SCSearchRoute();
+    multiSearchRoute = new SCMultiSearchRoute();
+    syntaxError = new SCSyntaxErrorResponse('Foo Message');
+    methodNotAllowedError = new SCMethodNotAllowedErrorResponse();
+    tooManyRequestsError = new SCTooManyRequestsErrorResponse();
+  });
 
   it('should reject GET, PUT with a valid search query', async function () {
-    // const expectedParams = JSON.parse(JSON.stringify(defaultParams));
     const {status} = await testApp.get('/search').set('Accept', 'application/json').send({
       query: 'Some search terms',
     });

backend/backend/test/routes/thing-update-route.spec.ts

@@ -13,23 +13,25 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {SCThingUpdateRoute} from '@openstapps/core';
+import {SCBook, SCThingUpdateRoute} from '@openstapps/core';
 import chaiAsPromised from 'chai-as-promised';
 import {bulkStorageMock, DEFAULT_TEST_TIMEOUT} from '../common.js';
 import {expect, use} from 'chai';
 import {testApp} from '../tests-setup.js';
-import {readFile} from 'fs/promises';
+import bookFile from '@openstapps/core/test/resources/indexable/Book.1.json' assert {type: 'json'};
 
 use(chaiAsPromised);
 
-const book = JSON.parse(
+const book = bookFile.instance as SCBook;
-  await readFile('node_modules/@openstapps/core/test/resources/indexable/Book.1.json', 'utf8'),
-).instance;
 
 describe('Thing update route', async function () {
   // increase timeout for the suite
   this.timeout(DEFAULT_TEST_TIMEOUT);
-  const thingUpdateRoute = new SCThingUpdateRoute();
+  let thingUpdateRoute: SCThingUpdateRoute;
+
+  before(function () {
+    thingUpdateRoute = new SCThingUpdateRoute();
+  });
 
   it('should update a thing', async function () {
     const thingUpdateRouteurlPath = thingUpdateRoute.urlPath

backend/backend/test/storage/elasticsearch/elasticsearch.spec.ts

@@ -39,7 +39,6 @@ import {Elasticsearch} from '../../../src/storage/elasticsearch/elasticsearch.js
 import {bulk, DEFAULT_TEST_TIMEOUT, getTransport, getIndex} from '../../common.js';
 import fs from 'fs';
 import {backendConfig} from '../../../src/config.js';
-import {readFile} from 'fs/promises';
 import {
   ACTIVE_INDICES_ALIAS,
   getIndexUID,
@@ -50,6 +49,11 @@ import {
 } from '../../../src/storage/elasticsearch/util/index.js';
 import cron from 'node-cron';
 import {query} from './query.js';
+import messageFile from '@openstapps/core/test/resources/indexable/Message.1.json' assert {type: 'json'};
+import bookFile from '@openstapps/core/test/resources/indexable/Book.1.json' assert {type: 'json'};
+
+const message = messageFile.instance as SCMessage;
+const book = bookFile.instance as SCBook;
 
 use(chaiAsPromised);
 
@@ -60,13 +64,6 @@ function searchResponse<T>(...hits: SearchHit<T>[]): SearchResponse<T> {
   return {hits: {hits}, took: 0, timed_out: false, _shards: {total: 1, failed: 0, successful: 1}};
 }
 
-const message = JSON.parse(
-  await readFile('node_modules/@openstapps/core/test/resources/indexable/Message.1.json', 'utf8'),
-);
-const book = JSON.parse(
-  await readFile('node_modules/@openstapps/core/test/resources/indexable/Book.1.json', 'utf8'),
-);
-
 describe('Elasticsearch', function () {
   // increase timeout for the suite
   this.timeout(DEFAULT_TEST_TIMEOUT);
@@ -74,8 +71,15 @@ describe('Elasticsearch', function () {
 
   before(function () {
     // eslint-disable-next-line no-console
-    console.log('before');
     sandbox.stub(fs, 'readFileSync').returns('{}');
+    sandbox.stub(backendConfig.internal.boostings.default[0], 'fields').value({
+      'academicTerms.acronym': {
+        'SS 2023': 1.05,
+        'WS 2023/24': 1.1,
+        'SoSe 2023': 1.05,
+        'WiSe 2023/24': 1.1,
+      },
+    });
   });
   after(function () {
     sandbox.restore();
@@ -445,7 +449,7 @@ describe('Elasticsearch', function () {
           _id: '',
           _index: '',
           _score: 0,
-          _source: message as SCMessage,
+          _source: message,
         };
         sandbox.stub(es.client, 'search').resolves(searchResponse(foundObject));
 
@@ -475,7 +479,7 @@ describe('Elasticsearch', function () {
         const object: SearchHit<SCMessage> = {
           _id: '',
           _index: oldIndex,
-          _source: message as SCMessage,
+          _source: message,
         };
         sandbox.stub(es.client, 'search').resolves(searchResponse<SCMessage>(object));
         sandbox.stub(es, 'prepareBulkWrite').resolves(index);
@@ -489,7 +493,7 @@ describe('Elasticsearch', function () {
         sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse);
 
         await es.init();
-        return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
+        return expect(es.post(message, bulk)).to.rejectedWith('creation');
       });
 
       it('should create a new object', async function () {
@@ -502,7 +506,7 @@ describe('Elasticsearch', function () {
         });
 
         await es.init();
-        await es.post(message as SCMessage, bulk);
+        await es.post(message, bulk);
 
         expect(createStub.called).to.be.true;
         expect(caughtParameter.document).to.be.eql({
@@ -527,7 +531,7 @@ describe('Elasticsearch', function () {
           _id: '',
           _index: getIndex(),
           _score: 0,
-          _source: message as SCMessage,
+          _source: message,
         };
         sandbox.stub(es.client, 'search').resolves(searchResponse());
 
@@ -541,7 +545,7 @@ describe('Elasticsearch', function () {
           _id: '',
           _index: getIndex(),
           _score: 0,
-          _source: message as SCMessage,
+          _source: message,
         };
         sandbox.stub(es.client, 'search').resolves(searchResponse(object));
         // @ts-expect-error unused
@@ -564,13 +568,13 @@ describe('Elasticsearch', function () {
         _id: '123',
         _index: getIndex(),
         _score: 0,
-        _source: message as SCMessage,
+        _source: message,
       };
       const objectBook: SearchHit<SCBook> = {
         _id: '321',
         _index: getIndex(),
         _score: 0,
-        _source: book as SCBook,
+        _source: book,
       };
       const fakeEsAggregations = {
         '@all': {

backend/backend/test/storage/elasticsearch/query.spec.ts

@@ -466,7 +466,7 @@ describe('Query', function () {
       const expectedFilter: QueryDslSpecificQueryContainer<'geo_distance'> = {
         geo_distance: {
           'distance': '1000m',
-          'geo.point.coordinates': {
+          'geo.point': {
             lat: 8.123,
             lon: 50.123,
           },
@@ -479,19 +479,40 @@ describe('Query', function () {
     it('should build geo filter for shapes and points', function () {
       const filter = buildFilter(searchFilters.geoPoint);
       const expectedFilter = {
+        bool: {
+          should: [
+            {
               geo_shape: {
                 'geo.polygon': {
                   relation: undefined,
                   shape: {
-              type: 'envelope',
                     coordinates: [
                       [50.123, 8.123],
                       [50.123, 8.123],
                     ],
+                    type: 'envelope',
                   },
                 },
                 'ignore_unmapped': true,
               },
+            },
+            {
+              geo_shape: {
+                'geo.point': {
+                  relation: undefined,
+                  shape: {
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                    type: 'envelope',
+                  },
+                },
+                'ignore_unmapped': true,
+              },
+            },
+          ],
+        },
       };
 
       expect(filter).to.be.eql(expectedFilter);
@@ -500,19 +521,40 @@ describe('Query', function () {
     it('should build geo filter for shapes only', function () {
       const filter = buildFilter(searchFilters.geoShape);
       const expectedFilter = {
+        bool: {
+          should: [
+            {
               geo_shape: {
                 'geo.polygon': {
                   relation: 'contains',
                   shape: {
-              type: 'envelope',
                     coordinates: [
                       [50.123, 8.123],
                       [50.123, 8.123],
                     ],
+                    type: 'envelope',
                   },
                 },
                 'ignore_unmapped': true,
               },
+            },
+            {
+              geo_shape: {
+                'geo.point': {
+                  relation: 'contains',
+                  shape: {
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                    type: 'envelope',
+                  },
+                },
+                'ignore_unmapped': true,
+              },
+            },
+          ],
+        },
       };
 
       expect(filter).to.be.eql(expectedFilter);
@@ -594,7 +636,7 @@ describe('Query', function () {
           'mode': 'avg',
           'order': 'desc',
           'unit': 'm',
-          'geo.point.coordinates': {
+          'geo.point': {
             lat: 50.123,
             lon: 8.123,
           },

backend/backend/tsconfig.json

@@ -2,7 +2,7 @@
   "extends": "@openstapps/tsconfig",
   "compilerOptions": {
     "resolveJsonModule": true,
-    "useUnknownInCatchVariables": false,
+    "useUnknownInCatchVariables": false
-    "allowJs": true
+  },
-  }
+  "exclude": ["lib", "app.js"]
 }

backend/database/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openstapps/database
 
+## 3.2.0
+
+### Patch Changes
+
+- 689ac68b: pin alpine version to 3.18 and add healthchecks
+
 ## 3.0.0
 
 ### Patch Changes

backend/database/Dockerfile

@@ -14,4 +14,6 @@ RUN chown elasticsearch:elasticsearch config/elasticsearch.yml
 
 USER elasticsearch
 
+HEALTHCHECK --interval=10s --timeout=10s --start-period=60s --retries=12 CMD curl --fail -s http://localhost:9200/ >/dev/null || exit 1
+
 CMD ["/usr/share/elasticsearch/bin/elasticsearch"]

backend/database/config/elasticsearch.yml

@@ -3,3 +3,4 @@ discovery.type: "single-node"
 cluster.routing.allocation.disk.threshold_enabled: false
 network.bind_host: 0.0.0.0
 xpack.security.enabled: false
+ingest.geoip.downloader.enabled: false

backend/database/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openstapps/database",
-  "version": "3.0.0",
+  "version": "3.2.0",
   "private": true,
   "files": [
     "config",

backend/proxy/config/default.cjs

@@ -16,7 +16,7 @@
 
 // ESM is not supported, and cts is not detected, so we use type-checked cjs instead.
 /** @type {import('../src/common').ConfigFile} */
-const configFile = {
+module.exports = {
   activeVersions: ['1\\.0\\.\\d+', '2\\.0\\.\\d+'],
   hiddenRoutes: ['/bulk'],
   logFormat: 'default',
@@ -31,5 +31,3 @@ const configFile = {
     dhparam: '/etc/nginx/certs/dhparam.pem',
   },
 };
-
-export default configFile;

backend/proxy/package.json

@@ -50,8 +50,8 @@
     "dockerode": "3.3.5",
     "is-cidr": "4.0.2",
     "mustache": "4.2.0",
-    "semver": "7.3.8",
+    "semver": "7.6.0",
-    "typescript": "5.1.6"
+    "typescript": "5.4.2"
   },
   "devDependencies": {
     "@openstapps/api-cli": "workspace:*",
@@ -65,7 +65,7 @@
     "@types/mustache": "4.2.2",
     "@types/node": "18.15.3",
     "@types/proxyquire": "1.3.28",
-    "@types/semver": "7.3.13",
+    "@types/semver": "7.5.8",
     "@types/sha1": "1.1.3",
     "@types/sinon": "10.0.14",
     "@types/sinon-chai": "3.2.9",
@@ -75,7 +75,7 @@
     "mocha-junit-reporter": "2.2.0",
     "sinon": "15.0.4",
     "sinon-chai": "3.7.0",
-    "ts-node": "10.9.1",
+    "ts-node": "10.9.2",
     "tsup": "6.7.0"
   },
   "tsup": {

backend/proxy/tsconfig.json

@@ -1,4 +1,4 @@
 {
   "extends": "@openstapps/tsconfig",
-  "exclude": ["./config/", "./lib/"]
+  "exclude": ["config", "lib", "app.js"]
 }

configuration/backend-config/test/menu.de.xml

@@ -5,6 +5,7 @@
     <item title="search" title.de="Suche" icon="search" route="/search"/>
     <item title="library catalog" title.de="Bibliothekskatalog" icon="local_library" route="/hebis-search"/>
     <item title="course catalog" title.de="Vorlesungsverzeichnis" icon="inventory_2" route="/catalog"/>
+    <item title="job postings" title.de="Jobangebote" icon="work" route="/jobs"/>
   </group>
   <group title="canteen" title.de="Mensa" icon="local_cafe" route="/canteen"/>
   <group title="campus map" title.de="Campus Karte" icon="map" route="/map"/>

configuration/backend-config/test/menu.xml

@@ -5,6 +5,7 @@
     <item title="search" title.de="Suche" icon="search" route="/search"/>
     <item title="library catalog" title.de="Bibliothekskatalog" icon="local_library" route="/hebis-search"/>
     <item title="course catalog" title.de="Vorlesungsverzeichnis" icon="inventory_2" route="/catalog"/>
+    <item title="job postings" title.de="Jobangebote" icon="work" route="/jobs"/>
   </group>
   <group title="canteen" title.de="Mensa" icon="local_cafe" route="/canteen"/>
   <group title="campus map" title.de="Campus Karte" icon="map" route="/map"/>

configuration/eslint-config/copyright-header-rule.js

@@ -1,4 +1,3 @@
-// @ts-check
 const fs = require("fs");
 const path = require("node:path");
 const child_process = require("child_process");

configuration/eslint-config/copyright-header-rule.spec.js

@@ -1,4 +1,3 @@
-// @ts-check
 "use strict"
 
 const rule = require('./copyright-header-rule')

configuration/eslint-config/index.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 /** @type {import('eslint').Linter.Config} */
 const config = {
   root: true,

configuration/eslint-config/package.json

@@ -18,16 +18,15 @@
   "devDependencies": {
     "@openstapps/tsconfig": "workspace:*",
     "@types/node": "18.15.3",
-    "eslint": "8.43.0",
+    "eslint": "8.57.0",
-    "typescript": "5.1.6"
+    "typescript": "5.4.2"
   },
   "peerDependencies": {
-    "@typescript-eslint/eslint-plugin": "5.60.1",
+    "@typescript-eslint/eslint-plugin": "7.2.0",
-    "@typescript-eslint/parser": "5.60.1",
+    "@typescript-eslint/parser": "7.2.0",
-    "eslint": "8.43.0",
+    "eslint": "8.57.0",
-    "eslint-config-prettier": "8.8.0",
+    "eslint-config-prettier": "9.1.0",
-    "eslint-plugin-jsdoc": "46.4.2",
+    "eslint-plugin-jsdoc": "48.2.1",
-    "eslint-plugin-prettier": "4.2.1",
+    "eslint-plugin-unicorn": "51.0.1"
-    "eslint-plugin-unicorn": "47.0.0"
   }
 }

configuration/prettier-config/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openstapps/prettier-config
 
+## 3.2.0
+
+### Patch Changes
+
+- dbb55850: Update Prettier to 3.1.1
+
 ## 3.0.0
 
 ### Major Changes
@@ -30,7 +36,7 @@
 
   ```js
   #!/usr/bin/env node
-  import './lib/app.js';
+  import "./lib/app.js";
   ```
 
 - 64caebaf: Migrate to ESM
@@ -69,11 +75,14 @@
 - 64caebaf: Migrated changelogs to changeset format
 
   ```js
-  import fs from 'fs';
+  import fs from "fs";
 
-  const path = 'packages/logger/CHANGELOG.md';
+  const path = "packages/logger/CHANGELOG.md";
 
-  fs.writeFileSync(path, fs.readFileSync(path, 'utf8').replace(/^#+\s+\[/gm, '## ['));
+  fs.writeFileSync(
+    path,
+    fs.readFileSync(path, "utf8").replace(/^#+\s+\[/gm, "## ["),
+  );
   ```
 
 - 98546a97: Migrate away from @openstapps/configuration
@@ -115,7 +124,7 @@
 
   ```js
   #!/usr/bin/env node
-  import './lib/app.js';
+  import "./lib/app.js";
   ```
 
 - 64caebaf: Migrate to ESM
@@ -154,11 +163,14 @@
 - 64caebaf: Migrated changelogs to changeset format
 
   ```js
-  import fs from 'fs';
+  import fs from "fs";
 
-  const path = 'packages/logger/CHANGELOG.md';
+  const path = "packages/logger/CHANGELOG.md";
 
-  fs.writeFileSync(path, fs.readFileSync(path, 'utf8').replace(/^#+\s+\[/gm, '## ['));
+  fs.writeFileSync(
+    path,
+    fs.readFileSync(path, "utf8").replace(/^#+\s+\[/gm, "## ["),
+  );
   ```
 
 - 98546a97: Migrate away from @openstapps/configuration

configuration/prettier-config/index.js

@@ -0,0 +1,15 @@
+/** @type {import('prettier').Config} */
+const config = {
+  tabWidth: 2,
+  printWidth: 110,
+  useTabs: false,
+  semi: true,
+  singleQuote: true,
+  quoteProps: 'consistent',
+  trailingComma: 'all',
+  bracketSpacing: false,
+  arrowParens: 'avoid',
+  endOfLine: 'lf'
+}
+
+export default config;

configuration/prettier-config/index.json

@@ -1,13 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/prettierrc",
-  "tabWidth": 2,
-  "printWidth": 110,
-  "useTabs": false,
-  "semi": true,
-  "singleQuote": true,
-  "quoteProps": "consistent",
-  "trailingComma": "all",
-  "bracketSpacing": false,
-  "arrowParens": "avoid",
-  "endOfLine": "lf"
-}

configuration/prettier-config/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openstapps/prettier-config",
   "description": "StApps Prettier Config",
-  "version": "3.0.0",
+  "version": "3.2.0",
   "type": "module",
   "license": "GPL-3.0-only",
   "repository": "git@gitlab.com:openstapps/prettier-config.git",
@@ -9,16 +9,19 @@
   "contributors": [
     "Rainer Killinger <mail-openstapps@killinger.co>"
   ],
-  "main": "index.json",
+  "main": "index.js",
   "files": [
-    "index.json",
+    "index.js",
     "CHANGELOG.md",
     "README.md"
   ],
   "scripts": {
-    "test": "npx prettier --config index.json --check \"test/*.js\""
+    "test": "prettier --config index.js --check \"test/*.js\""
+  },
+  "devDependencies": {
+    "prettier": "3.1.1"
   },
   "peerDependencies": {
-    "prettier": "2.8.6"
+    "prettier": "3.1.1"
   }
 }

configuration/projectmanagement/package.json

@@ -43,8 +43,8 @@
     "@openstapps/logger": "workspace:*",
     "@slack/web-api": "6.8.1",
     "commander": "10.0.0",
-    "date-fns": "2.30.0",
+    "date-fns": "3.6.0",
-    "glob": "10.2.7",
+    "glob": "10.3.10",
     "mustache": "4.2.0"
   },
   "devDependencies": {
@@ -53,7 +53,7 @@
     "@openstapps/tsconfig": "workspace:*",
     "@types/chai": "4.3.5",
     "@types/chai-as-promised": "7.1.5",
-    "@types/glob": "8.0.1",
+    "@types/glob": "8.1.0",
     "@types/mocha": "10.0.1",
     "@types/mustache": "4.2.2",
     "@types/node": "18.15.3",
@@ -63,9 +63,9 @@
     "chai-as-promised": "7.1.1",
     "mocha": "10.2.0",
     "mocha-junit-reporter": "2.2.0",
-    "ts-node": "10.9.1",
+    "ts-node": "10.9.2",
     "tsup": "6.7.0",
-    "typescript": "5.1.6"
+    "typescript": "5.4.2"
   },
   "tsup": {
     "entry": [

configuration/projectmanagement/tsconfig.json

@@ -1,3 +1,4 @@
 {
-  "extends": "@openstapps/tsconfig"
+  "extends": "@openstapps/tsconfig",
+  "exclude": ["lib", "app.js"]
 }

configuration/tsconfig/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openstapps/tsconfig
 
+## 3.3.0
+
+### Minor Changes
+
+- 688bc5f2: v3.3.0 changes
+
 ## 3.0.0
 
 ### Major Changes
@@ -30,7 +36,7 @@
 
   ```js
   #!/usr/bin/env node
-  import './lib/app.js';
+  import "./lib/app.js";
   ```
 
 - 64caebaf: Migrate to ESM
@@ -105,7 +111,7 @@
 
   ```js
   #!/usr/bin/env node
-  import './lib/app.js';
+  import "./lib/app.js";
   ```
 
 - 64caebaf: Migrate to ESM

configuration/tsconfig/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openstapps/tsconfig",
   "description": "The tsconfig for the openstapps project",
-  "version": "3.0.0",
+  "version": "3.3.0",
   "type": "commonjs",
   "license": "GPL-3.0-only",
   "repository": "git@gitlab.com:openstapps/eslint-config.git",

configuration/tsconfig/tsconfig.json

@@ -14,19 +14,26 @@
     "noFallthroughCasesInSwitch": true,
     "isolatedModules": true,
     "allowJs": true,
+    "checkJs": true,
     "resolveJsonModule": true,
     "allowSyntheticDefaultImports": true,
     "noImplicitAny": true,
     "noImplicitReturns": true,
-    "noUnusedLocals": true,
+    "noUnusedLocals": false,
-    "noUnusedParameters": true,
+    "noUnusedParameters": false,
     "outDir": "../../../lib/",
-    "lib": ["ES2022", "DOM"],
+    "lib": [
+      "ES2022",
+      "DOM"
+    ],
     "strict": true,
     "target": "ES2022"
   },
   "ts-node": {
     "transpileOnly": true
   },
-  "exclude": ["../../../app.js", "../../../lib/"]
+  "exclude": [
+    "../../../app.js",
+    "../../../lib/"
+  ]
 }

examples/minimal-connector/CHANGELOG.md

@@ -1,5 +1,39 @@
 # @openstapps/minimal-connector
 
+## 3.3.0
+
+### Patch Changes
+
+- Updated dependencies [688bc5f2]
+  - @openstapps/api@3.3.0
+  - @openstapps/core@3.3.0
+  - @openstapps/logger@3.0.0
+
+## 3.2.0
+
+### Patch Changes
+
+- Updated dependencies [912ae422]
+  - @openstapps/core@3.2.0
+  - @openstapps/api@3.2.0
+  - @openstapps/logger@3.0.0
+
+## 3.1.1
+
+### Patch Changes
+
+- Updated dependencies
+  - @openstapps/api@3.1.1
+  - @openstapps/core@3.1.1
+
+## 3.1.0
+
+### Patch Changes
+
+- Updated dependencies [06b8ca10]
+  - @openstapps/core@3.1.0
+  - @openstapps/api@3.1.0
+
 ## 3.0.0
 
 ### Major Changes

examples/minimal-connector/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openstapps/minimal-connector",
   "description": "This is a minimal connector which serves as an example",
-  "version": "3.0.0",
+  "version": "3.3.0",
   "private": true,
   "type": "module",
   "license": "GPL-3.0-only",
@@ -53,9 +53,9 @@
     "mocha": "10.2.0",
     "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
-    "ts-node": "10.9.1",
+    "ts-node": "10.9.2",
     "tsup": "6.7.0",
-    "typescript": "5.1.6"
+    "typescript": "5.4.2"
   },
   "tsup": {
     "entry": [

examples/minimal-connector/src/minimal-connector.ts

@@ -51,7 +51,7 @@ export class MinimalConnector extends Connector<SCMessage> {
   protected async fetchItems(): Promise<SCMessage[]> {
     return [
       {
-        audiences: ['students', 'employees'],
+        audiences: ['students', 'employees', 'guests'],
         categories: [],
         description: 'Some description 1',
         messageBody: 'Some message 1',
@@ -61,7 +61,7 @@ export class MinimalConnector extends Connector<SCMessage> {
         uid: createUUID({id: 'message_1'}, this.licensePlate),
       },
       {
-        audiences: ['students', 'employees'],
+        audiences: ['students', 'employees', 'guests'],
         categories: [],
         description: 'Some description 2',
         messageBody: 'Some message 2',
@@ -71,7 +71,7 @@ export class MinimalConnector extends Connector<SCMessage> {
         uid: '', // see Connetor.getItems()
       },
       {
-        audiences: ['students', 'employees'],
+        audiences: ['students', 'employees', 'guests'],
         categories: [],
         description: 'Some description 3',
         messageBody: 'Some message 3',

examples/minimal-connector/tsconfig.json

@@ -1,3 +1,4 @@
 {
-  "extends": "@openstapps/tsconfig"
+  "extends": "@openstapps/tsconfig",
+  "exclude": ["lib", "app.js"]
 }

examples/minimal-deployment/docker-compose.yml

@@ -1,44 +1,62 @@
 version: '3.7'
+
+x-development-variables: &development-variables
+  NODE_ENV: "development"
+  ALLOW_NO_TRANSPORT: "true"
+
 services:
   database:
-    image: registry.gitlab.com/openstapps/openstapps/database:2.0.0
+    image: registry.gitlab.com/openstapps/openstapps/database:3.0.0
-    volumes:
+    # If you need persistence for debugging purposes uncomment the following lines
-      - ./database:/usr/share/elasticsearch/data
+    #volumes:
+    #  - ./database:/usr/share/elasticsearch/data
     expose:
-      - "9200"
+      - 9200
+    ports:
+      - 127.0.0.1:9200:9200
+    environment:
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - discovery.type=single-node
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
     restart: unless-stopped
 
   backend:
-    image: registry.gitlab.com/openstapps/openstapps/backend:3.0.0-next.0
+    image: registry.gitlab.com/openstapps/openstapps/backend:3.1.0
     environment:
+      <<: *development-variables
       ES_ADDR: "http://database:9200"
       NODE_CONFIG_ENV: "elasticsearch"
-      ALLOW_NO_TRANSPORT: "true"
+      NODE_APP_INSTANCE: "f-u"
+      PROMETHEUS_MIDDLEWARE: "false"
     expose:
       - 3000
     ports:
-      - 3000:3000
+      - 127.0.0.1:3000:3000
-    links:
-      - "database"
     labels:
-      - stapps.version=1.0.0
+      - stapps.version=4.1.0
     restart: unless-stopped
     depends_on:
       - database
-
-  api:
-    image: registry.gitlab.com/openstapps/openstapps/api:3.0.0-next.0
     links:
-      - "backend"
+      - database
 
-  minimal-connector:
+  # api:
-    image: registry.gitlab.com/openstapps/minimal-connector:core-0.23
+  #   image: registry.gitlab.com/openstapps/openstapps/api:3.0.0
-    container_name: minimal-connector-0.23
+  #   links:
-    command: ["http://backend:3000", "minimal-connector", "f-u"]
+  #     - backend
 
-  app:
+  # minimal-connector:
-    image: registry.gitlab.com/openstapps/app/executable:core-0.23
+  #   image: registry.gitlab.com/openstapps/minimal-connector:core-0.23
-    expose:
+  #   container_name: minimal-connector-0.23
-      - 8100
+  #   command: ["http://backend:3000", "minimal-connector", "f-u"]
-    ports:
+
-      - 8100:8100
+  # app:
+  #   image: registry.gitlab.com/openstapps/app/executable:core-0.23
+  #   expose:
+  #     - 8100
+  #   ports:
+  #     - 8100:8100

examples/minimal-plugin/CHANGELOG.md

@@ -1,5 +1,45 @@
 # @openstapps/minimal-plugin
 
+## 3.3.0
+
+### Patch Changes
+
+- Updated dependencies [688bc5f2]
+  - @openstapps/api@3.3.0
+  - @openstapps/core@3.3.0
+  - @openstapps/api-plugin@3.3.0
+  - @openstapps/core-tools@3.3.0
+  - @openstapps/logger@3.0.0
+
+## 3.2.0
+
+### Patch Changes
+
+- Updated dependencies [912ae422]
+  - @openstapps/core@3.2.0
+  - @openstapps/api@3.2.0
+  - @openstapps/api-plugin@3.2.0
+  - @openstapps/core-tools@3.0.0
+  - @openstapps/logger@3.0.0
+
+## 3.1.1
+
+### Patch Changes
+
+- Updated dependencies
+  - @openstapps/api@3.1.1
+  - @openstapps/api-plugin@3.1.1
+  - @openstapps/core@3.1.1
+
+## 3.1.0
+
+### Patch Changes
+
+- Updated dependencies [06b8ca10]
+  - @openstapps/core@3.1.0
+  - @openstapps/api@3.1.0
+  - @openstapps/api-plugin@3.1.0
+
 ## 3.0.0
 
 ### Major Changes

examples/minimal-plugin/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openstapps/minimal-plugin",
   "description": "Minimal Plugin",
-  "version": "3.0.0",
+  "version": "3.3.0",
   "private": true,
   "type": "module",
   "license": "GPL-3.0-only",
@@ -35,7 +35,7 @@
     "@openstapps/logger": "workspace:*",
     "commander": "10.0.0",
     "express": "4.18.2",
-    "ts-node": "10.9.1"
+    "ts-node": "10.9.2"
   },
   "devDependencies": {
     "@openstapps/eslint-config": "workspace:*",
@@ -44,7 +44,7 @@
     "@types/express": "4.17.17",
     "@types/node": "18.15.3",
     "tsup": "6.7.0",
-    "typescript": "5.1.6"
+    "typescript": "5.4.2"
   },
   "tsup": {
     "entry": [

examples/minimal-plugin/tsconfig.json

@@ -1,3 +1,4 @@
 {
-  "extends": "@openstapps/tsconfig"
+  "extends": "@openstapps/tsconfig",
+  "exclude": ["lib", "app.js"]
 }

flake.lock

@@ -1,12 +1,30 @@
 {
   "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1709126324,
+        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1689752456,
+        "lastModified": 1709747860,
-        "narHash": "sha256-VOChdECcEI8ixz8QY+YC4JaNEFwQd1V8bA0G4B28Ki0=",
+        "narHash": "sha256-RT4zuBy579m+l8VyIQFOR66WXfcs4g1jntZUHjh6eoI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7f256d7da238cb627ef189d56ed590739f42f13b",
+        "rev": "58ae79ea707579c40102ddf62d84b902a987c58b",
         "type": "github"
       },
       "original": {
@@ -18,8 +36,24 @@
     },
     "root": {
       "inputs": {
+        "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -1,19 +1,25 @@
 {
   description = "A Nix-flake-based development environment for OpenStApps";
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+  inputs = {
-  outputs = { self, nixpkgs }:
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-    let
+    flake-utils.url = "github:numtide/flake-utils";
-      buildToolsVersion = "30.0.3";
+  };
+  outputs = {
+    self,
+    nixpkgs,
+    flake-utils,
+  }: let
+    aapt2buildToolsVersion = "33.0.2";
+  in
+    flake-utils.lib.eachDefaultSystem (system: let
+      pkgs = import nixpkgs {
+        inherit system;
         overlays = [
           (final: prev: rec {
-          nodejs = prev.nodejs-18_x;
+            fontMin = prev.python311.withPackages (ps: with ps; [brotli fonttools] ++ (with fonttools.optional-dependencies; [woff]));
-          pnpm = prev.nodePackages.pnpm;
-          chrome = prev.google-chrome;
-          firefox = prev.firefox;
-          webkit = prev.epiphany; # Safari-ish browser
             android = prev.androidenv.composeAndroidPackages {
-            buildToolsVersions = [ "${buildToolsVersion}" ];
+              buildToolsVersions = ["30.0.3" aapt2buildToolsVersion];
-            platformVersions = [ "32" ];
+              platformVersions = ["33"];
             };
             cypress = prev.cypress.overrideAttrs (cyPrev: rec {
               version = "13.2.0";
@@ -22,56 +28,44 @@
                 hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
               };
             });
+            nodejs = prev.nodejs_18;
           })
         ];
-      # TODO: aarch64-linux, x68_64-darwin, aarch64-darwin
-      supportedSystems = [ "x86_64-linux" ];
-      forEachSupportedSystem = f: nixpkgs.lib.genAttrs supportedSystems (system: f {
-        pkgs = import nixpkgs {
-          inherit overlays system;
         config = {
           allowUnfree = true;
           android_sdk.accept_license = true;
         };
       };
-      });
+      androidFhs = pkgs.buildFHSUserEnv {
-    in
+        name = "android-env";
-    {
+        targetPkgs = pkgs: with pkgs; [];
-      devShells = forEachSupportedSystem ({ pkgs }:
+        runScript = "bash";
-      let
+        profile = ''
-        python = (pkgs.python311.withPackages(ps: with ps; [ brotli fonttools ] ++ (with fonttools.optional-dependencies; [ ufo lxml unicode woff ])));
+          export ALLOW_NINJA_ENV=true
-      in
+          export USE_CCACHE=1
-      {
+          export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
-        default = (pkgs.buildFHSUserEnv {
+        '';
-          name = "StApps Dev";
+      };
-          targetPkgs = pkgs: with pkgs; [
+    in {
+      devShell = pkgs.mkShell rec {
+        nativeBuildInputs = [androidFhs];
+        buildInputs = with pkgs; [
           nodejs
-            pnpm
+          corepack
-            python
-            docker
           # tools
           curl
           jq
-            # browsers
+          fontMin
-            firefox
-            chrome
-            webkit
           cypress
           # android
           jdk17
           android.androidsdk
-            musl
         ];
-          runScript = "bash";
+        ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
-          profile = ''
+        ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
-            export CYPRESS_INSTALL_BINARY=0
+        GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
-            export CYPRESS_RUN_BINARY=${pkgs.cypress}/bin/Cypress
+        CYPRESS_INSTALL_BINARY = "0";
-            export ANDROID_SDK_ROOT=${pkgs.android.androidsdk}/libexec/android-sdk
+        CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
-            export ANDROID_JAVA_HOME=${pkgs.jdk.home}
-            export DOCKER_HOST=unix:///run/user/1000/docker.sock
-            { dockerd-rootless & } 2>/dev/null
-          '';
-        }).env;
-      });
       };
+    });
 }

frontend/app/.browserslistrc

@@ -5,10 +5,4 @@
 # You can see what browsers were selected by your queries by running:
 #   npx browserslist
 
-> 0.5%
+> 0.5% in DE and last 2 major versions and supports es6 and not dead
-last 2 versions
-Firefox ESR
-not dead
-not kaios 2.5
-not op_mini all
-not IE 9-11

frontend/app/.eslintrc.json

@@ -46,6 +46,7 @@
         "unicorn/no-nested-ternary": "off",
         "unicorn/better-regex": "off",
         "unicorn/no-non-null-assertion": "off",
+        "unicorn/consistent-function-scoping": ["error", {"checkArrowFunctions": false}],
         "jsdoc/no-types": "error",
         "jsdoc/require-param": "off",
         "jsdoc/require-param-description": "error",

frontend/app/.gitignore

@@ -37,6 +37,7 @@ resources/*/icon/
 resources/*/splash/
 android/app/src/main/res/**/*.png
 ios/App/App/Assets.xcassets/**/*.png
+AndroidManifest.xml.orig
 
 .DS_Store
 Thumbs.db

frontend/app/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @openstapps/app
 
+## 3.3.0
+
+### Minor Changes
+
+- 688bc5f2: v3.3.0 changes
+
+### Patch Changes
+
+- Updated dependencies [688bc5f2]
+  - @openstapps/api@3.2.0
+  - @openstapps/core@3.2.0
+  - @openstapps/collection-utils@3.0.0
+
+## 3.2.0
+
+### Patch Changes
+
+- 689ac68b: pin alpine version to 3.18 and add healthchecks
+- Updated dependencies [912ae422]
+  - @openstapps/core@4.0.0
+  - @openstapps/api@4.0.0
+  - @openstapps/collection-utils@3.0.0
+
+## 3.1.2
+
+### Patch Changes
+
+- Fix for Android users not being able to log in
+
+## 3.1.1
+
+### Patch Changes
+
+- Fix version history offered by backend
+- Updated dependencies
+  - @openstapps/api@3.1.1
+  - @openstapps/core@3.1.1
+
+## 3.1.1
+
+### Patch Changes
+
+- last minute deployment changes
+
+## 3.1.0
+
+### Minor Changes
+
+- 06b8ca10: Add job portal feature
+- 066e3744: Update to Capacitor 5.x
+
+### Patch Changes
+
+- 066e3744: Update logo flow to use capacitor-assets (single asset)
+- 066e3744: Replace NavigationBar/StatusBar plugins with native color setting
+- 066e3744: Hide splash screen only when app is ready
+- Updated dependencies [06b8ca10]
+  - @openstapps/core@3.1.0
+  - @openstapps/api@3.1.0
+
 ## 3.0.0
 
 ### Major Changes

frontend/app/Dockerfile

@@ -1,7 +1,7 @@
 # Creates a docker image with only the app as an executable unit
 # Dependencies need to be installed beforehand
 # Needs to be build beforehand
-FROM node:18-alpine
+FROM node:18-alpine3.18
 
 WORKDIR /app
 COPY www/ /app/www

frontend/app/ICONS.md

@@ -50,11 +50,11 @@ the config file.
 Icon font minification is done automatically, but requires you to
 follow a few simple rules:
 
-1. Use the tagged template literal for referencing icon names in
+1. Use the type-safe proxy for referencing icon names in
    TypeScript files and code
 
 ```ts
-SCIcon`icon_name`;
+SCIcon.icon_name;
 ```
 
 2. When using `ion-icon` in HTML, reference either icons that went through

frontend/app/README.md

@@ -13,25 +13,25 @@ The StApps 1.x.x (legacy app, but current app in stores) is written using Ionic
 
 There are (`npm`) scripts defined to get the app running as quickly as possible. Those scripts (shortcuts for docker commands) are called using the syntax `npm run + <script-name>`. So we have the following commands available:
 
-```
+```sh
 npm run docker:pull
 ```
 
 which pulls the up-to-date image ([Dockerfile](Dockerfile)) which contains all the tools needed for building, serving and deploying the app.
 
-```
+```sh
 npm run docker:enter
 ```
 
 which enters the container on docker builder image, where we can run `npm install` (to install the required npm packages) and `npm build` (to build the app: convert into executable files), but also any other arbitrary commands with the tools available in the docker image.
 
-```
+```sh
 npm run docker:build
 ```
 
 which runs `npm install` (to install the required npm packages) and `npm build` (to build the app: convert into executable files) in the docker container which runs on the docker builder image.
 
-```
+```sh
 npm run docker:serve
 ```
 
@@ -39,7 +39,7 @@ which serves the app for running it in the browser. It basically runs `ionic ser
 
 ## How to build and start the app using the default backend?
 
-```
+```sh
 npm run build
 npm run start
 ```
@@ -52,7 +52,7 @@ All the npm scripts are defined in `package.json` [file](package.json). It is re
 
 ## Most useful commands
 
-## Editing the Ionic Database from the browser
+### Editing the Ionic Database from the browser
 
 Add the following function using the browser console
 
@@ -86,39 +86,98 @@ addToIonicDB(
 
 You'll need to run _Chromium_ using
 
-```shell
+```sh
 pnpm chromium:no-cors
 ```
 
+### Help, I can't log in!
+
+Login services will often block hosts not coming from the production
+server.
+
+#### Web
+
+On the web you can circumvent this locally by using the `:virtual-host` scripts:
+
+```sh
+# Start the dev server on mobile.app.uni-frankfurt.de
+pnpm start:virtual-host
+# Run chromium with flags that redirect mobile.app.uni-frankfurt.de to localhost:8100
+pnpm chromium:virtual-host
+```
+
+#### Android
+
+On Android you will need to change the `custom_url_scheme` values
+to `de.unifrankfurt.app` in the following files:
+
+- `android/app/src/main/res/values/strings.xml`
+- `src/environment/environment.ts`
+
+Then start the app normally as you would
+
+```sh
+pnpm run:android
+```
+
+**This alone will not make auth work**, only the login flow.
+
+If you need to test login, you have to disable live reload:
+
+```sh
+pnpm ionic capacitor run android
+```
+
+_**CAUTION:** a remote chrome debugging session can in some
+cases hijack the device's ADB connection. If the connection
+fails for no obvious reason, close chrome and uninstall the
+app, then try again._
+
+#### iOS
+
+On Android you will need to change the `custom_url_scheme` value in
+`src/environment/environment.ts` as well as the `CFBundleURLTypes`
+in `ios/App/App/Info.plist`.
+
+- make sure to remove any `Info.plist.orig` as capacitor might override
+  the modified `Info.plist` with that.
+- make sure you have a valid device in XCode (Simulator or real device).
+
+After that, run
+
+```sh
+pnpm run:ios
+```
+
 ### Running the app
 
 Install the npm packages needed for running the app (as for any other node project which uses npm):
 
-```
+```sh
 npm install
 ```
 
 Check the code for linter issues:
 
-```
+```sh
 npm run lint
 ```
 
 Automatically fix linter issues (those where autofix is possible):
 
-```
+```sh
 npm run lint:fix
 ```
 
 Build the app (transpile etc.):
 
-```
+```sh
 npm run build
 ```
 
 Open the app in the browser:
 
-```
+```sh
 ionic serve
 ```
 
@@ -126,7 +185,7 @@ ionic serve
 
 Run the app for testing on an android device (with live reload in the webview / device, when files are changed):
 
-```
+```sh
 npm run build # if needed
 npm run resources:android # generate needed resources (icons and splashscreens)
 npm run docker:run:android # runs "ionic capacitor run android --livereload --external" on a selected device
@@ -146,7 +205,7 @@ Besides that, it is possible to monitor processes (and so the processes related
 
 Build the (debug) app for testing on an android device (creates an APK file in the android's build outputs path):
 
-```
+```sh
 npm run docker:build:android
 ```
 
@@ -156,13 +215,13 @@ The mentioned `docker:*:android` npm commands are executed in a docker container
 
 Execute unit tests:
 
-```
+```sh
 npm test
 ```
 
 Execute e2e tests:
 
-```
+```sh
 npm run e2e
 ```
 

frontend/app/android/app/build.gradle

@@ -1,6 +1,7 @@
 apply plugin: 'com.android.application'
 
 android {
+    namespace "de.anyschool.app"
     compileSdkVersion rootProject.ext.compileSdkVersion
     defaultConfig {
         applicationId "de.anyschool.app"

frontend/app/android/app/capacitor.build.gradle

@@ -2,13 +2,14 @@
 
 android {
   compileOptions {
-      sourceCompatibility JavaVersion.VERSION_11
+      sourceCompatibility JavaVersion.VERSION_17
-      targetCompatibility JavaVersion.VERSION_11
+      targetCompatibility JavaVersion.VERSION_17
   }
 }
 
 apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle"
 dependencies {
+    implementation project(':capacitor-community-screen-brightness')
     implementation project(':capacitor-app')
     implementation project(':capacitor-browser')
     implementation project(':capacitor-clipboard')
@@ -21,10 +22,9 @@ dependencies {
     implementation project(':capacitor-local-notifications')
     implementation project(':capacitor-network')
     implementation project(':capacitor-preferences')
+    implementation project(':capacitor-screen-orientation')
     implementation project(':capacitor-share')
     implementation project(':capacitor-splash-screen')
-    implementation project(':capacitor-status-bar')
-    implementation project(':hugotomazi-capacitor-navigation-bar')
     implementation project(':transistorsoft-capacitor-background-fetch')
     implementation project(':capacitor-secure-storage-plugin')
 

frontend/app/android/app/src/main/AndroidManifest.xml

@@ -1,7 +1,5 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version='1.0' encoding='utf-8' ?>
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
-    package="de.anyschool.app">
-
     <application
         android:allowBackup="true"
         android:icon="@mipmap/ic_launcher"
@@ -10,16 +8,14 @@
         android:supportsRtl="true"
         android:theme="@style/AppTheme"
         android:usesCleartextTraffic="true">
-
         <activity
-            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode"
+            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout"
-            android:name="de.anyschool.app.MainActivity"
-            android:label="@string/title_activity_main"
-            android:theme="@style/AppTheme.NoActionBarLaunch"
-            android:launchMode="singleTask"
             android:exported="true"
+            android:label="@string/title_activity_main"
+            android:launchMode="singleTask"
+            android:name="de.anyschool.app.MainActivity"
+            android:theme="@style/AppTheme.NoActionBarLaunch"
             android:windowSoftInputMode="adjustPan">
-
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
@@ -37,20 +33,14 @@
                 <data android:host="@string/app_host" android:scheme="https" />
             </intent-filter>
         </activity>
-
         <provider
-            android:name="androidx.core.content.FileProvider"
             android:authorities="${applicationId}.fileprovider"
             android:exported="false"
-            android:grantUriPermissions="true">
+            android:grantUriPermissions="true"
-            <meta-data
+            android:name="androidx.core.content.FileProvider">
-                android:name="android.support.FILE_PROVIDER_PATHS"
+            <meta-data android:name="android.support.FILE_PROVIDER_PATHS" android:resource="@xml/file_paths" />
-                android:resource="@xml/file_paths"></meta-data>
         </provider>
     </application>
-
-    <!-- Permissions -->
-
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
     <uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />

frontend/app/android/app/src/main/assets/capacitor.plugins.json

@@ -1,4 +1,8 @@
 [
+	{
+		"pkg": "@capacitor-community/screen-brightness",
+		"classpath": "com.elylucas.capscreenbrightness.ScreenBrightnessPlugin"
+	},
 	{
 		"pkg": "@capacitor/app",
 		"classpath": "com.capacitorjs.plugins.app.AppPlugin"
@@ -47,6 +51,10 @@
 		"pkg": "@capacitor/preferences",
 		"classpath": "com.capacitorjs.plugins.preferences.PreferencesPlugin"
 	},
+	{
+		"pkg": "@capacitor/screen-orientation",
+		"classpath": "com.capacitorjs.plugins.screenorientation.ScreenOrientationPlugin"
+	},
 	{
 		"pkg": "@capacitor/share",
 		"classpath": "com.capacitorjs.plugins.share.SharePlugin"
@@ -55,14 +63,6 @@
 		"pkg": "@capacitor/splash-screen",
 		"classpath": "com.capacitorjs.plugins.splashscreen.SplashScreenPlugin"
 	},
-	{
-		"pkg": "@capacitor/status-bar",
-		"classpath": "com.capacitorjs.plugins.statusbar.StatusBarPlugin"
-	},
-	{
-		"pkg": "@hugotomazi/capacitor-navigation-bar",
-		"classpath": "br.com.tombus.capacitor.plugin.navigationbar.NavigationBarPlugin"
-	},
 	{
 		"pkg": "@transistorsoft/capacitor-background-fetch",
 		"classpath": "com.transistorsoft.bgfetch.capacitor.BackgroundFetchPlugin"