feat: backend index diffing

Closes #223

backend/backend/package.json

@@ -106,6 +106,9 @@
     "entry": [
       "src/cli.ts"
     ],
+    "loader": {
+      ".groovy": "text"
+    },
     "sourcemap": true,
     "clean": true,
     "target": "es2022",

backend/backend/src/storage/elasticsearch/diff-index.groovy

@@ -0,0 +1,65 @@
+void traverse(def a, def b, ArrayList path, HashMap result) {
+  if (a instanceof Map && b instanceof Map) {
+    for (key in a.keySet()) {
+      path.add(key);
+      traverse(a.get(key), b.get(key), path, result);
+      path.remove(path.size() - 1);
+    }
+  } else if (a instanceof List && b instanceof List) {
+    int la = a.size();
+    int lb = b.size();
+    int max = la > lb ? la : lb;
+    for (int i = 0; i < max; i++) {
+      path.add(i);
+      if (i < la && i < lb) {
+        traverse(a[i], b[i], path, result);
+      } else if (i >= la) {
+        result.added.add(path.toArray());
+      } else {
+        result.removed.add(path.toArray());
+      }
+      path.remove(path.size() - 1);
+    }
+  } else if (a == null && b != null) {
+    result.removed.add(path.toArray());
+  } else if (a != null && b == null) {
+    result.added.add(path.toArray());
+  } else if (!a.equals(b)) {
+    result.changed.add(path.toArray());
+  }
+}
+
+def to;
+def from;
+
+for (state in states) {
+  if (state.index.equals(params.newIndex)) {
+    to = state.doc;
+  } else {
+    from = state.doc;
+  }
+}
+
+HashMap result = [
+  'added': [],
+  'removed': [],
+  'changed': []
+];
+
+traverse(to, from, new ArrayList(), result);
+
+if (to == null && from != null) {
+  result.status = 'removed';
+} else if (to != null && from == null) {
+  result.status = 'added';
+} else if (
+  result.added.size() == 0 &&
+  result.removed.size() == 0 &&
+  result.changed.size() == 0
+) {
+  result.status = 'unchanged';
+} else {
+  result.status = 'changed';
+}
+
+return result;

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -47,6 +47,7 @@ import {
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
 import {Feature, Point, Polygon} from 'geojson';
+import indexDiffScript from './diff-index.groovy';
 
 /**
  * A database interface for elasticsearch
@@ -239,6 +240,42 @@ export class Elasticsearch implements Database {
       .then(it => Object.entries(it).map(([name]) => name))
       .catch(() => [] as string[]);
 
+    if (activeIndices.length <= 1) {
+      const result = await this.client.transform.previewTransform({
+        source: {
+          index: [...activeIndices, index],
+          query: {match_all: {}},
+        },
+        dest: {index: 'compare'},
+        pivot: {
+          group_by: {
+            uid: {terms: {field: 'uid.raw'}},
+          },
+          aggregations: {
+            compare: {
+              scripted_metric: {
+                map_script: `
+                  state.index = doc['_index'];
+                  state.doc = params['_source'];`,
+                combine_script: `
+                  state.index = state.index[0];
+                  return state;
+                `,
+                reduce_script: {
+                  source: indexDiffScript,
+                  params: {
+                    newIndex: index,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      console.log(JSON.stringify(result.preview, null, 2))
+    }
+
     await this.client.indices.updateAliases({
       actions: [
         {

backend/backend/src/storage/elasticsearch/query/sort/price-sort.groovy

@@ -0,0 +1,26 @@
+// initialize the sort value with the maximum
+double price = Double.MAX_VALUE;
+
+// if we have any offers
+if (params._source.containsKey(params.field)) {
+  // iterate through all offers
+  for (offer in params._source[params.field]) {
+    // if this offer contains a role specific price
+    if (offer.containsKey('prices') && offer.prices.containsKey(params.universityRole)) {
+      // if the role specific price is smaller than the cheapest we found
+      if (offer.prices[params.universityRole] < price) {
+        // set the role specific price as cheapest for now
+        price = offer.prices[params.universityRole];
+      }
+    } else { // we have no role specific price for our role in this offer
+      // if the default price of this offer is lower than the cheapest we found
+      if (offer.price < price) {
+        // set this price as the cheapest
+        price = offer.price;
+      }
+    }
+  }
+}
+
+// return cheapest price for our role
+return price;

backend/backend/src/storage/elasticsearch/query/sort/price.ts

@@ -13,7 +13,8 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
-import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
+import {SCPriceSort} from '@openstapps/core';
+import priceSortScript from './price-sort.groovy';
 
 /**
  * Converts a price sort to elasticsearch syntax
@@ -23,47 +24,11 @@ export function buildPriceSort(sort: SCPriceSort): SortOptions {
   return {
     _script: {
       order: sort.order,
-      script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+      script: {
+        source: priceSortScript,
+        params: sort.arguments,
+      },
       type: 'number' as const,
     },
   };
 }
-
-/**
- * Provides a script for sorting search results by prices
- * @param universityRole User group which consumes university services
- * @param field Field in which wanted offers with prices are located
- */
-export function buildPriceSortScript(
-  universityRole: keyof SCSportCoursePriceGroup,
-  field: SCThingsField,
-): string {
-  return `
-  // initialize the sort value with the maximum
-  double price = Double.MAX_VALUE;
-
-  // if we have any offers
-  if (params._source.containsKey('${field}')) {
-    // iterate through all offers
-    for (offer in params._source.${field}) {
-      // if this offer contains a role specific price
-      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
-        // if the role specific price is smaller than the cheapest we found
-        if (offer.prices.${universityRole} < price) {
-          // set the role specific price as cheapest for now
-          price = offer.prices.${universityRole};
-        }
-      } else { // we have no role specific price for our role in this offer
-        // if the default price of this offer is lower than the cheapest we found
-        if (offer.price < price) {
-          // set this price as the cheapest
-          price = offer.price;
-        }
-      }
-    }
-  }
-
-  // return cheapest price for our role
-  return price;
-  `;
-}

backend/backend/src/types.d.ts

@@ -0,0 +1,6 @@
+declare module '*.groovy' {
+  const content: string;
+  export default content;
+}
+
+export {};

flake.nix

@@ -4,68 +4,86 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
   };
-  outputs = {
-    self,
-    nixpkgs,
-    flake-utils,
-  }: let
-    aapt2buildToolsVersion = "33.0.2";
-  in
-    flake-utils.lib.eachDefaultSystem (system: let
-      pkgs = import nixpkgs {
-        inherit system;
-        overlays = [
-          (final: prev: rec {
-            fontMin = prev.python311.withPackages (ps: with ps; [brotli fonttools] ++ (with fonttools.optional-dependencies; [woff]));
-            android = prev.androidenv.composeAndroidPackages {
-              buildToolsVersions = ["30.0.3" aapt2buildToolsVersion];
-              platformVersions = ["33"];
-            };
-            cypress = prev.cypress.overrideAttrs (cyPrev: rec {
-              version = "13.2.0";
-              src = prev.fetchzip {
-                url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
-                hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
+  outputs =
+    {
+      self,
+      nixpkgs,
+      flake-utils,
+    }:
+    let
+      aapt2buildToolsVersion = "33.0.2";
+    in
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [
+            (final: prev: rec {
+              fontMin = prev.python311.withPackages (
+                ps:
+                with ps;
+                [
+                  brotli
+                  fonttools
+                ]
+                ++ (with fonttools.optional-dependencies; [ woff ])
+              );
+              android = prev.androidenv.composeAndroidPackages {
+                buildToolsVersions = [
+                  "34.0.0"
+                  aapt2buildToolsVersion
+                ];
+                platformVersions = [ "34" ];
               };
-            });
-            nodejs = prev.nodejs_18;
-          })
-        ];
-        config = {
-          allowUnfree = true;
-          android_sdk.accept_license = true;
+              cypress = prev.cypress.overrideAttrs (cyPrev: rec {
+                version = "13.2.0";
+                src = prev.fetchzip {
+                  url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
+                  hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
+                };
+              });
+              nodejs = prev.nodejs_18;
+              corepack = prev.corepack_18;
+            })
+          ];
+          config = {
+            allowUnfree = true;
+            android_sdk.accept_license = true;
+          };
         };
-      };
-      androidFhs = pkgs.buildFHSUserEnv {
-        name = "android-env";
-        targetPkgs = pkgs: with pkgs; [];
-        runScript = "bash";
-        profile = ''
-          export ALLOW_NINJA_ENV=true
-          export USE_CCACHE=1
-          export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
-        '';
-      };
-    in {
-      devShell = pkgs.mkShell rec {
-        nativeBuildInputs = [androidFhs];
-        buildInputs = with pkgs; [
-          nodejs
-          corepack
-          # tools
-          curl
-          jq
-          fontMin
-          cypress
-          # android
-          jdk17
-          android.androidsdk
-        ];
-        ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
-        ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
-        GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
-        CYPRESS_INSTALL_BINARY = "0";
-        CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
-      };
-    });
+        androidFhs = pkgs.buildFHSUserEnv {
+          name = "android-env";
+          targetPkgs = pkgs: with pkgs; [ ];
+          runScript = "bash";
+          profile = ''
+            export ALLOW_NINJA_ENV=true
+            export USE_CCACHE=1
+            export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
+          '';
+        };
+      in
+      {
+        devShell = pkgs.mkShell rec {
+          nativeBuildInputs = [ androidFhs ];
+          buildInputs = with pkgs; [
+            nodejs
+            corepack
+            # tools
+            curl
+            jq
+            fontMin
+            cypress
+            # android
+            jdk17
+            android.androidsdk
+          ];
+          ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
+          ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
+          GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
+          CYPRESS_INSTALL_BINARY = "0";
+          CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
+        };
+      }
+    );
 }

frontend/app/src/app/modules/config/config.provider.spec.ts

@@ -67,7 +67,8 @@ describe('ConfigProvider', () => {
 
   it('should fetch app configuration', async () => {
     spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(sampleIndexResponse));
-    const result = await configProvider.fetch();
+    await configProvider.fetch();
+    const result = configProvider.config;
     expect(result).toEqual(sampleIndexResponse);
   });
 
@@ -110,7 +111,7 @@ describe('ConfigProvider', () => {
     expect(storageProviderSpy.has).toHaveBeenCalled();
     expect(storageProviderSpy.get).toHaveBeenCalledTimes(0);
     expect(configProvider.client.handshake).toHaveBeenCalled();
-    expect(await configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
+    expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
   });
 
   it('should throw error on failed initialisation', async () => {
@@ -192,4 +193,31 @@ describe('ConfigProvider', () => {
 
     expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
   });
+
+  it('should fetch new config from remote on init', async () => {
+    storageProviderSpy.has.and.returnValue(Promise.resolve(true));
+    storageProviderSpy.get.and.returnValue(Promise.resolve(sampleIndexResponse));
+    spyOn(configProvider, 'fetch');
+    await configProvider.init();
+
+    expect(configProvider.fetch).toHaveBeenCalled();
+  });
+
+  it('should update the local config with the one from remote', async () => {
+    storageProviderSpy.has.and.returnValue(Promise.resolve(true));
+    storageProviderSpy.get.and.returnValue(Promise.resolve(sampleIndexResponse));
+    const newConfig = structuredClone(sampleIndexResponse);
+    newConfig.app.name = 'New app name';
+    spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(newConfig));
+    await configProvider.init();
+
+    // Validate that the initial configuration is loaded
+    expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
+
+    // Fetch the new configuration from the remote
+    await configProvider.fetch();
+
+    // Validate that the new configuration is now set
+    expect(configProvider.getValue('name')).toEqual(newConfig.app.name);
+  });
 });

frontend/app/src/app/modules/config/config.provider.ts

@@ -83,17 +83,20 @@ export class ConfigProvider {
   /**
    * Fetches configuration from backend
    */
-  async fetch(): Promise<SCIndexResponse> {
+  async fetch(): Promise<void> {
     try {
       const isOffline = await firstValueFrom(this.internetConnectionService.offline$);
       if (isOffline) {
         throw new Error('Device is offline.');
       } else {
-        return await this.client.handshake(this.scVersion);
+        const fetchedConfig: SCIndexResponse = await this.client.handshake(this.scVersion);
+        await this.set(fetchedConfig);
+        this.logger.log(`Configuration updated from remote`);
       }
     } catch (error) {
       const error_ = error instanceof Error ? new ConfigFetchError(error.message) : new ConfigFetchError();
-      throw error_;
+      this.logger.warn(`Failed to fetch remote configuration:`, error_);
+      throw error_; // Rethrow the error to handle it in init()
     }
   }
 
@@ -121,40 +124,33 @@ export class ConfigProvider {
 
   /**
    * Initialises the ConfigProvider
-   * @throws ConfigInitError if no configuration could be loaded.
-   * @throws WrongConfigVersionInStorage if fetch failed and saved config has wrong SCVersion
+   * @throws ConfigInitError if no configuration could be loaded both locally and remote.
    */
   async init(): Promise<void> {
-    let loadError;
-    let fetchError;
-    // load saved configuration
     try {
+      // Attempt to load the configuration from local storage
       this.config = await this.loadLocal();
       this.firstSession = false;
       this.logger.log(`initialised configuration from storage`);
+
+      // Check if the stored configuration has the correct version
       if (this.config.backend.SCVersion.split('.')[0] !== this.scVersion.split('.')[0]) {
-        loadError = new WrongConfigVersionInStorage(this.scVersion, this.config.backend.SCVersion);
+        throw new WrongConfigVersionInStorage(this.scVersion, this.config.backend.SCVersion);
       }
-    } catch (error) {
-      loadError = error;
-    }
-    // fetch remote configuration from backend
-    try {
-      const fetchedConfig: SCIndexResponse = await this.fetch();
-      await this.set(fetchedConfig);
-      this.logger.log(`initialised configuration from remote`);
-    } catch (error) {
-      fetchError = error;
-    }
-    // check for occurred errors and throw them
-    if (loadError !== undefined && fetchError !== undefined) {
-      throw new ConfigInitError();
-    }
-    if (loadError !== undefined) {
+
+      // Fetch the remote configuration in a non-blocking manner
+      void this.fetch();
+    } catch (loadError) {
       this.logger.warn(loadError);
-    }
-    if (fetchError !== undefined) {
-      this.logger.warn(fetchError);
+
+      try {
+        // If local loading fails, immediately try to fetch the configuration from remote
+        await this.fetch();
+      } catch (fetchError) {
+        this.logger.warn(`Failed to fetch remote configuration:`, fetchError);
+        // If both local loading and remote fetching fail, throw ConfigInitError
+        throw new ConfigInitError();
+      }
     }
   }