feat: backend index diffing

.changeset/shaggy-plums-bow.md

@@ -1,5 +0,0 @@
----
-"@openstapps/app": patch
----
-
-Bug fixes and Android target sdk version is now 34

backend/backend/package.json

@@ -106,6 +106,9 @@
     "entry": [
       "src/cli.ts"
     ],
+    "loader": {
+      ".groovy": "text"
+    },
     "sourcemap": true,
     "clean": true,
     "target": "es2022",

backend/backend/src/storage/elasticsearch/diff-index.groovy

@@ -0,0 +1,65 @@
+void traverse(def a, def b, ArrayList path, HashMap result) {
+  if (a instanceof Map && b instanceof Map) {
+    for (key in a.keySet()) {
+      path.add(key);
+      traverse(a.get(key), b.get(key), path, result);
+      path.remove(path.size() - 1);
+    }
+  } else if (a instanceof List && b instanceof List) {
+    int la = a.size();
+    int lb = b.size();
+    int max = la > lb ? la : lb;
+    for (int i = 0; i < max; i++) {
+      path.add(i);
+      if (i < la && i < lb) {
+        traverse(a[i], b[i], path, result);
+      } else if (i >= la) {
+        result.added.add(path.toArray());
+      } else {
+        result.removed.add(path.toArray());
+      }
+      path.remove(path.size() - 1);
+    }
+  } else if (a == null && b != null) {
+    result.removed.add(path.toArray());
+  } else if (a != null && b == null) {
+    result.added.add(path.toArray());
+  } else if (!a.equals(b)) {
+    result.changed.add(path.toArray());
+  }
+}
+
+def to;
+def from;
+
+for (state in states) {
+  if (state.index.equals(params.newIndex)) {
+    to = state.doc;
+  } else {
+    from = state.doc;
+  }
+}
+
+HashMap result = [
+  'added': [],
+  'removed': [],
+  'changed': []
+];
+
+traverse(to, from, new ArrayList(), result);
+
+if (to == null && from != null) {
+  result.status = 'removed';
+} else if (to != null && from == null) {
+  result.status = 'added';
+} else if (
+  result.added.size() == 0 &&
+  result.removed.size() == 0 &&
+  result.changed.size() == 0
+) {
+  result.status = 'unchanged';
+} else {
+  result.status = 'changed';
+}
+
+return result;

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -47,6 +47,7 @@ import {
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
 import {Feature, Point, Polygon} from 'geojson';
+import indexDiffScript from './diff-index.groovy';
 
 /**
  * A database interface for elasticsearch
@@ -239,6 +240,42 @@ export class Elasticsearch implements Database {
       .then(it => Object.entries(it).map(([name]) => name))
       .catch(() => [] as string[]);
 
+    if (activeIndices.length <= 1) {
+      const result = await this.client.transform.previewTransform({
+        source: {
+          index: [...activeIndices, index],
+          query: {match_all: {}},
+        },
+        dest: {index: 'compare'},
+        pivot: {
+          group_by: {
+            uid: {terms: {field: 'uid.raw'}},
+          },
+          aggregations: {
+            compare: {
+              scripted_metric: {
+                map_script: `
+                  state.index = doc['_index'];
+                  state.doc = params['_source'];`,
+                combine_script: `
+                  state.index = state.index[0];
+                  return state;
+                `,
+                reduce_script: {
+                  source: indexDiffScript,
+                  params: {
+                    newIndex: index,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      console.log(JSON.stringify(result.preview, null, 2))
+    }
+
     await this.client.indices.updateAliases({
       actions: [
         {

backend/backend/src/storage/elasticsearch/query/sort/price-sort.groovy

@@ -0,0 +1,26 @@
+// initialize the sort value with the maximum
+double price = Double.MAX_VALUE;
+
+// if we have any offers
+if (params._source.containsKey(params.field)) {
+  // iterate through all offers
+  for (offer in params._source[params.field]) {
+    // if this offer contains a role specific price
+    if (offer.containsKey('prices') && offer.prices.containsKey(params.universityRole)) {
+      // if the role specific price is smaller than the cheapest we found
+      if (offer.prices[params.universityRole] < price) {
+        // set the role specific price as cheapest for now
+        price = offer.prices[params.universityRole];
+      }
+    } else { // we have no role specific price for our role in this offer
+      // if the default price of this offer is lower than the cheapest we found
+      if (offer.price < price) {
+        // set this price as the cheapest
+        price = offer.price;
+      }
+    }
+  }
+}
+
+// return cheapest price for our role
+return price;

backend/backend/src/storage/elasticsearch/query/sort/price.ts

@@ -13,7 +13,8 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
-import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
+import {SCPriceSort} from '@openstapps/core';
+import priceSortScript from './price-sort.groovy';
 
 /**
  * Converts a price sort to elasticsearch syntax
@@ -23,47 +24,11 @@ export function buildPriceSort(sort: SCPriceSort): SortOptions {
   return {
     _script: {
       order: sort.order,
-      script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+      script: {
+        source: priceSortScript,
+        params: sort.arguments,
+      },
       type: 'number' as const,
     },
   };
 }
-
-/**
- * Provides a script for sorting search results by prices
- * @param universityRole User group which consumes university services
- * @param field Field in which wanted offers with prices are located
- */
-export function buildPriceSortScript(
-  universityRole: keyof SCSportCoursePriceGroup,
-  field: SCThingsField,
-): string {
-  return `
-  // initialize the sort value with the maximum
-  double price = Double.MAX_VALUE;
-
-  // if we have any offers
-  if (params._source.containsKey('${field}')) {
-    // iterate through all offers
-    for (offer in params._source.${field}) {
-      // if this offer contains a role specific price
-      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
-        // if the role specific price is smaller than the cheapest we found
-        if (offer.prices.${universityRole} < price) {
-          // set the role specific price as cheapest for now
-          price = offer.prices.${universityRole};
-        }
-      } else { // we have no role specific price for our role in this offer
-        // if the default price of this offer is lower than the cheapest we found
-        if (offer.price < price) {
-          // set this price as the cheapest
-          price = offer.price;
-        }
-      }
-    }
-  }
-
-  // return cheapest price for our role
-  return price;
-  `;
-}

backend/backend/src/types.d.ts

@@ -0,0 +1,6 @@
+declare module '*.groovy' {
+  const content: string;
+  export default content;
+}
+
+export {};

frontend/app/package.json

@@ -97,7 +97,7 @@
     "form-data": "4.0.0",
     "geojson": "0.5.0",
     "ionic-appauth": "0.9.0",
-    "jsonpath-plus": "10.0.6",
+    "jsonpath-plus": "6.0.1",
     "maplibre-gl": "4.0.2",
     "material-symbols": "0.17.1",
     "moment": "2.30.1",

frontend/app/src/app/modules/auth/auth-helper.service.ts

@@ -81,6 +81,7 @@ export class AuthHelperService {
       user[key as keyof SCUserConfiguration] = JSONPath({
         path: this.userConfigurationMap[key as keyof SCUserConfiguration] as string,
         json: userInfo,
+        preventEval: true,
       })[0];
     }
     if (user.givenName && user.givenName.length > 0 && user.familyName && user.familyName.length > 0) {

frontend/app/src/app/modules/profile/id-cards.provider.ts

@@ -41,7 +41,7 @@ export class IdCardsProvider {
                 mergeMap(user => this.fetchFallbackIdCards(user)),
                 startWith([]),
               )
-          : of([]).pipe(tap({next: () => this.encryptedStorageProvider.delete('id-cards')})),
+          : of([]).pipe(tap(() => this.encryptedStorageProvider.delete('id-cards'))),
       ),
     );
   }
@@ -54,7 +54,7 @@ export class IdCardsProvider {
         },
         responseType: 'json',
       })
-      .pipe(tap({next: idCards => this.encryptedStorageProvider.set('id-cards', idCards)}));
+      .pipe(tap(idCards => this.encryptedStorageProvider.set('id-cards', idCards)));
   }
 
   private fetchFallbackIdCards(user: SCUserConfiguration): Observable<SCIdCard[]> {

pnpm-lock.yaml

@@ -837,8 +837,8 @@ importers:
         specifier: 0.9.0
         version: 0.9.0(rxjs@7.8.1)
       jsonpath-plus:
-        specifier: 10.0.6
-        version: 10.0.6
+        specifier: 6.0.1
+        version: 6.0.1
       maplibre-gl:
         specifier: 4.0.2
         version: 4.0.2
@@ -5522,7 +5522,7 @@ packages:
       object-assign: 4.1.1
       open: 8.4.0
       proxy-middleware: 0.15.0
-      send: 1.1.0
+      send: 0.18.0
       serve-index: 1.9.1
     transitivePeerDependencies:
       - supports-color
@@ -6971,24 +6971,6 @@ packages:
       '@jridgewell/resolve-uri': 3.1.1
       '@jridgewell/sourcemap-codec': 1.4.15
 
-  /@jsep-plugin/assignment@1.2.1(jsep@1.3.9):
-    resolution: {integrity: sha512-gaHqbubTi29aZpVbBlECRpmdia+L5/lh2BwtIJTmtxdbecEyyX/ejAOg7eQDGNvGOUmPY7Z2Yxdy9ioyH/VJeA==}
-    engines: {node: '>= 10.16.0'}
-    peerDependencies:
-      jsep: ^0.4.0||^1.0.0
-    dependencies:
-      jsep: 1.3.9
-    dev: false
-
-  /@jsep-plugin/regex@1.0.3(jsep@1.3.9):
-    resolution: {integrity: sha512-XfZgry4DwEZvSFtS/6Y+R48D7qJYJK6R9/yJFyUFHCIUMEEHuJ4X95TDgJp5QkmzfLYvapMPzskV5HpIDrREug==}
-    engines: {node: '>= 10.16.0'}
-    peerDependencies:
-      jsep: ^0.4.0||^1.0.0
-    dependencies:
-      jsep: 1.3.9
-    dev: false
-
   /@leichtgewicht/ip-codec@2.0.5:
     resolution: {integrity: sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==}
     dev: true
@@ -11984,11 +11966,6 @@ packages:
     resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
     engines: {node: '>= 0.8'}
 
-  /encodeurl@2.0.0:
-    resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==}
-    engines: {node: '>= 0.8'}
-    dev: true
-
   /encoding@0.1.13:
     resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==}
     requiresBuild: true
@@ -14943,11 +14920,6 @@ packages:
     resolution: {integrity: sha512-YtOli5Cmzy3q4dP26GraSOeAhqecewG04hoO8DY56CH4KJ9Fvv5qKWUCCo3HZob7esJQHCv6/+bnTy72xZZaVQ==}
     engines: {node: '>=12.0.0'}
 
-  /jsep@1.3.9:
-    resolution: {integrity: sha512-i1rBX5N7VPl0eYb6+mHNp52sEuaS2Wi8CDYx1X5sn9naevL78+265XJqy1qENEk7mRKwS06NHpUqiBwR7qeodw==}
-    engines: {node: '>= 10.16.0'}
-    dev: false
-
   /jsesc@0.5.0:
     resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==}
     hasBin: true
@@ -15039,14 +15011,9 @@ packages:
     engines: {'0': node >= 0.2.0}
     dev: true
 
-  /jsonpath-plus@10.0.6:
-    resolution: {integrity: sha512-Q0KCash90S0WQnPnE/W0uVXQSww4NkO34COfs+gbq0fk+Kv03FYpZ+uU2I7soLLaS4d/ywsm9PxplZsTMmfBmg==}
-    engines: {node: '>=18.0.0'}
-    hasBin: true
-    dependencies:
-      '@jsep-plugin/assignment': 1.2.1(jsep@1.3.9)
-      '@jsep-plugin/regex': 1.0.3(jsep@1.3.9)
-      jsep: 1.3.9
+  /jsonpath-plus@6.0.1:
+    resolution: {integrity: sha512-EvGovdvau6FyLexFH2OeXfIITlgIbgZoAZe3usiySeaIDm5QS+A10DKNpaPBBqqRSZr2HN6HVNXxtwUAr2apEw==}
+    engines: {node: '>=10.0.0'}
     dev: false
 
   /jsonpointer@5.0.1:
@@ -18961,26 +18928,6 @@ packages:
     transitivePeerDependencies:
       - supports-color
 
-  /send@1.1.0:
-    resolution: {integrity: sha512-v67WcEouB5GxbTWL/4NeToqcZiAWEq90N888fczVArY8A79J0L4FD7vj5hm3eUMua5EpoQ59wa/oovY6TLvRUA==}
-    engines: {node: '>= 18'}
-    dependencies:
-      debug: 4.3.6(supports-color@8.1.1)
-      destroy: 1.2.0
-      encodeurl: 2.0.0
-      escape-html: 1.0.3
-      etag: 1.8.1
-      fresh: 0.5.2
-      http-errors: 2.0.0
-      mime-types: 2.1.35
-      ms: 2.1.3
-      on-finished: 2.4.1
-      range-parser: 1.2.1
-      statuses: 2.0.1
-    transitivePeerDependencies:
-      - supports-color
-    dev: true
-
   /serialize-javascript@6.0.0:
     resolution: {integrity: sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==}
     dependencies: