feat: backend index diffing

backend/backend/package.json

@@ -106,6 +106,9 @@
     "entry": [
       "src/cli.ts"
     ],
+    "loader": {
+      ".groovy": "text"
+    },
     "sourcemap": true,
     "clean": true,
     "target": "es2022",

backend/backend/src/storage/elasticsearch/diff-index.groovy

@@ -0,0 +1,65 @@
+void traverse(def a, def b, ArrayList path, HashMap result) {
+  if (a instanceof Map && b instanceof Map) {
+    for (key in a.keySet()) {
+      path.add(key);
+      traverse(a.get(key), b.get(key), path, result);
+      path.remove(path.size() - 1);
+    }
+  } else if (a instanceof List && b instanceof List) {
+    int la = a.size();
+    int lb = b.size();
+    int max = la > lb ? la : lb;
+    for (int i = 0; i < max; i++) {
+      path.add(i);
+      if (i < la && i < lb) {
+        traverse(a[i], b[i], path, result);
+      } else if (i >= la) {
+        result.added.add(path.toArray());
+      } else {
+        result.removed.add(path.toArray());
+      }
+      path.remove(path.size() - 1);
+    }
+  } else if (a == null && b != null) {
+    result.removed.add(path.toArray());
+  } else if (a != null && b == null) {
+    result.added.add(path.toArray());
+  } else if (!a.equals(b)) {
+    result.changed.add(path.toArray());
+  }
+}
+
+def to;
+def from;
+
+for (state in states) {
+  if (state.index.equals(params.newIndex)) {
+    to = state.doc;
+  } else {
+    from = state.doc;
+  }
+}
+
+HashMap result = [
+  'added': [],
+  'removed': [],
+  'changed': []
+];
+
+traverse(to, from, new ArrayList(), result);
+
+if (to == null && from != null) {
+  result.status = 'removed';
+} else if (to != null && from == null) {
+  result.status = 'added';
+} else if (
+  result.added.size() == 0 &&
+  result.removed.size() == 0 &&
+  result.changed.size() == 0
+) {
+  result.status = 'unchanged';
+} else {
+  result.status = 'changed';
+}
+
+return result;

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -47,6 +47,7 @@ import {
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
 import {Feature, Point, Polygon} from 'geojson';
+import indexDiffScript from './diff-index.groovy';
 
 /**
  * A database interface for elasticsearch
@@ -239,6 +240,42 @@ export class Elasticsearch implements Database {
       .then(it => Object.entries(it).map(([name]) => name))
       .catch(() => [] as string[]);
 
+    if (activeIndices.length <= 1) {
+      const result = await this.client.transform.previewTransform({
+        source: {
+          index: [...activeIndices, index],
+          query: {match_all: {}},
+        },
+        dest: {index: 'compare'},
+        pivot: {
+          group_by: {
+            uid: {terms: {field: 'uid.raw'}},
+          },
+          aggregations: {
+            compare: {
+              scripted_metric: {
+                map_script: `
+                  state.index = doc['_index'];
+                  state.doc = params['_source'];`,
+                combine_script: `
+                  state.index = state.index[0];
+                  return state;
+                `,
+                reduce_script: {
+                  source: indexDiffScript,
+                  params: {
+                    newIndex: index,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      console.log(JSON.stringify(result.preview, null, 2))
+    }
+
     await this.client.indices.updateAliases({
       actions: [
         {

backend/backend/src/storage/elasticsearch/query/sort/price-sort.groovy

@@ -0,0 +1,26 @@
+// initialize the sort value with the maximum
+double price = Double.MAX_VALUE;
+
+// if we have any offers
+if (params._source.containsKey(params.field)) {
+  // iterate through all offers
+  for (offer in params._source[params.field]) {
+    // if this offer contains a role specific price
+    if (offer.containsKey('prices') && offer.prices.containsKey(params.universityRole)) {
+      // if the role specific price is smaller than the cheapest we found
+      if (offer.prices[params.universityRole] < price) {
+        // set the role specific price as cheapest for now
+        price = offer.prices[params.universityRole];
+      }
+    } else { // we have no role specific price for our role in this offer
+      // if the default price of this offer is lower than the cheapest we found
+      if (offer.price < price) {
+        // set this price as the cheapest
+        price = offer.price;
+      }
+    }
+  }
+}
+
+// return cheapest price for our role
+return price;

backend/backend/src/storage/elasticsearch/query/sort/price.ts

@@ -13,7 +13,8 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
-import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
+import {SCPriceSort} from '@openstapps/core';
+import priceSortScript from './price-sort.groovy';
 
 /**
  * Converts a price sort to elasticsearch syntax
@@ -23,47 +24,11 @@ export function buildPriceSort(sort: SCPriceSort): SortOptions {
   return {
     _script: {
       order: sort.order,
-      script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+      script: {
+        source: priceSortScript,
+        params: sort.arguments,
+      },
       type: 'number' as const,
     },
   };
 }
-
-/**
- * Provides a script for sorting search results by prices
- * @param universityRole User group which consumes university services
- * @param field Field in which wanted offers with prices are located
- */
-export function buildPriceSortScript(
-  universityRole: keyof SCSportCoursePriceGroup,
-  field: SCThingsField,
-): string {
-  return `
-  // initialize the sort value with the maximum
-  double price = Double.MAX_VALUE;
-
-  // if we have any offers
-  if (params._source.containsKey('${field}')) {
-    // iterate through all offers
-    for (offer in params._source.${field}) {
-      // if this offer contains a role specific price
-      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
-        // if the role specific price is smaller than the cheapest we found
-        if (offer.prices.${universityRole} < price) {
-          // set the role specific price as cheapest for now
-          price = offer.prices.${universityRole};
-        }
-      } else { // we have no role specific price for our role in this offer
-        // if the default price of this offer is lower than the cheapest we found
-        if (offer.price < price) {
-          // set this price as the cheapest
-          price = offer.price;
-        }
-      }
-    }
-  }
-
-  // return cheapest price for our role
-  return price;
-  `;
-}

backend/backend/src/types.d.ts

@@ -0,0 +1,6 @@
+declare module '*.groovy' {
+  const content: string;
+  export default content;
+}
+
+export {};