feat: backend index diffing

Closes #223

backend/backend/config/b-tu/elasticsearchrc.js

@@ -8,7 +8,8 @@ const config = {
     database: {
       name: 'elasticsearch',
       query: {
-        fields: ["name"]
+        minMatch: '60%',
+        queryType: 'query_string',
       },
     },
   },

backend/backend/config/default/backend/boostings.js

@@ -39,7 +39,7 @@ const boostings = {
       type: SCThingType.AcademicEvent,
     },
     {
-      factor: 2,
+      factor: 1.6,
       type: SCThingType.Building,
     },
     {
@@ -85,7 +85,7 @@ const boostings = {
   ],
   place: [
     {
-      factor: 3,
+      factor: 2,
       type: SCThingType.Building,
     },
     {

backend/backend/config/default/elasticsearchrc.js

@@ -17,21 +17,12 @@ const config = {
       name: 'elasticsearch',
       version: '8.4.2',
       query: {
-        type: 'best_fields',
+        minMatch: '75%',
-        fields: [
+        queryType: 'dis_max',
-          'identifiers^20',
+        matchBoosting: 1.3,
-          'name^10',
+        fuzziness: 'AUTO',
-          'translations.*.name^10',
+        cutoffFrequency: 0,
-          'alternateNames^10',
+        tieBreaker: 0,
-          'translations.*.alternateNames^10',
-          'description^2',
-          'translations.*.description^2',
-          'categories^5',
-        ],
-      },
-      searchAsYouTypeQuery: {
-        type: 'phrase_prefix',
-        fields: ['name.completion', 'name.completion._2gram', 'name.completion._3gram'],
       },
     },
   },

backend/backend/package.json

@@ -106,6 +106,9 @@
     "entry": [
       "src/cli.ts"
     ],
+    "loader": {
+      ".groovy": "text"
+    },
     "sourcemap": true,
     "clean": true,
     "target": "es2022",

backend/backend/src/storage/elasticsearch/diff-index.groovy

@@ -0,0 +1,65 @@
+void traverse(def a, def b, ArrayList path, HashMap result) {
+  if (a instanceof Map && b instanceof Map) {
+    for (key in a.keySet()) {
+      path.add(key);
+      traverse(a.get(key), b.get(key), path, result);
+      path.remove(path.size() - 1);
+    }
+  } else if (a instanceof List && b instanceof List) {
+    int la = a.size();
+    int lb = b.size();
+    int max = la > lb ? la : lb;
+    for (int i = 0; i < max; i++) {
+      path.add(i);
+      if (i < la && i < lb) {
+        traverse(a[i], b[i], path, result);
+      } else if (i >= la) {
+        result.added.add(path.toArray());
+      } else {
+        result.removed.add(path.toArray());
+      }
+      path.remove(path.size() - 1);
+    }
+  } else if (a == null && b != null) {
+    result.removed.add(path.toArray());
+  } else if (a != null && b == null) {
+    result.added.add(path.toArray());
+  } else if (!a.equals(b)) {
+    result.changed.add(path.toArray());
+  }
+}
+
+def to;
+def from;
+
+for (state in states) {
+  if (state.index.equals(params.newIndex)) {
+    to = state.doc;
+  } else {
+    from = state.doc;
+  }
+}
+
+HashMap result = [
+  'added': [],
+  'removed': [],
+  'changed': []
+];
+
+traverse(to, from, new ArrayList(), result);
+
+if (to == null && from != null) {
+  result.status = 'removed';
+} else if (to != null && from == null) {
+  result.status = 'added';
+} else if (
+  result.added.size() == 0 &&
+  result.removed.size() == 0 &&
+  result.changed.size() == 0
+) {
+  result.status = 'unchanged';
+} else {
+  result.status = 'changed';
+}
+
+return result;

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -20,8 +20,6 @@ import {
   IndicesGetAliasResponse,
   SearchHit,
   SearchResponse,
-  SearchTermSuggest,
-  SearchTermSuggestOption,
 } from '@elastic/elasticsearch/lib/api/types.js';
 import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
 import {Logger} from '@openstapps/logger';
@@ -49,9 +47,7 @@ import {
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
 import {Feature, Point, Polygon} from 'geojson';
-import {parseSuggestions} from './util/parse-suggestions.js';
+import indexDiffScript from './diff-index.groovy';
-import {buildScoringFunctions} from './query/boost/scoring-functions.js';
-import {buildFilter} from './query/filter.js';
 
 /**
  * A database interface for elasticsearch
@@ -244,6 +240,42 @@ export class Elasticsearch implements Database {
       .then(it => Object.entries(it).map(([name]) => name))
       .catch(() => [] as string[]);
 
+    if (activeIndices.length <= 1) {
+      const result = await this.client.transform.previewTransform({
+        source: {
+          index: [...activeIndices, index],
+          query: {match_all: {}},
+        },
+        dest: {index: 'compare'},
+        pivot: {
+          group_by: {
+            uid: {terms: {field: 'uid.raw'}},
+          },
+          aggregations: {
+            compare: {
+              scripted_metric: {
+                map_script: `
+                  state.index = doc['_index'];
+                  state.doc = params['_source'];`,
+                combine_script: `
+                  state.index = state.index[0];
+                  return state;
+                `,
+                reduce_script: {
+                  source: indexDiffScript,
+                  params: {
+                    newIndex: index,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+      console.log(JSON.stringify(result.preview, null, 2))
+    }
+
     await this.client.indices.updateAliases({
       actions: [
         {
@@ -360,39 +392,6 @@ export class Elasticsearch implements Database {
     throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.');
   }
 
-  public async searchAsYouType(parameters: SCSearchQuery): Promise<SCSearchResponse> {
-    const result = await this.client.search({
-      _source: 'name',
-      query: {
-        function_score: {
-          functions: buildScoringFunctions(this.config.internal.boostings, parameters.context),
-          query: {
-            bool: {
-              must: {
-                multi_match: {
-                  query: parameters.query,
-                  type: 'bool_prefix',
-                  fields: ['name.completion', 'name.completion._2gram', 'name.completion._3gram'],
-                },
-              },
-              should: [],
-              filter: parameters.filter === undefined ? undefined : buildFilter(parameters.filter),
-            },
-          },
-          score_mode: 'max',
-          boost_mode: 'multiply',
-        },
-      },
-      index: ACTIVE_INDICES_ALIAS,
-      allow_no_indices: true,
-      size: 5,
-    });
-
-    const suggestions = result.hits.hits.map(it => (it._source as any).name);
-    console.log(suggestions);
-    console.log(result.took);
-  }
-
   /**
    * Search all indexed data
    * @param parameters search query
@@ -402,23 +401,18 @@ export class Elasticsearch implements Database {
       throw new TypeError('Database is undefined. You have to configure the query build');
     }
 
-    const esConfig = this.config.internal.database as object as ElasticsearchConfig;
+    const esConfig: ElasticsearchConfig = {
+      name: this.config.internal.database.name as 'elasticsearch',
+      version: this.config.internal.database.version as string,
+      query: this.config.internal.database.query as
+        | ElasticsearchQueryDisMaxConfig
+        | ElasticsearchQueryQueryStringConfig
+        | undefined,
+    };
 
     const response: SearchResponse<SCThings> = await this.client.search({
       aggs: aggregations,
       query: buildQuery(parameters, this.config, esConfig),
-      suggest:
-        parameters.query === undefined
-          ? undefined
-          : {
-              text: parameters.query,
-              terms: {
-                term: {
-                  field: 'name',
-                  suggest_mode: 'missing',
-                },
-              },
-            },
       from: parameters.from,
       index: ACTIVE_INDICES_ALIAS,
       allow_no_indices: true,
@@ -438,7 +432,6 @@ export class Elasticsearch implements Database {
         response.aggregations === undefined
           ? []
           : parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>),
-      suggestions: response.suggest === undefined ? undefined : parseSuggestions(response.suggest),
       pagination: {
         count: response.hits.hits.length,
         offset: typeof parameters.from === 'number' ? parameters.from : 0,

backend/backend/src/storage/elasticsearch/query/query.ts

@@ -30,21 +30,84 @@ export const buildQuery = function buildQuery(
   defaultConfig: SCConfigFile,
   elasticsearchConfig: ElasticsearchConfig,
 ): QueryDslQueryContainer {
-  return {
+  // if config provides a minMatch parameter, we use query_string instead of a match query
+  let query;
+  if (elasticsearchConfig.query === undefined) {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: '90%',
+        query: typeof parameters.query === 'string' ? parameters.query : '*',
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'query_string') {
+    query = {
+      query_string: {
+        analyzer: 'search_german',
+        default_field: 'name',
+        minimum_should_match: elasticsearchConfig.query.minMatch,
+        query: typeof parameters.query === 'string' ? parameters.query : '*',
+      },
+    };
+  } else if (elasticsearchConfig.query.queryType === 'dis_max') {
+    if (typeof parameters.query === 'string' && parameters.query !== '*') {
+      query = {
+        dis_max: {
+          boost: 1.2,
+          queries: [
+            {
+              match: {
+                name: {
+                  boost: elasticsearchConfig.query.matchBoosting,
+                  fuzziness: elasticsearchConfig.query.fuzziness,
+                  query: parameters.query,
+                },
+              },
+            },
+            {
+              query_string: {
+                default_field: 'name',
+                minimum_should_match: elasticsearchConfig.query.minMatch,
+                query: parameters.query,
+              },
+            },
+          ],
+          tie_breaker: elasticsearchConfig.query.tieBreaker,
+        },
+      };
+    }
+  } else {
+    throw new Error(
+      'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
+    );
+  }
+
+  const functionScoreQuery: QueryDslQueryContainer = {
     function_score: {
       functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
       query: {
         bool: {
-          must:
+          minimum_should_match: 0, // if we have no should, nothing can match
-            parameters.query === undefined || parameters.query === '' || parameters.query === '*'
+          must: [],
-              ? {match_all: {}}
-              : {multi_match: {...elasticsearchConfig.query, query: parameters.query}},
           should: [],
-          filter: parameters.filter === undefined ? undefined : buildFilter(parameters.filter),
         },
       },
-      score_mode: 'max',
+      score_mode: 'multiply',
-      boost_mode: 'multiply',
     },
   };
+
+  const mustMatch = functionScoreQuery.function_score?.query?.bool?.must;
+
+  if (Array.isArray(mustMatch)) {
+    if (query !== undefined) {
+      mustMatch.push(query);
+    }
+
+    if (parameters.filter !== undefined) {
+      mustMatch.push(buildFilter(parameters.filter));
+    }
+  }
+
+  return functionScoreQuery;
 };

backend/backend/src/storage/elasticsearch/query/sort/price-sort.groovy

@@ -0,0 +1,26 @@
+// initialize the sort value with the maximum
+double price = Double.MAX_VALUE;
+
+// if we have any offers
+if (params._source.containsKey(params.field)) {
+  // iterate through all offers
+  for (offer in params._source[params.field]) {
+    // if this offer contains a role specific price
+    if (offer.containsKey('prices') && offer.prices.containsKey(params.universityRole)) {
+      // if the role specific price is smaller than the cheapest we found
+      if (offer.prices[params.universityRole] < price) {
+        // set the role specific price as cheapest for now
+        price = offer.prices[params.universityRole];
+      }
+    } else { // we have no role specific price for our role in this offer
+      // if the default price of this offer is lower than the cheapest we found
+      if (offer.price < price) {
+        // set this price as the cheapest
+        price = offer.price;
+      }
+    }
+  }
+}
+
+// return cheapest price for our role
+return price;

backend/backend/src/storage/elasticsearch/query/sort/price.ts

@@ -13,7 +13,8 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
-import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
+import {SCPriceSort} from '@openstapps/core';
+import priceSortScript from './price-sort.groovy';
 
 /**
  * Converts a price sort to elasticsearch syntax
@@ -23,47 +24,11 @@ export function buildPriceSort(sort: SCPriceSort): SortOptions {
   return {
     _script: {
       order: sort.order,
-      script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
+      script: {
+        source: priceSortScript,
+        params: sort.arguments,
+      },
       type: 'number' as const,
     },
   };
 }
-
-/**
- * Provides a script for sorting search results by prices
- * @param universityRole User group which consumes university services
- * @param field Field in which wanted offers with prices are located
- */
-export function buildPriceSortScript(
-  universityRole: keyof SCSportCoursePriceGroup,
-  field: SCThingsField,
-): string {
-  return `
-  // initialize the sort value with the maximum
-  double price = Double.MAX_VALUE;
-
-  // if we have any offers
-  if (params._source.containsKey('${field}')) {
-    // iterate through all offers
-    for (offer in params._source.${field}) {
-      // if this offer contains a role specific price
-      if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
-        // if the role specific price is smaller than the cheapest we found
-        if (offer.prices.${universityRole} < price) {
-          // set the role specific price as cheapest for now
-          price = offer.prices.${universityRole};
-        }
-      } else { // we have no role specific price for our role in this offer
-        // if the default price of this offer is lower than the cheapest we found
-        if (offer.price < price) {
-          // set this price as the cheapest
-          price = offer.price;
-        }
-      }
-    }
-  }
-
-  // return cheapest price for our role
-  return price;
-  `;
-}

backend/backend/src/storage/elasticsearch/types/elasticsearch-config.ts

@@ -13,7 +13,68 @@
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
 
-import {QueryDslMultiMatchQuery} from '@elastic/elasticsearch/lib/api/types.js';
+/**
+ * A configuration for using the Dis Max Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryDisMaxConfig {
+  /**
+   * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
+   */
+  cutoffFrequency: number;
+
+  /**
+   * The maximum allowed Levenshtein Edit Distance (or number of edits)
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
+   */
+  fuzziness: number | string;
+
+  /**
+   * Increase the importance (relevance score) of a field
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
+   */
+  matchBoosting: number;
+
+  /**
+   * Minimal number (or percentage) of words that should match in a query
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
+   */
+  minMatch: string;
+
+  /**
+   * Type of the query - in this case 'dis_max' which is a union of its subqueries
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
+   */
+  queryType: 'dis_max';
+
+  /**
+   * Changes behavior of default calculation of the score when multiple results match
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
+   */
+  tieBreaker: number;
+}
+
+/**
+ * A configuration for using Query String Query
+ *
+ * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
+ * explanation of what the parameters mean
+ */
+export interface ElasticsearchQueryQueryStringConfig {
+  /**
+   * Minimal number (or percentage) of words that should match in a query
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
+   */
+  minMatch: string;
+
+  /**
+   * Type of the query - in this case 'query_string' which uses a query parser in order to parse content
+   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
+   */
+  queryType: 'query_string';
+}
 
 /**
  * An config file for the elasticsearch database interface
@@ -44,12 +105,7 @@ export interface ElasticsearchConfig {
   /**
    * Configuration for using queries
    */
-  query: Omit<QueryDslMultiMatchQuery, 'query'>;
+  query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
-
-  /**
-   *
-   */
-  searchAsYouTypeQuery: Omit<QueryDslMultiMatchQuery, 'query'>;
 
   /**
    * Version of the used elasticsearch

backend/backend/src/storage/elasticsearch/util/parse-suggestions.ts

@@ -1,28 +0,0 @@
-import {
-  SearchSuggest,
-  SearchTermSuggest,
-  SearchTermSuggestOption,
-  SuggestionName,
-} from '@elastic/elasticsearch/lib/api/types.js';
-import {SCSearchSuggestions} from '@openstapps/core';
-
-/**
- * Parse ES Suggestions to SC Search Suggestions
- */
-export function parseSuggestions(suggest: Record<SuggestionName, SearchSuggest[]>): SCSearchSuggestions {
-  const termsSuggestions =
-    suggest.terms === undefined
-      ? []
-      : (suggest.terms as SearchTermSuggest[])
-          ?.map(
-            ({text, options}) =>
-              [
-                text,
-                (options as SearchTermSuggestOption[] | undefined)?.map(({text}) => text) ?? [],
-              ] as const,
-          )
-          .filter(([, suggestions]) => suggestions.length > 0) ?? [];
-  return {
-    terms: termsSuggestions.length === 0 ? undefined : Object.fromEntries(termsSuggestions),
-  };
-}

backend/backend/src/types.d.ts

@@ -0,0 +1,6 @@
+declare module '*.groovy' {
+  const content: string;
+  export default content;
+}
+
+export {};

flake.nix

@@ -4,68 +4,86 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
   };
-  outputs = {
+  outputs =
-    self,
+    {
-    nixpkgs,
+      self,
-    flake-utils,
+      nixpkgs,
-  }: let
+      flake-utils,
-    aapt2buildToolsVersion = "33.0.2";
+    }:
-  in
+    let
-    flake-utils.lib.eachDefaultSystem (system: let
+      aapt2buildToolsVersion = "33.0.2";
-      pkgs = import nixpkgs {
+    in
-        inherit system;
+    flake-utils.lib.eachDefaultSystem (
-        overlays = [
+      system:
-          (final: prev: rec {
+      let
-            fontMin = prev.python311.withPackages (ps: with ps; [brotli fonttools] ++ (with fonttools.optional-dependencies; [woff]));
+        pkgs = import nixpkgs {
-            android = prev.androidenv.composeAndroidPackages {
+          inherit system;
-              buildToolsVersions = ["30.0.3" aapt2buildToolsVersion];
+          overlays = [
-              platformVersions = ["33"];
+            (final: prev: rec {
-            };
+              fontMin = prev.python311.withPackages (
-            cypress = prev.cypress.overrideAttrs (cyPrev: rec {
+                ps:
-              version = "13.2.0";
+                with ps;
-              src = prev.fetchzip {
+                [
-                url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
+                  brotli
-                hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
+                  fonttools
+                ]
+                ++ (with fonttools.optional-dependencies; [ woff ])
+              );
+              android = prev.androidenv.composeAndroidPackages {
+                buildToolsVersions = [
+                  "34.0.0"
+                  aapt2buildToolsVersion
+                ];
+                platformVersions = [ "34" ];
               };
-            });
+              cypress = prev.cypress.overrideAttrs (cyPrev: rec {
-            nodejs = prev.nodejs_18;
+                version = "13.2.0";
-          })
+                src = prev.fetchzip {
-        ];
+                  url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
-        config = {
+                  hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
-          allowUnfree = true;
+                };
-          android_sdk.accept_license = true;
+              });
+              nodejs = prev.nodejs_18;
+              corepack = prev.corepack_18;
+            })
+          ];
+          config = {
+            allowUnfree = true;
+            android_sdk.accept_license = true;
+          };
         };
-      };
+        androidFhs = pkgs.buildFHSUserEnv {
-      androidFhs = pkgs.buildFHSUserEnv {
+          name = "android-env";
-        name = "android-env";
+          targetPkgs = pkgs: with pkgs; [ ];
-        targetPkgs = pkgs: with pkgs; [];
+          runScript = "bash";
-        runScript = "bash";
+          profile = ''
-        profile = ''
+            export ALLOW_NINJA_ENV=true
-          export ALLOW_NINJA_ENV=true
+            export USE_CCACHE=1
-          export USE_CCACHE=1
+            export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
-          export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
+          '';
-        '';
+        };
-      };
+      in
-    in {
+      {
-      devShell = pkgs.mkShell rec {
+        devShell = pkgs.mkShell rec {
-        nativeBuildInputs = [androidFhs];
+          nativeBuildInputs = [ androidFhs ];
-        buildInputs = with pkgs; [
+          buildInputs = with pkgs; [
-          nodejs
+            nodejs
-          corepack
+            corepack
-          # tools
+            # tools
-          curl
+            curl
-          jq
+            jq
-          fontMin
+            fontMin
-          cypress
+            cypress
-          # android
+            # android
-          jdk17
+            jdk17
-          android.androidsdk
+            android.androidsdk
-        ];
+          ];
-        ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
+          ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
-        ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
+          ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
-        GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
+          GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
-        CYPRESS_INSTALL_BINARY = "0";
+          CYPRESS_INSTALL_BINARY = "0";
-        CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
+          CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
-      };
+        };
-    });
+      }
+    );
 }

frontend/app/src/app/modules/config/config.provider.spec.ts

@@ -67,7 +67,8 @@ describe('ConfigProvider', () => {
 
   it('should fetch app configuration', async () => {
     spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(sampleIndexResponse));
-    const result = await configProvider.fetch();
+    await configProvider.fetch();
+    const result = configProvider.config;
     expect(result).toEqual(sampleIndexResponse);
   });
 
@@ -110,7 +111,7 @@ describe('ConfigProvider', () => {
     expect(storageProviderSpy.has).toHaveBeenCalled();
     expect(storageProviderSpy.get).toHaveBeenCalledTimes(0);
     expect(configProvider.client.handshake).toHaveBeenCalled();
-    expect(await configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
+    expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
   });
 
   it('should throw error on failed initialisation', async () => {
@@ -192,4 +193,31 @@ describe('ConfigProvider', () => {
 
     expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
   });
+
+  it('should fetch new config from remote on init', async () => {
+    storageProviderSpy.has.and.returnValue(Promise.resolve(true));
+    storageProviderSpy.get.and.returnValue(Promise.resolve(sampleIndexResponse));
+    spyOn(configProvider, 'fetch');
+    await configProvider.init();
+
+    expect(configProvider.fetch).toHaveBeenCalled();
+  });
+
+  it('should update the local config with the one from remote', async () => {
+    storageProviderSpy.has.and.returnValue(Promise.resolve(true));
+    storageProviderSpy.get.and.returnValue(Promise.resolve(sampleIndexResponse));
+    const newConfig = structuredClone(sampleIndexResponse);
+    newConfig.app.name = 'New app name';
+    spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(newConfig));
+    await configProvider.init();
+
+    // Validate that the initial configuration is loaded
+    expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
+
+    // Fetch the new configuration from the remote
+    await configProvider.fetch();
+
+    // Validate that the new configuration is now set
+    expect(configProvider.getValue('name')).toEqual(newConfig.app.name);
+  });
 });

frontend/app/src/app/modules/config/config.provider.ts

@@ -83,17 +83,20 @@ export class ConfigProvider {
   /**
    * Fetches configuration from backend
    */
-  async fetch(): Promise<SCIndexResponse> {
+  async fetch(): Promise<void> {
     try {
       const isOffline = await firstValueFrom(this.internetConnectionService.offline$);
       if (isOffline) {
         throw new Error('Device is offline.');
       } else {
-        return await this.client.handshake(this.scVersion);
+        const fetchedConfig: SCIndexResponse = await this.client.handshake(this.scVersion);
+        await this.set(fetchedConfig);
+        this.logger.log(`Configuration updated from remote`);
       }
     } catch (error) {
       const error_ = error instanceof Error ? new ConfigFetchError(error.message) : new ConfigFetchError();
-      throw error_;
+      this.logger.warn(`Failed to fetch remote configuration:`, error_);
+      throw error_; // Rethrow the error to handle it in init()
     }
   }
 
@@ -121,40 +124,33 @@ export class ConfigProvider {
 
   /**
    * Initialises the ConfigProvider
-   * @throws ConfigInitError if no configuration could be loaded.
+   * @throws ConfigInitError if no configuration could be loaded both locally and remote.
-   * @throws WrongConfigVersionInStorage if fetch failed and saved config has wrong SCVersion
    */
   async init(): Promise<void> {
-    let loadError;
-    let fetchError;
-    // load saved configuration
     try {
+      // Attempt to load the configuration from local storage
       this.config = await this.loadLocal();
       this.firstSession = false;
       this.logger.log(`initialised configuration from storage`);
+
+      // Check if the stored configuration has the correct version
       if (this.config.backend.SCVersion.split('.')[0] !== this.scVersion.split('.')[0]) {
-        loadError = new WrongConfigVersionInStorage(this.scVersion, this.config.backend.SCVersion);
+        throw new WrongConfigVersionInStorage(this.scVersion, this.config.backend.SCVersion);
       }
-    } catch (error) {
+
-      loadError = error;
+      // Fetch the remote configuration in a non-blocking manner
-    }
+      void this.fetch();
-    // fetch remote configuration from backend
+    } catch (loadError) {
-    try {
-      const fetchedConfig: SCIndexResponse = await this.fetch();
-      await this.set(fetchedConfig);
-      this.logger.log(`initialised configuration from remote`);
-    } catch (error) {
-      fetchError = error;
-    }
-    // check for occurred errors and throw them
-    if (loadError !== undefined && fetchError !== undefined) {
-      throw new ConfigInitError();
-    }
-    if (loadError !== undefined) {
       this.logger.warn(loadError);
-    }
+
-    if (fetchError !== undefined) {
+      try {
-      this.logger.warn(fetchError);
+        // If local loading fails, immediately try to fetch the configuration from remote
+        await this.fetch();
+      } catch (fetchError) {
+        this.logger.warn(`Failed to fetch remote configuration:`, fetchError);
+        // If both local loading and remote fetching fail, throw ConfigInitError
+        throw new ConfigInitError();
+      }
     }
   }
 

frontend/app/src/app/modules/data/list/search-page.component.ts

@@ -24,7 +24,6 @@ import {
   SCSearchQuery,
   SCSearchSort,
   SCThings,
-  SCSearchSuggestions,
 } from '@openstapps/core';
 import {NGXLogger} from 'ngx-logger';
 import {combineLatest, Subject} from 'rxjs';
@@ -111,8 +110,6 @@ export class SearchPageComponent implements OnInit {
    */
   items: Promise<SCThings[]>;
 
-  suggestions: SCSearchSuggestions | undefined;
-
   /**
    * Page size of queries
    */
@@ -222,7 +219,6 @@ export class SearchPageComponent implements OnInit {
 
     try {
       const result = await this.dataProvider.search(searchOptions);
-      this.suggestions = result.suggestions;
       this.singleTypeResponse = result.facets.find(facet => facet.field === 'type')?.buckets.length === 1;
       if (append) {
         // append results
@@ -287,12 +283,6 @@ export class SearchPageComponent implements OnInit {
     this.contextMenuService.updateContextFilter(facets);
   }
 
-  applySuggestion(target: string, suggestion: string) {
-    this.queryText = this.queryText.replaceAll(target, suggestion);
-    this.suggestions = undefined;
-    this.searchStringChanged(this.queryText);
-  }
-
   ngOnInit(defaultListeners = true) {
     this.initialize();
     this.contextMenuService.setContextSort({

frontend/app/src/app/modules/data/list/search-page.html

@@ -69,23 +69,7 @@
 </ion-header>
 
 <ion-content class="content">
-  <div class="suggestions">
-    @if (suggestions?.terms; as terms) {
-      <span>{{ 'search.SUGGESTIONS' | translate }}: </span>
-      @for (suggestion of terms | keyvalue; track suggestion) {
-        @for (term of suggestion.value; track term) {
-          @if ($index == 0) {
-            <b (click)="applySuggestion(suggestion.key, term)" class="suggestion">{{ term }}</b>
-          } @else {
-            <span (click)="applySuggestion(suggestion.key, term)" class="suggestion">{{ term }}</span>
-          }
-        }
-      }
-    }
-  </div>
-
   <div
-    class="hint"
     [class.no-results]="!showDefaultData && !items && !loading"
     [style.display]="!showDefaultData && !items && !loading ? 'block' : 'none'"
   >

frontend/app/src/app/modules/data/list/search-page.scss

@@ -46,7 +46,7 @@ ion-content {
   --background: var(--ion-background-color);
 }
 
-.content > .hint {
+.content > div {
   height: 100%;
 
   ion-label.centered-message-container {
@@ -60,19 +60,3 @@ ion-content {
 ion-header {
   background: var(--ion-color-primary);
 }
-
-.suggestions {
-  padding: var(--spacing-md);
-  padding-block-end: 0;
-}
-
-.suggestion {
-  cursor: pointer;
-}
-
-.suggestion + .suggestion::before {
-  cursor: text;
-  content: ',';
-  padding-inline-end: 0.25ch;
-  font-weight: normal;
-}

frontend/app/src/app/modules/feedback/feedback-page.html

@@ -80,13 +80,12 @@
         </ion-item>
         <ion-item>
           <ion-checkbox
-            class="ion-text-wrap"
             color="primary"
             label-placement="end"
             justify="start"
             [(ngModel)]="termsAgree"
             name="termsAgree"
-            >{{ 'feedback.form.termsAgree.0' | translate }}</ion-checkbox
+            ><span class="ion-text-wrap">{{ 'feedback.form.termsAgree.0' | translate }}</span></ion-checkbox
           >
         </ion-item>
         <ion-item lines="none">
@@ -104,7 +103,9 @@
             justify="start"
             [(ngModel)]="protocolDataAgree"
             name="protocolDataAgree"
-            >{{ 'feedback.form.protocolDataAgree' | translate }}</ion-checkbox
+            ><span class="ion-text-wrap">{{
+              'feedback.form.protocolDataAgree' | translate
+            }}</span></ion-checkbox
           >
         </ion-item>
         <ion-card>

frontend/app/src/assets/i18n/de.json

@@ -422,8 +422,7 @@
       "placeholder": "Veranstaltungen, Personen, Orte und mehr"
     },
     "instruction": "Finde alle Informationen rund ums Studium und den Campus",
-    "nothing_found": "Keine Ergebnisse",
+    "nothing_found": "Keine Ergebnisse"
-    "SUGGESTIONS": "Meintest du"
   },
   "hebisSearch": {
     "title": "Bibliothekssuche",

frontend/app/src/assets/i18n/en.json

@@ -422,8 +422,7 @@
       "placeholder": "Events, places, persons and more"
     },
     "instruction": "Find all information related to your studies and campus",
-    "nothing_found": "No results",
+    "nothing_found": "No results"
-    "SUGGESTIONS": "Did you mean"
   },
   "hebisSearch": {
     "title": "Library Search",

frontend/app/src/environments/environment.ts

@@ -18,7 +18,7 @@
 // The list of which env maps to which file can be found in `.angular-cli.json`.
 
 export const environment = {
-  backend_url: 'http://localhost:3000',
+  backend_url: 'https://mobile.server.uni-frankfurt.de',
   app_host: 'mobile.app.uni-frankfurt.de',
   custom_url_scheme: 'de.anyschool.app',
   backend_version: '999.0.0',

packages/api-cli/README.md

@@ -23,7 +23,7 @@ node ./lib/cli.js e2e http://localhost:3000
 Example to clone the full database
 
 ```shell
-node app.js copy "*" -a "999.0.0" https://mobile.server.uni-frankfurt.de http://localhost:3000 100
+node app.js copy "*" https://mobile.app.uni-frankfurt.de http://localhost:3000 100
 ```
 
 ### Program arguments

packages/core/package.json

@@ -97,8 +97,6 @@
             "date",
             "validatable",
             "filterable",
-            "suggestable",
-            "completable",
             "inheritTags",
             "minLength",
             "pattern",

packages/core/src/protocol/search/result.ts

@@ -24,11 +24,6 @@ export interface SCSearchResult {
    */
   data: SCThings[];
 
-  /**
-   * Suggestions for query corrections
-   */
-  suggestions?: SCSearchSuggestions;
-
   /**
    * Facets (aggregations over all matching data)
    */
@@ -45,18 +40,6 @@ export interface SCSearchResult {
   stats: SCSearchResultSearchEngineStats;
 }
 
-/**
- * Seach suggestions
- *
- * Not to be confused with search-as-you-type suggestions
- */
-export interface SCSearchSuggestions {
-  /**
-   * Suggestions for query terms that might have been misspelled
-   */
-  terms?: Record<string, string[]>;
-}
-
 /**
  * Stores information about Pagination
  */

packages/core/src/things/abstract/thing.ts

@@ -63,7 +63,7 @@ export interface SCThingWithoutReferences {
   /**
    * Alternate names of the thing
    * @filterable
-   * @text
+   * @keyword
    */
   alternateNames?: string[];
 
@@ -92,8 +92,6 @@ export interface SCThingWithoutReferences {
    * @filterable
    * @minLength 1
    * @sortable ducet
-   * @completable
-   * @suggestable
    * @text
    */
   name: string;
@@ -242,8 +240,6 @@ export interface SCThingTranslatableProperties {
    * Translation of the name of the thing
    * @sortable ducet
    * @text
-   * @suggestable
-   * @completable
    */
   name?: string;
   /**

packages/es-mapping-generator/src/config/fieldmap.ts

@@ -43,27 +43,6 @@ export const fieldmap: ElasticsearchFieldmap = {
     },
     ignore: ['price'],
   },
-  suggestable: {
-    default: {
-      trigram: {
-        type: 'text',
-        analyzer: 'trigram',
-      },
-      reverse: {
-        type: 'text',
-        analyzer: 'reverse',
-      },
-    },
-    ignore: [],
-  },
-  completable: {
-    default: {
-      completion: {
-        type: 'search_as_you_type',
-      },
-    },
-    ignore: [],
-  },
 };
 
 export const filterableTagName = 'filterable';

packages/es-mapping-generator/src/config/settings.ts

@@ -19,27 +19,4 @@ export const settings: IndicesPutTemplateRequest['settings'] = {
   'max_result_window': 30_000,
   'number_of_replicas': 0,
   'number_of_shards': 1,
-  'index': {
-    analysis: {
-      analyzer: {
-        trigram: {
-          type: 'custom',
-          tokenizer: 'standard',
-          filter: ['lowercase', 'shingle'],
-        },
-        reverse: {
-          type: 'custom',
-          tokenizer: 'standard',
-          filter: ['lowercase', 'reverse'],
-        },
-      },
-      filter: {
-        shingle: {
-          type: 'shingle',
-          min_shingle_size: 2,
-          max_shingle_size: 3,
-        },
-      },
-    },
-  },
 };