mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-22 09:32:41 +00:00
Compare commits
1 Commits
bb1f596bfc
...
226-improv
| Author | SHA1 | Date | |
|---|---|---|---|
|
bc0e219158
|
@@ -8,8 +8,7 @@ const config = {
|
|||||||
database: {
|
database: {
|
||||||
name: 'elasticsearch',
|
name: 'elasticsearch',
|
||||||
query: {
|
query: {
|
||||||
minMatch: '60%',
|
fields: ["name"]
|
||||||
queryType: 'query_string',
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ const boostings = {
|
|||||||
type: SCThingType.AcademicEvent,
|
type: SCThingType.AcademicEvent,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
factor: 1.6,
|
factor: 2,
|
||||||
type: SCThingType.Building,
|
type: SCThingType.Building,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -85,7 +85,7 @@ const boostings = {
|
|||||||
],
|
],
|
||||||
place: [
|
place: [
|
||||||
{
|
{
|
||||||
factor: 2,
|
factor: 3,
|
||||||
type: SCThingType.Building,
|
type: SCThingType.Building,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -17,12 +17,21 @@ const config = {
|
|||||||
name: 'elasticsearch',
|
name: 'elasticsearch',
|
||||||
version: '8.4.2',
|
version: '8.4.2',
|
||||||
query: {
|
query: {
|
||||||
minMatch: '75%',
|
type: 'best_fields',
|
||||||
queryType: 'dis_max',
|
fields: [
|
||||||
matchBoosting: 1.3,
|
'identifiers^20',
|
||||||
fuzziness: 'AUTO',
|
'name^10',
|
||||||
cutoffFrequency: 0,
|
'translations.*.name^10',
|
||||||
tieBreaker: 0,
|
'alternateNames^10',
|
||||||
|
'translations.*.alternateNames^10',
|
||||||
|
'description^2',
|
||||||
|
'translations.*.description^2',
|
||||||
|
'categories^5',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
searchAsYouTypeQuery: {
|
||||||
|
type: 'phrase_prefix',
|
||||||
|
fields: ['name.completion', 'name.completion._2gram', 'name.completion._3gram'],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ import {
|
|||||||
IndicesGetAliasResponse,
|
IndicesGetAliasResponse,
|
||||||
SearchHit,
|
SearchHit,
|
||||||
SearchResponse,
|
SearchResponse,
|
||||||
|
SearchTermSuggest,
|
||||||
|
SearchTermSuggestOption,
|
||||||
} from '@elastic/elasticsearch/lib/api/types.js';
|
} from '@elastic/elasticsearch/lib/api/types.js';
|
||||||
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
|
import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
|
||||||
import {Logger} from '@openstapps/logger';
|
import {Logger} from '@openstapps/logger';
|
||||||
@@ -47,6 +49,9 @@ import {
|
|||||||
import {noUndefined} from './util/no-undefined.js';
|
import {noUndefined} from './util/no-undefined.js';
|
||||||
import {retryCatch, RetryOptions} from './util/retry.js';
|
import {retryCatch, RetryOptions} from './util/retry.js';
|
||||||
import {Feature, Point, Polygon} from 'geojson';
|
import {Feature, Point, Polygon} from 'geojson';
|
||||||
|
import {parseSuggestions} from './util/parse-suggestions.js';
|
||||||
|
import {buildScoringFunctions} from './query/boost/scoring-functions.js';
|
||||||
|
import {buildFilter} from './query/filter.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A database interface for elasticsearch
|
* A database interface for elasticsearch
|
||||||
@@ -355,6 +360,39 @@ export class Elasticsearch implements Database {
|
|||||||
throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.');
|
throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async searchAsYouType(parameters: SCSearchQuery): Promise<SCSearchResponse> {
|
||||||
|
const result = await this.client.search({
|
||||||
|
_source: 'name',
|
||||||
|
query: {
|
||||||
|
function_score: {
|
||||||
|
functions: buildScoringFunctions(this.config.internal.boostings, parameters.context),
|
||||||
|
query: {
|
||||||
|
bool: {
|
||||||
|
must: {
|
||||||
|
multi_match: {
|
||||||
|
query: parameters.query,
|
||||||
|
type: 'bool_prefix',
|
||||||
|
fields: ['name.completion', 'name.completion._2gram', 'name.completion._3gram'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
should: [],
|
||||||
|
filter: parameters.filter === undefined ? undefined : buildFilter(parameters.filter),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
score_mode: 'max',
|
||||||
|
boost_mode: 'multiply',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
index: ACTIVE_INDICES_ALIAS,
|
||||||
|
allow_no_indices: true,
|
||||||
|
size: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
const suggestions = result.hits.hits.map(it => (it._source as any).name);
|
||||||
|
console.log(suggestions);
|
||||||
|
console.log(result.took);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Search all indexed data
|
* Search all indexed data
|
||||||
* @param parameters search query
|
* @param parameters search query
|
||||||
@@ -364,18 +402,23 @@ export class Elasticsearch implements Database {
|
|||||||
throw new TypeError('Database is undefined. You have to configure the query build');
|
throw new TypeError('Database is undefined. You have to configure the query build');
|
||||||
}
|
}
|
||||||
|
|
||||||
const esConfig: ElasticsearchConfig = {
|
const esConfig = this.config.internal.database as object as ElasticsearchConfig;
|
||||||
name: this.config.internal.database.name as 'elasticsearch',
|
|
||||||
version: this.config.internal.database.version as string,
|
|
||||||
query: this.config.internal.database.query as
|
|
||||||
| ElasticsearchQueryDisMaxConfig
|
|
||||||
| ElasticsearchQueryQueryStringConfig
|
|
||||||
| undefined,
|
|
||||||
};
|
|
||||||
|
|
||||||
const response: SearchResponse<SCThings> = await this.client.search({
|
const response: SearchResponse<SCThings> = await this.client.search({
|
||||||
aggs: aggregations,
|
aggs: aggregations,
|
||||||
query: buildQuery(parameters, this.config, esConfig),
|
query: buildQuery(parameters, this.config, esConfig),
|
||||||
|
suggest:
|
||||||
|
parameters.query === undefined
|
||||||
|
? undefined
|
||||||
|
: {
|
||||||
|
text: parameters.query,
|
||||||
|
terms: {
|
||||||
|
term: {
|
||||||
|
field: 'name',
|
||||||
|
suggest_mode: 'missing',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
from: parameters.from,
|
from: parameters.from,
|
||||||
index: ACTIVE_INDICES_ALIAS,
|
index: ACTIVE_INDICES_ALIAS,
|
||||||
allow_no_indices: true,
|
allow_no_indices: true,
|
||||||
@@ -395,6 +438,7 @@ export class Elasticsearch implements Database {
|
|||||||
response.aggregations === undefined
|
response.aggregations === undefined
|
||||||
? []
|
? []
|
||||||
: parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>),
|
: parseAggregations(response.aggregations as Record<AggregateName, AggregationsMultiTermsBucket>),
|
||||||
|
suggestions: response.suggest === undefined ? undefined : parseSuggestions(response.suggest),
|
||||||
pagination: {
|
pagination: {
|
||||||
count: response.hits.hits.length,
|
count: response.hits.hits.length,
|
||||||
offset: typeof parameters.from === 'number' ? parameters.from : 0,
|
offset: typeof parameters.from === 'number' ? parameters.from : 0,
|
||||||
|
|||||||
@@ -30,84 +30,21 @@ export const buildQuery = function buildQuery(
|
|||||||
defaultConfig: SCConfigFile,
|
defaultConfig: SCConfigFile,
|
||||||
elasticsearchConfig: ElasticsearchConfig,
|
elasticsearchConfig: ElasticsearchConfig,
|
||||||
): QueryDslQueryContainer {
|
): QueryDslQueryContainer {
|
||||||
// if config provides a minMatch parameter, we use query_string instead of a match query
|
return {
|
||||||
let query;
|
|
||||||
if (elasticsearchConfig.query === undefined) {
|
|
||||||
query = {
|
|
||||||
query_string: {
|
|
||||||
analyzer: 'search_german',
|
|
||||||
default_field: 'name',
|
|
||||||
minimum_should_match: '90%',
|
|
||||||
query: typeof parameters.query === 'string' ? parameters.query : '*',
|
|
||||||
},
|
|
||||||
};
|
|
||||||
} else if (elasticsearchConfig.query.queryType === 'query_string') {
|
|
||||||
query = {
|
|
||||||
query_string: {
|
|
||||||
analyzer: 'search_german',
|
|
||||||
default_field: 'name',
|
|
||||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
|
||||||
query: typeof parameters.query === 'string' ? parameters.query : '*',
|
|
||||||
},
|
|
||||||
};
|
|
||||||
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
|
|
||||||
if (typeof parameters.query === 'string' && parameters.query !== '*') {
|
|
||||||
query = {
|
|
||||||
dis_max: {
|
|
||||||
boost: 1.2,
|
|
||||||
queries: [
|
|
||||||
{
|
|
||||||
match: {
|
|
||||||
name: {
|
|
||||||
boost: elasticsearchConfig.query.matchBoosting,
|
|
||||||
fuzziness: elasticsearchConfig.query.fuzziness,
|
|
||||||
query: parameters.query,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
query_string: {
|
|
||||||
default_field: 'name',
|
|
||||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
|
||||||
query: parameters.query,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
tie_breaker: elasticsearchConfig.query.tieBreaker,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw new Error(
|
|
||||||
'Unsupported query type. Check your config file and reconfigure your elasticsearch query',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const functionScoreQuery: QueryDslQueryContainer = {
|
|
||||||
function_score: {
|
function_score: {
|
||||||
functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
|
functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context),
|
||||||
query: {
|
query: {
|
||||||
bool: {
|
bool: {
|
||||||
minimum_should_match: 0, // if we have no should, nothing can match
|
must:
|
||||||
must: [],
|
parameters.query === undefined || parameters.query === '' || parameters.query === '*'
|
||||||
|
? {match_all: {}}
|
||||||
|
: {multi_match: {...elasticsearchConfig.query, query: parameters.query}},
|
||||||
should: [],
|
should: [],
|
||||||
|
filter: parameters.filter === undefined ? undefined : buildFilter(parameters.filter),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
score_mode: 'multiply',
|
score_mode: 'max',
|
||||||
|
boost_mode: 'multiply',
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const mustMatch = functionScoreQuery.function_score?.query?.bool?.must;
|
|
||||||
|
|
||||||
if (Array.isArray(mustMatch)) {
|
|
||||||
if (query !== undefined) {
|
|
||||||
mustMatch.push(query);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parameters.filter !== undefined) {
|
|
||||||
mustMatch.push(buildFilter(parameters.filter));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return functionScoreQuery;
|
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -13,68 +13,7 @@
|
|||||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
import {QueryDslMultiMatchQuery} from '@elastic/elasticsearch/lib/api/types.js';
|
||||||
* A configuration for using the Dis Max Query
|
|
||||||
*
|
|
||||||
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
|
|
||||||
* explanation of what the parameters mean
|
|
||||||
*/
|
|
||||||
export interface ElasticsearchQueryDisMaxConfig {
|
|
||||||
/**
|
|
||||||
* Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
|
|
||||||
*/
|
|
||||||
cutoffFrequency: number;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The maximum allowed Levenshtein Edit Distance (or number of edits)
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
|
|
||||||
*/
|
|
||||||
fuzziness: number | string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Increase the importance (relevance score) of a field
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
|
|
||||||
*/
|
|
||||||
matchBoosting: number;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Minimal number (or percentage) of words that should match in a query
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
|
||||||
*/
|
|
||||||
minMatch: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type of the query - in this case 'dis_max' which is a union of its subqueries
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
|
|
||||||
*/
|
|
||||||
queryType: 'dis_max';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Changes behavior of default calculation of the score when multiple results match
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
|
|
||||||
*/
|
|
||||||
tieBreaker: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A configuration for using Query String Query
|
|
||||||
*
|
|
||||||
* See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
|
|
||||||
* explanation of what the parameters mean
|
|
||||||
*/
|
|
||||||
export interface ElasticsearchQueryQueryStringConfig {
|
|
||||||
/**
|
|
||||||
* Minimal number (or percentage) of words that should match in a query
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
|
|
||||||
*/
|
|
||||||
minMatch: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Type of the query - in this case 'query_string' which uses a query parser in order to parse content
|
|
||||||
* @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
|
|
||||||
*/
|
|
||||||
queryType: 'query_string';
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An config file for the elasticsearch database interface
|
* An config file for the elasticsearch database interface
|
||||||
@@ -105,7 +44,12 @@ export interface ElasticsearchConfig {
|
|||||||
/**
|
/**
|
||||||
* Configuration for using queries
|
* Configuration for using queries
|
||||||
*/
|
*/
|
||||||
query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;
|
query: Omit<QueryDslMultiMatchQuery, 'query'>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
searchAsYouTypeQuery: Omit<QueryDslMultiMatchQuery, 'query'>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Version of the used elasticsearch
|
* Version of the used elasticsearch
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
import {
|
||||||
|
SearchSuggest,
|
||||||
|
SearchTermSuggest,
|
||||||
|
SearchTermSuggestOption,
|
||||||
|
SuggestionName,
|
||||||
|
} from '@elastic/elasticsearch/lib/api/types.js';
|
||||||
|
import {SCSearchSuggestions} from '@openstapps/core';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse ES Suggestions to SC Search Suggestions
|
||||||
|
*/
|
||||||
|
export function parseSuggestions(suggest: Record<SuggestionName, SearchSuggest[]>): SCSearchSuggestions {
|
||||||
|
const termsSuggestions =
|
||||||
|
suggest.terms === undefined
|
||||||
|
? []
|
||||||
|
: (suggest.terms as SearchTermSuggest[])
|
||||||
|
?.map(
|
||||||
|
({text, options}) =>
|
||||||
|
[
|
||||||
|
text,
|
||||||
|
(options as SearchTermSuggestOption[] | undefined)?.map(({text}) => text) ?? [],
|
||||||
|
] as const,
|
||||||
|
)
|
||||||
|
.filter(([, suggestions]) => suggestions.length > 0) ?? [];
|
||||||
|
return {
|
||||||
|
terms: termsSuggestions.length === 0 ? undefined : Object.fromEntries(termsSuggestions),
|
||||||
|
};
|
||||||
|
}
|
||||||
142
flake.nix
142
flake.nix
@@ -4,86 +4,68 @@
|
|||||||
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||||
flake-utils.url = "github:numtide/flake-utils";
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
};
|
};
|
||||||
outputs =
|
outputs = {
|
||||||
{
|
self,
|
||||||
self,
|
nixpkgs,
|
||||||
nixpkgs,
|
flake-utils,
|
||||||
flake-utils,
|
}: let
|
||||||
}:
|
aapt2buildToolsVersion = "33.0.2";
|
||||||
let
|
in
|
||||||
aapt2buildToolsVersion = "33.0.2";
|
flake-utils.lib.eachDefaultSystem (system: let
|
||||||
in
|
pkgs = import nixpkgs {
|
||||||
flake-utils.lib.eachDefaultSystem (
|
inherit system;
|
||||||
system:
|
overlays = [
|
||||||
let
|
(final: prev: rec {
|
||||||
pkgs = import nixpkgs {
|
fontMin = prev.python311.withPackages (ps: with ps; [brotli fonttools] ++ (with fonttools.optional-dependencies; [woff]));
|
||||||
inherit system;
|
android = prev.androidenv.composeAndroidPackages {
|
||||||
overlays = [
|
buildToolsVersions = ["30.0.3" aapt2buildToolsVersion];
|
||||||
(final: prev: rec {
|
platformVersions = ["33"];
|
||||||
fontMin = prev.python311.withPackages (
|
};
|
||||||
ps:
|
cypress = prev.cypress.overrideAttrs (cyPrev: rec {
|
||||||
with ps;
|
version = "13.2.0";
|
||||||
[
|
src = prev.fetchzip {
|
||||||
brotli
|
url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
|
||||||
fonttools
|
hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
|
||||||
]
|
|
||||||
++ (with fonttools.optional-dependencies; [ woff ])
|
|
||||||
);
|
|
||||||
android = prev.androidenv.composeAndroidPackages {
|
|
||||||
buildToolsVersions = [
|
|
||||||
"34.0.0"
|
|
||||||
aapt2buildToolsVersion
|
|
||||||
];
|
|
||||||
platformVersions = [ "34" ];
|
|
||||||
};
|
};
|
||||||
cypress = prev.cypress.overrideAttrs (cyPrev: rec {
|
});
|
||||||
version = "13.2.0";
|
nodejs = prev.nodejs_18;
|
||||||
src = prev.fetchzip {
|
})
|
||||||
url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
|
];
|
||||||
hash = "sha256-9o0nprGcJhudS1LNm+T7Vf0Dwd1RBauYKI+w1FBQ3ZM=";
|
config = {
|
||||||
};
|
allowUnfree = true;
|
||||||
});
|
android_sdk.accept_license = true;
|
||||||
nodejs = prev.nodejs_18;
|
|
||||||
corepack = prev.corepack_18;
|
|
||||||
})
|
|
||||||
];
|
|
||||||
config = {
|
|
||||||
allowUnfree = true;
|
|
||||||
android_sdk.accept_license = true;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
androidFhs = pkgs.buildFHSUserEnv {
|
};
|
||||||
name = "android-env";
|
androidFhs = pkgs.buildFHSUserEnv {
|
||||||
targetPkgs = pkgs: with pkgs; [ ];
|
name = "android-env";
|
||||||
runScript = "bash";
|
targetPkgs = pkgs: with pkgs; [];
|
||||||
profile = ''
|
runScript = "bash";
|
||||||
export ALLOW_NINJA_ENV=true
|
profile = ''
|
||||||
export USE_CCACHE=1
|
export ALLOW_NINJA_ENV=true
|
||||||
export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
|
export USE_CCACHE=1
|
||||||
'';
|
export LD_LIBRARY_PATH=/usr/lib:/usr/lib32
|
||||||
};
|
'';
|
||||||
in
|
};
|
||||||
{
|
in {
|
||||||
devShell = pkgs.mkShell rec {
|
devShell = pkgs.mkShell rec {
|
||||||
nativeBuildInputs = [ androidFhs ];
|
nativeBuildInputs = [androidFhs];
|
||||||
buildInputs = with pkgs; [
|
buildInputs = with pkgs; [
|
||||||
nodejs
|
nodejs
|
||||||
corepack
|
corepack
|
||||||
# tools
|
# tools
|
||||||
curl
|
curl
|
||||||
jq
|
jq
|
||||||
fontMin
|
fontMin
|
||||||
cypress
|
cypress
|
||||||
# android
|
# android
|
||||||
jdk17
|
jdk17
|
||||||
android.androidsdk
|
android.androidsdk
|
||||||
];
|
];
|
||||||
ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
|
ANDROID_JAVA_HOME = "${pkgs.jdk.home}";
|
||||||
ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
|
ANDROID_SDK_ROOT = "${pkgs.android.androidsdk}/libexec/android-sdk";
|
||||||
GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
|
GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${aapt2buildToolsVersion}/aapt2";
|
||||||
CYPRESS_INSTALL_BINARY = "0";
|
CYPRESS_INSTALL_BINARY = "0";
|
||||||
CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
|
CYPRESS_RUN_BINARY = "${pkgs.cypress}/bin/Cypress";
|
||||||
};
|
};
|
||||||
}
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,8 +67,7 @@ describe('ConfigProvider', () => {
|
|||||||
|
|
||||||
it('should fetch app configuration', async () => {
|
it('should fetch app configuration', async () => {
|
||||||
spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(sampleIndexResponse));
|
spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(sampleIndexResponse));
|
||||||
await configProvider.fetch();
|
const result = await configProvider.fetch();
|
||||||
const result = configProvider.config;
|
|
||||||
expect(result).toEqual(sampleIndexResponse);
|
expect(result).toEqual(sampleIndexResponse);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -111,7 +110,7 @@ describe('ConfigProvider', () => {
|
|||||||
expect(storageProviderSpy.has).toHaveBeenCalled();
|
expect(storageProviderSpy.has).toHaveBeenCalled();
|
||||||
expect(storageProviderSpy.get).toHaveBeenCalledTimes(0);
|
expect(storageProviderSpy.get).toHaveBeenCalledTimes(0);
|
||||||
expect(configProvider.client.handshake).toHaveBeenCalled();
|
expect(configProvider.client.handshake).toHaveBeenCalled();
|
||||||
expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
|
expect(await configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should throw error on failed initialisation', async () => {
|
it('should throw error on failed initialisation', async () => {
|
||||||
@@ -193,31 +192,4 @@ describe('ConfigProvider', () => {
|
|||||||
|
|
||||||
expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
|
expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should fetch new config from remote on init', async () => {
|
|
||||||
storageProviderSpy.has.and.returnValue(Promise.resolve(true));
|
|
||||||
storageProviderSpy.get.and.returnValue(Promise.resolve(sampleIndexResponse));
|
|
||||||
spyOn(configProvider, 'fetch');
|
|
||||||
await configProvider.init();
|
|
||||||
|
|
||||||
expect(configProvider.fetch).toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should update the local config with the one from remote', async () => {
|
|
||||||
storageProviderSpy.has.and.returnValue(Promise.resolve(true));
|
|
||||||
storageProviderSpy.get.and.returnValue(Promise.resolve(sampleIndexResponse));
|
|
||||||
const newConfig = structuredClone(sampleIndexResponse);
|
|
||||||
newConfig.app.name = 'New app name';
|
|
||||||
spyOn(configProvider.client, 'handshake').and.returnValue(Promise.resolve(newConfig));
|
|
||||||
await configProvider.init();
|
|
||||||
|
|
||||||
// Validate that the initial configuration is loaded
|
|
||||||
expect(configProvider.getValue('name')).toEqual(sampleIndexResponse.app.name);
|
|
||||||
|
|
||||||
// Fetch the new configuration from the remote
|
|
||||||
await configProvider.fetch();
|
|
||||||
|
|
||||||
// Validate that the new configuration is now set
|
|
||||||
expect(configProvider.getValue('name')).toEqual(newConfig.app.name);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -83,20 +83,17 @@ export class ConfigProvider {
|
|||||||
/**
|
/**
|
||||||
* Fetches configuration from backend
|
* Fetches configuration from backend
|
||||||
*/
|
*/
|
||||||
async fetch(): Promise<void> {
|
async fetch(): Promise<SCIndexResponse> {
|
||||||
try {
|
try {
|
||||||
const isOffline = await firstValueFrom(this.internetConnectionService.offline$);
|
const isOffline = await firstValueFrom(this.internetConnectionService.offline$);
|
||||||
if (isOffline) {
|
if (isOffline) {
|
||||||
throw new Error('Device is offline.');
|
throw new Error('Device is offline.');
|
||||||
} else {
|
} else {
|
||||||
const fetchedConfig: SCIndexResponse = await this.client.handshake(this.scVersion);
|
return await this.client.handshake(this.scVersion);
|
||||||
await this.set(fetchedConfig);
|
|
||||||
this.logger.log(`Configuration updated from remote`);
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const error_ = error instanceof Error ? new ConfigFetchError(error.message) : new ConfigFetchError();
|
const error_ = error instanceof Error ? new ConfigFetchError(error.message) : new ConfigFetchError();
|
||||||
this.logger.warn(`Failed to fetch remote configuration:`, error_);
|
throw error_;
|
||||||
throw error_; // Rethrow the error to handle it in init()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,33 +121,40 @@ export class ConfigProvider {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialises the ConfigProvider
|
* Initialises the ConfigProvider
|
||||||
* @throws ConfigInitError if no configuration could be loaded both locally and remote.
|
* @throws ConfigInitError if no configuration could be loaded.
|
||||||
|
* @throws WrongConfigVersionInStorage if fetch failed and saved config has wrong SCVersion
|
||||||
*/
|
*/
|
||||||
async init(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
|
let loadError;
|
||||||
|
let fetchError;
|
||||||
|
// load saved configuration
|
||||||
try {
|
try {
|
||||||
// Attempt to load the configuration from local storage
|
|
||||||
this.config = await this.loadLocal();
|
this.config = await this.loadLocal();
|
||||||
this.firstSession = false;
|
this.firstSession = false;
|
||||||
this.logger.log(`initialised configuration from storage`);
|
this.logger.log(`initialised configuration from storage`);
|
||||||
|
|
||||||
// Check if the stored configuration has the correct version
|
|
||||||
if (this.config.backend.SCVersion.split('.')[0] !== this.scVersion.split('.')[0]) {
|
if (this.config.backend.SCVersion.split('.')[0] !== this.scVersion.split('.')[0]) {
|
||||||
throw new WrongConfigVersionInStorage(this.scVersion, this.config.backend.SCVersion);
|
loadError = new WrongConfigVersionInStorage(this.scVersion, this.config.backend.SCVersion);
|
||||||
}
|
}
|
||||||
|
} catch (error) {
|
||||||
// Fetch the remote configuration in a non-blocking manner
|
loadError = error;
|
||||||
void this.fetch();
|
}
|
||||||
} catch (loadError) {
|
// fetch remote configuration from backend
|
||||||
|
try {
|
||||||
|
const fetchedConfig: SCIndexResponse = await this.fetch();
|
||||||
|
await this.set(fetchedConfig);
|
||||||
|
this.logger.log(`initialised configuration from remote`);
|
||||||
|
} catch (error) {
|
||||||
|
fetchError = error;
|
||||||
|
}
|
||||||
|
// check for occurred errors and throw them
|
||||||
|
if (loadError !== undefined && fetchError !== undefined) {
|
||||||
|
throw new ConfigInitError();
|
||||||
|
}
|
||||||
|
if (loadError !== undefined) {
|
||||||
this.logger.warn(loadError);
|
this.logger.warn(loadError);
|
||||||
|
}
|
||||||
try {
|
if (fetchError !== undefined) {
|
||||||
// If local loading fails, immediately try to fetch the configuration from remote
|
this.logger.warn(fetchError);
|
||||||
await this.fetch();
|
|
||||||
} catch (fetchError) {
|
|
||||||
this.logger.warn(`Failed to fetch remote configuration:`, fetchError);
|
|
||||||
// If both local loading and remote fetching fail, throw ConfigInitError
|
|
||||||
throw new ConfigInitError();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import {
|
|||||||
SCSearchQuery,
|
SCSearchQuery,
|
||||||
SCSearchSort,
|
SCSearchSort,
|
||||||
SCThings,
|
SCThings,
|
||||||
|
SCSearchSuggestions,
|
||||||
} from '@openstapps/core';
|
} from '@openstapps/core';
|
||||||
import {NGXLogger} from 'ngx-logger';
|
import {NGXLogger} from 'ngx-logger';
|
||||||
import {combineLatest, Subject} from 'rxjs';
|
import {combineLatest, Subject} from 'rxjs';
|
||||||
@@ -110,6 +111,8 @@ export class SearchPageComponent implements OnInit {
|
|||||||
*/
|
*/
|
||||||
items: Promise<SCThings[]>;
|
items: Promise<SCThings[]>;
|
||||||
|
|
||||||
|
suggestions: SCSearchSuggestions | undefined;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Page size of queries
|
* Page size of queries
|
||||||
*/
|
*/
|
||||||
@@ -219,6 +222,7 @@ export class SearchPageComponent implements OnInit {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await this.dataProvider.search(searchOptions);
|
const result = await this.dataProvider.search(searchOptions);
|
||||||
|
this.suggestions = result.suggestions;
|
||||||
this.singleTypeResponse = result.facets.find(facet => facet.field === 'type')?.buckets.length === 1;
|
this.singleTypeResponse = result.facets.find(facet => facet.field === 'type')?.buckets.length === 1;
|
||||||
if (append) {
|
if (append) {
|
||||||
// append results
|
// append results
|
||||||
@@ -283,6 +287,12 @@ export class SearchPageComponent implements OnInit {
|
|||||||
this.contextMenuService.updateContextFilter(facets);
|
this.contextMenuService.updateContextFilter(facets);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
applySuggestion(target: string, suggestion: string) {
|
||||||
|
this.queryText = this.queryText.replaceAll(target, suggestion);
|
||||||
|
this.suggestions = undefined;
|
||||||
|
this.searchStringChanged(this.queryText);
|
||||||
|
}
|
||||||
|
|
||||||
ngOnInit(defaultListeners = true) {
|
ngOnInit(defaultListeners = true) {
|
||||||
this.initialize();
|
this.initialize();
|
||||||
this.contextMenuService.setContextSort({
|
this.contextMenuService.setContextSort({
|
||||||
|
|||||||
@@ -69,7 +69,23 @@
|
|||||||
</ion-header>
|
</ion-header>
|
||||||
|
|
||||||
<ion-content class="content">
|
<ion-content class="content">
|
||||||
|
<div class="suggestions">
|
||||||
|
@if (suggestions?.terms; as terms) {
|
||||||
|
<span>{{ 'search.SUGGESTIONS' | translate }}: </span>
|
||||||
|
@for (suggestion of terms | keyvalue; track suggestion) {
|
||||||
|
@for (term of suggestion.value; track term) {
|
||||||
|
@if ($index == 0) {
|
||||||
|
<b (click)="applySuggestion(suggestion.key, term)" class="suggestion">{{ term }}</b>
|
||||||
|
} @else {
|
||||||
|
<span (click)="applySuggestion(suggestion.key, term)" class="suggestion">{{ term }}</span>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div
|
<div
|
||||||
|
class="hint"
|
||||||
[class.no-results]="!showDefaultData && !items && !loading"
|
[class.no-results]="!showDefaultData && !items && !loading"
|
||||||
[style.display]="!showDefaultData && !items && !loading ? 'block' : 'none'"
|
[style.display]="!showDefaultData && !items && !loading ? 'block' : 'none'"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ ion-content {
|
|||||||
--background: var(--ion-background-color);
|
--background: var(--ion-background-color);
|
||||||
}
|
}
|
||||||
|
|
||||||
.content > div {
|
.content > .hint {
|
||||||
height: 100%;
|
height: 100%;
|
||||||
|
|
||||||
ion-label.centered-message-container {
|
ion-label.centered-message-container {
|
||||||
@@ -60,3 +60,19 @@ ion-content {
|
|||||||
ion-header {
|
ion-header {
|
||||||
background: var(--ion-color-primary);
|
background: var(--ion-color-primary);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.suggestions {
|
||||||
|
padding: var(--spacing-md);
|
||||||
|
padding-block-end: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestion {
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.suggestion + .suggestion::before {
|
||||||
|
cursor: text;
|
||||||
|
content: ',';
|
||||||
|
padding-inline-end: 0.25ch;
|
||||||
|
font-weight: normal;
|
||||||
|
}
|
||||||
|
|||||||
@@ -80,12 +80,13 @@
|
|||||||
</ion-item>
|
</ion-item>
|
||||||
<ion-item>
|
<ion-item>
|
||||||
<ion-checkbox
|
<ion-checkbox
|
||||||
|
class="ion-text-wrap"
|
||||||
color="primary"
|
color="primary"
|
||||||
label-placement="end"
|
label-placement="end"
|
||||||
justify="start"
|
justify="start"
|
||||||
[(ngModel)]="termsAgree"
|
[(ngModel)]="termsAgree"
|
||||||
name="termsAgree"
|
name="termsAgree"
|
||||||
><span class="ion-text-wrap">{{ 'feedback.form.termsAgree.0' | translate }}</span></ion-checkbox
|
>{{ 'feedback.form.termsAgree.0' | translate }}</ion-checkbox
|
||||||
>
|
>
|
||||||
</ion-item>
|
</ion-item>
|
||||||
<ion-item lines="none">
|
<ion-item lines="none">
|
||||||
@@ -103,9 +104,7 @@
|
|||||||
justify="start"
|
justify="start"
|
||||||
[(ngModel)]="protocolDataAgree"
|
[(ngModel)]="protocolDataAgree"
|
||||||
name="protocolDataAgree"
|
name="protocolDataAgree"
|
||||||
><span class="ion-text-wrap">{{
|
>{{ 'feedback.form.protocolDataAgree' | translate }}</ion-checkbox
|
||||||
'feedback.form.protocolDataAgree' | translate
|
|
||||||
}}</span></ion-checkbox
|
|
||||||
>
|
>
|
||||||
</ion-item>
|
</ion-item>
|
||||||
<ion-card>
|
<ion-card>
|
||||||
|
|||||||
@@ -422,7 +422,8 @@
|
|||||||
"placeholder": "Veranstaltungen, Personen, Orte und mehr"
|
"placeholder": "Veranstaltungen, Personen, Orte und mehr"
|
||||||
},
|
},
|
||||||
"instruction": "Finde alle Informationen rund ums Studium und den Campus",
|
"instruction": "Finde alle Informationen rund ums Studium und den Campus",
|
||||||
"nothing_found": "Keine Ergebnisse"
|
"nothing_found": "Keine Ergebnisse",
|
||||||
|
"SUGGESTIONS": "Meintest du"
|
||||||
},
|
},
|
||||||
"hebisSearch": {
|
"hebisSearch": {
|
||||||
"title": "Bibliothekssuche",
|
"title": "Bibliothekssuche",
|
||||||
|
|||||||
@@ -422,7 +422,8 @@
|
|||||||
"placeholder": "Events, places, persons and more"
|
"placeholder": "Events, places, persons and more"
|
||||||
},
|
},
|
||||||
"instruction": "Find all information related to your studies and campus",
|
"instruction": "Find all information related to your studies and campus",
|
||||||
"nothing_found": "No results"
|
"nothing_found": "No results",
|
||||||
|
"SUGGESTIONS": "Did you mean"
|
||||||
},
|
},
|
||||||
"hebisSearch": {
|
"hebisSearch": {
|
||||||
"title": "Library Search",
|
"title": "Library Search",
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
// The list of which env maps to which file can be found in `.angular-cli.json`.
|
// The list of which env maps to which file can be found in `.angular-cli.json`.
|
||||||
|
|
||||||
export const environment = {
|
export const environment = {
|
||||||
backend_url: 'https://mobile.server.uni-frankfurt.de',
|
backend_url: 'http://localhost:3000',
|
||||||
app_host: 'mobile.app.uni-frankfurt.de',
|
app_host: 'mobile.app.uni-frankfurt.de',
|
||||||
custom_url_scheme: 'de.anyschool.app',
|
custom_url_scheme: 'de.anyschool.app',
|
||||||
backend_version: '999.0.0',
|
backend_version: '999.0.0',
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ node ./lib/cli.js e2e http://localhost:3000
|
|||||||
Example to clone the full database
|
Example to clone the full database
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
node app.js copy "*" https://mobile.app.uni-frankfurt.de http://localhost:3000 100
|
node app.js copy "*" -a "999.0.0" https://mobile.server.uni-frankfurt.de http://localhost:3000 100
|
||||||
```
|
```
|
||||||
|
|
||||||
### Program arguments
|
### Program arguments
|
||||||
|
|||||||
@@ -97,6 +97,8 @@
|
|||||||
"date",
|
"date",
|
||||||
"validatable",
|
"validatable",
|
||||||
"filterable",
|
"filterable",
|
||||||
|
"suggestable",
|
||||||
|
"completable",
|
||||||
"inheritTags",
|
"inheritTags",
|
||||||
"minLength",
|
"minLength",
|
||||||
"pattern",
|
"pattern",
|
||||||
|
|||||||
0
packages/core/src/protocol/search/completion.ts
Normal file
0
packages/core/src/protocol/search/completion.ts
Normal file
@@ -24,6 +24,11 @@ export interface SCSearchResult {
|
|||||||
*/
|
*/
|
||||||
data: SCThings[];
|
data: SCThings[];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Suggestions for query corrections
|
||||||
|
*/
|
||||||
|
suggestions?: SCSearchSuggestions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Facets (aggregations over all matching data)
|
* Facets (aggregations over all matching data)
|
||||||
*/
|
*/
|
||||||
@@ -40,6 +45,18 @@ export interface SCSearchResult {
|
|||||||
stats: SCSearchResultSearchEngineStats;
|
stats: SCSearchResultSearchEngineStats;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seach suggestions
|
||||||
|
*
|
||||||
|
* Not to be confused with search-as-you-type suggestions
|
||||||
|
*/
|
||||||
|
export interface SCSearchSuggestions {
|
||||||
|
/**
|
||||||
|
* Suggestions for query terms that might have been misspelled
|
||||||
|
*/
|
||||||
|
terms?: Record<string, string[]>;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stores information about Pagination
|
* Stores information about Pagination
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ export interface SCThingWithoutReferences {
|
|||||||
/**
|
/**
|
||||||
* Alternate names of the thing
|
* Alternate names of the thing
|
||||||
* @filterable
|
* @filterable
|
||||||
* @keyword
|
* @text
|
||||||
*/
|
*/
|
||||||
alternateNames?: string[];
|
alternateNames?: string[];
|
||||||
|
|
||||||
@@ -92,6 +92,8 @@ export interface SCThingWithoutReferences {
|
|||||||
* @filterable
|
* @filterable
|
||||||
* @minLength 1
|
* @minLength 1
|
||||||
* @sortable ducet
|
* @sortable ducet
|
||||||
|
* @completable
|
||||||
|
* @suggestable
|
||||||
* @text
|
* @text
|
||||||
*/
|
*/
|
||||||
name: string;
|
name: string;
|
||||||
@@ -240,6 +242,8 @@ export interface SCThingTranslatableProperties {
|
|||||||
* Translation of the name of the thing
|
* Translation of the name of the thing
|
||||||
* @sortable ducet
|
* @sortable ducet
|
||||||
* @text
|
* @text
|
||||||
|
* @suggestable
|
||||||
|
* @completable
|
||||||
*/
|
*/
|
||||||
name?: string;
|
name?: string;
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -43,6 +43,27 @@ export const fieldmap: ElasticsearchFieldmap = {
|
|||||||
},
|
},
|
||||||
ignore: ['price'],
|
ignore: ['price'],
|
||||||
},
|
},
|
||||||
|
suggestable: {
|
||||||
|
default: {
|
||||||
|
trigram: {
|
||||||
|
type: 'text',
|
||||||
|
analyzer: 'trigram',
|
||||||
|
},
|
||||||
|
reverse: {
|
||||||
|
type: 'text',
|
||||||
|
analyzer: 'reverse',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ignore: [],
|
||||||
|
},
|
||||||
|
completable: {
|
||||||
|
default: {
|
||||||
|
completion: {
|
||||||
|
type: 'search_as_you_type',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ignore: [],
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
export const filterableTagName = 'filterable';
|
export const filterableTagName = 'filterable';
|
||||||
|
|||||||
@@ -19,4 +19,27 @@ export const settings: IndicesPutTemplateRequest['settings'] = {
|
|||||||
'max_result_window': 30_000,
|
'max_result_window': 30_000,
|
||||||
'number_of_replicas': 0,
|
'number_of_replicas': 0,
|
||||||
'number_of_shards': 1,
|
'number_of_shards': 1,
|
||||||
|
'index': {
|
||||||
|
analysis: {
|
||||||
|
analyzer: {
|
||||||
|
trigram: {
|
||||||
|
type: 'custom',
|
||||||
|
tokenizer: 'standard',
|
||||||
|
filter: ['lowercase', 'shingle'],
|
||||||
|
},
|
||||||
|
reverse: {
|
||||||
|
type: 'custom',
|
||||||
|
tokenizer: 'standard',
|
||||||
|
filter: ['lowercase', 'reverse'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
filter: {
|
||||||
|
shingle: {
|
||||||
|
type: 'shingle',
|
||||||
|
min_shingle_size: 2,
|
||||||
|
max_shingle_size: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user