diff --git a/backend/backend/config/b-tu/elasticsearchrc.js b/backend/backend/config/b-tu/elasticsearchrc.js index db469267..1621f91f 100644 --- a/backend/backend/config/b-tu/elasticsearchrc.js +++ b/backend/backend/config/b-tu/elasticsearchrc.js @@ -8,8 +8,7 @@ const config = { database: { name: 'elasticsearch', query: { - minMatch: '60%', - queryType: 'query_string', + fields: ["name"] }, }, }, diff --git a/backend/backend/config/default/backend/boostings.js b/backend/backend/config/default/backend/boostings.js index e3d90b91..592260a1 100644 --- a/backend/backend/config/default/backend/boostings.js +++ b/backend/backend/config/default/backend/boostings.js @@ -39,7 +39,7 @@ const boostings = { type: SCThingType.AcademicEvent, }, { - factor: 1.6, + factor: 2, type: SCThingType.Building, }, { @@ -85,7 +85,7 @@ const boostings = { ], place: [ { - factor: 2, + factor: 3, type: SCThingType.Building, }, { diff --git a/backend/backend/config/default/elasticsearchrc.js b/backend/backend/config/default/elasticsearchrc.js index 9444a640..69bfcd77 100644 --- a/backend/backend/config/default/elasticsearchrc.js +++ b/backend/backend/config/default/elasticsearchrc.js @@ -17,12 +17,21 @@ const config = { name: 'elasticsearch', version: '8.4.2', query: { - minMatch: '75%', - queryType: 'dis_max', - matchBoosting: 1.3, - fuzziness: 'AUTO', - cutoffFrequency: 0, - tieBreaker: 0, + type: 'best_fields', + fields: [ + 'identifiers^20', + 'name^10', + 'translations.*.name^10', + 'alternateNames^10', + 'translations.*.alternateNames^10', + 'description^2', + 'translations.*.description^2', + 'categories^5', + ], + }, + searchAsYouTypeQuery: { + type: 'phrase_prefix', + fields: ['name.completion', 'name.completion._2gram', 'name.completion._3gram'], }, }, }, diff --git a/backend/backend/src/storage/elasticsearch/elasticsearch.ts b/backend/backend/src/storage/elasticsearch/elasticsearch.ts index dad86eef..b098160e 100644 --- a/backend/backend/src/storage/elasticsearch/elasticsearch.ts +++ b/backend/backend/src/storage/elasticsearch/elasticsearch.ts @@ -20,6 +20,8 @@ import { IndicesGetAliasResponse, SearchHit, SearchResponse, + SearchTermSuggest, + SearchTermSuggestOption, } from '@elastic/elasticsearch/lib/api/types.js'; import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core'; import {Logger} from '@openstapps/logger'; @@ -47,6 +49,9 @@ import { import {noUndefined} from './util/no-undefined.js'; import {retryCatch, RetryOptions} from './util/retry.js'; import {Feature, Point, Polygon} from 'geojson'; +import {parseSuggestions} from './util/parse-suggestions.js'; +import {buildScoringFunctions} from './query/boost/scoring-functions.js'; +import {buildFilter} from './query/filter.js'; /** * A database interface for elasticsearch @@ -355,6 +360,39 @@ export class Elasticsearch implements Database { throw new Error('You tried to PUT an non-existing object. PUT is only supported on existing objects.'); } + public async searchAsYouType(parameters: SCSearchQuery): Promise { + const result = await this.client.search({ + _source: 'name', + query: { + function_score: { + functions: buildScoringFunctions(this.config.internal.boostings, parameters.context), + query: { + bool: { + must: { + multi_match: { + query: parameters.query, + type: 'bool_prefix', + fields: ['name.completion', 'name.completion._2gram', 'name.completion._3gram'], + }, + }, + should: [], + filter: parameters.filter === undefined ? undefined : buildFilter(parameters.filter), + }, + }, + score_mode: 'max', + boost_mode: 'multiply', + }, + }, + index: ACTIVE_INDICES_ALIAS, + allow_no_indices: true, + size: 5, + }); + + const suggestions = result.hits.hits.map(it => (it._source as any).name); + console.log(suggestions); + console.log(result.took); + } + /** * Search all indexed data * @param parameters search query @@ -364,18 +402,23 @@ export class Elasticsearch implements Database { throw new TypeError('Database is undefined. You have to configure the query build'); } - const esConfig: ElasticsearchConfig = { - name: this.config.internal.database.name as 'elasticsearch', - version: this.config.internal.database.version as string, - query: this.config.internal.database.query as - | ElasticsearchQueryDisMaxConfig - | ElasticsearchQueryQueryStringConfig - | undefined, - }; + const esConfig = this.config.internal.database as object as ElasticsearchConfig; const response: SearchResponse = await this.client.search({ aggs: aggregations, query: buildQuery(parameters, this.config, esConfig), + suggest: + parameters.query === undefined + ? undefined + : { + text: parameters.query, + terms: { + term: { + field: 'name', + suggest_mode: 'missing', + }, + }, + }, from: parameters.from, index: ACTIVE_INDICES_ALIAS, allow_no_indices: true, @@ -395,6 +438,7 @@ export class Elasticsearch implements Database { response.aggregations === undefined ? [] : parseAggregations(response.aggregations as Record), + suggestions: response.suggest === undefined ? undefined : parseSuggestions(response.suggest), pagination: { count: response.hits.hits.length, offset: typeof parameters.from === 'number' ? parameters.from : 0, diff --git a/backend/backend/src/storage/elasticsearch/query/query.ts b/backend/backend/src/storage/elasticsearch/query/query.ts index 3832cab5..ea29fbc9 100644 --- a/backend/backend/src/storage/elasticsearch/query/query.ts +++ b/backend/backend/src/storage/elasticsearch/query/query.ts @@ -30,84 +30,21 @@ export const buildQuery = function buildQuery( defaultConfig: SCConfigFile, elasticsearchConfig: ElasticsearchConfig, ): QueryDslQueryContainer { - // if config provides a minMatch parameter, we use query_string instead of a match query - let query; - if (elasticsearchConfig.query === undefined) { - query = { - query_string: { - analyzer: 'search_german', - default_field: 'name', - minimum_should_match: '90%', - query: typeof parameters.query === 'string' ? parameters.query : '*', - }, - }; - } else if (elasticsearchConfig.query.queryType === 'query_string') { - query = { - query_string: { - analyzer: 'search_german', - default_field: 'name', - minimum_should_match: elasticsearchConfig.query.minMatch, - query: typeof parameters.query === 'string' ? parameters.query : '*', - }, - }; - } else if (elasticsearchConfig.query.queryType === 'dis_max') { - if (typeof parameters.query === 'string' && parameters.query !== '*') { - query = { - dis_max: { - boost: 1.2, - queries: [ - { - match: { - name: { - boost: elasticsearchConfig.query.matchBoosting, - fuzziness: elasticsearchConfig.query.fuzziness, - query: parameters.query, - }, - }, - }, - { - query_string: { - default_field: 'name', - minimum_should_match: elasticsearchConfig.query.minMatch, - query: parameters.query, - }, - }, - ], - tie_breaker: elasticsearchConfig.query.tieBreaker, - }, - }; - } - } else { - throw new Error( - 'Unsupported query type. Check your config file and reconfigure your elasticsearch query', - ); - } - - const functionScoreQuery: QueryDslQueryContainer = { + return { function_score: { functions: buildScoringFunctions(defaultConfig.internal.boostings, parameters.context), query: { bool: { - minimum_should_match: 0, // if we have no should, nothing can match - must: [], + must: + parameters.query === undefined || parameters.query === '' || parameters.query === '*' + ? {match_all: {}} + : {multi_match: {...elasticsearchConfig.query, query: parameters.query}}, should: [], + filter: parameters.filter === undefined ? undefined : buildFilter(parameters.filter), }, }, - score_mode: 'multiply', + score_mode: 'max', + boost_mode: 'multiply', }, }; - - const mustMatch = functionScoreQuery.function_score?.query?.bool?.must; - - if (Array.isArray(mustMatch)) { - if (query !== undefined) { - mustMatch.push(query); - } - - if (parameters.filter !== undefined) { - mustMatch.push(buildFilter(parameters.filter)); - } - } - - return functionScoreQuery; }; diff --git a/backend/backend/src/storage/elasticsearch/types/elasticsearch-config.ts b/backend/backend/src/storage/elasticsearch/types/elasticsearch-config.ts index 52476d94..cdbc0a20 100644 --- a/backend/backend/src/storage/elasticsearch/types/elasticsearch-config.ts +++ b/backend/backend/src/storage/elasticsearch/types/elasticsearch-config.ts @@ -13,68 +13,7 @@ * this program. If not, see . */ -/** - * A configuration for using the Dis Max Query - * - * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further - * explanation of what the parameters mean - */ -export interface ElasticsearchQueryDisMaxConfig { - /** - * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear - */ - cutoffFrequency: number; - - /** - * The maximum allowed Levenshtein Edit Distance (or number of edits) - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness - */ - fuzziness: number | string; - - /** - * Increase the importance (relevance score) of a field - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html - */ - matchBoosting: number; - - /** - * Minimal number (or percentage) of words that should match in a query - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html - */ - minMatch: string; - - /** - * Type of the query - in this case 'dis_max' which is a union of its subqueries - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html - */ - queryType: 'dis_max'; - - /** - * Changes behavior of default calculation of the score when multiple results match - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker - */ - tieBreaker: number; -} - -/** - * A configuration for using Query String Query - * - * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further - * explanation of what the parameters mean - */ -export interface ElasticsearchQueryQueryStringConfig { - /** - * Minimal number (or percentage) of words that should match in a query - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html - */ - minMatch: string; - - /** - * Type of the query - in this case 'query_string' which uses a query parser in order to parse content - * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html - */ - queryType: 'query_string'; -} +import {QueryDslMultiMatchQuery} from '@elastic/elasticsearch/lib/api/types.js'; /** * An config file for the elasticsearch database interface @@ -105,7 +44,12 @@ export interface ElasticsearchConfig { /** * Configuration for using queries */ - query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig; + query: Omit; + + /** + * + */ + searchAsYouTypeQuery: Omit; /** * Version of the used elasticsearch diff --git a/backend/backend/src/storage/elasticsearch/util/parse-suggestions.ts b/backend/backend/src/storage/elasticsearch/util/parse-suggestions.ts new file mode 100644 index 00000000..dec4a611 --- /dev/null +++ b/backend/backend/src/storage/elasticsearch/util/parse-suggestions.ts @@ -0,0 +1,28 @@ +import { + SearchSuggest, + SearchTermSuggest, + SearchTermSuggestOption, + SuggestionName, +} from '@elastic/elasticsearch/lib/api/types.js'; +import {SCSearchSuggestions} from '@openstapps/core'; + +/** + * Parse ES Suggestions to SC Search Suggestions + */ +export function parseSuggestions(suggest: Record): SCSearchSuggestions { + const termsSuggestions = + suggest.terms === undefined + ? [] + : (suggest.terms as SearchTermSuggest[]) + ?.map( + ({text, options}) => + [ + text, + (options as SearchTermSuggestOption[] | undefined)?.map(({text}) => text) ?? [], + ] as const, + ) + .filter(([, suggestions]) => suggestions.length > 0) ?? []; + return { + terms: termsSuggestions.length === 0 ? undefined : Object.fromEntries(termsSuggestions), + }; +} diff --git a/frontend/app/src/app/modules/data/list/search-page.component.ts b/frontend/app/src/app/modules/data/list/search-page.component.ts index 9d41c9de..61006f79 100644 --- a/frontend/app/src/app/modules/data/list/search-page.component.ts +++ b/frontend/app/src/app/modules/data/list/search-page.component.ts @@ -29,6 +29,7 @@ import { SCSearchQuery, SCSearchSort, SCThings, + SCSearchSuggestions, } from '@openstapps/core'; import {NGXLogger} from 'ngx-logger'; import {combineLatest, Subject} from 'rxjs'; @@ -117,6 +118,8 @@ export class SearchPageComponent implements OnInit { */ items: Promise; + suggestions: SCSearchSuggestions | undefined; + /** * Page size of queries */ @@ -227,6 +230,7 @@ export class SearchPageComponent implements OnInit { try { const result = await this.dataProvider.search(searchOptions); + this.suggestions = result.suggestions; this.singleTypeResponse = result.facets.find(facet => facet.field === 'type')?.buckets.length === 1; if (append) { // append results @@ -291,6 +295,12 @@ export class SearchPageComponent implements OnInit { this.contextMenuService.updateContextFilter(facets); } + applySuggestion(target: string, suggestion: string) { + this.queryText = this.queryText.replaceAll(target, suggestion); + this.suggestions = undefined; + this.searchStringChanged(this.queryText); + } + ngOnInit(defaultListeners = true) { this.initialize(); this.contextMenuService.setContextSort({ diff --git a/frontend/app/src/app/modules/data/list/search-page.html b/frontend/app/src/app/modules/data/list/search-page.html index e45a8748..e716d659 100644 --- a/frontend/app/src/app/modules/data/list/search-page.html +++ b/frontend/app/src/app/modules/data/list/search-page.html @@ -66,7 +66,23 @@ +
+ @if (suggestions?.terms; as terms) { + {{ 'search.SUGGESTIONS' | translate }}: + @for (suggestion of terms | keyvalue; track suggestion) { + @for (term of suggestion.value; track term) { + @if ($index == 0) { + {{ term }} + } @else { + {{ term }} + } + } + } + } +
+
diff --git a/frontend/app/src/app/modules/data/list/search-page.scss b/frontend/app/src/app/modules/data/list/search-page.scss index 9dde415a..c0a66c33 100644 --- a/frontend/app/src/app/modules/data/list/search-page.scss +++ b/frontend/app/src/app/modules/data/list/search-page.scss @@ -46,7 +46,7 @@ ion-content { --background: var(--ion-background-color); } -.content > div { +.content > .hint { height: 100%; ion-label.centered-message-container { @@ -60,3 +60,19 @@ ion-content { ion-header { background: var(--ion-color-primary); } + +.suggestions { + padding: var(--spacing-md); + padding-block-end: 0; +} + +.suggestion { + cursor: pointer; +} + +.suggestion + .suggestion::before { + cursor: text; + content: ','; + padding-inline-end: 0.25ch; + font-weight: normal; +} diff --git a/frontend/app/src/assets/i18n/de.json b/frontend/app/src/assets/i18n/de.json index 00264829..e1f1e069 100644 --- a/frontend/app/src/assets/i18n/de.json +++ b/frontend/app/src/assets/i18n/de.json @@ -425,7 +425,8 @@ "placeholder": "Veranstaltungen, Personen, Orte und mehr" }, "instruction": "Finde alle Informationen rund ums Studium und den Campus", - "nothing_found": "Keine Ergebnisse" + "nothing_found": "Keine Ergebnisse", + "SUGGESTIONS": "Meintest du" }, "hebisSearch": { "title": "Bibliothekssuche", diff --git a/frontend/app/src/assets/i18n/en.json b/frontend/app/src/assets/i18n/en.json index 1ca91e79..230a8eff 100644 --- a/frontend/app/src/assets/i18n/en.json +++ b/frontend/app/src/assets/i18n/en.json @@ -425,7 +425,8 @@ "placeholder": "Events, places, persons and more" }, "instruction": "Find all information related to your studies and campus", - "nothing_found": "No results" + "nothing_found": "No results", + "SUGGESTIONS": "Did you mean" }, "hebisSearch": { "title": "Library Search", diff --git a/frontend/app/src/environments/environment.ts b/frontend/app/src/environments/environment.ts index b89ada44..4f2093f8 100644 --- a/frontend/app/src/environments/environment.ts +++ b/frontend/app/src/environments/environment.ts @@ -18,7 +18,7 @@ // The list of which env maps to which file can be found in `.angular-cli.json`. export const environment = { - backend_url: 'https://mobile.server.uni-frankfurt.de', + backend_url: 'http://localhost:3000', app_host: 'mobile.app.uni-frankfurt.de', custom_url_scheme: 'de.anyschool.app', backend_version: '999.0.0', diff --git a/packages/api-cli/README.md b/packages/api-cli/README.md index 483bfb37..2e1fc7e5 100644 --- a/packages/api-cli/README.md +++ b/packages/api-cli/README.md @@ -23,7 +23,7 @@ node ./lib/cli.js e2e http://localhost:3000 Example to clone the full database ```shell -node app.js copy "*" https://mobile.app.uni-frankfurt.de http://localhost:3000 100 +node app.js copy "*" -a "999.0.0" https://mobile.server.uni-frankfurt.de http://localhost:3000 100 ``` ### Program arguments diff --git a/packages/core/package.json b/packages/core/package.json index 6682331a..17f928b4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -97,6 +97,8 @@ "date", "validatable", "filterable", + "suggestable", + "completable", "inheritTags", "minLength", "pattern", diff --git a/packages/core/src/protocol/search/completion.ts b/packages/core/src/protocol/search/completion.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/core/src/protocol/search/result.ts b/packages/core/src/protocol/search/result.ts index 26764eec..fa811e2c 100644 --- a/packages/core/src/protocol/search/result.ts +++ b/packages/core/src/protocol/search/result.ts @@ -24,6 +24,11 @@ export interface SCSearchResult { */ data: SCThings[]; + /** + * Suggestions for query corrections + */ + suggestions?: SCSearchSuggestions; + /** * Facets (aggregations over all matching data) */ @@ -40,6 +45,18 @@ export interface SCSearchResult { stats: SCSearchResultSearchEngineStats; } +/** + * Seach suggestions + * + * Not to be confused with search-as-you-type suggestions + */ +export interface SCSearchSuggestions { + /** + * Suggestions for query terms that might have been misspelled + */ + terms?: Record; +} + /** * Stores information about Pagination */ diff --git a/packages/core/src/things/abstract/thing.ts b/packages/core/src/things/abstract/thing.ts index b88b269d..c4c72acd 100644 --- a/packages/core/src/things/abstract/thing.ts +++ b/packages/core/src/things/abstract/thing.ts @@ -63,7 +63,7 @@ export interface SCThingWithoutReferences { /** * Alternate names of the thing * @filterable - * @keyword + * @text */ alternateNames?: string[]; @@ -92,6 +92,8 @@ export interface SCThingWithoutReferences { * @filterable * @minLength 1 * @sortable ducet + * @completable + * @suggestable * @text */ name: string; @@ -240,6 +242,8 @@ export interface SCThingTranslatableProperties { * Translation of the name of the thing * @sortable ducet * @text + * @suggestable + * @completable */ name?: string; /** diff --git a/packages/es-mapping-generator/src/config/fieldmap.ts b/packages/es-mapping-generator/src/config/fieldmap.ts index 6ff1fb28..ae9a0a7b 100644 --- a/packages/es-mapping-generator/src/config/fieldmap.ts +++ b/packages/es-mapping-generator/src/config/fieldmap.ts @@ -43,6 +43,27 @@ export const fieldmap: ElasticsearchFieldmap = { }, ignore: ['price'], }, + suggestable: { + default: { + trigram: { + type: 'text', + analyzer: 'trigram', + }, + reverse: { + type: 'text', + analyzer: 'reverse', + }, + }, + ignore: [], + }, + completable: { + default: { + completion: { + type: 'search_as_you_type', + }, + }, + ignore: [], + }, }; export const filterableTagName = 'filterable'; diff --git a/packages/es-mapping-generator/src/config/settings.ts b/packages/es-mapping-generator/src/config/settings.ts index 24ec9660..0e8b4676 100644 --- a/packages/es-mapping-generator/src/config/settings.ts +++ b/packages/es-mapping-generator/src/config/settings.ts @@ -19,4 +19,27 @@ export const settings: IndicesPutTemplateRequest['settings'] = { 'max_result_window': 30_000, 'number_of_replicas': 0, 'number_of_shards': 1, + 'index': { + analysis: { + analyzer: { + trigram: { + type: 'custom', + tokenizer: 'standard', + filter: ['lowercase', 'shingle'], + }, + reverse: { + type: 'custom', + tokenizer: 'standard', + filter: ['lowercase', 'reverse'], + }, + }, + filter: { + shingle: { + type: 'shingle', + min_shingle_size: 2, + max_shingle_size: 3, + }, + }, + }, + }, };