mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-20 16:42:56 +00:00
feat: add backend
This commit is contained in:
Anselm Stordeur
Rainer Killinger
408
src/storage/elasticsearch/query.ts
Normal file
408
src/storage/elasticsearch/query.ts
Normal file
@@ -0,0 +1,408 @@
|
||||
/*
|
||||
* Copyright (C) 2019 StApps
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
SCBackendConfigurationSearchBoosting,
|
||||
SCConfigFile,
|
||||
SCSearchBooleanFilter,
|
||||
SCSearchFilter,
|
||||
SCSearchQuery,
|
||||
SCSearchSort,
|
||||
SCSportCoursePriceGroup,
|
||||
SCThingsField,
|
||||
SCThingTypes,
|
||||
} from '@openstapps/core';
|
||||
import { ElasticsearchConfig, ScriptSort } from './common';
|
||||
import {
|
||||
ESBooleanFilter,
|
||||
ESBooleanFilterArguments,
|
||||
ESDucetSort,
|
||||
ESFunctionScoreQuery,
|
||||
ESFunctionScoreQueryFunction,
|
||||
ESGeoDistanceFilter,
|
||||
ESGeoDistanceFilterArguments,
|
||||
ESGeoDistanceSort,
|
||||
ESGeoDistanceSortArguments,
|
||||
ESTermFilter,
|
||||
ESTypeFilter,
|
||||
} from './common';
|
||||
|
||||
/**
|
||||
* Builds a boolean filter. Returns an elasticsearch boolean filter
|
||||
*/
|
||||
export function buildBooleanFilter(booleanFilter: SCSearchBooleanFilter): ESBooleanFilterArguments<any> {
|
||||
|
||||
const result: ESBooleanFilterArguments<any> = {
|
||||
minimum_should_match: 0,
|
||||
must: [],
|
||||
must_not: [],
|
||||
should: [],
|
||||
};
|
||||
|
||||
if (booleanFilter.arguments.operation === 'and') {
|
||||
result.must = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
|
||||
}
|
||||
|
||||
if (booleanFilter.arguments.operation === 'or') {
|
||||
result.should = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
|
||||
result.minimum_should_match = 1;
|
||||
}
|
||||
|
||||
if (booleanFilter.arguments.operation === 'not') {
|
||||
result.must_not = booleanFilter.arguments.filters.map((filter) => buildFilter(filter));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts Array of Filters to elasticsearch query-syntax
|
||||
* @param filter
|
||||
*/
|
||||
export function buildFilter(filter: SCSearchFilter): ESTermFilter | ESGeoDistanceFilter | ESBooleanFilter<any> {
|
||||
|
||||
switch (filter.type) {
|
||||
case 'value':
|
||||
const filterObj: { [field: string]: string } = {};
|
||||
filterObj[filter.arguments.field + '.raw'] = filter.arguments.value;
|
||||
return {
|
||||
term: filterObj,
|
||||
};
|
||||
case 'availability':
|
||||
const startRangeFilter: { [field: string]: { lte: string } } = {};
|
||||
startRangeFilter[filter.arguments.fromField] = {
|
||||
lte: 'now',
|
||||
};
|
||||
|
||||
const endRangeFilter: { [field: string]: { gte: string } } = {};
|
||||
endRangeFilter[filter.arguments.toField] = {
|
||||
gte: 'now',
|
||||
};
|
||||
|
||||
return {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
range: startRangeFilter,
|
||||
},
|
||||
{
|
||||
range: endRangeFilter,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
bool: {
|
||||
must_not: [
|
||||
{
|
||||
exists: {
|
||||
field: filter.arguments.fromField,
|
||||
},
|
||||
},
|
||||
{
|
||||
exists: {
|
||||
field: filter.arguments.toField,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
case 'distance':
|
||||
const geoObject: ESGeoDistanceFilterArguments = {
|
||||
distance: filter.arguments.distanceInM + 'm',
|
||||
};
|
||||
geoObject[filter.arguments.field] = {
|
||||
lat: filter.arguments.lat,
|
||||
lon: filter.arguments.lon,
|
||||
};
|
||||
return {
|
||||
geo_distance: geoObject,
|
||||
};
|
||||
case 'boolean':
|
||||
return {
|
||||
bool: buildBooleanFilter(filter),
|
||||
};
|
||||
default:
|
||||
throw new Error('Unknown Filter type');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds scorings functions from boosting config
|
||||
* @param boosting
|
||||
* @returns
|
||||
*/
|
||||
export function buildFunctions(boosting: SCBackendConfigurationSearchBoosting[]): ESFunctionScoreQueryFunction[] {
|
||||
|
||||
const functions: ESFunctionScoreQueryFunction[] = [];
|
||||
|
||||
// add a good scoring subset from config file
|
||||
boosting.forEach((boostingForOneSCType) => {
|
||||
const typeFilter: ESTypeFilter = {
|
||||
type: {
|
||||
value: boostingForOneSCType.type,
|
||||
},
|
||||
};
|
||||
|
||||
functions.push({
|
||||
filter: typeFilter,
|
||||
weight: boostingForOneSCType.factor,
|
||||
});
|
||||
|
||||
if (typeof boostingForOneSCType.fields !== 'undefined') {
|
||||
|
||||
const fields = boostingForOneSCType.fields;
|
||||
|
||||
Object.keys(boostingForOneSCType.fields).forEach((fieldName) => {
|
||||
|
||||
const boostingForOneField = fields[fieldName];
|
||||
|
||||
Object.keys(boostingForOneField).forEach((value) => {
|
||||
const factor = boostingForOneField[value];
|
||||
|
||||
// build term filter
|
||||
const termFilter: ESTermFilter = {
|
||||
term: {},
|
||||
};
|
||||
termFilter.term[fieldName + '.raw'] = value;
|
||||
|
||||
functions.push({
|
||||
filter: {
|
||||
bool: {
|
||||
must: [
|
||||
typeFilter,
|
||||
termFilter,
|
||||
],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
weight: factor,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return functions;
|
||||
}
|
||||
/**
|
||||
* Builds body for Elasticsearch requests
|
||||
* @param params
|
||||
* @param defaultConfig
|
||||
* @returns ElasticsearchQuery (body of a search-request)
|
||||
*/
|
||||
export function buildQuery(
|
||||
params: SCSearchQuery,
|
||||
defaultConfig: SCConfigFile,
|
||||
elasticsearchConfig: ElasticsearchConfig,
|
||||
): ESFunctionScoreQuery {
|
||||
|
||||
// if a sort is used it, we may have to narrow down the types so the sort is executable
|
||||
let typeFiltersToAppend: ESTypeFilter[] = [];
|
||||
|
||||
if (typeof params.sort !== 'undefined') {
|
||||
params.sort.forEach((sort) => {
|
||||
// types that the sort is supported on
|
||||
const types: SCThingTypes[] = [];
|
||||
|
||||
defaultConfig.backend.sortableFields
|
||||
.filter((sortableField) => {
|
||||
return sortableField.fieldName === sort.arguments.field && sortableField.sortTypes.indexOf(sort.type) > -1;
|
||||
})
|
||||
.forEach((sortableField) => {
|
||||
if (typeof sortableField.onlyOnTypes !== 'undefined') {
|
||||
sortableField.onlyOnTypes.forEach((scType) => {
|
||||
if (types.indexOf(scType) === -1) {
|
||||
types.push(scType);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (types.length > 0) {
|
||||
typeFiltersToAppend = types.map((type) => {
|
||||
return {
|
||||
type: {
|
||||
value: type,
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// if config provides an minMatch parameter we use query_string instead of match query
|
||||
let query;
|
||||
if (typeof elasticsearchConfig.query === 'undefined') {
|
||||
query = {
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: '90%',
|
||||
query: (typeof params.query !== 'string') ? '*' : params.query,
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'query_string') {
|
||||
query = {
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.minMatch,
|
||||
query: (typeof params.query !== 'string') ? '*' : params.query,
|
||||
},
|
||||
};
|
||||
} else if (elasticsearchConfig.query.queryType === 'dis_max') {
|
||||
if (params.query !== '*') {
|
||||
query = {
|
||||
dis_max: {
|
||||
boost: 1.2,
|
||||
queries: [
|
||||
{
|
||||
match: {
|
||||
name: {
|
||||
boost: elasticsearchConfig.query.matchBoosting,
|
||||
cutoff_frequency: elasticsearchConfig.query.cutoffFrequency,
|
||||
fuzziness: elasticsearchConfig.query.fuzziness,
|
||||
query: (typeof params.query !== 'string') ? '*' : params.query,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
query_string: {
|
||||
analyzer: 'search_german',
|
||||
default_field: 'name',
|
||||
minimum_should_match: elasticsearchConfig.query.fuzziness,
|
||||
query: (typeof params.query !== 'string') ? '*' : params.query,
|
||||
},
|
||||
},
|
||||
],
|
||||
tie_breaker: elasticsearchConfig.query.tieBreaker,
|
||||
},
|
||||
|
||||
};
|
||||
} else {
|
||||
throw new Error('Query Type is not supported. Check your config file and reconfigure your elasticsearch query');
|
||||
}
|
||||
}
|
||||
|
||||
const functionScoreQuery: ESFunctionScoreQuery = {
|
||||
function_score: {
|
||||
functions: buildFunctions(defaultConfig.internal.boostings),
|
||||
query: {
|
||||
bool: {
|
||||
minimum_should_match: 0, // if we have no should, nothing can match
|
||||
must: [],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
score_mode: 'multiply',
|
||||
},
|
||||
};
|
||||
|
||||
const mustMatch = functionScoreQuery.function_score.query.bool.must;
|
||||
|
||||
if (Array.isArray(mustMatch)) {
|
||||
if (typeof query !== 'undefined') {
|
||||
mustMatch.push(query);
|
||||
}
|
||||
|
||||
if (typeof params.filter !== 'undefined') {
|
||||
mustMatch.push(buildFilter(params.filter));
|
||||
}
|
||||
|
||||
// add type filters for sorts
|
||||
mustMatch.push.apply(mustMatch, typeFiltersToAppend);
|
||||
}
|
||||
return functionScoreQuery;
|
||||
}
|
||||
|
||||
/**
|
||||
* converts query to
|
||||
* @param params
|
||||
* @param sortableFields
|
||||
* @returns an array of sort queries
|
||||
*/
|
||||
export function buildSort(
|
||||
sorts: SCSearchSort[],
|
||||
): Array<ESDucetSort | ESGeoDistanceSort | ScriptSort> {
|
||||
return sorts.map((sort) => {
|
||||
switch (sort.type) {
|
||||
case 'ducet':
|
||||
const ducetSort: ESDucetSort = {};
|
||||
ducetSort[sort.arguments.field + '.sort'] = sort.order;
|
||||
return ducetSort;
|
||||
case 'distance':
|
||||
const args: ESGeoDistanceSortArguments = {
|
||||
mode: 'avg',
|
||||
order: sort.order,
|
||||
unit: 'm',
|
||||
};
|
||||
|
||||
args[sort.arguments.field] = {
|
||||
lat: sort.arguments.lat,
|
||||
lon: sort.arguments.lon,
|
||||
};
|
||||
|
||||
return {
|
||||
_geo_distance: args,
|
||||
};
|
||||
case 'price':
|
||||
return {
|
||||
_script: {
|
||||
order: sort.order,
|
||||
script: buildPriceSortScript(sort.arguments.universityRole, sort.arguments.field),
|
||||
type: 'number' as 'number',
|
||||
},
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function buildPriceSortScript(universityRole: keyof SCSportCoursePriceGroup, field: SCThingsField): string {
|
||||
return `
|
||||
// initialize the sort value with the maximum
|
||||
double price = Double.MAX_VALUE;
|
||||
|
||||
// if we have any offers
|
||||
if (params._source.containsKey('${field}')) {
|
||||
// iterate through all offers
|
||||
for (offer in params._source.${field}) {
|
||||
// if this offer contains a role specific price
|
||||
if (offer.containsKey('prices') && offer.prices.containsKey('${universityRole}')) {
|
||||
// if the role specific price is smaller than the cheapest we found
|
||||
if (offer.prices.${universityRole} < price) {
|
||||
// set the role specific price as cheapest for now
|
||||
price = offer.prices.${universityRole};
|
||||
}
|
||||
} else { // we have no role specific price for our role in this offer
|
||||
// if the default price of this offer is lower than the cheapest we found
|
||||
if (offer.price < price) {
|
||||
// set this price as the cheapest
|
||||
price = offer.price;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// return cheapest price for our role
|
||||
return price;
|
||||
`;
|
||||
}
|
||||
Reference in New Issue
Block a user