openstapps/src/storage/elasticsearch/common.ts

/*
 * Copyright (C) 2019 StApps
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
import {SCThingType} from '@openstapps/core';
import {SCThing} from '@openstapps/core';
import {
  ESAggMatchAllFilter,
  ESAggTypeFilter, ESNestedAggregation,
  ESTermsFilter,
} from '@openstapps/core-tools/lib/mappings/aggregation-definitions';
// we only have the @types package because some things type definitions are still missing from the official
// @elastic/elasticsearch package
// tslint:disable-next-line:no-implicit-dependencies
import {NameList} from 'elasticsearch';

/**
 * An elasticsearch aggregation bucket
 */
interface Bucket {
  /**
   * Number of documents in the aggregation bucket
   */
  doc_count: number;

  /**
   * Text representing the documents in the bucket
   */
  key: string;
}

/**
 * An elasticsearch aggregation response
 */
export interface AggregationResponse {
  /**
   * The individual aggregations
   */
  [field: string]: BucketAggregation | NestedAggregation;
}

/**
 * An elasticsearch bucket aggregation
 */
export interface BucketAggregation {
  /**
   * Buckets in an aggregation
   */
  buckets: Bucket[];

  /**
   * Number of documents in an aggregation
   */
  doc_count?: number;
}

/**
 * Checks if the type is a BucketAggregation
 * @param agg the type to check
 */
export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
  return typeof agg !== 'number';
}

/**
 * An aggregation that contains more aggregations nested inside
 */
export interface NestedAggregation {
  /**
   * Number of documents in an aggregation
   */
  doc_count: number;

  /**
   * Any nested responses
   */
  [name: string]: BucketAggregation | number;
}

/**
 * Checks if the type is a NestedAggregation
 * @param agg the type to check
 */
export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
  return typeof (agg as BucketAggregation).buckets === 'undefined';
}

/**
 * An elasticsearch bucket aggregation
 * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
 */
export interface AggregationSchema {
  [aggregationName: string]: ESTermsFilter | ESNestedAggregation;
}

/**
 * A configuration for using the Dis Max Query
 *
 * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-dis-max-query.html for further
 * explanation of what the parameters mean
 */
export interface ElasticsearchQueryDisMaxConfig {
  /**
   * Relative (to a total number of documents) or absolute number to exclude meaningless matches that frequently appear
   */
  cutoffFrequency: number;

  /**
   * The maximum allowed Levenshtein Edit Distance (or number of edits)
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/common-options.html#fuzziness
   */
  fuzziness: number | string;

  /**
   * Increase the importance (relevance score) of a field
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-boost.html
   */
  matchBoosting: number;

  /**
   * Minimal number (or percentage) of words that should match in a query
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
   */
  minMatch: string;

  /**
   * Type of the query - in this case 'dis_max' which is a union of its subqueries
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-dis-max-query.html
   */
  queryType: 'dis_max';

  /**
   * Changes behavior of default calculation of the score when multiple results match
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-multi-match-query.html#tie-breaker
   */
  tieBreaker: number;
}

/**
 * A configuration for using Query String Query
 *
 * See https://www.elastic.co/guide/en/elasticsearch/reference/5.5/query-dsl-query-string-query.html for further
 * explanation of what the parameters mean
 */
export interface ElasticsearchQueryQueryStringConfig {
  /**
   * Minimal number (or percentage) of words that should match in a query
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
   */
  minMatch: string;

  /**
   * Type of the query - in this case 'query_string' which uses a query parser in order to parse content
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-query-string-query.html
   */
  queryType: 'query_string';
}

/**
 * A hit in an elasticsearch search result
 * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-fields.html
 */
export interface ElasticsearchObject<T extends SCThing> {
  /**
   * Unique identifier of a document (object)
   */
  _id: string;

  /**
   * The index to which the document belongs
   */
  _index: string;

  /**
   * Relevancy of the document to a query
   */
  _score: number;

  /**
   * The original JSON representing the body of the document
   */
  _source: T;

  /**
   * The document's mapping type
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping-type-field.html
   */
  _type: string;

  /**
   * Version of the document
   */
  _version?: number;

  /**
   * Used to index the same field in different ways for different purposes
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/multi-fields.html
   */
  fields?: NameList;

  /**
   * Used to highlight search results on one or more fields
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-highlighting.html
   */
  // tslint:disable-next-line: no-any
  highlight?: any;

  /**
   * Used in when nested/children documents match the query
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-request-inner-hits.html
   */
  // tslint:disable-next-line: no-any
  inner_hits?: any;

  /**
   * Queries that matched for documents in results
   */
  matched_queries?: string[];

  /**
   * Sorting definition
   */
  sort?: string[];
}

/**
 * An config file for the elasticsearch database interface
 *
 * The config file extends the SCConfig file by further defining how the database property
 */
export interface ElasticsearchConfigFile {
  /**
   * Configuration that is not visible to clients
   */
  internal: {
    /**
     * Database configuration
     */
    database: ElasticsearchConfig;
  };
}

/**
 * An elasticsearch configuration
 */
export interface ElasticsearchConfig {
  /**
   * Name of the database
   */
  name: 'elasticsearch';

  /**
   * Configuration for using queries
   */
  query?: ElasticsearchQueryDisMaxConfig | ElasticsearchQueryQueryStringConfig;

  /**
   * Version of the used elasticsearch
   */
  version: string;
}

/**
 * An elasticsearch term filter
 */
export type ESTermFilter = {
  /**
   * Definition of a term to match
   */
  term: {
    [fieldName: string]: string;
  };
} | {
  /**
   * Definition of terms to match (or)
   */
  terms: {
    [fieldName: string]: string[];
  };
};

export interface ESGenericRange<T> {
  /**
   * Greater than field
   */
  gt?: T;

  /**
   * Greater or equal than field
   */
  gte?: T;

  /**
   * Less than field
   */
  lt?: T;

  /**
   * Less or equal than field
   */
  lte?: T;
}

interface ESGenericRangeFilter<G, T extends ESGenericRange<G>> {
  /**
   * Range filter definition
   */
  range: {
    [fieldName: string]: T;
  };
}

export interface ESDateRange extends ESGenericRange<string> {
  /**
   * Optional date format override
   */
  format?: string;

  /**
   * Optional timezone specifier
   */
  time_zone?: string;
}

export type ESNumericRangeFilter = ESGenericRangeFilter<number, ESGenericRange<number>>;
export type ESDateRangeFilter = ESGenericRangeFilter<string, ESDateRange>;
export type ESRangeFilter = ESNumericRangeFilter | ESDateRangeFilter;

/**
 * Checks if the parameter is of type ESTermsFilter
 * @param agg the value to check
 */
export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
  return typeof (agg as ESTermsFilter).terms !== 'undefined';
}

/**
 * Checks if the parameter is of type ESTermsFilter
 * @param agg the value to check
 */
export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
  return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
}

/**
 * Checks if the parameter is of type
 *
 * @param filter the filter to narrow the type of
 */
export function isESAggMatchAllFilter(filter: ESAggTypeFilter | ESAggMatchAllFilter): filter is ESAggMatchAllFilter {
  return filter.hasOwnProperty('match_all');
}

/**
 * An elasticsearch type filter
 */
export interface ESTypeFilter {
  /**
   * Type filter definition
   */
  type: {
    /**
     * Type name (SCThingType) to filter with
     */
    value: SCThingType;
  };
}

/**
 * Filter arguments for an elasticsearch geo distance filter
 */
export interface ESGeoDistanceFilterArguments {
  /**
   * The radius of the circle centred on the specified location
   */
  distance: string;

  [fieldName: string]: {
    /**
     * Latitude
     */
    lat: number;

    /**
     * Longitude
     */
    lon: number;
  } | string;
}

/**
 * An elasticsearch geo distance filter
 */
export interface ESGeoDistanceFilter {
  /**
   * @see ESGeoDistanceFilterArguments
   */
  geo_distance: ESGeoDistanceFilterArguments;
}

/**
 * Filter arguments for an elasticsearch boolean filter
 * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-bool-query.html
 */
export interface ESBooleanFilterArguments<T> {
  /**
   * Minimal number (or percentage) of words that should match in a query
   * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-minimum-should-match.html
   */
  minimum_should_match?: number;

  /**
   * The clause (query) must appear in matching documents and will contribute to the score.
   */
  must?: T[];

  /**
   * The clause (query) must not appear in the matching documents.
   */
  must_not?: T[];

  /**
   * The clause (query) should appear in the matching document.
   */
  should?: T[];
}

/**
 * An elasticsearch boolean filter
 */
export interface ESBooleanFilter<T> {
  /**
   * @see ESBooleanFilterArguments
   */
  bool: ESBooleanFilterArguments<T>;
}

/**
 * An elasticsearch function score query
 * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/query-dsl-function-score-query.html
 */
export interface ESFunctionScoreQuery {
  /**
   * Function score definition
   */
  function_score: {
    /**
     * Functions that compute score for query results (documents)
     * @see ESFunctionScoreQueryFunction
     */
    functions: ESFunctionScoreQueryFunction[];

    /**
     * @see ESBooleanFilter
     */
    query: ESBooleanFilter<unknown>;

    /**
     * Specifies how the computed scores are combined
     */
    score_mode: 'multiply';
  };
}

/**
 * An function for an elasticsearch functions score query
 */
export interface ESFunctionScoreQueryFunction {
  /**
   * Function is applied only if a document matches the given filtering query
   */
  filter: ESTermFilter | ESTypeFilter | ESBooleanFilter<ESTermFilter | ESTypeFilter>;

  /**
   * Weight (importance) of the filter
   */
  weight: number;
}

/**
 * An elasticsearch generic sort
 */
export interface ESGenericSort {
  [field: string]: string;
}

/**
 * Sort arguments for an elasticsearch geo distance sort
 */
export interface ESGeoDistanceSortArguments {
  /**
   * What value to pick for sorting
   */
  mode: 'avg' | 'max' | 'median' | 'min';

  /**
   * Order
   */
  order: 'asc' | 'desc';

  /**
   * Value unit
   */
  unit: 'm';

  [field: string]: {
    /**
     * Latitude
     */
    lat: number;

    /**
     * Longitude
     */
    lon: number;
  } | string;
}

/**
 * An elasticsearch geo distance sort
 */
export interface ESGeoDistanceSort {
  /**
   * @see ESGeoDistanceFilterArguments
   */
  _geo_distance: ESGeoDistanceSortArguments;
}

/**
 * An elasticsearch script sort
 */
export interface ScriptSort {
  /**
   * A script
   */
  _script: {
    /**
     * Order
     */
    order: 'asc' | 'desc';

    /**
     * The custom script used for sorting
     */
    script: string;

    /**
     * What type is being sorted
     */
    type: 'number' | 'string';
  };
}