/* * Copyright (C) 2019-2022 Open StApps * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, version 3. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ import {asyncPool} from '@krlwlfrt/async-pool/lib/async-pool'; import { isThing, SCAssociatedThingWithoutReferences, SCBulkResponse, SCBulkRoute, SCLicensePlate, SCNamespaces, SCThings, SCThingType, SCThingUpdateResponse, SCThingUpdateRoute, } from '@openstapps/core'; import moment from 'moment'; import clone = require('rfdc'); import {v5} from 'uuid'; import {Bulk} from './bulk'; import {Client} from './client'; import {EmptyBulkError, NamespaceNotDefinedError} from './errors'; /** * StApps-API client */ export class ConnectorClient extends Client { /** * The default timeout for the bulk to expire */ static readonly BULK_TIMEOUT = 3600; /** * The limit of how many items should be indexed concurrently */ static readonly ITEM_CONCURRENT_LIMIT = 5; /** * Instance of multi search request route */ private readonly bulkRoute = new SCBulkRoute(); /** * Instance of multi search request route */ private readonly thingUpdateRoute = new SCThingUpdateRoute(); /** * Make a UUID from a UID and a namespace ID * * Note: valid namespace IDs are license plates of StApps universities. * See documentation of `NAMESPACES` for valid namespace IDs.* * * @param uid UID to make UUID from * @param namespaceId Namespace ID to use to make UUID */ static makeUUID(uid: string, namespaceId: SCLicensePlate): string { if (typeof SCNamespaces[namespaceId] === 'undefined') { throw new NamespaceNotDefinedError(namespaceId); } return v5(uid.toString(), SCNamespaces[namespaceId]!); } /** * Remove fields from a thing that are references * * This effectively turns a thing into a thing without references, e.g. SCDish into SCDishWithoutReferences. * * @param thing Thing to remove references from */ static removeReferences(thing: THING): SCAssociatedThingWithoutReferences { // eslint-disable-next-line @typescript-eslint/no-explicit-any const thingWithoutReferences = clone()(thing); delete thingWithoutReferences.origin; // iterate over all properties for (const key in thingWithoutReferences) { /* istanbul ignore if */ if (!thingWithoutReferences.hasOwnProperty(key)) { continue; } const property = thingWithoutReferences[key]; // check if property itself is a thing if (isThing(property)) { // delete said property delete thingWithoutReferences[key]; continue; } // check if property is an array if (Array.isArray(property)) { if (property.every(isThing)) { // delete property if every item in it is a thing delete thingWithoutReferences[key]; } else { // check every item in array for (const item of property) { if (['boolean', 'number', 'string'].includes(typeof item)) { // skip primitives continue; } // check every property for (const itemKey in item) { /* istanbul ignore if */ if (!item.hasOwnProperty(itemKey)) { continue; } if (isThing(item[itemKey])) { // delete properties that are things delete item[itemKey]; } } } } } else if (typeof property === 'object') { // iterate over all properties in nested objects for (const nestedKey in property) { if (isThing(property[nestedKey])) { // delete properties that are things delete property[nestedKey]; } } } } return thingWithoutReferences as SCAssociatedThingWithoutReferences; } /** * Recursively deletes all undefined properties from an object instance * * @param object Object to delete undefined properties from */ static removeUndefinedProperties(object: object): void { // return atomic data types and arrays (recursion anchor) if (typeof object !== 'object' || Array.isArray(object)) { return; } // check each key for (const key in object) { /* istanbul ignore if */ if (!object.hasOwnProperty(key)) { continue; } const indexedObject = object as {[k: string]: unknown}; if (typeof indexedObject[key] === 'undefined') { // delete undefined keyss delete indexedObject[key]; } else { // check recursive ConnectorClient.removeUndefinedProperties(indexedObject[key] as object); } } return; } /** * Request a bulk transfer to the backend * * This uses the Bulk API supplied by the backend and returns an object that can be used * just like the client itself, while handling the information necessary in bulk transfers. * * @param type StAppsCore thing type * @param source Source identifier (should be unique per actual data source) * @param timeout Timeout in seconds when the bulk should expire */ async bulk(type: SCThingType, source: string, timeout?: number): Promise> { // set default value for timeout to one hour const bulkTimeout = typeof timeout !== 'number' ? ConnectorClient.BULK_TIMEOUT : timeout; const bulkData = await this.invokeRoute(this.bulkRoute, undefined, { expiration: moment().add(bulkTimeout, 'seconds').format(), source: source, type: type, }); return new Bulk(type, this, bulkData); } /** * Index a list of things * * Note that source is optional but is set to `'stapps-api'` in that case. * This will override any previous bulk that you indexed with that source. * * @param things List of things to index * @param source Source of the things * @param timeout Timeout of the bulk in seconds * @see ConnectorClient.bulk */ async index(things: T[], source?: string, timeout?: number): Promise { // check that number of things is not zero if (things.length === 0) { throw new EmptyBulkError(); } // set default source if none is given const thingSource = typeof source === 'undefined' ? 'stapps-api' : source; // request a new bulk const bulk = await this.bulk(things[0].type, thingSource, timeout); // add items to the bulk - 5 concurrently await asyncPool(ConnectorClient.ITEM_CONCURRENT_LIMIT, things, thing => bulk.add(thing)); // close bulk await bulk.done(); } /** * Update an existing StAppsCore thing * * @param thing StAppsCore thing to update */ async update(thing: SCThings): Promise { return this.invokeRoute( this.thingUpdateRoute, { TYPE: encodeURIComponent(thing.type), UID: encodeURIComponent(thing.uid), }, thing, ); } }