/* * Copyright (C) 2019 StApps * This program is free software: you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation, version 3. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program. If not, see . */ import {asyncPool} from '@krlwlfrt/async-pool/lib/async-pool'; import { isThing, SCAssociatedThingWithoutReferences, SCBulkResponse, SCBulkRoute, SCLicensePlate, SCNamespaces, SCThings, SCThingType, SCThingUpdateResponse, SCThingUpdateRoute, } from '@openstapps/core'; import clone = require('fast-clone'); import moment from 'moment'; import {v5} from 'uuid'; import {Bulk} from './bulk'; import {Client} from './client'; import {EmptyBulkError, NamespaceNotDefinedError} from './errors'; /** * StApps-API client */ export class ConnectorClient extends Client { /** * The default timeout for the bulk to expire */ static readonly BULK_TIMEOUT = 3600; /** * The limit of how many items should be indexed concurrently */ static readonly ITEM_CONCURRENT_LIMIT = 5; /** * Instance of multi search request route */ private readonly bulkRoute = new SCBulkRoute(); /** * Instance of multi search request route */ private readonly thingUpdateRoute = new SCThingUpdateRoute(); /** * Make a UUID from a UID and a namespace ID * * *Note: valid namespace IDs are license plates of StApps universities. * See documentation of `NAMESPACES` for valid namespace IDs.* * * @param uid UID to make UUID from * @param namespaceId Namespace ID to use to make UUID */ static makeUUID(uid: string, namespaceId: SCLicensePlate): string { if (typeof SCNamespaces[namespaceId] === 'undefined') { throw new NamespaceNotDefinedError(namespaceId); } return v5(uid.toString(), SCNamespaces[namespaceId]!); } /** * Remove fields from a thing that are references * * This effectively turns a thing into a thing without references, e.g. SCDish into SCDishWithoutReferences. * * @param thing Thing to remove references from */ static removeReferences(thing: THING): SCAssociatedThingWithoutReferences { // tslint:disable-next-line:no-any const thingWithoutReferences = clone(thing); delete thingWithoutReferences.origin; // iterate over all properties for (const key in thingWithoutReferences) { /* istanbul ignore if */ if (!thingWithoutReferences.hasOwnProperty(key)) { continue; } const property = thingWithoutReferences[key]; // check if property itself is a thing if (isThing(property)) { // delete said property delete thingWithoutReferences[key]; continue; } // check if property is an array if (Array.isArray(property)) { if (property.every(isThing)) { // delete property if every item in it is a thing delete thingWithoutReferences[key]; } else { // check every item in array for (const item of property) { if (['boolean', 'number', 'string'].indexOf(typeof item) >= 0) { // skip primitives continue; } // check every property for (const itemKey in item) { /* istanbul ignore if */ if (!item.hasOwnProperty(itemKey)) { continue; } if (isThing(item[itemKey])) { // delete properties that are things delete item[itemKey]; } } } } } else if (typeof property === 'object') { // iterate over all properties in nested objects for (const nestedKey in property) { if (isThing(property[nestedKey])) { // delete properties that are things delete property[nestedKey]; } } } } return thingWithoutReferences as SCAssociatedThingWithoutReferences; } /** * Recursively deletes all undefined properties from an object instance * * @param obj Object to delete undefined properties from */ static removeUndefinedProperties(obj: object): void { // return atomic data types and arrays (recursion anchor) if (typeof obj !== 'object' || Array.isArray(obj)) { return; } // check each key for (const key in obj) { /* istanbul ignore if */ if (!obj.hasOwnProperty(key)) { continue; } const indexedObj = obj as { [k: string]: unknown; }; if (typeof indexedObj[key] === 'undefined') { // delete undefined keyss delete indexedObj[key]; } else { // check recursive ConnectorClient.removeUndefinedProperties(indexedObj[key] as object); } } return; } /** * Request a bulk transfer to the backend * * This uses the Bulk API supplied by the backend and returns an object that can be used * just like the client itself, while handling the information necessary in bulk transfers. * * @param type StAppsCore thing type * @param source Source identifier (should be unique per actual data source) * @param timeout Timeout in seconds when the bulk should expire */ async bulk(type: SCThingType, source: string, timeout?: number): Promise> { let bulkTimeout: number; // set default value for timeout to one hour if (typeof timeout !== 'number') { bulkTimeout = ConnectorClient.BULK_TIMEOUT; } else { bulkTimeout = timeout; } const bulkData = await this.invokeRoute(this.bulkRoute, undefined, { expiration: moment() .add(bulkTimeout, 'seconds') .format(), source: source, type: type, }); return new Bulk(type, this, bulkData); } /** * Index a list of things * * Note that source is optional but is set to `'stapps-api'` in that case. * This will override any previous bulk that you indexed with that source. * * @param things List of things to index * @param source Source of the things * @param timeout Timeout of the bulk in seconds * @see ConnectorClient.bulk */ async index(things: T[], source?: string, timeout?: number): Promise { // check that number of things is not zero if (things.length === 0) { throw new EmptyBulkError(); } let thingSource: string; // set default source if none is given if (typeof source === 'undefined') { thingSource = 'stapps-api'; } else { thingSource = source; } // request a new bulk const bulk = await this.bulk(things[0].type, thingSource, timeout); // add items to the bulk - 5 concurrently await asyncPool(ConnectorClient.ITEM_CONCURRENT_LIMIT, things, (thing) => bulk.add(thing)); // close bulk await bulk.done(); } /** * Update an existing StAppsCore thing * * @param thing StAppsCore thing to update */ async update(thing: SCThings): Promise { return this.invokeRoute(this.thingUpdateRoute, { TYPE: encodeURIComponent(thing.type), UID: encodeURIComponent(thing.uid), }, thing); } }