mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-21 17:12:43 +00:00
feat: add api
This commit is contained in:
83
src/bulk.ts
Normal file
83
src/bulk.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
SCBulkAddResponse,
|
||||
SCBulkAddRoute,
|
||||
SCBulkDoneResponse,
|
||||
SCBulkDoneRoute,
|
||||
SCBulkResponse,
|
||||
SCThings,
|
||||
SCThingTypes,
|
||||
} from '@openstapps/core';
|
||||
import {Client} from './client';
|
||||
import {BulkWithMultipleTypesError} from './errors';
|
||||
|
||||
/**
|
||||
* A bulk
|
||||
*
|
||||
* **!!! Bulk should only be instantiated by Client !!!**
|
||||
*/
|
||||
export class Bulk<T extends SCThings> {
|
||||
/**
|
||||
* Instance of multi search request route
|
||||
*/
|
||||
private readonly bulkAddRoute = new SCBulkAddRoute();
|
||||
|
||||
/**
|
||||
* Instance of multi search request route
|
||||
*/
|
||||
private readonly bulkDoneRoute = new SCBulkDoneRoute();
|
||||
|
||||
/**
|
||||
* **!!! Bulk should only be instantiated by Client !!!**
|
||||
*
|
||||
* @see Client.bulk
|
||||
*/
|
||||
constructor(
|
||||
private type: SCThingTypes,
|
||||
private client: Client,
|
||||
private bulkResponse: SCBulkResponse,
|
||||
) {
|
||||
// noop
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a thing to the bulk
|
||||
*
|
||||
* @param thing Thing to add to the bulk
|
||||
*/
|
||||
async add(thing: T): Promise<SCBulkAddResponse> {
|
||||
// check that thing has same type as bulk
|
||||
if (this.type !== thing.type) {
|
||||
throw new BulkWithMultipleTypesError(thing);
|
||||
}
|
||||
|
||||
return this.client.invokeRoute<SCBulkAddResponse>(this.bulkAddRoute, {
|
||||
UID: encodeURIComponent(this.bulkResponse.uid),
|
||||
}, thing);
|
||||
}
|
||||
|
||||
/**
|
||||
* Declare this bulk transfer as done
|
||||
*
|
||||
* This will activate the index in the backend and possibly delete old data. There are many potential processing steps
|
||||
* required in the backend so it might take a few seconds before the callback is called.
|
||||
*/
|
||||
async done(): Promise<SCBulkDoneResponse> {
|
||||
return this.client.invokeRoute<SCBulkDoneResponse>(this.bulkDoneRoute, {
|
||||
UID: this.bulkResponse.uid,
|
||||
});
|
||||
}
|
||||
}
|
||||
77
src/cli.ts
Executable file
77
src/cli.ts
Executable file
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {Logger} from '@openstapps/logger';
|
||||
import * as commander from 'commander';
|
||||
import {readFileSync} from 'fs';
|
||||
import {join} from 'path';
|
||||
import {copy} from './copy';
|
||||
import {HttpClient} from './httpClient';
|
||||
|
||||
const logger = new Logger();
|
||||
const pkgJson = JSON.parse(readFileSync(join(__dirname, '..', 'package.json')).toString());
|
||||
|
||||
const client = new HttpClient();
|
||||
|
||||
let actionDone = false;
|
||||
|
||||
process.on('unhandledRejection', (error) => {
|
||||
logger.error('unhandledRejection', error);
|
||||
});
|
||||
|
||||
commander
|
||||
.command('copy <from> <to>')
|
||||
.version(pkgJson.version)
|
||||
.description('Copy data from one instance to another')
|
||||
.option(
|
||||
'-s, --bulkSource [bulkSource]',
|
||||
'The source identifier for the bulk to use with the target instance [copy]',
|
||||
'copy',
|
||||
)
|
||||
.option('-t, --type <type>', 'The type to request from the source instance')
|
||||
// TODO: remove
|
||||
.option('-a, --appVersion [version]', 'The App version to use [unset by default]')
|
||||
.option('-b, --batchSize <amount>', 'Number of items per batch')
|
||||
.allowUnknownOption(false)
|
||||
.action((from, to, copyCommand) => {
|
||||
actionDone = true;
|
||||
|
||||
if (typeof copyCommand.type === 'undefined') {
|
||||
logger.error('<type> argument is required');
|
||||
copyCommand.outputHelp();
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
logger.info('Copying ' + copyCommand.type + ' objects from ' + from + ' to ' + to);
|
||||
|
||||
copy(client, {
|
||||
batchSize: parseInt(copyCommand.batchSize, 10),
|
||||
from: from,
|
||||
source: copyCommand.bulkSource,
|
||||
to: to,
|
||||
type: copyCommand.type,
|
||||
version: copyCommand.appVersion,
|
||||
}).then(() => {
|
||||
logger.ok('Done');
|
||||
}, (err) => {
|
||||
throw err;
|
||||
});
|
||||
});
|
||||
|
||||
commander
|
||||
.parse(process.argv);
|
||||
|
||||
if (!actionDone) {
|
||||
commander.outputHelp();
|
||||
}
|
||||
274
src/client.ts
Normal file
274
src/client.ts
Normal file
@@ -0,0 +1,274 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
SCAbstractRoute,
|
||||
SCFeedbackRequest,
|
||||
SCFeedbackResponse,
|
||||
SCFeedbackRoute,
|
||||
SCIndexRequest,
|
||||
SCIndexResponse,
|
||||
SCIndexRoute,
|
||||
SCInternalServerErrorResponse,
|
||||
SCMultiSearchRequest,
|
||||
SCMultiSearchResponse,
|
||||
SCMultiSearchRoute,
|
||||
SCNotFoundErrorResponse,
|
||||
SCRequests,
|
||||
SCSearchRequest,
|
||||
SCSearchResponse,
|
||||
SCSearchRoute,
|
||||
SCThings,
|
||||
} from '@openstapps/core';
|
||||
import {ApiError, CoreVersionIncompatibleError, OutOfRangeError} from './errors';
|
||||
import {HttpClientHeaders, HttpClientInterface} from './httpClientInterface';
|
||||
|
||||
/**
|
||||
* StApps-API client
|
||||
*/
|
||||
export class Client {
|
||||
/**
|
||||
* Instance of feedback request route
|
||||
*/
|
||||
private readonly feedbackRoute = new SCFeedbackRoute();
|
||||
|
||||
/**
|
||||
* Instance of index route
|
||||
*/
|
||||
private readonly indexRoute = new SCIndexRoute();
|
||||
|
||||
/**
|
||||
* Instance of multi search request route
|
||||
*/
|
||||
private readonly multiSearchRoute = new SCMultiSearchRoute();
|
||||
|
||||
/**
|
||||
* Instance of search request route
|
||||
*/
|
||||
private readonly searchRoute = new SCSearchRoute();
|
||||
|
||||
/**
|
||||
* Default headers
|
||||
*
|
||||
* TODO: remove headers
|
||||
*/
|
||||
protected readonly headers: HttpClientHeaders = {};
|
||||
|
||||
/**
|
||||
* Create a new search request with altered pagination parameters to move to the next result window
|
||||
*
|
||||
* @param searchRequest Last search request
|
||||
* @param searchResponse Search response for supplied search request
|
||||
* @throws OutOfRangeError Throws an error if the next window is beyond the total number of results
|
||||
*/
|
||||
static nextWindow(searchRequest: SCSearchRequest, searchResponse: SCSearchResponse): SCSearchRequest {
|
||||
// calculate next from
|
||||
const from = searchResponse.pagination.offset + searchResponse.pagination.count;
|
||||
|
||||
// throw an error if the next window is beyond the total number of results
|
||||
if (from >= searchResponse.pagination.total) {
|
||||
throw new OutOfRangeError(searchRequest);
|
||||
}
|
||||
|
||||
// return a search request with the next window
|
||||
return {
|
||||
...searchRequest,
|
||||
from,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate a new StApps-API client to communicate with a StApps-backend.
|
||||
*
|
||||
* @param httpClient HTTP client to use
|
||||
* @param url URL of the backend
|
||||
* @param version App version to use when requesting data *(only necessary if URL is ambiguous)*
|
||||
*
|
||||
* TODO: remove headers/version
|
||||
*/
|
||||
constructor(protected httpClient: HttpClientInterface, protected url: string, protected version?: string) {
|
||||
if (typeof version === 'string') {
|
||||
// set header to tell proxy to select the correct backend
|
||||
this.headers = {
|
||||
'X-StApps-Version': this.version,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send feedback
|
||||
*
|
||||
* @param feedback Feedback to send
|
||||
*/
|
||||
async feedback(feedback: SCFeedbackRequest): Promise<SCFeedbackResponse> {
|
||||
return await this.invokeRoute<SCFeedbackResponse>(this.feedbackRoute, undefined, feedback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a thing by its UID
|
||||
*
|
||||
* @param uid UID of the thing to fetch
|
||||
*/
|
||||
async getThing(uid: string): Promise<SCThings> {
|
||||
const response = await this.search({
|
||||
filter: {
|
||||
arguments: {
|
||||
field: 'uid',
|
||||
value: uid,
|
||||
},
|
||||
type: 'value',
|
||||
},
|
||||
size: 1,
|
||||
});
|
||||
|
||||
if (response.data.length === 1 && response.data[0].uid === uid) {
|
||||
return response.data[0];
|
||||
}
|
||||
|
||||
throw new SCInternalServerErrorResponse(new SCNotFoundErrorResponse(true), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a handshake with the backend and check StAppsCore version
|
||||
*
|
||||
* @param coreVersion StAppsCore version to check
|
||||
*/
|
||||
async handshake(coreVersion: string): Promise<SCIndexResponse> {
|
||||
const request: SCIndexRequest = {};
|
||||
|
||||
const response = await this.invokeRoute<SCIndexResponse>(this.indexRoute, undefined, request);
|
||||
|
||||
if (response.backend.SCVersion.split('.')[0] !== coreVersion.split('.')[0]) {
|
||||
throw new CoreVersionIncompatibleError(coreVersion, response.backend.SCVersion);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke a route
|
||||
*
|
||||
* @param route Route to invoke
|
||||
* @param parameters Parameters for the URL fragment
|
||||
* @param body Body for the request
|
||||
*/
|
||||
async invokeRoute<T>(route: SCAbstractRoute,
|
||||
parameters?: { [k: string]: string },
|
||||
body?: SCRequests): Promise<T> {
|
||||
// make the request
|
||||
const response = await this.httpClient.request({
|
||||
body: body,
|
||||
// TODO: remove headers
|
||||
headers: this.headers,
|
||||
method: route.method,
|
||||
url: new URL(this.url + route.getUrlFragment(parameters)),
|
||||
});
|
||||
|
||||
if (response.statusCode === route.statusCodeSuccess) {
|
||||
return response.body as T;
|
||||
}
|
||||
|
||||
throw new ApiError(response.body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a multi search request to the backend
|
||||
*
|
||||
* All results will be returned for requests where no size is set.
|
||||
*
|
||||
* @param multiSearchRequest Multi search request
|
||||
*/
|
||||
async multiSearch(multiSearchRequest: SCMultiSearchRequest): Promise<SCMultiSearchResponse> {
|
||||
const preFlightRequest: SCMultiSearchRequest = {};
|
||||
let preFlightNecessary = false;
|
||||
|
||||
// gather search requests where size is not set
|
||||
Object.keys(multiSearchRequest).forEach((key) => {
|
||||
const searchRequest = multiSearchRequest[key];
|
||||
|
||||
if (typeof searchRequest.size === 'undefined') {
|
||||
preFlightRequest[key] = {
|
||||
...searchRequest,
|
||||
};
|
||||
preFlightRequest[key].size = 0;
|
||||
preFlightNecessary = true;
|
||||
}
|
||||
});
|
||||
|
||||
if (preFlightNecessary) {
|
||||
// copy multi search request
|
||||
multiSearchRequest = {
|
||||
...multiSearchRequest,
|
||||
};
|
||||
|
||||
// make pre flight request
|
||||
const preFlightResponse = await this.invokeRoute<SCMultiSearchResponse>(
|
||||
this.multiSearchRoute,
|
||||
undefined,
|
||||
preFlightRequest,
|
||||
);
|
||||
|
||||
// set size for multi search requests that were in pre flight request
|
||||
Object.keys(preFlightRequest).forEach((key) => {
|
||||
multiSearchRequest[key].size = preFlightResponse[key].pagination.total;
|
||||
});
|
||||
}
|
||||
|
||||
// actually invoke the route
|
||||
return await this.invokeRoute<SCMultiSearchResponse>(this.multiSearchRoute, undefined, multiSearchRequest);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a search request to the backend
|
||||
*
|
||||
* All results will be returned if no size is set in the request.
|
||||
*
|
||||
* @param searchRequest Search request
|
||||
*/
|
||||
async search(searchRequest: SCSearchRequest): Promise<SCSearchResponse> {
|
||||
let size: number | undefined = searchRequest.size;
|
||||
|
||||
if (typeof size === 'undefined') {
|
||||
const preFlightResponse = await this.invokeRoute<SCSearchResponse>(this.searchRoute, undefined, {
|
||||
...searchRequest,
|
||||
size: 0,
|
||||
});
|
||||
|
||||
size = preFlightResponse.pagination.total;
|
||||
}
|
||||
|
||||
return await this.invokeRoute<SCSearchResponse>(this.searchRoute, undefined, {
|
||||
...searchRequest,
|
||||
size,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the next search results
|
||||
*
|
||||
* @param searchRequest Last search request
|
||||
* @param searchResponse Search response for supplied search request
|
||||
*/
|
||||
async searchNext(searchRequest: SCSearchRequest, searchResponse: SCSearchResponse): Promise<{
|
||||
searchRequest: SCSearchRequest;
|
||||
searchResponse: SCSearchResponse;
|
||||
}> {
|
||||
const nextSearchRequest = Client.nextWindow(searchRequest, searchResponse);
|
||||
|
||||
return {
|
||||
searchRequest: nextSearchRequest,
|
||||
searchResponse: await this.search(nextSearchRequest),
|
||||
};
|
||||
}
|
||||
}
|
||||
139
src/connectorClient.ts
Normal file
139
src/connectorClient.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {
|
||||
SCBulkResponse,
|
||||
SCBulkRoute,
|
||||
SCLicensePlate,
|
||||
SCNamespaces,
|
||||
SCThings,
|
||||
SCThingTypes,
|
||||
SCThingUpdateResponse,
|
||||
SCThingUpdateRoute,
|
||||
} from '@openstapps/core';
|
||||
import {asyncPool} from 'async-pool-native/dist/async-pool';
|
||||
import * as moment from 'moment';
|
||||
import {Bulk} from './bulk';
|
||||
import {Client} from './client';
|
||||
import {EmptyBulkError, NamespaceNotDefinedError} from './errors';
|
||||
|
||||
/* tslint:disable:no-var-requires */
|
||||
|
||||
/**
|
||||
* The package @types/uuid unfortunately doesn't expose the browser versions of the hashing functions.
|
||||
* That's why we need to use a little trickery to get to it.
|
||||
*/
|
||||
const v35 = require('uuid/lib/v35');
|
||||
const sha1Browser = require('uuid/lib/sha1-browser');
|
||||
const v5 = v35('v5', 0x50, sha1Browser);
|
||||
|
||||
/* tslint:enable */
|
||||
|
||||
export class ConnectorClient extends Client {
|
||||
/**
|
||||
* Instance of multi search request route
|
||||
*/
|
||||
private readonly bulkRoute = new SCBulkRoute();
|
||||
|
||||
/**
|
||||
* Instance of multi search request route
|
||||
*/
|
||||
private readonly thingUpdateRoute = new SCThingUpdateRoute();
|
||||
|
||||
/**
|
||||
* Make a UUID from a UID and a namespace ID
|
||||
*
|
||||
* *Note: valid namespace IDs are license plates of StApps universities.
|
||||
* See documentation of `NAMESPACES` for valid namespace IDs.*
|
||||
*
|
||||
* @param uid UID to make UUID from
|
||||
* @param namespaceId Namespace ID to use to make UUID
|
||||
*/
|
||||
static makeUUID(uid: string, namespaceId: SCLicensePlate): string {
|
||||
if (typeof SCNamespaces[namespaceId] === 'undefined') {
|
||||
throw new NamespaceNotDefinedError(namespaceId);
|
||||
}
|
||||
|
||||
return v5(uid.toString(), SCNamespaces[namespaceId]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Request a bulk transfer to the backend
|
||||
*
|
||||
* This uses the Bulk API supplied by the backend and returns an object that can be used
|
||||
* just like the client itself, while handling the information necessary in bulk transfers.
|
||||
*
|
||||
* @param type StAppsCore thing type
|
||||
* @param source Source identifier (should be unique per actual data source)
|
||||
* @param timeout Timeout in seconds when the bulk should expire
|
||||
*/
|
||||
async bulk<T extends SCThings>(type: SCThingTypes, source: string, timeout?: number): Promise<Bulk<T>> {
|
||||
// set default value for timeout to one hour
|
||||
if (typeof timeout !== 'number') {
|
||||
timeout = 3600;
|
||||
}
|
||||
|
||||
const bulkData = await this.invokeRoute<SCBulkResponse>(this.bulkRoute, undefined, {
|
||||
expiration: moment().add(timeout, 'seconds').format(),
|
||||
source: source,
|
||||
type: type,
|
||||
});
|
||||
|
||||
return new Bulk(type, this, bulkData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a list of things
|
||||
*
|
||||
* Note that source is optional but is set to `'stapps-api'` in that case.
|
||||
* This will override any previous bulk that you indexed with that source.
|
||||
*
|
||||
* @param things List of things to index
|
||||
* @param source Source of the things
|
||||
* @param timeout Timeout of the bulk in seconds
|
||||
* @see ConnectorClient.bulk
|
||||
*/
|
||||
async index<T extends SCThings>(things: T[], source?: string, timeout?: number): Promise<void> {
|
||||
// check that number of things is not zero
|
||||
if (things.length === 0) {
|
||||
throw new EmptyBulkError();
|
||||
}
|
||||
|
||||
// set default source if none is given
|
||||
if (typeof source === 'undefined') {
|
||||
source = 'stapps-api';
|
||||
}
|
||||
|
||||
// request a new bulk
|
||||
const bulk = await this.bulk(things[0].type, source, timeout);
|
||||
|
||||
// add items to the bulk - 5 concurrently
|
||||
await asyncPool(5, things, (thing) => bulk.add(thing));
|
||||
|
||||
// close bulk
|
||||
await bulk.done();
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing StAppsCore thing
|
||||
*
|
||||
* @param thing StAppsCore thing to update
|
||||
*/
|
||||
async update(thing: SCThings): Promise<SCThingUpdateResponse> {
|
||||
return this.invokeRoute<SCThingUpdateResponse>(this.thingUpdateRoute, {
|
||||
TYPE: encodeURIComponent(thing.type),
|
||||
UID: encodeURIComponent(thing.uid),
|
||||
}, thing);
|
||||
}
|
||||
}
|
||||
114
src/copy.ts
Normal file
114
src/copy.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCSearchRequest, SCThingTypes} from '@openstapps/core';
|
||||
import {asyncPool} from 'async-pool-native/dist/async-pool';
|
||||
import {Bar} from 'cli-progress';
|
||||
import {Client} from './client';
|
||||
import {ConnectorClient} from './connectorClient';
|
||||
import {OutOfRangeError} from './errors';
|
||||
import {HttpClientInterface} from './httpClientInterface';
|
||||
|
||||
/**
|
||||
* Options to set up copying data from one backend to another
|
||||
*/
|
||||
export interface CopyOptions {
|
||||
/**
|
||||
* Batch size to copy at once
|
||||
*/
|
||||
batchSize: number;
|
||||
|
||||
/**
|
||||
* URL of the backend to copy from
|
||||
*/
|
||||
from: string;
|
||||
|
||||
/**
|
||||
* Source identifier
|
||||
*/
|
||||
source: string;
|
||||
|
||||
/**
|
||||
* URL of the backend to copy to
|
||||
*/
|
||||
to: string;
|
||||
|
||||
/**
|
||||
* StAppsCore type to copy
|
||||
*/
|
||||
type: SCThingTypes;
|
||||
|
||||
/**
|
||||
* StApps version identifier to copy data for
|
||||
*/
|
||||
version: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy data for a StAppsCore type from one backend to another
|
||||
*
|
||||
* @param client HTTP client
|
||||
* @param options Map of options
|
||||
*/
|
||||
export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
|
||||
|
||||
const apiIn = new Client(client, options.from, options.version);
|
||||
const apiOut = new ConnectorClient(client, options.to);
|
||||
|
||||
// open a bulk
|
||||
const bulk = await apiOut.bulk(options.type, options.source);
|
||||
|
||||
let searchRequest: SCSearchRequest = {
|
||||
filter: {
|
||||
arguments: {
|
||||
field: 'type',
|
||||
value: options.type,
|
||||
},
|
||||
type: 'value',
|
||||
},
|
||||
size: 0,
|
||||
};
|
||||
|
||||
let searchResponse = await apiIn.search(searchRequest);
|
||||
|
||||
searchRequest.size = options.batchSize;
|
||||
|
||||
const progressBar = new Bar({});
|
||||
progressBar.start(searchResponse.pagination.total, 0);
|
||||
|
||||
let outOfRange = false;
|
||||
|
||||
do {
|
||||
try {
|
||||
({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
|
||||
|
||||
await asyncPool(5, searchResponse.data, (item) => {
|
||||
progressBar.increment(1);
|
||||
|
||||
return bulk.add(item);
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof OutOfRangeError) {
|
||||
outOfRange = true;
|
||||
} else {
|
||||
progressBar.stop();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
} while (!outOfRange);
|
||||
|
||||
await bulk.done();
|
||||
|
||||
progressBar.stop();
|
||||
}
|
||||
133
src/errors.ts
Normal file
133
src/errors.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCErrorResponse, SCLicensePlate, SCSearchRequest, SCThings} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* An error that can occur in the StApps API
|
||||
*/
|
||||
export class ApiError extends Error {
|
||||
/**
|
||||
* Instantiate a new error
|
||||
*
|
||||
* @param data Representation of an error that happened in the backend
|
||||
*/
|
||||
constructor(protected data: Partial<SCErrorResponse>) {
|
||||
super(data.message);
|
||||
|
||||
if (typeof data.name === 'string') {
|
||||
this.name = data.name;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add additional data to the output of the error
|
||||
*/
|
||||
toString(): string {
|
||||
let str = super.toString();
|
||||
|
||||
// add additional data
|
||||
if (typeof this.data.additionalData !== 'undefined') {
|
||||
str += '\n\n' + JSON.stringify(this.data.additionalData);
|
||||
}
|
||||
|
||||
// add "remote" stack trace
|
||||
if (typeof this.data.stack !== 'undefined') {
|
||||
str += '\n\n' + this.data.stack;
|
||||
}
|
||||
|
||||
return str;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error that is thrown if the next window is beyond the total number of results
|
||||
*/
|
||||
export class OutOfRangeError extends ApiError {
|
||||
/**
|
||||
* Instantiate a new error
|
||||
*
|
||||
* @param searchRequest Search request where window is out of range
|
||||
*/
|
||||
constructor(searchRequest: SCSearchRequest) {
|
||||
super({
|
||||
additionalData: searchRequest,
|
||||
message: 'The next window is beyond the total number of results.',
|
||||
name: 'OutOfRangeError',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error that is thrown when a bulk is filled with things of multiple types
|
||||
*/
|
||||
export class BulkWithMultipleTypesError extends ApiError {
|
||||
/**
|
||||
* Instantiate a new error
|
||||
*
|
||||
* @param offendingThing Thing that has a different type than the previous things
|
||||
*/
|
||||
constructor(offendingThing: SCThings) {
|
||||
super({
|
||||
additionalData: offendingThing,
|
||||
message: 'A bulk can only contain one type of things!',
|
||||
name: 'BulkWithMultipleTypesError',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error that is thrown when a bulk is empty
|
||||
*/
|
||||
export class EmptyBulkError extends ApiError {
|
||||
/**
|
||||
* Instantiate a new error
|
||||
*/
|
||||
constructor() {
|
||||
super({
|
||||
message: 'You can not fill a bulk with zero things.',
|
||||
name: 'EmptyBulkError',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error that is thrown when there is no namespace for a license plate
|
||||
*/
|
||||
export class NamespaceNotDefinedError extends ApiError {
|
||||
/**
|
||||
* Instantiate a new error
|
||||
*/
|
||||
constructor(namespaceId: SCLicensePlate) {
|
||||
super({
|
||||
message: `'${namespaceId}' has no namespace defined`,
|
||||
name: 'NamespaceNotDefinedError',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error that is thrown when API and backend StAppsCore versions are incompatible
|
||||
*/
|
||||
export class CoreVersionIncompatibleError extends ApiError {
|
||||
/**
|
||||
* Instantiate a new error
|
||||
*/
|
||||
constructor(localVersion: string, remoteVersion: string) {
|
||||
super({
|
||||
message: `Local StAppsCore version ${localVersion} is incompatible to remote version ${remoteVersion}.`,
|
||||
name: 'CoreVersionIncompatibleError',
|
||||
});
|
||||
}
|
||||
}
|
||||
71
src/httpClient.ts
Normal file
71
src/httpClient.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import * as request from 'request';
|
||||
|
||||
/**
|
||||
* Request options that requires a url
|
||||
*/
|
||||
export interface RequestOptions extends request.CoreOptions {
|
||||
url: URL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Response with generic for the type of body that is returned from the request
|
||||
*/
|
||||
export interface Response<TYPE_OF_BODY> extends request.Response {
|
||||
body: TYPE_OF_BODY;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTTP client that is based on request
|
||||
*/
|
||||
export class HttpClient {
|
||||
/**
|
||||
* Make a request
|
||||
* @param requestConfig Configuration of the request
|
||||
*/
|
||||
request<TYPE_OF_BODY>(
|
||||
requestConfig: RequestOptions,
|
||||
): Promise<Response<TYPE_OF_BODY>> {
|
||||
const params: request.CoreOptions = {
|
||||
body: {},
|
||||
followAllRedirects: true,
|
||||
json: true,
|
||||
method: 'GET',
|
||||
};
|
||||
|
||||
if (typeof requestConfig.body !== 'undefined') {
|
||||
params.body = requestConfig.body;
|
||||
}
|
||||
|
||||
if (typeof requestConfig.headers !== 'undefined') {
|
||||
params.headers = requestConfig.headers;
|
||||
}
|
||||
|
||||
if (requestConfig.method !== 'GET') {
|
||||
params.method = requestConfig.method;
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
request(requestConfig.url.toString(), params, (err, response) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
resolve(response);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
88
src/httpClientInterface.ts
Normal file
88
src/httpClientInterface.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (C) 2018 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCErrorResponse, SCRequests, SCResponses} from '@openstapps/core';
|
||||
|
||||
/**
|
||||
* A HTTP client that can send requests for the StApps API
|
||||
*/
|
||||
export interface HttpClientInterface {
|
||||
/**
|
||||
* Send request
|
||||
*
|
||||
* @param request Request to send
|
||||
*/
|
||||
request<T extends SCResponses>(
|
||||
request: HttpClientRequest,
|
||||
): Promise<HttpClientResponse<T>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A map of headers
|
||||
*/
|
||||
export interface HttpClientHeaders {
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
/**
|
||||
* A HTTP client request
|
||||
*/
|
||||
export interface HttpClientRequest {
|
||||
/**
|
||||
* Body of the request
|
||||
*
|
||||
* A body to send with the request.
|
||||
*/
|
||||
body?: SCRequests;
|
||||
|
||||
/**
|
||||
* Headers of the request
|
||||
*
|
||||
* A key-value-map of headers to send with the request.
|
||||
*/
|
||||
headers?: HttpClientHeaders;
|
||||
|
||||
/**
|
||||
* Method of the request
|
||||
*
|
||||
* Should default to 'GET' if nothing is specified.
|
||||
*/
|
||||
method?: 'GET' | 'POST' | 'PUT';
|
||||
|
||||
/**
|
||||
* URL of the request
|
||||
*
|
||||
* The url to send the request to.
|
||||
*/
|
||||
url: URL;
|
||||
}
|
||||
|
||||
/**
|
||||
* A HTTP client response
|
||||
*/
|
||||
export interface HttpClientResponse<T extends SCResponses> {
|
||||
body: T | SCErrorResponse;
|
||||
|
||||
/**
|
||||
* Headers of the response
|
||||
*
|
||||
* A key-value-map of headers of the response
|
||||
*/
|
||||
headers: HttpClientHeaders;
|
||||
|
||||
/**
|
||||
* Status code of the response
|
||||
*/
|
||||
statusCode: number;
|
||||
}
|
||||
Reference in New Issue
Block a user