mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-17 23:22:54 +00:00
113 lines
2.8 KiB
TypeScript
113 lines
2.8 KiB
TypeScript
/*
|
|
* Copyright (C) 2018-2022 Open StApps
|
|
* This program is free software: you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License as published by the Free
|
|
* Software Foundation, version 3.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
import {SCSearchRequest, SCThingType} from '@openstapps/core';
|
|
import {Bar} from 'cli-progress';
|
|
import {Client, ConnectorClient, OutOfRangeError, HttpClientInterface} from '@openstapps/api';
|
|
|
|
/**
|
|
* Options to set up copying data from one backend to another
|
|
*/
|
|
export interface CopyOptions {
|
|
/**
|
|
* Batch size to copy at once
|
|
*/
|
|
batchSize: number;
|
|
|
|
/**
|
|
* URL of the backend to copy from
|
|
*/
|
|
from: string;
|
|
|
|
/**
|
|
* Source identifier
|
|
*/
|
|
source: string;
|
|
|
|
/**
|
|
* URL of the backend to copy to
|
|
*/
|
|
to: string;
|
|
|
|
/**
|
|
* StAppsCore type to copy
|
|
*/
|
|
type: SCThingType;
|
|
|
|
/**
|
|
* StApps version identifier to copy data for
|
|
*/
|
|
version: string;
|
|
}
|
|
|
|
/**
|
|
* Copy data for a StAppsCore type from one backend to another
|
|
* @param client HTTP client
|
|
* @param options Map of options
|
|
*/
|
|
export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
|
|
const apiIn = new Client(client, options.from, options.version);
|
|
const apiOut = new ConnectorClient(client, options.to);
|
|
|
|
// open a bulk
|
|
const bulk = await apiOut.bulk(options.type, options.source);
|
|
|
|
let searchRequest: SCSearchRequest = {
|
|
filter: {
|
|
arguments: {
|
|
field: 'type',
|
|
value: options.type,
|
|
},
|
|
type: 'value',
|
|
},
|
|
size: 0,
|
|
};
|
|
|
|
let searchResponse = await apiIn.search(searchRequest);
|
|
|
|
searchRequest.size = options.batchSize;
|
|
|
|
const progressBar = new Bar({});
|
|
progressBar.start(searchResponse.pagination.total, 0);
|
|
|
|
let outOfRange = false;
|
|
|
|
do {
|
|
try {
|
|
({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
|
|
const things = searchResponse.data;
|
|
|
|
await Promise.all(
|
|
Array.from({length: ConnectorClient.ITEM_CONCURRENT_LIMIT}).map(async () => {
|
|
for (let item = things.shift(); item !== undefined; item = things.shift()) {
|
|
progressBar.increment(1);
|
|
await bulk.add(item);
|
|
}
|
|
}),
|
|
);
|
|
} catch (error) {
|
|
if (error instanceof OutOfRangeError) {
|
|
outOfRange = true;
|
|
} else {
|
|
progressBar.stop();
|
|
throw error;
|
|
}
|
|
}
|
|
} while (!outOfRange);
|
|
|
|
await bulk.done();
|
|
|
|
progressBar.stop();
|
|
}
|