Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-20 00:23:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: copy connector

Browse Source
This commit is contained in:
Thea Schöbl
2023-11-07 12:48:21 +01:00
parent 62d5ea4275
commit 780916eb35
35 changed files with 1121 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/api-cli/src/app.ts
View File

@@ -18,7 +18,7 @@ import {Command} from 'commander';
import {URL} from 'url';
import waitOn from 'wait-on';
import {HttpClient} from '@openstapps/api';
import {copy} from './copy.js';
import {copy} from '../../../backend/copy-connector/src/copy.js';
import {version} from '../package.json';
// eslint-disable-next-line unicorn/prevent-abbreviations
import {e2eRun} from './e2e.js';

112
packages/api-cli/src/copy.ts
View File

@@ -1,112 +0,0 @@
/*
* Copyright (C) 2018-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCSearchRequest, SCThingType} from '@openstapps/core';
import {Bar} from 'cli-progress';
import {Client, ConnectorClient, OutOfRangeError, HttpClientInterface} from '@openstapps/api';
/**
* Options to set up copying data from one backend to another
*/
export interface CopyOptions {
/**
* Batch size to copy at once
*/
batchSize: number;
/**
* URL of the backend to copy from
*/
from: string;
/**
* Source identifier
*/
source: string;
/**
* URL of the backend to copy to
*/
to: string;
/**
* StAppsCore type to copy
*/
type: SCThingType;
/**
* StApps version identifier to copy data for
*/
version: string;
}
/**
* Copy data for a StAppsCore type from one backend to another
* @param client HTTP client
* @param options Map of options
*/
export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
const apiIn = new Client(client, options.from, options.version);
const apiOut = new ConnectorClient(client, options.to);
// open a bulk
const bulk = await apiOut.bulk(options.type, options.source);
let searchRequest: SCSearchRequest = {
filter: {
arguments: {
field: 'type',
value: options.type,
},
type: 'value',
},
size: 0,
};
let searchResponse = await apiIn.search(searchRequest);
searchRequest.size = options.batchSize;
const progressBar = new Bar({});
progressBar.start(searchResponse.pagination.total, 0);
let outOfRange = false;
do {
try {
({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
const things = searchResponse.data;
await Promise.all(
Array.from({length: ConnectorClient.ITEM_CONCURRENT_LIMIT}).map(async () => {
for (let item = things.shift(); item !== undefined; item = things.shift()) {
progressBar.increment(1);
await bulk.add(item);
}
}),
);
} catch (error) {
if (error instanceof OutOfRangeError) {
outOfRange = true;
} else {
progressBar.stop();
throw error;
}
}
} while (!outOfRange);
await bulk.done();
progressBar.stop();
}

228
packages/api-cli/test/copy.spec.ts
View File

@@ -1,228 +0,0 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
SCBulkAddResponse,
SCBulkAddRoute,
SCBulkDoneResponse,
SCBulkDoneRoute,
SCBulkResponse,
SCBulkRoute,
SCSearchRequest,
SCSearchResponse,
SCSearchRoute,
SCThingType,
} from '@openstapps/core';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import chaiSpies from 'chai-spies';
import {ApiError, HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
import {copy} from '../src/copy.js';
/**
* Recursive Partial
*
* @see https://stackoverflow.com/a/51365037
*/
export type RecursivePartial<T> = {
[P in keyof T]?: T[P] extends Array<infer U>
? Array<RecursivePartial<U>>
: T[P] extends object
? RecursivePartial<T[P]>
: T[P];
};
chai.should();
chai.use(chaiSpies);
chai.use(chaiAsPromised);
const sandbox = chai.spy.sandbox();
const bulkRoute = new SCBulkRoute();
const bulkAddRoute = new SCBulkAddRoute();
const bulkDoneRoute = new SCBulkDoneRoute();
const searchRoute = new SCSearchRoute();
const httpClient = new HttpClient();
describe('Copy', function () {
afterEach(function () {
sandbox.restore();
});
it('should copy', async function () {
type responses = HttpClientResponse<
SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse
>;
sandbox.on(
httpClient,
'request',
async (request: HttpClientRequest): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === 'http://foo.bar' + searchRoute.getUrlPath().toString()) {
const body = request.body as SCSearchRequest;
let count = 0;
if (typeof body.size === 'number' && body.size > 0) {
count = 1;
}
return {
body: {
data: [
{
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: new Date(Date.now()).toISOString(),
name: 'bar',
},
type: SCThingType.Dish,
uid: 'foo',
},
],
facets: [],
pagination: {
count: count,
offset: 0,
total: 1,
},
stats: {
time: 1,
},
},
statusCode: searchRoute.statusCodeSuccess,
};
} else if (request.url.toString() === 'http://localhost' + bulkRoute.getUrlPath().toString()) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
} else if (
request.url.toString() ===
'http://localhost' +
bulkAddRoute
.getUrlPath({
UID: 'foo',
})
.toString()
) {
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
},
);
await copy(httpClient, {
batchSize: 5,
from: 'http://foo.bar',
source: 'stapps-copy',
to: 'http://localhost',
type: SCThingType.Dish,
version: 'foo.bar.foobar',
});
});
it('should fail to copy', async function () {
type responses = HttpClientResponse<
SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse
>;
sandbox.on(
httpClient,
'request',
async (request: HttpClientRequest): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === 'http://foo.bar' + searchRoute.getUrlPath().toString()) {
const body = request.body as SCSearchRequest;
if (typeof body.size === 'number' && body.size > 0) {
throw new ApiError({});
}
return {
body: {
data: [
{
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: new Date(Date.now()).toISOString(),
name: 'bar',
},
type: SCThingType.Dish,
uid: 'foo',
},
],
facets: [],
pagination: {
count: 0,
offset: 0,
total: 1,
},
stats: {
time: 1,
},
},
statusCode: searchRoute.statusCodeSuccess,
};
} else if (request.url.toString() === 'http://localhost' + bulkRoute.getUrlPath().toString()) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
} else if (
request.url.toString() ===
'http://localhost' +
bulkAddRoute
.getUrlPath({
UID: 'foo',
})
.toString()
) {
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
},
);
await copy(httpClient, {
batchSize: 5,
from: 'http://foo.bar',
source: 'stapps-copy',
to: 'http://localhost',
type: SCThingType.Dish,
version: 'foo.bar.foobar',
}).should.be.rejectedWith(ApiError);
});
});

2
packages/api-cli/test/e2e.spec.ts
View File

@@ -30,7 +30,7 @@ import chaiSpies from 'chai-spies';
import {existsSync, mkdirSync, rmdirSync, unlinkSync} from 'fs';
import {createFileSync} from 'fs-extra';
import {HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
import {RecursivePartial} from './copy.spec.js';
import {RecursivePartial} from '../../../backend/copy-connector/test/copy.spec.js';
import {expect} from 'chai';
import path from 'path';
import {fileURLToPath} from 'url';