Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-01-21 17:12:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: copy connector

Browse Source
This commit is contained in:
Thea Schöbl
2023-11-07 12:48:21 +01:00
parent 62d5ea4275
commit 780916eb35
35 changed files with 1121 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/api-cli/src/app.ts
View File

@@ -18,7 +18,7 @@ import {Command} from 'commander';
import {URL} from 'url';
import waitOn from 'wait-on';
import {HttpClient} from '@openstapps/api';
import {copy} from './copy.js';
import {copy} from '../../../backend/copy-connector/src/copy.js';
import {version} from '../package.json';
// eslint-disable-next-line unicorn/prevent-abbreviations
import {e2eRun} from './e2e.js';

112
packages/api-cli/src/copy.ts
View File

@@ -1,112 +0,0 @@
/*
* Copyright (C) 2018-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCSearchRequest, SCThingType} from '@openstapps/core';
import {Bar} from 'cli-progress';
import {Client, ConnectorClient, OutOfRangeError, HttpClientInterface} from '@openstapps/api';
/**
* Options to set up copying data from one backend to another
*/
export interface CopyOptions {
/**
* Batch size to copy at once
*/
batchSize: number;
/**
* URL of the backend to copy from
*/
from: string;
/**
* Source identifier
*/
source: string;
/**
* URL of the backend to copy to
*/
to: string;
/**
* StAppsCore type to copy
*/
type: SCThingType;
/**
* StApps version identifier to copy data for
*/
version: string;
}
/**
* Copy data for a StAppsCore type from one backend to another
* @param client HTTP client
* @param options Map of options
*/
export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
const apiIn = new Client(client, options.from, options.version);
const apiOut = new ConnectorClient(client, options.to);
// open a bulk
const bulk = await apiOut.bulk(options.type, options.source);
let searchRequest: SCSearchRequest = {
filter: {
arguments: {
field: 'type',
value: options.type,
},
type: 'value',
},
size: 0,
};
let searchResponse = await apiIn.search(searchRequest);
searchRequest.size = options.batchSize;
const progressBar = new Bar({});
progressBar.start(searchResponse.pagination.total, 0);
let outOfRange = false;
do {
try {
({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
const things = searchResponse.data;
await Promise.all(
Array.from({length: ConnectorClient.ITEM_CONCURRENT_LIMIT}).map(async () => {
for (let item = things.shift(); item !== undefined; item = things.shift()) {
progressBar.increment(1);
await bulk.add(item);
}
}),
);
} catch (error) {
if (error instanceof OutOfRangeError) {
outOfRange = true;
} else {
progressBar.stop();
throw error;
}
}
} while (!outOfRange);
await bulk.done();
progressBar.stop();
}

228
packages/api-cli/test/copy.spec.ts
View File

@@ -1,228 +0,0 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
SCBulkAddResponse,
SCBulkAddRoute,
SCBulkDoneResponse,
SCBulkDoneRoute,
SCBulkResponse,
SCBulkRoute,
SCSearchRequest,
SCSearchResponse,
SCSearchRoute,
SCThingType,
} from '@openstapps/core';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import chaiSpies from 'chai-spies';
import {ApiError, HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
import {copy} from '../src/copy.js';
/**
* Recursive Partial
*
* @see https://stackoverflow.com/a/51365037
*/
export type RecursivePartial<T> = {
[P in keyof T]?: T[P] extends Array<infer U>
? Array<RecursivePartial<U>>
: T[P] extends object
? RecursivePartial<T[P]>
: T[P];
};
chai.should();
chai.use(chaiSpies);
chai.use(chaiAsPromised);
const sandbox = chai.spy.sandbox();
const bulkRoute = new SCBulkRoute();
const bulkAddRoute = new SCBulkAddRoute();
const bulkDoneRoute = new SCBulkDoneRoute();
const searchRoute = new SCSearchRoute();
const httpClient = new HttpClient();
describe('Copy', function () {
afterEach(function () {
sandbox.restore();
});
it('should copy', async function () {
type responses = HttpClientResponse<
SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse
>;
sandbox.on(
httpClient,
'request',
async (request: HttpClientRequest): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === 'http://foo.bar' + searchRoute.getUrlPath().toString()) {
const body = request.body as SCSearchRequest;
let count = 0;
if (typeof body.size === 'number' && body.size > 0) {
count = 1;
}
return {
body: {
data: [
{
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: new Date(Date.now()).toISOString(),
name: 'bar',
},
type: SCThingType.Dish,
uid: 'foo',
},
],
facets: [],
pagination: {
count: count,
offset: 0,
total: 1,
},
stats: {
time: 1,
},
},
statusCode: searchRoute.statusCodeSuccess,
};
} else if (request.url.toString() === 'http://localhost' + bulkRoute.getUrlPath().toString()) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
} else if (
request.url.toString() ===
'http://localhost' +
bulkAddRoute
.getUrlPath({
UID: 'foo',
})
.toString()
) {
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
},
);
await copy(httpClient, {
batchSize: 5,
from: 'http://foo.bar',
source: 'stapps-copy',
to: 'http://localhost',
type: SCThingType.Dish,
version: 'foo.bar.foobar',
});
});
it('should fail to copy', async function () {
type responses = HttpClientResponse<
SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse
>;
sandbox.on(
httpClient,
'request',
async (request: HttpClientRequest): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === 'http://foo.bar' + searchRoute.getUrlPath().toString()) {
const body = request.body as SCSearchRequest;
if (typeof body.size === 'number' && body.size > 0) {
throw new ApiError({});
}
return {
body: {
data: [
{
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: new Date(Date.now()).toISOString(),
name: 'bar',
},
type: SCThingType.Dish,
uid: 'foo',
},
],
facets: [],
pagination: {
count: 0,
offset: 0,
total: 1,
},
stats: {
time: 1,
},
},
statusCode: searchRoute.statusCodeSuccess,
};
} else if (request.url.toString() === 'http://localhost' + bulkRoute.getUrlPath().toString()) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
} else if (
request.url.toString() ===
'http://localhost' +
bulkAddRoute
.getUrlPath({
UID: 'foo',
})
.toString()
) {
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
},
);
await copy(httpClient, {
batchSize: 5,
from: 'http://foo.bar',
source: 'stapps-copy',
to: 'http://localhost',
type: SCThingType.Dish,
version: 'foo.bar.foobar',
}).should.be.rejectedWith(ApiError);
});
});

2
packages/api-cli/test/e2e.spec.ts
View File

@@ -30,7 +30,7 @@ import chaiSpies from 'chai-spies';
import {existsSync, mkdirSync, rmdirSync, unlinkSync} from 'fs';
import {createFileSync} from 'fs-extra';
import {HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
import {RecursivePartial} from './copy.spec.js';
import {RecursivePartial} from '../../../backend/copy-connector/test/copy.spec.js';
import {expect} from 'chai';
import path from 'path';
import {fileURLToPath} from 'url';

7
packages/core-validator/src/index.ts
View File

@@ -1,6 +1,7 @@
import Ajv, {AnySchema} from 'ajv';
import addFormats from 'ajv-formats';
import schema, {SchemaMap} from '@openstapps/core/schema.json';
import schema from '@openstapps/core/schema.json' assert {type: 'json'};
import type {SchemaMap} from '@openstapps/core/schema.json';
export type RemoveNeverProperties<T> = {
[K in Exclude<
@@ -32,6 +33,7 @@ export class Validator {
this.ajv = new Ajv.default({
schemas: [schema, ...additionalSchemas],
verbose: true,
keywords: ['elasticsearch'],
allowUnionTypes: true,
});
addFormats.default(this.ajv, {
@@ -54,7 +56,8 @@ export class Validator {
*/
public validate<T extends SchemaMap[keyof SchemaMap]>(instance: unknown, schema: NameOf<T>): instance is T;
public validate(instance: unknown, schema: Ajv.Schema): boolean;
public validate(instance: unknown, schema: string): boolean;
public validate(instance: unknown, schema: Ajv.Schema | string): boolean {
return this.ajv.validate(schema, instance);
return this.ajv.validate(typeof schema === 'string' ? `#/definitions/${schema}` : schema, instance);
}
}

2
packages/core/package.json
View File

@@ -34,7 +34,7 @@
"import": "./lib/schema.json",
"types": "./lib/schema.json.d.ts"
},
"./elasticsearch-mappings.json": {
"./elasticsearch.json": {
"import": "./lib/elasticsearch.json",
"types": "./lib/elasticsearch.json.d.ts"
}

1
packages/es-mapping-generator/package.json
View File

@@ -16,7 +16,6 @@
],
"scripts": {
"build": "tsup-node --dts",
"dev": "tsup --watch --onSuccess \"node lib/index.js\"",
"format": "prettier . -c --ignore-path ../../.gitignore",
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
"lint": "eslint --ext .ts src/",

9
packages/es-mapping-generator/schema/mappings.d.ts vendored
View File

@@ -1,9 +1,6 @@
import {IndicesPutTemplateRequest, SearchRequest} from '@elastic/elasticsearch/lib/api/types.js';
export interface ElasticsearchConfig {
mappings: Record<string, IndicesPutTemplateRequest>;
search: Partial<SearchRequest>;
}
export const mappings: Record<string, IndicesPutTemplateRequest>;
export const search: Partial<SearchRequest>;
declare const elasticsearchConfig: ElasticsearchConfig;
export default elasticsearchConfig;
export default {mappings, search};