Theaninova/openstapps
1
0
Fork 0
mirror of https://gitlab.com/openstapps/openstapps.git synced 2026-02-25 18:32:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: split api into api, api-cli & api-plugin

Browse Source
This commit is contained in:
Thea Schöbl
2023-06-02 16:41:25 +02:00
parent 495a63977c
commit b21833de40
205 changed files with 1981 additions and 1492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
packages/api/.editorconfig
View File

@@ -1,15 +0,0 @@
# editorconfig.org
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.md]
trim_trailing_whitespace = false

2
packages/api/.eslintignore
View File

@@ -1,2 +0,0 @@
resources
openapi

3
packages/api/.eslintrc.json
View File

@@ -1,3 +0,0 @@
{
"extends": "@openstapps"
}

91
packages/api/.gitignore vendored
View File

@@ -1,91 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# parcel-bundler cache (https://parceljs.org/)
.cache
# next.js build output
.next
# nuxt.js build output
.nuxt
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
#DynamoDB Local files
.dynamodb/
########## end of https://github.com/github/gitignore/blob/master/Node.gitignore
# ignore ide files
.idea
.vscode
# ignore lib
lib
# ignore docs
docs

11
packages/api/.npmignore
View File

@@ -1,11 +0,0 @@
# Ignore all files/folders by default
# See https://stackoverflow.com/a/29932318
/*
# Except these files/folders
!lib
lib/tsconfig.tsbuildinfo
!LICENSE
!package.json
!package-lock.json
!README.md
!src

8
packages/api/Dockerfile
View File

@@ -1,8 +0,0 @@
FROM registry.gitlab.com/openstapps/projectmanagement/node
ADD . /app
WORKDIR /app
ENTRYPOINT ["node", "lib/cli.js"]
CMD ["--help"]

62
packages/api/README.md
View File

@@ -4,64 +4,4 @@ Node.js library to interact with the StApps backend service
## Use this as a standalone program
To get some data into a local `backend-node`-instance, you can run this
as a standalone program to copy data of a remote `backend-node`-instance
into your local one.
Example to copy all Events of the b-tu instance:
```shell
npm install
npm run build
node ./lib/cli.js copy Event https://stappsbe01.innocampus.tu-berlin.de http://localhost:3000 100
```
Example to index all items from @openstapps/core test files to a backend:
```shell
npm install
npm run build
node ./lib/cli.js e2e http://localhost:3000
```
### Program arguments
```shell
node ./lib/cli.js copy <type> <from> <to> <batchSize>
node ./lib/cli.js e2e <to>
```
#### Options
The source identifier for the bulk to use with the target instance (default is 'copy')
```shell
-s, --bulkSource <bulkScource>
```
The App version to use (unset by default)
```shell
-a, --appVersion <version>
```
The only available option for `e2e` command. File path to json test files each containing a SCThing.
```shell
-s, --samples <path>
```
### Example execution
with docker when backend is running on `localhost:3000`:
```shell
docker run --net=host registry.gitlab.com/openstapps/api/cli copy Place https://stappsbe01.innocampus.tu-berlin.de http://localhost:3000 100
```
Or using `e2e` command:
```shell
docker run --net=host registry.gitlab.com/openstapps/api/cli e2e http://localhost:3000
```
See `@openstapps/api-cli`

2
packages/api/app.js
View File

@@ -1,2 +0,0 @@
#!/usr/bin/env node
import './lib/app.js';

39
packages/api/package.json
View File

@@ -17,9 +17,6 @@
],
"main": "lib/index.js",
"types": "lib/index.d.ts",
"bin": {
"openstapps-api": "app.js"
},
"scripts": {
"build": "tsup --dts",
"docs": "typedoc --json ./docs/docs.json --options ../../typedoc.base.json src/index.ts",
@@ -30,56 +27,32 @@
"test": "c8 mocha"
},
"dependencies": {
"@krlwlfrt/async-pool": "0.7.0",
"@openstapps/core": "workspace:*",
"@openstapps/core-tools": "workspace:*",
"@openstapps/logger": "workspace:*",
"@types/cli-progress": "3.11.0",
"@types/express": "4.17.17",
"@types/morgan": "1.9.4",
"@types/node": "18.15.3",
"@types/traverse": "0.6.32",
"@types/uuid": "8.3.4",
"@types/wait-on": "5.3.1",
"body-parser": "1.20.2",
"cli-progress": "3.12.0",
"commander": "10.0.0",
"express": "4.18.2",
"got": "12.6.0",
"json-schema": "0.4.0",
"moment": "2.29.4",
"morgan": "1.10.0",
"rfdc": "1.3.0",
"traverse": "0.6.7",
"uuid": "8.3.2",
"wait-on": "6.0.1"
"uuid": "8.3.2"
},
"devDependencies": {
"@openstapps/eslint-config": "workspace:*",
"@openstapps/nyc-config": "workspace:*",
"@openstapps/prettier-config": "workspace:*",
"@openstapps/tsconfig": "workspace:*",
"@types/body-parser": "1.19.2",
"@types/chai": "4.3.5",
"@types/chai-as-promised": "7.1.5",
"@types/chai-spies": "1.0.3",
"@types/cli-progress": "3.11.0",
"@types/express": "4.17.17",
"@types/fs-extra": "9.0.13",
"@types/json-schema": "7.0.11",
"@types/mocha": "10.0.1",
"@types/traverse": "0.6.32",
"undici": "5.22.1",
"c8": "7.14.0",
"chai": "4.3.7",
"chai-as-promised": "7.1.1",
"chai-spies": "1.0.0",
"conventional-changelog-cli": "2.2.2",
"fs-extra": "10.1.0",
"date-fns": "2.30.0",
"mocha": "10.2.0",
"nock": "13.3.1",
"traverse": "0.6.7",
"ts-node": "10.9.1",
"tsup": "6.7.0",
"typedoc": "0.24.7",
"typescript": "4.8.4"
"typescript": "4.9.5"
},
"peerDependencies": {
"@openstapps/core": "workspace:*"

138
packages/api/src/app.ts
View File

@@ -1,138 +0,0 @@
/*
* Copyright (C) 2018-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCThingType} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
import {Command} from 'commander';
import {readFileSync} from 'fs';
import path from 'path';
import {URL} from 'url';
import waitOn from 'wait-on';
import {copy} from './copy.js';
// eslint-disable-next-line unicorn/prevent-abbreviations
import {e2eRun} from './e2e.js';
import {HttpClient} from './http-client.js';
process.on('unhandledRejection', async error => {
await Logger.error('unhandledRejection', error);
});
// eslint-disable-next-line unicorn/prefer-module
const packageJson = JSON.parse(readFileSync(path.join(__dirname, '..', 'package.json')).toString());
const client = new HttpClient();
const commander = new Command();
const helpAndExit = (help: string) => {
// eslint-disable-next-line no-console
console.log(help);
process.exit(-1);
};
commander
.command('e2e <to>')
.version(packageJson.version)
.description(
'Run in end to end test mode. Indexing and afterwards retrieving all test files from @openstapp/core to the backend',
)
.option(
'-s --samples [path]',
'Path to @openstapp/core test files',
'./node_modules/@openstapps/core/test/resources/indexable',
)
.option('-w --waiton [resource]', 'wait-on resource parameter see "www.npmjs.com/wait-on"')
// eslint-disable-next-line unicorn/prevent-abbreviations
.action(async (to, e2eCommand) => {
let toURL = '';
// validate url
try {
toURL = new URL(to).toString();
} catch (error) {
await Logger.error('expected parameter <to> to be valid url', error);
helpAndExit(e2eCommand.helpInformation());
}
try {
if (typeof e2eCommand.waiton === 'string') {
Logger.info(`Waiting for availibilty of resource: ${e2eCommand.waiton}`);
await waitOn({
resources: [e2eCommand.waiton],
timeout: 300_000,
});
Logger.info(`Resource became available`);
}
await e2eRun(client, {to: toURL, samplesLocation: e2eCommand.samples});
Logger.ok('Done');
} catch (error) {
await Logger.error(error);
}
});
commander
.command('copy <type> <from> <to> <batchSize>')
.version(packageJson.version)
.description('Copy data from one instance to another')
.option(
'-s, --bulkSource <bulkSource>',
'The source identifier for the bulk to use with the target instance [copy]',
'copy',
)
// TODO: remove
.option('-a, --appVersion <version>', 'The App version to use [unset by default]')
.allowUnknownOption(false)
.action(async (type, from, to, batchSize, copyCommand) => {
// validate type
if (typeof type !== 'string') {
await Logger.error('expected parameter "type" to be of type: string');
copyCommand.help();
helpAndExit(copyCommand.helpInformation());
}
let fromURL = '';
let toURL = '';
// validate urls
try {
fromURL = new URL(from).toString();
toURL = new URL(to).toString();
} catch (error) {
await Logger.error('expected parameters "from" and "to" to be valid urls', error);
helpAndExit(copyCommand.helpInformation());
}
// validate batchSize
if (Number.isNaN(Number.parseInt(batchSize, 10))) {
await Logger.error('expected parameter "batchSize" to be of type: number');
helpAndExit(copyCommand.helpInformation());
}
Logger.info(`Copying ${type} objects from ${fromURL} to ${toURL}`);
copy(client, {
batchSize: Number.parseInt(batchSize, 10),
from: fromURL,
source: copyCommand.bulkSource,
to: toURL,
type: type as SCThingType,
version: copyCommand.appVersion,
}).then(
() => {
Logger.ok('Done');
},
error => {
throw error;
},
);
});
commander.parse(process.argv);

2
packages/api/src/bulk.ts
View File

@@ -42,7 +42,6 @@ export class Bulk<T extends SCThings> {
/**
* **!!! Bulk should only be instantiated by Client !!!**
*
* @see Client.bulk
*/
constructor(
@@ -55,7 +54,6 @@ export class Bulk<T extends SCThings> {
/**
* Add a thing to the bulk
*
* @param thing Thing to add to the bulk
*/
async add(thing: T): Promise<SCBulkAddResponse> {

19
packages/api/src/client.ts
View File

@@ -67,7 +67,6 @@ export class Client {
/**
* Create a new search request with altered pagination parameters to move to the next result window
*
* @param searchRequest Last search request
* @param searchResponse Search response for supplied search request
* @throws OutOfRangeError Throws an error if the next window is beyond the total number of results
@@ -90,7 +89,6 @@ export class Client {
/**
* Instantiate a new StApps-API client to communicate with a StApps-backend.
*
* @param httpClient HTTP client to use
* @param url URL of the backend
* @param version App version to use when requesting data *(only necessary if URL is ambiguous)*
@@ -112,7 +110,6 @@ export class Client {
/**
* Get a thing by its UID
*
* @param uid UID of the thing to fetch
*/
async getThing(uid: string): Promise<SCThings> {
@@ -136,7 +133,6 @@ export class Client {
/**
* Make a handshake with the backend and check StAppsCore version
*
* @param coreVersion StAppsCore version to check
*/
async handshake(coreVersion: string): Promise<SCIndexResponse> {
@@ -155,22 +151,21 @@ export class Client {
/**
* Invoke a plugin route
*
* @param name name of the plugin
* @param parameters Parameters for the URL fragment
* @param body Body for the request
*/
async invokePlugin<T>(name: string, parameters?: {[k: string]: string}, body?: SCRequests): Promise<T> {
if (typeof this.supportedFeatures === 'undefined') {
if (this.supportedFeatures === undefined) {
const request: SCIndexRequest = {};
const response = await this.invokeRoute<SCIndexResponse>(this.indexRoute, undefined, request);
if (typeof response?.app?.features !== 'undefined') {
if (response?.app?.features !== undefined) {
/* istanbul ignore next */
this.supportedFeatures = response?.app?.features;
}
}
const pluginInfo: SCFeatureConfigurationPlugin | undefined = this.supportedFeatures?.plugins?.[name];
if (typeof pluginInfo === 'undefined') {
if (pluginInfo === undefined) {
throw new PluginNotAvailableError(name);
}
@@ -182,7 +177,6 @@ export class Client {
/**
* Invoke a route
*
* @param route Route to invoke
* @param parameters Parameters for the URL fragment
* @param body Body for the request
@@ -212,7 +206,6 @@ export class Client {
* Send a multi search request to the backend
*
* All results will be returned for requests where no size is set.
*
* @param multiSearchRequest Multi search request
*/
async multiSearch(multiSearchRequest: SCMultiSearchRequest): Promise<SCMultiSearchResponse> {
@@ -223,7 +216,7 @@ export class Client {
for (const key of Object.keys(multiSearchRequest)) {
const searchRequest = multiSearchRequest[key];
if (typeof searchRequest.size === 'undefined') {
if (searchRequest.size === undefined) {
preFlightRequest[key] = {
...searchRequest,
};
@@ -265,13 +258,12 @@ export class Client {
* Send a search request to the backend
*
* All results will be returned if no size is set in the request.
*
* @param searchRequest Search request
*/
async search(searchRequest: SCSearchRequest): Promise<SCSearchResponse> {
let size: number | undefined = searchRequest.size;
if (typeof size === 'undefined') {
if (size === undefined) {
const preFlightResponse = await this.invokeRoute<SCSearchResponse>(this.searchRoute, undefined, {
...searchRequest,
size: 0,
@@ -288,7 +280,6 @@ export class Client {
/**
* Get the next search results
*
* @param searchRequest Last search request
* @param searchResponse Search response for supplied search request
*/

33
packages/api/src/connector-client.ts
View File

@@ -12,7 +12,6 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {asyncPool} from '@krlwlfrt/async-pool/lib/async-pool.js';
import {
isThing,
SCAssociatedThingWithoutReferences,
@@ -25,8 +24,6 @@ import {
SCThingUpdateResponse,
SCThingUpdateRoute,
} from '@openstapps/core';
import moment from 'moment';
import clone from 'rfdc';
import {v5} from 'uuid';
import {Bulk} from './bulk.js';
import {Client} from './client.js';
@@ -61,12 +58,11 @@ export class ConnectorClient extends Client {
*
* Note: valid namespace IDs are license plates of StApps universities.
* See documentation of `NAMESPACES` for valid namespace IDs.*
*
* @param uid UID to make UUID from
* @param namespaceId Namespace ID to use to make UUID
*/
static makeUUID(uid: string, namespaceId: SCLicensePlate): string {
if (typeof SCNamespaces[namespaceId] === 'undefined') {
if (SCNamespaces[namespaceId] === undefined) {
throw new NamespaceNotDefinedError(namespaceId);
}
@@ -77,12 +73,10 @@ export class ConnectorClient extends Client {
* Remove fields from a thing that are references
*
* This effectively turns a thing into a thing without references, e.g. SCDish into SCDishWithoutReferences.
*
* @param thing Thing to remove references from
*/
static removeReferences<THING extends SCThings>(thing: THING): SCAssociatedThingWithoutReferences<THING> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const thingWithoutReferences = clone()<any>(thing);
const thingWithoutReferences = JSON.parse(JSON.stringify(thing));
delete thingWithoutReferences.origin;
@@ -145,7 +139,6 @@ export class ConnectorClient extends Client {
/**
* Recursively deletes all undefined properties from an object instance
*
* @param object Object to delete undefined properties from
*/
static removeUndefinedProperties(object: object): void {
@@ -163,7 +156,7 @@ export class ConnectorClient extends Client {
const indexedObject = object as {[k: string]: unknown};
if (typeof indexedObject[key] === 'undefined') {
if (indexedObject[key] === undefined) {
// delete undefined keyss
delete indexedObject[key];
} else {
@@ -180,17 +173,17 @@ export class ConnectorClient extends Client {
*
* This uses the Bulk API supplied by the backend and returns an object that can be used
* just like the client itself, while handling the information necessary in bulk transfers.
*
* @param type StAppsCore thing type
* @param source Source identifier (should be unique per actual data source)
* @param timeout Timeout in seconds when the bulk should expire
*/
async bulk<T extends SCThings>(type: SCThingType, source: string, timeout?: number): Promise<Bulk<T>> {
// set default value for timeout to one hour
const bulkTimeout = typeof timeout === 'number' ? timeout : ConnectorClient.BULK_TIMEOUT;
// set the default value for timeout to one hour
const expiration = new Date();
expiration.setSeconds(expiration.getSeconds() + (timeout ?? ConnectorClient.BULK_TIMEOUT));
const bulkData = await this.invokeRoute<SCBulkResponse>(this.bulkRoute, undefined, {
expiration: moment().add(bulkTimeout, 'seconds').format(),
expiration: expiration.toISOString(),
source: source,
type: type,
});
@@ -203,7 +196,6 @@ export class ConnectorClient extends Client {
*
* Note that source is optional but is set to `'stapps-api'` in that case.
* This will override any previous bulk that you indexed with that source.
*
* @param things List of things to index
* @param source Source of the things
* @param timeout Timeout of the bulk in seconds
@@ -216,13 +208,19 @@ export class ConnectorClient extends Client {
}
// set the default source if none is given
const thingSource = typeof source === 'undefined' ? 'stapps-api' : source;
const thingSource = source === undefined ? 'stapps-api' : source;
// request a new bulk
const bulk = await this.bulk(things[0].type, thingSource, timeout);
// add items to the bulk - 5 concurrently
await asyncPool(ConnectorClient.ITEM_CONCURRENT_LIMIT, things, thing => bulk.add(thing));
await Promise.all(
Array.from({length: ConnectorClient.ITEM_CONCURRENT_LIMIT}).map(async () => {
for (let thing = things.pop(); thing !== undefined; thing = things.pop()) {
await bulk.add(thing);
}
}),
);
// close bulk
await bulk.done();
@@ -230,7 +228,6 @@ export class ConnectorClient extends Client {
/**
* Update an existing StAppsCore thing
*
* @param thing StAppsCore thing to update
*/
async update(thing: SCThings): Promise<SCThingUpdateResponse> {

113
packages/api/src/copy.ts
View File

@@ -1,113 +0,0 @@
/*
* Copyright (C) 2018-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {asyncPool} from '@krlwlfrt/async-pool/lib/async-pool.js';
import {SCSearchRequest, SCThingType} from '@openstapps/core';
import {Bar} from 'cli-progress';
import {Client} from './client.js';
import {ConnectorClient} from './connector-client.js';
import {OutOfRangeError} from './errors.js';
import {HttpClientInterface} from './http-client-interface.js';
/**
* Options to set up copying data from one backend to another
*/
export interface CopyOptions {
/**
* Batch size to copy at once
*/
batchSize: number;
/**
* URL of the backend to copy from
*/
from: string;
/**
* Source identifier
*/
source: string;
/**
* URL of the backend to copy to
*/
to: string;
/**
* StAppsCore type to copy
*/
type: SCThingType;
/**
* StApps version identifier to copy data for
*/
version: string;
}
/**
* Copy data for a StAppsCore type from one backend to another
*
* @param client HTTP client
* @param options Map of options
*/
export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
const apiIn = new Client(client, options.from, options.version);
const apiOut = new ConnectorClient(client, options.to);
// open a bulk
const bulk = await apiOut.bulk(options.type, options.source);
let searchRequest: SCSearchRequest = {
filter: {
arguments: {
field: 'type',
value: options.type,
},
type: 'value',
},
size: 0,
};
let searchResponse = await apiIn.search(searchRequest);
searchRequest.size = options.batchSize;
const progressBar = new Bar({});
progressBar.start(searchResponse.pagination.total, 0);
let outOfRange = false;
do {
try {
({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
await asyncPool(ConnectorClient.ITEM_CONCURRENT_LIMIT, searchResponse.data, async item => {
progressBar.increment(1);
return bulk.add(item);
});
} catch (error) {
if (error instanceof OutOfRangeError) {
outOfRange = true;
} else {
progressBar.stop();
throw error;
}
}
} while (!outOfRange);
await bulk.done();
progressBar.stop();
}

166
packages/api/src/e2e.ts
View File

@@ -1,166 +0,0 @@
/*
* Copyright (C) 2019-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/* eslint-disable unicorn/prevent-abbreviations */
import {SCSearchRequest, SCThings, SCThingType} from '@openstapps/core';
import {Logger} from '@openstapps/logger';
import {deepStrictEqual} from 'assert';
import {readdir, readFile} from 'fs';
import path from 'path';
import {promisify} from 'util';
import {ConnectorClient} from './connector-client.js';
import {HttpClientInterface} from './http-client-interface.js';
const localItemMap: Map<string, SCThings> = new Map();
const remoteItemMap: Map<string, SCThings> = new Map();
/**
* Options to set up indexing core test files to backend
*/
export interface E2EOptions {
/**
* File path of the directory containing core test files
*/
samplesLocation: string;
/**
* URL of the backend to index to
*/
to: string;
}
/**
* Function that can be used for integration tests.
* Adds all the SCThings that getItemsFromSamples() returns to the backend.
* Afterwards retrieves the items from backend and checks for differences with original ones.
*/
export async function e2eRun(client: HttpClientInterface, options: E2EOptions): Promise<void> {
localItemMap.clear();
remoteItemMap.clear();
const api = new ConnectorClient(client, options.to);
try {
await indexSamples(api, options);
Logger.info(`All samples have been indexed via the backend`);
await retrieveItems(api);
Logger.info(`All samples have been retrieved from the backend`);
compareItems();
} catch (error) {
throw error;
}
}
/**
* Retieves all samples previously index using the api
*/
async function retrieveItems(api: ConnectorClient): Promise<void> {
const singleItemSearchRequest: SCSearchRequest = {
filter: {
arguments: {
field: 'uid',
value: 'replace-me',
},
type: 'value',
},
};
for (const uid of localItemMap.keys()) {
singleItemSearchRequest.filter!.arguments.value = uid;
const searchResonse = await api.search(singleItemSearchRequest);
if (searchResonse.data.length !== 1) {
throw new Error(
`Search for single SCThing with uid: ${uid} returned ${searchResonse.data.length} results`,
);
}
remoteItemMap.set(uid, searchResonse.data[0]);
}
}
/**
* Compares all samples (local and remote) with the same uid and throws if they're not deep equal
*/
function compareItems() {
for (const localThing of localItemMap.values()) {
/* istanbul ignore next retrieveItems will throw before*/
if (!remoteItemMap.has(localThing.uid)) {
throw new Error(`Did not retrieve expected SCThing with uid: ${localThing.uid}`);
}
const remoteThing = remoteItemMap.get(localThing.uid);
deepStrictEqual(remoteThing, localThing, `Unexpected difference between original and retrieved sample`);
}
Logger.info(
`All samples retrieved from the backend are the same (deep equal) as the original ones submitted`,
);
}
/**
* Function to add all the SCThings that getItemsFromSamples() returns to the backend
*/
async function indexSamples(api: ConnectorClient, options: E2EOptions): Promise<void> {
try {
const items = await getItemsFromSamples(options.samplesLocation);
if (items.length === 0) {
throw new Error('Could not index samples. None were retrieved from the file system.');
}
// sort items by type
const itemMap: Map<SCThingType, SCThings[]> = new Map();
for (const item of items) {
if (!itemMap.has(item.type)) {
itemMap.set(item.type, []);
}
const itemsOfSameType = itemMap.get(item.type) as SCThings[];
itemsOfSameType.push(item);
itemMap.set(item.type, itemsOfSameType);
localItemMap.set(item.uid, item);
}
// add items depending on their type property with one type per bulk
for (const type of itemMap.keys()) {
await api.index(itemMap.get(type) as SCThings[], 'stapps-core-sample-data');
}
} catch (error) {
throw error;
}
}
/**
* Get all SCThings from the predefined core test json files
*
* @param samplesDirectory Filepath to the directory containing to the core test json files
* @returns an Array of all the SCThings specified for test usage
*/
export async function getItemsFromSamples<T extends SCThings>(samplesDirectory: string): Promise<T[]> {
const readDirPromised = promisify(readdir);
const readFilePromised = promisify(readFile);
const things: T[] = [];
try {
const fileNames = await readDirPromised(samplesDirectory);
for (const fileName of fileNames) {
const filePath = path.join(samplesDirectory, fileName);
if (filePath.endsWith('.json')) {
const fileContent = await readFilePromised(filePath, {encoding: 'utf8'});
const schemaObject = JSON.parse(fileContent);
if (schemaObject.errorNames.length === 0 && typeof schemaObject.instance.type === 'string') {
things.push(schemaObject.instance);
}
}
}
} catch (error) {
throw error;
}
return things;
}

7
packages/api/src/errors.ts
View File

@@ -20,7 +20,6 @@ import {SCErrorResponse, SCLicensePlate, SCSearchRequest, SCThings} from '@opens
export class ApiError extends Error {
/**
* Instantiate a new error
*
* @param data Representation of an error that happened in the backend
*/
constructor(protected data: Partial<SCErrorResponse>) {
@@ -38,12 +37,12 @@ export class ApiError extends Error {
let string_ = super.toString();
// add additional data
if (typeof this.data.additionalData !== 'undefined') {
if (this.data.additionalData !== undefined) {
string_ += `\n\n${JSON.stringify(this.data.additionalData)}`;
}
// add "remote" stack trace
if (typeof this.data.stack !== 'undefined') {
if (this.data.stack !== undefined) {
string_ += `\n\n${this.data.stack}`;
}
@@ -57,7 +56,6 @@ export class ApiError extends Error {
export class OutOfRangeError extends ApiError {
/**
* Instantiate a new error
*
* @param searchRequest Search request where window is out of range
*/
constructor(searchRequest: SCSearchRequest) {
@@ -75,7 +73,6 @@ export class OutOfRangeError extends ApiError {
export class BulkWithMultipleTypesError extends ApiError {
/**
* Instantiate a new error
*
* @param offendingThing Thing that has a different type than the previous things
*/
constructor(offendingThing: SCThings) {

1
packages/api/src/http-client-interface.ts
View File

@@ -20,7 +20,6 @@ import {SCErrorResponse, SCRequests, SCResponses} from '@openstapps/core';
export interface HttpClientInterface {
/**
* Send request
*
* @param request Request to send
*/
request<T extends SCResponses>(request: HttpClientRequest): Promise<HttpClientResponse<T>>;

73
packages/api/src/http-client.ts
View File

@@ -12,75 +12,34 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import got, {OptionsOfJSONResponseBody, Response as GotResponse} from 'got';
/**
* Request options that requires a url
* Note: adjust request options of got library for backward compatibility
*/
export interface RequestOptions extends Omit<OptionsOfJSONResponseBody, 'json' | 'body'> {
/**
* Body of the request
*/
// TODO: Use a specific type?
// eslint-disable-next-line @typescript-eslint/no-explicit-any
body?: any;
/**
* Target URL of the request
*/
url: URL;
}
/**
* Response with generic for the type of body that is returned from the request
*/
export interface Response<TYPE_OF_BODY> extends GotResponse {
/**
* Typed body of the response
*/
body: TYPE_OF_BODY;
}
import type {HttpClientInterface, HttpClientRequest, HttpClientResponse} from './http-client-interface.js';
import type {SCResponses} from '@openstapps/core';
/**
* HTTP client that is based on request
*/
export class HttpClient {
export class HttpClient implements HttpClientInterface {
/**
* Make a request
*
* @param requestConfig Configuration of the request
*/
async request<TYPE_OF_BODY>(requestConfig: RequestOptions): Promise<Response<TYPE_OF_BODY>> {
const parameters: OptionsOfJSONResponseBody = {
followRedirect: true,
method: 'GET',
responseType: 'json',
async request<T extends SCResponses>(requestConfig: HttpClientRequest): Promise<HttpClientResponse<T>> {
const parameters: RequestInit = {
method: requestConfig.method ?? 'GET',
headers: requestConfig.headers ?? {},
};
if (typeof requestConfig.body !== 'undefined') {
parameters.json = requestConfig.body;
if (requestConfig.body !== undefined) {
(parameters.headers as Record<string, string>)['Content-Type'] = 'application/json';
parameters.body = JSON.stringify(requestConfig.body);
}
if (typeof requestConfig.headers !== 'undefined') {
parameters.headers = requestConfig.headers;
}
const response = await fetch(requestConfig.url, parameters);
if (typeof requestConfig.method !== 'undefined') {
parameters.method = requestConfig.method;
}
let response: Response<TYPE_OF_BODY>;
try {
response = await got(requestConfig.url.toString(), parameters);
} catch (error) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (typeof (error as any).response === 'undefined') {
throw error;
}
// if there is a response (e.g. response with statusCode 404 etc.) provide it
// eslint-disable-next-line @typescript-eslint/no-explicit-any
response = (error as any).response as Response<TYPE_OF_BODY>;
}
return response;
return {
body: await response.json().catch(() => response.text().catch(() => undefined)),
headers: response.headers,
statusCode: response.status,
};
}
}

4
packages/api/src/index.ts
View File

@@ -1,10 +1,6 @@
export * from './bulk.js';
export * from './client.js';
export * from './connector-client.js';
export * from './copy.js';
export * from './e2e.js';
export * from './errors.js';
export * from './http-client.js';
export * from './http-client-interface.js';
export * from './plugin.js';
export * from './plugin-client.js';

70
packages/api/src/plugin-client.ts
View File

@@ -1,70 +0,0 @@
/*
* Copyright (C) 2019-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCPluginRegisterRequest, SCPluginRegisterRoute} from '@openstapps/core';
import {ConnectorClient} from './connector-client.js';
import {Plugin} from './plugin.js';
/**
* The PluginClient for registering and unregistering HTTP Plugins
*
* It contains a lot of the boilerplate for creating plugins, and thus simplifies the creation of such.
*/
export class PluginClient extends ConnectorClient {
/**
* Register a plugin in the backend
*
* **This method automatically calls [[Plugin.start]]**
* You need to call this method before you can do anything with the plugin. If you want to register the plugin again,
* you might first want to inform yourself how the backend behaves in such cases TODO: add docs for this
*
* @param plugin The instance of the plugin you want to register
*/
async registerPlugin(plugin: Plugin) {
const request: SCPluginRegisterRequest = {
action: 'add',
plugin: {
address: plugin.fullUrl,
name: plugin.name,
requestSchema: plugin.requestSchema,
responseSchema: plugin.responseSchema,
route: plugin.route,
},
};
await this.invokeRoute(new SCPluginRegisterRoute(), undefined, request);
// start the plugin we just registered
plugin.start();
}
/**
* Unregister a plugin from the backend
*
* **This method automatically calls [[Plugin.stop]]**
* If you want to unregister your plugin for some reason, you can do so by calling this method.
* Use with caution.*
*
* @param plugin The instance of the plugin you want to register
*/
async unregisterPlugin(plugin: Plugin) {
const request: SCPluginRegisterRequest = {
action: 'remove',
route: plugin.route,
};
// stop the plugin we want to unregister
plugin.stop();
await this.invokeRoute(new SCPluginRegisterRoute(), undefined, request);
}
}

254
packages/api/src/plugin.ts
View File

@@ -1,254 +0,0 @@
/*
* Copyright (C) 2019-2022 Open StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Converter} from '@openstapps/core-tools';
import {Logger} from '@openstapps/logger';
import bodyParser from 'body-parser';
import express from 'express';
import * as http from 'http';
import * as http2 from 'http2';
import {JSONSchema7} from 'json-schema';
import morgan from 'morgan';
import ErrnoException = NodeJS.ErrnoException;
/**
* The Plugin for creating HTTP backend plugins
*
* It contains a lot of the boilerplate for creating plugins, and thus simplifies the creation of such.
* To create your own plugin, you need to extend this class and implement the [[Plugin.onRouteInvoke]] method
*/
export abstract class Plugin {
/**
* The express instance
*/
private readonly app = express();
/**
* The HTTP server
*/
private readonly server: http.Server;
/**
* Whether the server is active or not
*
* When active is false, it will return 404 on all routes.
*/
protected active = false;
/**
* The full URL of the plugin
*
* The full URL of the plugin consists out of URL:PORT
*/
public get fullUrl() {
return `${this.url}:${this.port}`;
}
/**
* The port on which the plugin will listen on
*/
public port: string | number | false;
/**
* The schema of the request interfaces defined by the user
*/
public readonly requestSchema: JSONSchema7 = {};
/**
* The schema of the response interfaces defined by the user
*/
public readonly responseSchema: JSONSchema7 = {};
/**
* Normalize a port into a number, string, or false.
*
* @param value the port you want to normalize
*/
protected static normalizePort(value: string) {
const portNumber = Number.parseInt(value, 10);
/* istanbul ignore next */
if (Number.isNaN(portNumber)) {
// named pipe
/* istanbul ignore next */
return value;
}
/* istanbul ignore next */
if (portNumber >= 0) {
// port number
return portNumber;
}
/* istanbul ignore next */
return false;
}
/**
* Create an instance of the PluginClient
*
* Don't forget to call [[PluginClient.registerPlugin]]!
* Refer to the examples for how to use the schemas. TODO: examples
*
* @param port The port of the plugin
* @param name The name of the plugin
* @param url The url of the plugin without the port or anything else, for example `http://localhost`
* @param route The desired route that will be registered in the backend
* @param backendUrl The url of the backend
* @param converter If you want to use an already existing converter, you can pass it here
* @param requestName the name of the request schema
* @param responseName the name of the response schema
* @param version the version. You should retrieve it from the package.json
*/
constructor(
port: number,
public name: string,
public url: string,
public route: string,
protected backendUrl: string,
converter: Converter,
requestName: string,
responseName: string,
version: string,
) {
this.app.use(bodyParser.json());
this.port = Plugin.normalizePort(
/* istanbul ignore next */
typeof process.env.PORT === 'undefined' ? port.toString() : process.env.PORT,
);
this.app.set('port', this.port);
// setup express
this.server = http.createServer(this.app);
this.server.listen(this.port);
/* istanbul ignore next */
this.server.on('error', error => {
/* istanbul ignore next */
this.onError(error);
});
this.server.on('listening', () => {
this.onListening();
});
this.requestSchema = converter.getSchema(requestName, version);
this.responseSchema = converter.getSchema(responseName, version);
this.app.use(morgan('dev'));
this.app.set('env', process.env.NODE_ENV);
this.app.all('*', async (request: express.Request, response: express.Response) => {
if (this.active) {
await this.onRouteInvoke(request, response);
} else {
response.status(http2.constants.HTTP_STATUS_NOT_FOUND);
response.send();
}
});
}
/**
* Event listener for HTTP server "error" event.
*
* @param error The error that occurred
*/
/* istanbul ignore next */
private onError(error: ErrnoException) {
if (error.syscall !== 'listen') {
throw error;
}
const bind = typeof this.port === 'string' ? `Pipe ${this.port}` : `Port ${this.port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES': {
// tslint:disable-next-line:no-floating-promises
Logger.error(`${bind} requires elevated privileges`);
process.exit(1);
break;
}
case 'EADDRINUSE': {
// tslint:disable-next-line:no-floating-promises
Logger.error(`${bind} is already in use`);
process.exit(1);
break;
}
default: {
throw error;
}
}
}
/**
* Event listener for HTTP server "listening" event.
*/
private onListening() {
const addr = this.server.address();
/* istanbul ignore next */
const bind = typeof addr === 'string' ? `pipe ${addr}` : addr === null ? 'null' : `port ${addr.port}`;
Logger.ok(`Listening on ${bind}`);
}
/**
* When the route gets invoked
*
* Override this method for your own plugin
*
* @param request An express Request from the backend
* @param response An express Response to the backend for you to send back data
*/
protected abstract onRouteInvoke(request: express.Request, response: express.Response): Promise<void>;
/**
* Closes the server
*
* This will stop the plugin from listening to any requests at all, and is currently an irreversible process.
* This means, that the instantiated plugin is basically useless afterwards.
*/
public async close() {
return new Promise((resolve, reject) => {
this.server.close(error => {
/* istanbul ignore next */
if (typeof error !== 'undefined') {
/* istanbul ignore next */
reject(error);
}
resolve(undefined);
});
});
}
/**
* Start the plugin
*
* **THIS METHOD GETS CALLED AUTOMATICALLY WITH [[PluginClient.registerPlugin]]**
* If the plugin is not started, it will return 404 on any route
*/
public start() {
this.active = true;
}
/**
* Stop the plugin
*
* **THIS METHOD GETS CALLED AUTOMATICALLY WITH [[PluginClient.unregisterPlugin]]**
* If the plugin is not started, it will return 404 on any route
*/
public stop() {
// you can't unregister routes from express. So this is a workaround.
this.active = false;
}
}

15
packages/api/test/bulk.spec.ts
View File

@@ -24,8 +24,8 @@ import {expect} from 'chai';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import chaiSpies from 'chai-spies';
import moment from 'moment';
import {HttpClient, Bulk, Client, BulkWithMultipleTypesError} from '../src/index.js';
import {addHours} from 'date-fns';
chai.should();
chai.use(chaiSpies);
@@ -50,9 +50,10 @@ describe('Bulk', function () {
});
expect(client.invokeRoute).not.to.have.been.called();
const expiration = addHours(new Date(), 1).toISOString();
const bulk = new Bulk(SCThingType.Dish, client, {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'foo',
state: 'in progress',
type: SCThingType.Dish,
@@ -63,7 +64,7 @@ describe('Bulk', function () {
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: moment().format(),
indexed: new Date().toISOString(),
name: 'bar',
type: SCThingOriginType.Remote,
},
@@ -84,7 +85,7 @@ describe('Bulk', function () {
it('should fail add', async function () {
const bulk = new Bulk(SCThingType.Dish, client, {
expiration: moment().add(3600, 'seconds').format(),
expiration: addHours(new Date(), 1).toISOString(),
source: 'foo',
state: 'in progress',
type: SCThingType.Dish,
@@ -97,7 +98,7 @@ describe('Bulk', function () {
messageBody: 'Lorem ipsum.',
name: 'foobar',
origin: {
indexed: moment().format(),
indexed: new Date().toISOString(),
name: 'bar',
type: SCThingOriginType.Remote,
},
@@ -111,7 +112,7 @@ describe('Bulk', function () {
it('should construct', function () {
expect(() => {
return new Bulk(SCThingType.Dish, client, {
expiration: moment().add(3600, 'seconds').format(),
expiration: addHours(new Date(), 1).toISOString(),
source: 'foo',
state: 'in progress',
type: SCThingType.Dish,
@@ -128,7 +129,7 @@ describe('Bulk', function () {
expect(client.invokeRoute).not.to.have.been.called();
const bulk = new Bulk(SCThingType.Dish, client, {
expiration: moment().add(3600, 'seconds').format(),
expiration: addHours(new Date(), 1).toISOString(),
source: 'foo',
state: 'in progress',
type: SCThingType.Dish,

34
packages/api/test/connector-client.spec.ts
View File

@@ -32,8 +32,6 @@ import chai from 'chai';
import {expect} from 'chai';
import chaiAsPromised from 'chai-as-promised';
import chaiSpies from 'chai-spies';
import clone = require('rfdc');
import moment from 'moment';
import traverse from 'traverse';
import {
ConnectorClient,
@@ -82,10 +80,12 @@ describe('ConnectorClient', function () {
});
it('should bulk', async function () {
sandbox.on(httpClient, 'request', async (): Promise<HttpClientResponse<SCBulkResponse>> => {
let expiration = 'ERROR';
sandbox.on(httpClient, 'request', async (response): Promise<HttpClientResponse<SCBulkResponse>> => {
expiration = response?.body?.expiration ?? 'ERROR';
return {
body: {
expiration: moment().add(1800, 'seconds').format(),
expiration,
source: 'foo',
state: 'in progress',
type: SCThingType.Message,
@@ -103,7 +103,7 @@ describe('ConnectorClient', function () {
expect(httpClient.request).to.have.been.first.called.with({
body: {
expiration: moment().add(1800, 'seconds').format(),
expiration,
source: 'foo',
type: SCThingType.Message,
},
@@ -116,10 +116,12 @@ describe('ConnectorClient', function () {
});
it('should bulk without timeout', async function () {
sandbox.on(httpClient, 'request', async (): Promise<HttpClientResponse<SCBulkResponse>> => {
let expiration = 'ERROR';
sandbox.on(httpClient, 'request', async (request): Promise<HttpClientResponse<SCBulkResponse>> => {
expiration = request?.body?.expiration ?? 'ERROR';
return {
body: {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'foo',
state: 'in progress',
type: SCThingType.Message,
@@ -137,7 +139,7 @@ describe('ConnectorClient', function () {
expect(httpClient.request).to.have.been.first.called.with({
body: {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'foo',
type: SCThingType.Message,
},
@@ -181,14 +183,17 @@ describe('ConnectorClient', function () {
type responses = SCBulkResponse | SCBulkAddResponse | SCBulkDoneResponse;
let expiration = 'ERROR';
sandbox.on(
httpClient,
'request',
async (request: HttpClientRequest): Promise<HttpClientResponse<responses>> => {
if (request.url.toString() === new URL('http://localhost' + bulkRoute.getUrlPath()).toString()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expiration = (request?.body as any)?.expiration ?? 'ERROR';
return {
body: {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'copy',
state: 'in progress',
type: SCThingType.Message,
@@ -226,7 +231,7 @@ describe('ConnectorClient', function () {
expect(httpClient.request).to.have.been.first.called.with({
body: {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'copy',
type: SCThingType.Message,
},
@@ -275,14 +280,17 @@ describe('ConnectorClient', function () {
type responses = SCBulkResponse | SCBulkAddResponse | SCBulkDoneResponse;
let expiration = 'ERROR';
sandbox.on(
httpClient,
'request',
async (request: HttpClientRequest): Promise<HttpClientResponse<responses>> => {
if (request.url.toString() === new URL('http://localhost' + bulkRoute.getUrlPath()).toString()) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expiration = (request?.body as any)?.expiration ?? 'ERROR';
return {
body: {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'stapps-api',
state: 'in progress',
type: SCThingType.Message,
@@ -320,7 +328,7 @@ describe('ConnectorClient', function () {
expect(httpClient.request).to.have.been.first.called.with({
body: {
expiration: moment().add(3600, 'seconds').format(),
expiration,
source: 'stapps-api',
type: SCThingType.Message,
},
@@ -370,7 +378,7 @@ describe('ConnectorClient', function () {
);
for (const testInstance of testInstances) {
const checkInstance = clone()(testInstance);
const checkInstance = JSON.parse(JSON.stringify(testInstance));
const testInstanceWithoutReferences = ConnectorClient.removeReferences(testInstance);
expect(doesContainThings(testInstanceWithoutReferences)).to.be.equal(

212
packages/api/test/copy.spec.ts
View File

@@ -1,212 +0,0 @@
/*
* Copyright (C) 2018 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {
SCBulkAddResponse,
SCBulkAddRoute,
SCBulkDoneResponse,
SCBulkDoneRoute,
SCBulkResponse,
SCBulkRoute,
SCSearchRequest,
SCSearchResponse,
SCSearchRoute,
SCThingType,
} from '@openstapps/core';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import chaiSpies from 'chai-spies';
import moment from 'moment';
import {copy, ApiError, HttpClient, RequestOptions, Response} from '../src/index.js';
import {RecursivePartial} from './client.spec.js';
chai.should();
chai.use(chaiSpies);
chai.use(chaiAsPromised);
const sandbox = chai.spy.sandbox();
const bulkRoute = new SCBulkRoute();
const bulkAddRoute = new SCBulkAddRoute();
const bulkDoneRoute = new SCBulkDoneRoute();
const searchRoute = new SCSearchRoute();
const httpClient = new HttpClient();
describe('Copy', function () {
afterEach(function () {
sandbox.restore();
});
it('should copy', async function () {
type responses = Response<SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse>;
sandbox.on(
httpClient,
'request',
async (request: RequestOptions): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === 'http://foo.bar' + searchRoute.getUrlPath().toString()) {
const body = request.body as SCSearchRequest;
let count = 0;
if (typeof body.size === 'number' && body.size > 0) {
count = 1;
}
return {
body: {
data: [
{
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: moment().format(),
name: 'bar',
},
type: SCThingType.Dish,
uid: 'foo',
},
],
facets: [],
pagination: {
count: count,
offset: 0,
total: 1,
},
stats: {
time: 1,
},
},
statusCode: searchRoute.statusCodeSuccess,
};
} else if (request.url.toString() === 'http://localhost' + bulkRoute.getUrlPath().toString()) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
} else if (
request.url.toString() ===
'http://localhost' +
bulkAddRoute
.getUrlPath({
UID: 'foo',
})
.toString()
) {
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
},
);
await copy(httpClient, {
batchSize: 5,
from: 'http://foo.bar',
source: 'stapps-copy',
to: 'http://localhost',
type: SCThingType.Dish,
version: 'foo.bar.foobar',
});
});
it('should fail to copy', async function () {
type responses = Response<SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse>;
sandbox.on(
httpClient,
'request',
async (request: RequestOptions): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === 'http://foo.bar' + searchRoute.getUrlPath().toString()) {
const body = request.body as SCSearchRequest;
if (typeof body.size === 'number' && body.size > 0) {
throw new ApiError({});
}
return {
body: {
data: [
{
categories: ['main dish'],
name: 'foobar',
origin: {
indexed: moment().format(),
name: 'bar',
},
type: SCThingType.Dish,
uid: 'foo',
},
],
facets: [],
pagination: {
count: 0,
offset: 0,
total: 1,
},
stats: {
time: 1,
},
},
statusCode: searchRoute.statusCodeSuccess,
};
} else if (request.url.toString() === 'http://localhost' + bulkRoute.getUrlPath().toString()) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
} else if (
request.url.toString() ===
'http://localhost' +
bulkAddRoute
.getUrlPath({
UID: 'foo',
})
.toString()
) {
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
},
);
await copy(httpClient, {
batchSize: 5,
from: 'http://foo.bar',
source: 'stapps-copy',
to: 'http://localhost',
type: SCThingType.Dish,
version: 'foo.bar.foobar',
}).should.be.rejectedWith(ApiError);
});
});

211
packages/api/test/e2e.spec.ts
View File

@@ -1,211 +0,0 @@
/*
* Copyright (C) 2019 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
// eslint-disable-next-line unicorn/prevent-abbreviations
import {
SCBulkAddResponse,
SCBulkAddRoute,
SCBulkDoneResponse,
SCBulkDoneRoute,
SCBulkResponse,
SCBulkRoute,
SCSearchResponse,
SCSearchRoute,
SCThings,
} from '@openstapps/core';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import chaiSpies from 'chai-spies';
import clone = require('rfdc');
import {existsSync, mkdirSync, rmdirSync, unlinkSync} from 'fs';
import {createFileSync} from 'fs-extra';
// eslint-disable-next-line unicorn/prevent-abbreviations
import {e2eRun, getItemsFromSamples, ApiError, HttpClient, RequestOptions, Response} from '../src/index.js';
import {RecursivePartial} from './client.spec.js';
import {expect} from 'chai';
import path from 'path';
import {fileURLToPath} from 'url';
chai.should();
chai.use(chaiSpies);
chai.use(chaiAsPromised);
const sandbox = chai.spy.sandbox();
const bulkRoute = new SCBulkRoute();
const bulkAddRoute = new SCBulkAddRoute();
const bulkDoneRoute = new SCBulkDoneRoute();
const searchRoute = new SCSearchRoute();
const httpClient = new HttpClient();
const storedThings: Map<string, SCThings> = new Map();
describe('e2e Connector', function () {
afterEach(function () {
sandbox.restore();
});
it('should get core test samples', async function () {
const items = await getItemsFromSamples('./node_modules/@openstapps/core/test/resources');
expect(items).to.not.be.empty;
});
it('should fail to get core test samples', async function () {
await chai.expect(getItemsFromSamples('./non-existent-directory')).to.be.rejectedWith(Error);
});
it('should run e2e simulation', async function () {
type responses = Response<SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse>;
let failOnCompare = false;
let failOnLookup = false;
sandbox.on(
httpClient,
'request',
async (request: RequestOptions): Promise<RecursivePartial<responses>> => {
if (request.url.toString() === `http://localhost${bulkRoute.getUrlPath().toString()}`) {
return {
body: {
state: 'in progress',
uid: 'foo',
},
statusCode: bulkRoute.statusCodeSuccess,
};
}
if (
request.url.toString() === `http://localhost${bulkAddRoute.getUrlPath({UID: 'foo'}).toString()}`
) {
storedThings.set(request.body.uid, clone()(request.body));
return {
body: {},
statusCode: bulkAddRoute.statusCodeSuccess,
};
}
if (
request.url.toString() === `http://localhost${bulkDoneRoute.getUrlPath({UID: 'foo'}).toString()}`
) {
return {
body: {},
statusCode: bulkDoneRoute.statusCodeSuccess,
};
}
if (request.url.toString() === `http://localhost${searchRoute.getUrlPath().toString()}`) {
const thing = storedThings.get(request.body.filter.arguments.value);
if (failOnCompare) {
thing!.origin!.modified = 'altered';
}
const returnThing = failOnLookup ? [] : [thing];
const returnBody = {
data: returnThing,
facets: [],
pagination: {
count: returnThing.length,
offset: 0,
total: returnThing.length,
},
stats: {
time: 42,
},
};
return {
body: returnBody,
statusCode: searchRoute.statusCodeSuccess,
};
}
return {
body: {},
statusCode: searchRoute.statusCodeSuccess,
};
},
);
// tslint:disable-next-line: max-line-length
await e2eRun(httpClient, {
to: 'http://localhost',
samplesLocation: './node_modules/@openstapps/core/test/resources',
});
failOnLookup = true;
failOnCompare = false;
// tslint:disable-next-line: max-line-length
await e2eRun(httpClient, {
to: 'http://localhost',
samplesLocation: './node_modules/@openstapps/core/test/resources',
}).should.be.rejectedWith('Search for single SCThing with uid');
failOnLookup = false;
failOnCompare = true;
// tslint:disable-next-line: max-line-length
await e2eRun(httpClient, {
to: 'http://localhost',
samplesLocation: './node_modules/@openstapps/core/test/resources',
}).should.be.rejectedWith('Unexpected difference');
});
it('should fail to index', async function () {
type responses = Response<SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse>;
sandbox.on(httpClient, 'request', async (): Promise<RecursivePartial<responses>> => {
return {
body: {},
statusCode: Number.MAX_SAFE_INTEGER,
};
});
// tslint:disable-next-line: max-line-length
return e2eRun(httpClient, {
to: 'http://localhost',
samplesLocation: './node_modules/@openstapps/core/test/resources',
}).should.be.rejectedWith(ApiError);
});
it('should fail to index directory without data', async function () {
const emptyDirectoryPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'emptyDir');
if (!existsSync(emptyDirectoryPath)) {
mkdirSync(emptyDirectoryPath);
}
await e2eRun(httpClient, {
to: 'http://localhost',
samplesLocation: emptyDirectoryPath,
}).should.be.rejectedWith('Could not index samples. None were retrieved from the file system.');
rmdirSync(emptyDirectoryPath);
});
it('should fail to index directory without json data', async function () {
const somewhatFilledDirectoryPath = path.join(
path.dirname(fileURLToPath(import.meta.url)),
'somewhatFilledDir',
);
if (!existsSync(somewhatFilledDirectoryPath)) {
mkdirSync(somewhatFilledDirectoryPath);
}
const nonJsonFile = path.join(somewhatFilledDirectoryPath, 'nonjson.txt');
createFileSync(nonJsonFile);
await e2eRun(httpClient, {
to: 'http://localhost',
samplesLocation: somewhatFilledDirectoryPath,
}).should.be.rejectedWith('Could not index samples. None were retrieved from the file system.');
unlinkSync(nonJsonFile);
rmdirSync(somewhatFilledDirectoryPath);
});
});

65
packages/api/test/http-client.spec.ts
View File

@@ -13,14 +13,18 @@
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {expect} from 'chai';
import nock from 'nock';
import {Interceptable, MockAgent, setGlobalDispatcher} from 'undici';
import {HttpClient} from '../src/index.js';
// TODO: use after each to clean up the nock (then there is no need for numerated resource links)
const mockAgent = new MockAgent();
setGlobalDispatcher(mockAgent);
describe('HttpClient', function () {
afterEach(function () {
nock.cleanAll();
let mockPool: Interceptable;
beforeEach(function () {
mockPool?.close();
mockPool = mockAgent.get('http://www.example.com');
});
it('should construct', function () {
@@ -32,32 +36,32 @@ describe('HttpClient', function () {
it('should request', async function () {
const client = new HttpClient();
nock('http://www.example.com').get('/resource').reply(200, 'foo');
mockPool.intercept({path: '/resource', method: 'GET'}).reply(200, {foo: 'bar'});
const response = await client.request({
url: new URL('http://www.example.com/resource'),
});
expect(response.body).to.be.equal('foo');
expect(response.body).to.be.deep.equal({foo: 'bar'});
});
it('should request with body', async function () {
const client = new HttpClient();
nock('http://www.example.com').get('/resource').reply(200, 'foo');
mockPool.intercept({path: '/resource', method: 'GET'}).reply(200, {foo: 'foo'});
const response = await client.request({
url: new URL('http://www.example.com/resource'),
});
expect(response.body).to.be.equal('foo');
expect(response.body).to.be.deep.equal({foo: 'foo'});
});
it('should request with error', async function () {
const client = new HttpClient();
let caughtError;
nock('http://www.example.com').get('/resource').replyWithError('foo');
mockPool.intercept({path: '/resource', method: 'GET'}).replyWithError(new Error('foo'));
try {
await client.request({
@@ -76,7 +80,15 @@ describe('HttpClient', function () {
it('should request with headers', async function () {
const client = new HttpClient();
nock('http://www.example.com').get('/resource').reply(200, 'foo');
mockPool
.intercept({
path: '/resource',
method: 'GET',
headers: {
'X-StApps-Version': 'foo.bar.foobar',
},
})
.reply(200, {foo: 'bar'});
const response = await client.request({
headers: {
@@ -85,32 +97,55 @@ describe('HttpClient', function () {
url: new URL('http://www.example.com/resource'),
});
expect(response.body).to.be.equal('foo');
expect(response.body).to.be.deep.equal({foo: 'bar'});
});
it('should request with body', async function () {
const client = new HttpClient();
mockPool
.intercept({
path: '/resource',
method: 'POST',
headers: {
'content-type': 'application/json',
},
body: JSON.stringify({bar: 'baz'}),
})
.reply(200, {foo: 'foo'});
const response = await client.request({
body: {bar: 'baz'},
method: 'POST',
url: new URL('http://www.example.com/resource'),
});
expect(response.body).to.be.deep.equal({foo: 'foo'});
});
it('should request with method GET', async function () {
const client = new HttpClient();
nock('http://www.example.com').get('/resource').reply(200, 'foo');
mockPool.intercept({path: '/resource', method: 'GET'}).reply(200, {foo: 'foo'});
const response = await client.request({
method: 'GET',
url: new URL('http://www.example.com/resource'),
});
expect(response.body).to.be.equal('foo');
expect(response.body).to.be.deep.equal({foo: 'foo'});
});
it('should request with method POST', async function () {
const client = new HttpClient();
nock('http://www.example.com').post('/resource').reply(200, 'foo');
mockPool.intercept({path: '/resource', method: 'POST'}).reply(200, {foo: 'foo'});
const response = await client.request({
method: 'POST',
url: new URL('http://www.example.com/resource'),
});
expect(response.body).to.be.equal('foo');
expect(response.body).to.be.deep.equal({foo: 'foo'});
});
});

123
packages/api/test/plugin-client.spec.ts
View File

@@ -1,123 +0,0 @@
/*
* Copyright (C) 2019 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SCPluginRegisterRequest, SCPluginRegisterResponse, SCPluginRegisterRoute} from '@openstapps/core';
import chai from 'chai';
import {expect} from 'chai';
import chaiSpies from 'chai-spies';
import {HttpClient, HttpClientResponse, PluginClient} from '../src/index.js';
import {TestPlugin} from './plugin-resources/test-plugin.js';
chai.use(chaiSpies);
const sandbox = chai.spy.sandbox();
const httpClient = new HttpClient();
const pluginRegisterRoute = new SCPluginRegisterRoute();
const pluginClient = new PluginClient(httpClient, 'http://localhost');
describe('PluginClient', function () {
this.timeout(10_000);
let plugin: TestPlugin;
beforeEach(async function () {
plugin = new TestPlugin(
4000,
'',
'',
'',
'',
{
getSchema: () => {
/***/
},
} as never,
'',
'',
'',
);
});
afterEach(async function () {
await plugin.close();
sandbox.restore();
});
it('should register the plugin', async function () {
sandbox.on(httpClient, 'request', async (): Promise<HttpClientResponse<SCPluginRegisterResponse>> => {
return {
body: {
success: true,
},
headers: {},
statusCode: pluginRegisterRoute.statusCodeSuccess,
};
});
expect(httpClient.request).not.to.have.been.called();
await pluginClient.registerPlugin(plugin);
const request: SCPluginRegisterRequest = {
action: 'add',
plugin: {
address: plugin.fullUrl,
name: plugin.name,
requestSchema: plugin.requestSchema,
responseSchema: plugin.responseSchema,
route: plugin.route,
},
};
expect(httpClient.request).to.have.been.first.called.with({
body: request,
headers: {
'Content-Type': 'application/json',
},
method: pluginRegisterRoute.method,
url: new URL(`http://localhost${pluginRegisterRoute.getUrlPath()}`),
});
});
it('should unregister the plugin', async function () {
sandbox.on(httpClient, 'request', async (): Promise<HttpClientResponse<SCPluginRegisterResponse>> => {
return {
body: {
success: true,
},
headers: {},
statusCode: pluginRegisterRoute.statusCodeSuccess,
};
});
expect(httpClient.request).not.to.have.been.called();
await pluginClient.unregisterPlugin(plugin);
const request: SCPluginRegisterRequest = {
action: 'remove',
route: plugin.route,
};
expect(httpClient.request).to.have.been.first.called.with({
body: request,
headers: {
'Content-Type': 'application/json',
},
method: pluginRegisterRoute.method,
url: new URL(`http://localhost${pluginRegisterRoute.getUrlPath()}`),
});
});
});

25
packages/api/test/plugin-resources/test-plugin-response.ts
View File

@@ -1,25 +0,0 @@
/*
* Copyright (C) 2021 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The Response Interface
*
* @validatable
*/
export interface TestPluginResponse {
/**
* Query dummy
*/
query: string;
}

32
packages/api/test/plugin-resources/test-plugin.ts
View File

@@ -1,32 +0,0 @@
/*
* Copyright (C) 2019 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import * as express from 'express';
import {Plugin} from '../../src/plugin.js';
/**
* A test plugin we use for all the tests
*
* It can be constructed without any parameter at all, or with all parameters if we want to test it
* It also serves as kind of a minimal plugin
*/
export class TestPlugin extends Plugin {
// tslint:disable-next-line: completed-docs prefer-function-over-method
protected async onRouteInvoke(_req: express.Request, res: express.Response): Promise<void> {
res.json({});
return undefined;
}
}

161
packages/api/test/plugin.spec.ts
View File

@@ -1,161 +0,0 @@
/*
* Copyright (C) 2019 StApps
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {Converter} from '@openstapps/core-tools';
import chai from 'chai';
import {expect} from 'chai';
import chaiSpies from 'chai-spies';
import {HttpClient} from '../src/index.js';
import {TestPlugin} from './plugin-resources/test-plugin.js';
import path from 'path';
import {readFile} from 'fs/promises';
import {fileURLToPath} from 'url';
chai.use(chaiSpies);
process.on('unhandledRejection', error => {
throw error;
});
const sandbox = chai.spy.sandbox();
const httpClient = new HttpClient();
const dirname = path.dirname(fileURLToPath(import.meta.url));
describe('Plugin', function () {
this.timeout(20_000);
let testPlugin: TestPlugin;
beforeEach(function () {
testPlugin = new TestPlugin(
4000,
'',
'',
'',
'',
{
getSchema: () => {
/***/
},
} as never,
'',
'',
'',
);
});
afterEach(async function () {
await testPlugin.close();
sandbox.restore();
});
it('should construct', async function () {
const converter = new Converter(
dirname,
path.resolve(dirname, 'plugin-resources', 'test-plugin-response.ts'),
);
sandbox.on(converter, 'getSchema', schemaName => {
return {$id: schemaName};
});
const constructTestPlugin = new TestPlugin(
4001,
'A',
'http://B',
'/C', // this doesn't matter for our tests, it's only something that affects the backend
'http://D',
converter,
'PluginTestRequest',
'PluginTestResponse',
JSON.parse(await readFile(path.resolve(dirname, '..', 'package.json'), 'utf8')).version,
);
expect(constructTestPlugin.port).to.be.equal(4001);
expect(constructTestPlugin.name).to.be.equal('A');
expect(constructTestPlugin.url).to.be.equal('http://B');
expect(constructTestPlugin.route).to.be.equal('/C');
// @ts-expect-error private property
expect(constructTestPlugin.backendUrl).to.be.equal('http://D');
// schemas are already covered, together with the directory and version
// @ts-expect-error private property
expect(constructTestPlugin.active).to.be.equal(false);
expect(constructTestPlugin.requestSchema.$id).to.be.equal('PluginTestRequest');
expect(constructTestPlugin.responseSchema.$id).to.be.equal('PluginTestResponse');
sandbox.on(constructTestPlugin, 'onRouteInvoke');
await httpClient.request({
url: new URL('http://localhost:4001'),
});
// onRouteInvoke is a protected method, but we need to access it from the outside to test it
// @ts-expect-error protected method
expect(constructTestPlugin.onRouteInvoke).not.to.have.been.called();
await constructTestPlugin.close();
sandbox.restore(constructTestPlugin, 'onRouteInvoke');
});
it('should have full url', async function () {
const constructTestPlugin = new TestPlugin(
4001,
'',
'http://B',
'',
'',
{
getSchema: () => {
/***/
},
} as never,
'',
'',
'',
);
expect(constructTestPlugin.fullUrl).to.be.equal('http://B:4001');
await constructTestPlugin.close();
});
it('should start', async function () {
testPlugin.start();
sandbox.on(testPlugin, 'onRouteInvoke');
await httpClient.request({
url: new URL('http://localhost:4000'),
});
// onRouteInvoke is a protected method, but we need to access it from the outside to test it
// @ts-expect-error protected method
expect(testPlugin.onRouteInvoke).to.have.been.called();
});
it('should stop', async function () {
// simulate a normal use case by first starting the plugin and then stopping it
testPlugin.start();
testPlugin.stop();
sandbox.on(testPlugin, 'onRouteInvoke');
const response = await httpClient.request({
url: new URL('http://localhost:4000'),
});
await expect(response.statusCode).to.be.equal(404);
// onRouteInvoke is a protected method, but we need to access it from the outside to test it
// @ts-expect-error protected method
expect(testPlugin.onRouteInvoke).not.to.have.been.called();
});
});