mirror of
https://gitlab.com/openstapps/openstapps.git
synced 2026-01-21 00:52:55 +00:00
feat: e2e connector
This commit is contained in:
@@ -1,234 +0,0 @@
|
||||
# @openstapps/api-cli
|
||||
|
||||
## 3.0.0
|
||||
|
||||
### Major Changes
|
||||
|
||||
- 64caebaf: Split API into API, API-CLI and API-Plugin
|
||||
|
||||
Plugins are now required to use `api-plugin`.
|
||||
Consumers of `api` can benefit from a slimmer package with
|
||||
no NodeJS dependencies.
|
||||
|
||||
- 64caebaf: Move project to a turbo monorepo & pnpm
|
||||
|
||||
Internal dependencies are now defined using `"@openstapps/package": "workspace:*"`
|
||||
|
||||
- Removed extraneous files from packages
|
||||
- `.npmrc`
|
||||
- `.npmignore`
|
||||
- `.mailmap`
|
||||
- `.gitignore`
|
||||
- `CONTRIBUTING.md`
|
||||
- `LICENSE` (Project license file is added upon publishing, see [pnpm.io](https://pnpm.io/cli/publish))
|
||||
- `package-lock.json`
|
||||
- `.editorconfig`
|
||||
- `.eslintrc.json` (moved eslint config to `package.json`)
|
||||
- `.eslintignore`
|
||||
- `.gitlab-ci.yml` (Most workflows are workspace-level)
|
||||
- `.gitlab/**` (issue templates etc. are now workspace-level)
|
||||
- `.dockerignore` (Docker files are determined by which files are deployed with `pnpm deploy`, as per `package.json/files`)
|
||||
- TSConfig has been moved to its own package (You can now use `"extends": "@openstapps/tsconfig"`)
|
||||
- Removed ESLint and Prettier peer dependency hell by injecting them through the `.pnpmfile.cjs`
|
||||
- Added syncpack for keeping dependency versions in sync (and consistent key ordering in `package.json`)
|
||||
- Replaced conventional changelog with changesets
|
||||
- Apps with binaries now use a top level `app.js`
|
||||
|
||||
```js
|
||||
#!/usr/bin/env node
|
||||
import './lib/app.js';
|
||||
```
|
||||
|
||||
- 64caebaf: Migrate integration tests from docker-compose solution to a shell script
|
||||
|
||||
`api-cli` no longer builds as a Docker container as a result.
|
||||
|
||||
- 64caebaf: Migrate to ESM
|
||||
|
||||
CommonJS is no longer supported in any capacity. To use the new
|
||||
version, you will need to migrate your package to ESM.
|
||||
We recommend using `tsup` and `Node 18`.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "module"
|
||||
}
|
||||
```
|
||||
|
||||
- 64caebaf: Migrate package to Node 18
|
||||
|
||||
- Consumers of this package will need to migrate to Node 18 or
|
||||
higher.
|
||||
- Packages have been migrated from promisified `readFile` or
|
||||
`readFileSync` towards `fs/promises`
|
||||
- Packages use native `flatMap` now
|
||||
|
||||
- 64caebaf: Migrate build system to `tsup`
|
||||
|
||||
All packages now use an `index.ts` file to expose contents.
|
||||
|
||||
You will need to migrate paths from `import foo from '@scope/package/lib/foo` to `import foo from '@scope/package'`
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 64caebaf: Migrate tests to C8/Chai/Mocha
|
||||
|
||||
- `@testdeck` OOP testing has been removed.
|
||||
- Tests have been unified
|
||||
- CommonJS module mocking has been replaced through
|
||||
refactoring of tests, as ES Modules cannot be mocked
|
||||
(do yourself a favor and don't try to mock them)
|
||||
- C8 now replaces NYC as a native coverage tool
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 98546a97: Migrate away from @openstapps/configuration
|
||||
- 23481d0d: Update to TypeScript 5.1.6
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [1f62b5c5]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [23481d0d]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [0a7e6af1]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- @openstapps/core-tools@3.0.0
|
||||
- @openstapps/api@3.0.0
|
||||
- @openstapps/eslint-config@3.0.0
|
||||
- @openstapps/logger@3.0.0
|
||||
- @openstapps/core@3.0.0
|
||||
|
||||
## 3.0.0-next.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 23481d0d: Update to TypeScript 5.1.6
|
||||
- Updated dependencies [23481d0d]
|
||||
- @openstapps/eslint-config@3.0.0-next.4
|
||||
- @openstapps/core-tools@3.0.0-next.4
|
||||
- @openstapps/logger@3.0.0-next.4
|
||||
- @openstapps/core@3.0.0-next.4
|
||||
- @openstapps/api@3.0.0-next.4
|
||||
|
||||
## 3.0.0-next.0
|
||||
|
||||
### Major Changes
|
||||
|
||||
- 64caebaf: Split API into API, API-CLI and API-Plugin
|
||||
|
||||
Plugins are now required to use `api-plugin`.
|
||||
Consumers of `api` can benefit from a slimmer package with
|
||||
no NodeJS dependencies.
|
||||
|
||||
- 64caebaf: Move project to a turbo monorepo & pnpm
|
||||
|
||||
Internal dependencies are now defined using `"@openstapps/package": "workspace:*"`
|
||||
|
||||
- Removed extraneous files from packages
|
||||
- `.npmrc`
|
||||
- `.npmignore`
|
||||
- `.mailmap`
|
||||
- `.gitignore`
|
||||
- `CONTRIBUTING.md`
|
||||
- `LICENSE` (Project license file is added upon publishing, see [pnpm.io](https://pnpm.io/cli/publish))
|
||||
- `package-lock.json`
|
||||
- `.editorconfig`
|
||||
- `.eslintrc.json` (moved eslint config to `package.json`)
|
||||
- `.eslintignore`
|
||||
- `.gitlab-ci.yml` (Most workflows are workspace-level)
|
||||
- `.gitlab/**` (issue templates etc. are now workspace-level)
|
||||
- `.dockerignore` (Docker files are determined by which files are deployed with `pnpm deploy`, as per `package.json/files`)
|
||||
- TSConfig has been moved to its own package (You can now use `"extends": "@openstapps/tsconfig"`)
|
||||
- Removed ESLint and Prettier peer dependency hell by injecting them through the `.pnpmfile.cjs`
|
||||
- Added syncpack for keeping dependency versions in sync (and consistent key ordering in `package.json`)
|
||||
- Replaced conventional changelog with changesets
|
||||
- Apps with binaries now use a top level `app.js`
|
||||
|
||||
```js
|
||||
#!/usr/bin/env node
|
||||
import './lib/app.js';
|
||||
```
|
||||
|
||||
- 64caebaf: Migrate integration tests from docker-compose solution to a shell script
|
||||
|
||||
`api-cli` no longer builds as a Docker container as a result.
|
||||
|
||||
- 64caebaf: Migrate to ESM
|
||||
|
||||
CommonJS is no longer supported in any capacity. To use the new
|
||||
version, you will need to migrate your package to ESM.
|
||||
We recommend using `tsup` and `Node 18`.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "module"
|
||||
}
|
||||
```
|
||||
|
||||
- 64caebaf: Migrate package to Node 18
|
||||
|
||||
- Consumers of this package will need to migrate to Node 18 or
|
||||
higher.
|
||||
- Packages have been migrated from promisified `readFile` or
|
||||
`readFileSync` towards `fs/promises`
|
||||
- Packages use native `flatMap` now
|
||||
|
||||
- 64caebaf: Migrate build system to `tsup`
|
||||
|
||||
All packages now use an `index.ts` file to expose contents.
|
||||
|
||||
You will need to migrate paths from `import foo from '@scope/package/lib/foo` to `import foo from '@scope/package'`
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 64caebaf: Migrate tests to C8/Chai/Mocha
|
||||
|
||||
- `@testdeck` OOP testing has been removed.
|
||||
- Tests have been unified
|
||||
- CommonJS module mocking has been replaced through
|
||||
refactoring of tests, as ES Modules cannot be mocked
|
||||
(do yourself a favor and don't try to mock them)
|
||||
- C8 now replaces NYC as a native coverage tool
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 98546a97: Migrate away from @openstapps/configuration
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [0a7e6af1]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [64caebaf]
|
||||
- Updated dependencies [98546a97]
|
||||
- @openstapps/core-tools@3.0.0-next.0
|
||||
- @openstapps/api@3.0.0-next.0
|
||||
- @openstapps/eslint-config@3.0.0-next.0
|
||||
- @openstapps/logger@3.0.0-next.0
|
||||
- @openstapps/core@3.0.0-next.0
|
||||
@@ -1,12 +0,0 @@
|
||||
FROM registry.gitlab.com/openstapps/openstapps/node-base
|
||||
|
||||
USER root
|
||||
RUN apk add --update python3 py3-pip make g++ gcompat
|
||||
|
||||
USER node
|
||||
ADD --chown=node:node . /app
|
||||
ENV NODE_ENV=production
|
||||
WORKDIR /app
|
||||
|
||||
EXPOSE 3000
|
||||
ENTRYPOINT ["node", "app.js"]
|
||||
@@ -1,69 +0,0 @@
|
||||
# @openstapps/api-cli
|
||||
|
||||
To get some data into a local `backend-node`-instance, you can run this
|
||||
as a standalone program to copy data of a remote `backend-node`-instance
|
||||
into your local one.
|
||||
|
||||
Example to copy all Events of the b-tu instance:
|
||||
|
||||
```shell
|
||||
npm install
|
||||
npm run build
|
||||
node ./lib/cli.js copy Event https://stappsbe01.innocampus.tu-berlin.de http://localhost:3000 100
|
||||
```
|
||||
|
||||
Example to index all items from @openstapps/core test files to a backend:
|
||||
|
||||
```shell
|
||||
npm install
|
||||
npm run build
|
||||
node ./lib/cli.js e2e http://localhost:3000
|
||||
```
|
||||
|
||||
Example to clone the full database
|
||||
|
||||
```shell
|
||||
node app.js copy "*" https://mobile.app.uni-frankfurt.de http://localhost:3000 100
|
||||
```
|
||||
|
||||
### Program arguments
|
||||
|
||||
```shell
|
||||
node ./lib/cli.js copy <type> <from> <to> <batchSize>
|
||||
|
||||
node ./lib/cli.js e2e <to>
|
||||
```
|
||||
|
||||
#### Options
|
||||
|
||||
The source identifier for the bulk to use with the target instance (default is 'copy')
|
||||
|
||||
```shell
|
||||
-s, --bulkSource <bulkScource>
|
||||
```
|
||||
|
||||
The App version to use (unset by default)
|
||||
|
||||
```shell
|
||||
-a, --appVersion <version>
|
||||
```
|
||||
|
||||
The only available option for `e2e` command. File path to json test files each containing a SCThing.
|
||||
|
||||
```shell
|
||||
-s, --samples <path>
|
||||
```
|
||||
|
||||
### Example execution
|
||||
|
||||
with docker when backend is running on `localhost:3000`:
|
||||
|
||||
```shell
|
||||
docker run --net=host registry.gitlab.com/openstapps/api/cli copy Place https://stappsbe01.innocampus.tu-berlin.de http://localhost:3000 100
|
||||
```
|
||||
|
||||
Or using `e2e` command:
|
||||
|
||||
```shell
|
||||
docker run --net=host registry.gitlab.com/openstapps/api/cli e2e http://localhost:3000
|
||||
```
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import './lib/app.js';
|
||||
@@ -1,89 +0,0 @@
|
||||
{
|
||||
"name": "@openstapps/api-cli",
|
||||
"description": "CLI client for @openstapps/api",
|
||||
"version": "3.0.0",
|
||||
"type": "module",
|
||||
"license": "GPL-3.0-only",
|
||||
"repository": "git@gitlab.com:openstapps/api.git",
|
||||
"author": "Karl-Philipp Wulfert <krlwlfrt@gmail.com>",
|
||||
"contributors": [
|
||||
"André Michael Thomas Bierlein",
|
||||
"Anselm Stordeur <anselmstordeur@gmail.com>",
|
||||
"Jovan Krunić <jovan.krunic@gmail.com>",
|
||||
"Michel Jonathan Schmitz",
|
||||
"Rainer Killinger <mail-openstapps@killinger.co>",
|
||||
"Roman Klopsch",
|
||||
"Thea Schöbl <dev@theaninova.de>"
|
||||
],
|
||||
"bin": {
|
||||
"openstapps-api": "app.js"
|
||||
},
|
||||
"files": [
|
||||
"app.js",
|
||||
"lib",
|
||||
"Dockerfile",
|
||||
"README.md",
|
||||
"CHANGELOG.md"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsup-node",
|
||||
"deploy": "pnpm --prod --filter=@openstapps/api-cli deploy ../../.deploy/api-cli",
|
||||
"format": "prettier . -c --ignore-path ../../.gitignore",
|
||||
"format:fix": "prettier --write . --ignore-path ../../.gitignore",
|
||||
"lint": "tsc --noEmit && eslint --ext .ts src/",
|
||||
"lint:fix": "eslint --fix --ext .ts src/",
|
||||
"test": "c8 mocha --exit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@openstapps/api": "workspace:*",
|
||||
"@openstapps/core": "workspace:*",
|
||||
"@openstapps/logger": "workspace:*",
|
||||
"@types/cli-progress": "3.11.0",
|
||||
"@types/express": "4.17.17",
|
||||
"@types/fs-extra": "9.0.13",
|
||||
"@types/json-schema": "7.0.14",
|
||||
"@types/junit-report-builder": "3.0.0",
|
||||
"@types/mocha": "10.0.1",
|
||||
"@types/node": "18.15.3",
|
||||
"@types/wait-on": "5.3.1",
|
||||
"cli-progress": "3.12.0",
|
||||
"commander": "10.0.0",
|
||||
"fs-extra": "10.1.0",
|
||||
"junit-report-builder": "3.0.1",
|
||||
"wait-on": "6.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@openstapps/prettier-config": "workspace:*",
|
||||
"@openstapps/eslint-config": "workspace:*",
|
||||
"@openstapps/tsconfig": "workspace:*",
|
||||
"@types/chai": "4.3.5",
|
||||
"@types/chai-as-promised": "7.1.5",
|
||||
"@types/chai-spies": "1.0.3",
|
||||
"c8": "7.14.0",
|
||||
"chai": "4.3.7",
|
||||
"chai-as-promised": "7.1.1",
|
||||
"chai-spies": "1.0.0",
|
||||
"mocha": "10.2.0",
|
||||
"mocha-junit-reporter": "2.2.0",
|
||||
"nock": "13.3.1",
|
||||
"ts-node": "10.9.1",
|
||||
"tsup": "6.7.0",
|
||||
"typescript": "5.1.6"
|
||||
},
|
||||
"tsup": {
|
||||
"entry": [
|
||||
"src/app.ts",
|
||||
"src/index.ts"
|
||||
],
|
||||
"sourcemap": true,
|
||||
"clean": true,
|
||||
"format": "esm",
|
||||
"outDir": "lib"
|
||||
},
|
||||
"prettier": "@openstapps/prettier-config",
|
||||
"eslintConfig": {
|
||||
"extends": [
|
||||
"@openstapps"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2022 Open StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import {SCThingType} from '@openstapps/core';
|
||||
import {Logger} from '@openstapps/logger';
|
||||
import {Command} from 'commander';
|
||||
import {URL} from 'url';
|
||||
import waitOn from 'wait-on';
|
||||
import {HttpClient} from '@openstapps/api';
|
||||
import {copy} from '../../../backend/copy-connector/src/copy.js';
|
||||
import {version} from '../package.json';
|
||||
// eslint-disable-next-line unicorn/prevent-abbreviations
|
||||
import {e2eRun} from './e2e.js';
|
||||
|
||||
process.on('unhandledRejection', async error => {
|
||||
await Logger.error('unhandledRejection', error);
|
||||
});
|
||||
|
||||
const client = new HttpClient();
|
||||
const commander = new Command();
|
||||
const helpAndExit = (help: string) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(help);
|
||||
process.exit(-1);
|
||||
};
|
||||
|
||||
commander
|
||||
.command('e2e <to>')
|
||||
.version(version)
|
||||
.description(
|
||||
'Run in end to end test mode. Indexing and afterwards retrieving all test files from @openstapp/core to the backend',
|
||||
)
|
||||
.option(
|
||||
'-s --samples [path]',
|
||||
'Path to @openstapp/core test files',
|
||||
'./node_modules/@openstapps/core/test/resources/indexable',
|
||||
)
|
||||
.option('-w --waiton [resource]', 'wait-on resource parameter see "www.npmjs.com/wait-on"')
|
||||
.option('-r --reportPath [reportPath]', 'JUnit Report Path')
|
||||
// eslint-disable-next-line unicorn/prevent-abbreviations
|
||||
.action(async (to, e2eCommand) => {
|
||||
let toURL = '';
|
||||
// validate url
|
||||
try {
|
||||
toURL = new URL(to).toString();
|
||||
} catch (error) {
|
||||
await Logger.error('expected parameter <to> to be valid url', error);
|
||||
helpAndExit(e2eCommand.helpInformation());
|
||||
}
|
||||
|
||||
try {
|
||||
if (typeof e2eCommand.waiton === 'string') {
|
||||
Logger.info(`Waiting for availibilty of resource: ${e2eCommand.waiton}`);
|
||||
await waitOn({
|
||||
resources: [e2eCommand.waiton],
|
||||
timeout: 300_000,
|
||||
});
|
||||
Logger.info(`Resource became available`);
|
||||
}
|
||||
await e2eRun(client, {
|
||||
to: toURL,
|
||||
samplesLocation: e2eCommand.samples,
|
||||
reportLocation: e2eCommand.reportPath,
|
||||
});
|
||||
Logger.ok('Done');
|
||||
} catch (error) {
|
||||
await Logger.error(error);
|
||||
}
|
||||
});
|
||||
|
||||
commander
|
||||
.command('copy <type> <from> <to> <batchSize>')
|
||||
.version(version)
|
||||
.description('Copy data from one instance to another')
|
||||
.option(
|
||||
'-s, --bulkSource <bulkSource>',
|
||||
'The source identifier for the bulk to use with the target instance [copy]',
|
||||
'copy',
|
||||
)
|
||||
.option('-a, --appVersion <version>', 'The App version to use [unset by default]', version)
|
||||
.allowUnknownOption(false)
|
||||
.action(async (type, from, to, batchSize, copyCommand) => {
|
||||
// validate type
|
||||
if (typeof type !== 'string') {
|
||||
await Logger.error('expected parameter "type" to be of type: string');
|
||||
copyCommand.help();
|
||||
helpAndExit(copyCommand.helpInformation());
|
||||
}
|
||||
|
||||
let fromURL = '';
|
||||
let toURL = '';
|
||||
|
||||
// validate urls
|
||||
try {
|
||||
fromURL = new URL(from).toString();
|
||||
toURL = new URL(to).toString();
|
||||
} catch (error) {
|
||||
await Logger.error('expected parameters "from" and "to" to be valid urls', error);
|
||||
helpAndExit(copyCommand.helpInformation());
|
||||
}
|
||||
|
||||
// validate batchSize
|
||||
if (Number.isNaN(Number.parseInt(batchSize, 10))) {
|
||||
await Logger.error('expected parameter "batchSize" to be of type: number');
|
||||
helpAndExit(copyCommand.helpInformation());
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a single type from one backend to another
|
||||
*/
|
||||
async function copySingleType(type: string) {
|
||||
Logger.info(`Copying ${type} objects from ${fromURL} to ${toURL}`);
|
||||
|
||||
await copy(client, {
|
||||
batchSize: Number.parseInt(batchSize, 10),
|
||||
from: fromURL,
|
||||
source: copyCommand.bulkSource,
|
||||
to: toURL,
|
||||
type: type as SCThingType,
|
||||
version: copyCommand.appVersion,
|
||||
});
|
||||
Logger.ok('done');
|
||||
}
|
||||
|
||||
if (type === '*') {
|
||||
const types = Object.values(SCThingType);
|
||||
for (const [i, type] of types.entries()) {
|
||||
Logger.info(`Copying ${type} (${i + 1} / ${types.length})`);
|
||||
try {
|
||||
await copySingleType(type);
|
||||
} catch (error) {
|
||||
await Logger.error(error);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await copySingleType(type);
|
||||
}
|
||||
});
|
||||
|
||||
commander.parse(process.argv);
|
||||
@@ -1,228 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2022 Open StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
/* eslint-disable unicorn/prevent-abbreviations */
|
||||
|
||||
import {SCSearchRequest, SCThings, SCThingType} from '@openstapps/core';
|
||||
import {Logger} from '@openstapps/logger';
|
||||
import {deepStrictEqual} from 'assert';
|
||||
import {readdir, readFile} from 'fs';
|
||||
import path from 'path';
|
||||
import {promisify} from 'util';
|
||||
import {ConnectorClient, HttpClientInterface} from '@openstapps/api';
|
||||
import junit from 'junit-report-builder';
|
||||
|
||||
const localItemMap: Map<string, SCThings> = new Map();
|
||||
const remoteItemMap: Map<string, SCThings> = new Map();
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
async function runTest(name: string, scope: () => Promise<void>, suite: junit.TestSuite, errors: string[]) {
|
||||
const testCase = suite.testCase().name(name);
|
||||
process.stdout.addListener('data', testCase.standardOutput);
|
||||
process.stderr.addListener('data', testCase.standardError);
|
||||
|
||||
const start = performance.now();
|
||||
await scope().catch(async error => {
|
||||
await Logger.error(error);
|
||||
testCase.failure(error.message);
|
||||
errors.push(error.message);
|
||||
});
|
||||
process.stdout.removeListener('data', testCase.standardOutput);
|
||||
process.stderr.removeListener('data', testCase.standardError);
|
||||
|
||||
const end = performance.now();
|
||||
testCase.time((end - start) / 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Options to set up indexing core test files to backend
|
||||
*/
|
||||
export interface E2EOptions {
|
||||
/**
|
||||
* File path of the directory containing core test files
|
||||
*/
|
||||
samplesLocation: string;
|
||||
|
||||
/**
|
||||
* URL of the backend to index to
|
||||
*/
|
||||
to: string;
|
||||
|
||||
/**
|
||||
* Location of the report
|
||||
*/
|
||||
reportLocation?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Function that can be used for integration tests.
|
||||
* Adds all the SCThings that getItemsFromSamples() returns to the backend.
|
||||
* Afterward, retrieves the items from backend and checks for differences with original ones.
|
||||
*/
|
||||
export async function e2eRun(client: HttpClientInterface, options: E2EOptions): Promise<string[]> {
|
||||
localItemMap.clear();
|
||||
remoteItemMap.clear();
|
||||
const builder = junit.newBuilder();
|
||||
const errors: string[] = [];
|
||||
|
||||
const api = new ConnectorClient(client, options.to);
|
||||
try {
|
||||
const indexSuite = builder.testSuite().name('e2e index');
|
||||
await indexSamples(api, options, indexSuite, errors);
|
||||
Logger.info(`All samples have been indexed via the backend`);
|
||||
|
||||
const retrieveSuite = builder.testSuite().name('e2e retrieve');
|
||||
await retrieveItems(api, retrieveSuite, errors);
|
||||
Logger.info(`All samples have been retrieved from the backend`);
|
||||
|
||||
const compareSuite = builder.testSuite().name('e2e compare');
|
||||
await compareItems(compareSuite, errors);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (options.reportLocation) {
|
||||
builder.writeTo(options.reportLocation);
|
||||
}
|
||||
await (errors.length > 0
|
||||
? Logger.error(`\n${errors.length} failed test cases`)
|
||||
: Logger.ok('All tests passed.'));
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves all samples previously index using the api
|
||||
*/
|
||||
async function retrieveItems(api: ConnectorClient, suite: junit.TestSuite, errors: string[]): Promise<void> {
|
||||
const singleItemSearchRequest: SCSearchRequest = {
|
||||
filter: {
|
||||
arguments: {
|
||||
field: 'uid',
|
||||
value: 'replace-me',
|
||||
},
|
||||
type: 'value',
|
||||
},
|
||||
};
|
||||
for (const {uid, type} of localItemMap.values()) {
|
||||
await runTest(
|
||||
`Should find ${type} (${uid})`,
|
||||
async () => {
|
||||
singleItemSearchRequest.filter!.arguments.value = uid;
|
||||
const searchResponse = await api.search(singleItemSearchRequest);
|
||||
if (searchResponse.data.length !== 1) {
|
||||
throw new Error(
|
||||
`Search for single SCThing with uid: ${uid} returned ${searchResponse.data.length} results`,
|
||||
);
|
||||
}
|
||||
remoteItemMap.set(uid, searchResponse.data[0]);
|
||||
},
|
||||
suite,
|
||||
errors,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares all samples (local and remote) with the same uid and throws if they're not deep equal
|
||||
*/
|
||||
async function compareItems(suite: junit.TestSuite, errors: string[]) {
|
||||
for (const localThing of localItemMap.values()) {
|
||||
await runTest(
|
||||
`Should be the same for ${localThing.type} (${localThing.uid})`,
|
||||
async () => {
|
||||
/* istanbul ignore next retrieveItems will throw before*/
|
||||
if (!remoteItemMap.has(localThing.uid)) {
|
||||
throw new Error(`Did not retrieve expected SCThing with uid: ${localThing.uid}`);
|
||||
}
|
||||
const remoteThing = remoteItemMap.get(localThing.uid);
|
||||
deepStrictEqual(
|
||||
remoteThing,
|
||||
localThing,
|
||||
`Unexpected difference between original and retrieved sample`,
|
||||
);
|
||||
},
|
||||
suite,
|
||||
errors,
|
||||
);
|
||||
}
|
||||
Logger.info(`All samples retrieved from the backend have been compared`);
|
||||
}
|
||||
/**
|
||||
* Function to add all the SCThings that getItemsFromSamples() returns to the backend
|
||||
*/
|
||||
async function indexSamples(
|
||||
api: ConnectorClient,
|
||||
options: E2EOptions,
|
||||
suite: junit.TestSuite,
|
||||
errors: string[],
|
||||
): Promise<void> {
|
||||
const items = await getItemsFromSamples(options.samplesLocation);
|
||||
|
||||
if (items.length === 0) {
|
||||
throw new Error('Could not index samples. None were retrieved from the file system.');
|
||||
}
|
||||
|
||||
// sort items by type
|
||||
const itemMap: Map<SCThingType, SCThings[]> = new Map();
|
||||
for (const item of items) {
|
||||
if (!itemMap.has(item.type)) {
|
||||
itemMap.set(item.type, []);
|
||||
}
|
||||
const itemsOfSameType = itemMap.get(item.type) as SCThings[];
|
||||
itemsOfSameType.push(item);
|
||||
itemMap.set(item.type, itemsOfSameType);
|
||||
localItemMap.set(item.uid, item);
|
||||
}
|
||||
// add items depending on their type property with one type per bulk
|
||||
for (const type of itemMap.keys()) {
|
||||
await runTest(
|
||||
`Should index ${type}`,
|
||||
async () => api.index(itemMap.get(type) as SCThings[], 'stapps-core-sample-data'),
|
||||
suite,
|
||||
errors,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all SCThings from the predefined core test json files
|
||||
* @param samplesDirectory Filepath to the directory containing to the core test json files
|
||||
* @returns an Array of all the SCThings specified for test usage
|
||||
*/
|
||||
export async function getItemsFromSamples<T extends SCThings>(samplesDirectory: string): Promise<T[]> {
|
||||
const readDirPromised = promisify(readdir);
|
||||
const readFilePromised = promisify(readFile);
|
||||
|
||||
const things: T[] = [];
|
||||
try {
|
||||
const fileNames = await readDirPromised(samplesDirectory);
|
||||
for (const fileName of fileNames) {
|
||||
const filePath = path.join(samplesDirectory, fileName);
|
||||
if (filePath.endsWith('.json')) {
|
||||
const fileContent = await readFilePromised(filePath, {encoding: 'utf8'});
|
||||
const schemaObject = JSON.parse(fileContent);
|
||||
if (schemaObject.errorNames.length === 0 && typeof schemaObject.instance.type === 'string') {
|
||||
things.push(schemaObject.instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
return things;
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019 StApps
|
||||
* This program is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, version 3.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
// eslint-disable-next-line unicorn/prevent-abbreviations
|
||||
import {
|
||||
SCBulkAddResponse,
|
||||
SCBulkAddRoute,
|
||||
SCBulkDoneResponse,
|
||||
SCBulkDoneRoute,
|
||||
SCBulkResponse,
|
||||
SCBulkRoute,
|
||||
SCSearchResponse,
|
||||
SCSearchRoute,
|
||||
SCThings,
|
||||
} from '@openstapps/core';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import chaiSpies from 'chai-spies';
|
||||
import {existsSync, mkdirSync, rmdirSync, unlinkSync} from 'fs';
|
||||
import {createFileSync} from 'fs-extra';
|
||||
import {HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
|
||||
import {RecursivePartial} from '../../../backend/copy-connector/test/copy.spec.js';
|
||||
import {expect} from 'chai';
|
||||
import path from 'path';
|
||||
import {fileURLToPath} from 'url';
|
||||
// eslint-disable-next-line unicorn/prevent-abbreviations
|
||||
import {e2eRun, getItemsFromSamples} from '../src/e2e.js';
|
||||
|
||||
chai.should();
|
||||
chai.use(chaiSpies);
|
||||
chai.use(chaiAsPromised);
|
||||
|
||||
const sandbox = chai.spy.sandbox();
|
||||
|
||||
const bulkRoute = new SCBulkRoute();
|
||||
const bulkAddRoute = new SCBulkAddRoute();
|
||||
const bulkDoneRoute = new SCBulkDoneRoute();
|
||||
|
||||
const searchRoute = new SCSearchRoute();
|
||||
|
||||
const httpClient = new HttpClient();
|
||||
|
||||
const storedThings: Map<string, SCThings> = new Map();
|
||||
|
||||
describe('e2e Connector', function () {
|
||||
afterEach(function () {
|
||||
sandbox.restore();
|
||||
});
|
||||
|
||||
it('should get core test samples', async function () {
|
||||
const items = await getItemsFromSamples('./node_modules/@openstapps/core/test/resources');
|
||||
expect(items).to.not.be.empty;
|
||||
});
|
||||
|
||||
it('should fail to get core test samples', async function () {
|
||||
await chai.expect(getItemsFromSamples('./non-existent-directory')).to.be.rejectedWith(Error);
|
||||
});
|
||||
|
||||
it('should run e2e simulation', async function () {
|
||||
type responses = HttpClientResponse<
|
||||
SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse | SCSearchResponse
|
||||
>;
|
||||
|
||||
let failOnCompare = false;
|
||||
let failOnLookup = false;
|
||||
|
||||
sandbox.on(
|
||||
httpClient,
|
||||
'request',
|
||||
async (request: HttpClientRequest): Promise<RecursivePartial<responses>> => {
|
||||
if (request.url.toString() === `http://localhost${bulkRoute.getUrlPath().toString()}`) {
|
||||
return {
|
||||
body: {
|
||||
state: 'in progress',
|
||||
uid: 'foo',
|
||||
},
|
||||
statusCode: bulkRoute.statusCodeSuccess,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
request.url.toString() === `http://localhost${bulkAddRoute.getUrlPath({UID: 'foo'}).toString()}`
|
||||
) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
storedThings.set((request.body as any).uid, JSON.parse(JSON.stringify(request.body)));
|
||||
|
||||
return {
|
||||
body: {},
|
||||
statusCode: bulkAddRoute.statusCodeSuccess,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
request.url.toString() === `http://localhost${bulkDoneRoute.getUrlPath({UID: 'foo'}).toString()}`
|
||||
) {
|
||||
return {
|
||||
body: {},
|
||||
statusCode: bulkDoneRoute.statusCodeSuccess,
|
||||
};
|
||||
}
|
||||
|
||||
if (request.url.toString() === `http://localhost${searchRoute.getUrlPath().toString()}`) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const thing = storedThings.get((request.body as any).filter.arguments.value);
|
||||
if (failOnCompare) {
|
||||
thing!.origin!.modified = 'altered';
|
||||
}
|
||||
const returnThing = failOnLookup ? [] : [thing];
|
||||
const returnBody = {
|
||||
data: returnThing,
|
||||
facets: [],
|
||||
pagination: {
|
||||
count: returnThing.length,
|
||||
offset: 0,
|
||||
total: returnThing.length,
|
||||
},
|
||||
stats: {
|
||||
time: 42,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
body: returnBody,
|
||||
statusCode: searchRoute.statusCodeSuccess,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
body: {},
|
||||
statusCode: searchRoute.statusCodeSuccess,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// tslint:disable-next-line: max-line-length
|
||||
await e2eRun(httpClient, {
|
||||
to: 'http://localhost',
|
||||
samplesLocation: './node_modules/@openstapps/core/test/resources',
|
||||
}).should.eventually.have.length(0);
|
||||
|
||||
failOnLookup = true;
|
||||
failOnCompare = false;
|
||||
// tslint:disable-next-line: max-line-length
|
||||
await e2eRun(httpClient, {
|
||||
to: 'http://localhost',
|
||||
samplesLocation: './node_modules/@openstapps/core/test/resources',
|
||||
}).should.eventually.include(
|
||||
'Search for single SCThing with uid: 184b717a-d020-46f5-995c-03023670cc62 returned 0 results',
|
||||
);
|
||||
|
||||
failOnLookup = false;
|
||||
failOnCompare = true;
|
||||
// tslint:disable-next-line: max-line-length
|
||||
await e2eRun(httpClient, {
|
||||
to: 'http://localhost',
|
||||
samplesLocation: './node_modules/@openstapps/core/test/resources',
|
||||
}).should.eventually.include('Unexpected difference between original and retrieved sample');
|
||||
});
|
||||
|
||||
it('should fail to index', async function () {
|
||||
type responses = HttpClientResponse<SCBulkAddResponse | SCBulkDoneResponse | SCBulkResponse>;
|
||||
|
||||
sandbox.on(httpClient, 'request', async (): Promise<RecursivePartial<responses>> => {
|
||||
return {
|
||||
body: {},
|
||||
statusCode: Number.MAX_SAFE_INTEGER,
|
||||
};
|
||||
});
|
||||
|
||||
// tslint:disable-next-line: max-line-length
|
||||
return e2eRun(httpClient, {
|
||||
to: 'http://localhost',
|
||||
samplesLocation: './node_modules/@openstapps/core/test/resources',
|
||||
}).should.eventually.include('');
|
||||
});
|
||||
|
||||
it('should fail to index directory without data', async function () {
|
||||
const emptyDirectoryPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'emptyDir');
|
||||
if (!existsSync(emptyDirectoryPath)) {
|
||||
mkdirSync(emptyDirectoryPath);
|
||||
}
|
||||
await e2eRun(httpClient, {
|
||||
to: 'http://localhost',
|
||||
samplesLocation: emptyDirectoryPath,
|
||||
}).should.be.rejectedWith('Could not index samples. None were retrieved from the file system.');
|
||||
rmdirSync(emptyDirectoryPath);
|
||||
});
|
||||
|
||||
it('should fail to index directory without json data', async function () {
|
||||
const somewhatFilledDirectoryPath = path.join(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
'somewhatFilledDir',
|
||||
);
|
||||
if (!existsSync(somewhatFilledDirectoryPath)) {
|
||||
mkdirSync(somewhatFilledDirectoryPath);
|
||||
}
|
||||
const nonJsonFile = path.join(somewhatFilledDirectoryPath, 'nonjson.txt');
|
||||
createFileSync(nonJsonFile);
|
||||
await e2eRun(httpClient, {
|
||||
to: 'http://localhost',
|
||||
samplesLocation: somewhatFilledDirectoryPath,
|
||||
}).should.be.rejectedWith('Could not index samples. None were retrieved from the file system.');
|
||||
unlinkSync(nonJsonFile);
|
||||
rmdirSync(somewhatFilledDirectoryPath);
|
||||
});
|
||||
});
|
||||
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"extends": "@openstapps/tsconfig"
|
||||
}
|
||||
Reference in New Issue
Block a user