refactor: app deployment

packages/api-cli/Dockerfile

@@ -4,7 +4,7 @@ USER node
 ENV NODE_ENV=production
 WORKDIR /app
 
-COPY --chown=node:node pruned .
+COPY --chown=node:node . .
 
+EXPOSE 3000
 ENTRYPOINT ["node", "app.js"]
-CMD ["--help"]

packages/api-cli/package.json

@@ -21,11 +21,12 @@
   "files": [
     "app.js",
     "lib",
+    "Dockerfile",
     "README.md"
   ],
   "scripts": {
     "build": "tsup-node",
-    "build:docker": "docker build -t openstapps:api-cli .",
+    "deploy": "pnpm --prod --filter=@openstapps/api-cli deploy ../../.deploy/api-cli",
     "format": "prettier . -c --ignore-path ../../.gitignore",
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
     "lint": "tsc --noEmit && eslint --ext .ts src/",
@@ -41,6 +42,7 @@
     "@types/cli-progress": "3.11.0",
     "@types/express": "4.17.17",
     "@types/fs-extra": "9.0.13",
+    "@types/junit-report-builder": "3.0.0",
     "@types/json-schema": "7.0.11",
     "@types/mocha": "10.0.1",
     "@types/node": "18.15.3",
@@ -48,6 +50,7 @@
     "cli-progress": "3.12.0",
     "commander": "10.0.0",
     "fs-extra": "10.1.0",
+    "junit-report-builder": "3.0.1",
     "wait-on": "6.0.1"
   },
   "devDependencies": {
@@ -61,6 +64,7 @@
     "chai-as-promised": "7.1.1",
     "chai-spies": "1.0.0",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",

packages/api-cli/src/app.ts

@@ -53,6 +53,7 @@ commander
     './node_modules/@openstapps/core/test/resources/indexable',
   )
   .option('-w --waiton [resource]', 'wait-on resource parameter see "www.npmjs.com/wait-on"')
+  .option('-r --reportPath [reportPath]', 'JUnit Report Path')
   // eslint-disable-next-line unicorn/prevent-abbreviations
   .action(async (to, e2eCommand) => {
     let toURL = '';
@@ -73,7 +74,11 @@ commander
         });
         Logger.info(`Resource became available`);
       }
-      await e2eRun(client, {to: toURL, samplesLocation: e2eCommand.samples});
+      await e2eRun(client, {
+        to: toURL,
+        samplesLocation: e2eCommand.samples,
+        reportLocation: e2eCommand.reportPath,
+      });
       Logger.ok('Done');
     } catch (error) {
       await Logger.error(error);

packages/api-cli/src/e2e.ts

@@ -21,10 +21,29 @@ import {readdir, readFile} from 'fs';
 import path from 'path';
 import {promisify} from 'util';
 import {ConnectorClient, HttpClientInterface} from '@openstapps/api';
+import junit from 'junit-report-builder';
 
 const localItemMap: Map<string, SCThings> = new Map();
 const remoteItemMap: Map<string, SCThings> = new Map();
 
+async function runTest(name: string, scope: () => Promise<void>, suite: junit.TestSuite, errors: string[]) {
+  const testCase = suite.testCase().name(name);
+  process.stdout.addListener('data', testCase.standardOutput);
+  process.stderr.addListener('data', testCase.standardError);
+
+  const start = performance.now();
+  await scope().catch(async error => {
+    await Logger.error(error);
+    testCase.failure(error.message);
+    errors.push(error.message);
+  });
+  process.stdout.removeListener('data', testCase.standardOutput);
+  process.stderr.removeListener('data', testCase.standardError);
+
+  const end = performance.now();
+  testCase.time((end - start) / 1000);
+}
+
 /**
  * Options to set up indexing core test files to backend
  */
@@ -38,34 +57,54 @@ export interface E2EOptions {
    * URL of the backend to index to
    */
   to: string;
+
+  /**
+   * Location of the report
+   */
+  reportLocation?: string;
 }
 
 /**
  * Function that can be used for integration tests.
  * Adds all the SCThings that getItemsFromSamples() returns to the backend.
- * Afterwards retrieves the items from backend and checks for differences with original ones.
+ * Afterward, retrieves the items from backend and checks for differences with original ones.
  */
-export async function e2eRun(client: HttpClientInterface, options: E2EOptions): Promise<void> {
+export async function e2eRun(client: HttpClientInterface, options: E2EOptions): Promise<string[]> {
   localItemMap.clear();
   remoteItemMap.clear();
+  const builder = junit.newBuilder();
+  const errors: string[] = [];
 
   const api = new ConnectorClient(client, options.to);
   try {
-    await indexSamples(api, options);
+    const indexSuite = builder.testSuite().name('e2e index');
+    await indexSamples(api, options, indexSuite, errors);
     Logger.info(`All samples have been indexed via the backend`);
 
-    await retrieveItems(api);
+    const retrieveSuite = builder.testSuite().name('e2e retrieve');
+    await retrieveItems(api, retrieveSuite, errors);
     Logger.info(`All samples have been retrieved from the backend`);
-    compareItems();
+
+    const compareSuite = builder.testSuite().name('e2e compare');
+    await compareItems(compareSuite, errors);
   } catch (error) {
     throw error;
   }
+
+  if (options.reportLocation) {
+    builder.writeTo(options.reportLocation);
+  }
+  await (errors.length > 0
+    ? Logger.error(`\n${errors.length} failed test cases`)
+    : Logger.ok('All tests passed.'));
+
+  return errors;
 }
 
 /**
- * Retieves all samples previously index using the api
+ * Retrieves all samples previously index using the api
  */
-async function retrieveItems(api: ConnectorClient): Promise<void> {
+async function retrieveItems(api: ConnectorClient, suite: junit.TestSuite, errors: string[]): Promise<void> {
   const singleItemSearchRequest: SCSearchRequest = {
     filter: {
       arguments: {
@@ -76,61 +115,83 @@ async function retrieveItems(api: ConnectorClient): Promise<void> {
     },
   };
   for (const uid of localItemMap.keys()) {
-    singleItemSearchRequest.filter!.arguments.value = uid;
-    const searchResonse = await api.search(singleItemSearchRequest);
-    if (searchResonse.data.length !== 1) {
-      throw new Error(
-        `Search for single SCThing with uid: ${uid} returned ${searchResonse.data.length} results`,
-      );
-    }
-    remoteItemMap.set(uid, searchResonse.data[0]);
+    await runTest(
+      `Should find ${uid}`,
+      async () => {
+        singleItemSearchRequest.filter!.arguments.value = uid;
+        const searchResponse = await api.search(singleItemSearchRequest);
+        if (searchResponse.data.length !== 1) {
+          throw new Error(
+            `Search for single SCThing with uid: ${uid} returned ${searchResponse.data.length} results`,
+          );
+        }
+        remoteItemMap.set(uid, searchResponse.data[0]);
+      },
+      suite,
+      errors,
+    );
   }
 }
 
 /**
  * Compares all samples (local and remote) with the same uid and throws if they're not deep equal
  */
-function compareItems() {
+async function compareItems(suite: junit.TestSuite, errors: string[]) {
   for (const localThing of localItemMap.values()) {
-    /* istanbul ignore next  retrieveItems will throw before*/
-    if (!remoteItemMap.has(localThing.uid)) {
-      throw new Error(`Did not retrieve expected SCThing with uid: ${localThing.uid}`);
-    }
-    const remoteThing = remoteItemMap.get(localThing.uid);
-    deepStrictEqual(remoteThing, localThing, `Unexpected difference between original and retrieved sample`);
+    await runTest(
+      `Should be the same for ${localThing.uid}`,
+      async () => {
+        /* istanbul ignore next retrieveItems will throw before*/
+        if (!remoteItemMap.has(localThing.uid)) {
+          throw new Error(`Did not retrieve expected SCThing with uid: ${localThing.uid}`);
+        }
+        const remoteThing = remoteItemMap.get(localThing.uid);
+        deepStrictEqual(
+          remoteThing,
+          localThing,
+          `Unexpected difference between original and retrieved sample`,
+        );
+      },
+      suite,
+      errors,
+    );
   }
-  Logger.info(
-    `All samples retrieved from the backend are the same (deep equal) as the original ones submitted`,
-  );
+  Logger.info(`All samples retrieved from the backend have been compared`);
 }
 /**
  * Function to add all the SCThings that getItemsFromSamples() returns to the backend
  */
-async function indexSamples(api: ConnectorClient, options: E2EOptions): Promise<void> {
-  try {
-    const items = await getItemsFromSamples(options.samplesLocation);
+async function indexSamples(
+  api: ConnectorClient,
+  options: E2EOptions,
+  suite: junit.TestSuite,
+  errors: string[],
+): Promise<void> {
+  const items = await getItemsFromSamples(options.samplesLocation);
 
-    if (items.length === 0) {
-      throw new Error('Could not index samples. None were retrieved from the file system.');
-    }
+  if (items.length === 0) {
+    throw new Error('Could not index samples. None were retrieved from the file system.');
+  }
 
-    // sort items by type
-    const itemMap: Map<SCThingType, SCThings[]> = new Map();
-    for (const item of items) {
-      if (!itemMap.has(item.type)) {
-        itemMap.set(item.type, []);
-      }
-      const itemsOfSameType = itemMap.get(item.type) as SCThings[];
-      itemsOfSameType.push(item);
-      itemMap.set(item.type, itemsOfSameType);
-      localItemMap.set(item.uid, item);
+  // sort items by type
+  const itemMap: Map<SCThingType, SCThings[]> = new Map();
+  for (const item of items) {
+    if (!itemMap.has(item.type)) {
+      itemMap.set(item.type, []);
     }
-    // add items depending on their type property with one type per bulk
-    for (const type of itemMap.keys()) {
-      await api.index(itemMap.get(type) as SCThings[], 'stapps-core-sample-data');
-    }
-  } catch (error) {
-    throw error;
+    const itemsOfSameType = itemMap.get(item.type) as SCThings[];
+    itemsOfSameType.push(item);
+    itemMap.set(item.type, itemsOfSameType);
+    localItemMap.set(item.uid, item);
+  }
+  // add items depending on their type property with one type per bulk
+  for (const type of itemMap.keys()) {
+    await runTest(
+      `Should index ${type}`,
+      async () => api.index(itemMap.get(type) as SCThings[], 'stapps-core-sample-data'),
+      suite,
+      errors,
+    );
   }
 }
 

packages/api-cli/test/e2e.spec.ts

@@ -29,7 +29,7 @@ import chaiAsPromised from 'chai-as-promised';
 import chaiSpies from 'chai-spies';
 import {existsSync, mkdirSync, rmdirSync, unlinkSync} from 'fs';
 import {createFileSync} from 'fs-extra';
-import {ApiError, HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
+import {HttpClient, HttpClientRequest, HttpClientResponse} from '@openstapps/api';
 import {RecursivePartial} from './copy.spec.js';
 import {expect} from 'chai';
 import path from 'path';
@@ -147,7 +147,7 @@ describe('e2e Connector', function () {
     await e2eRun(httpClient, {
       to: 'http://localhost',
       samplesLocation: './node_modules/@openstapps/core/test/resources',
-    });
+    }).should.eventually.have.length(0);
 
     failOnLookup = true;
     failOnCompare = false;
@@ -155,7 +155,9 @@ describe('e2e Connector', function () {
     await e2eRun(httpClient, {
       to: 'http://localhost',
       samplesLocation: './node_modules/@openstapps/core/test/resources',
-    }).should.be.rejectedWith('Search for single SCThing with uid');
+    }).should.eventually.include(
+      'Search for single SCThing with uid: 184b717a-d020-46f5-995c-03023670cc62 returned 0 results',
+    );
 
     failOnLookup = false;
     failOnCompare = true;
@@ -163,7 +165,7 @@ describe('e2e Connector', function () {
     await e2eRun(httpClient, {
       to: 'http://localhost',
       samplesLocation: './node_modules/@openstapps/core/test/resources',
-    }).should.be.rejectedWith('Unexpected difference');
+    }).should.eventually.include('Unexpected difference between original and retrieved sample');
   });
 
   it('should fail to index', async function () {
@@ -180,7 +182,7 @@ describe('e2e Connector', function () {
     return e2eRun(httpClient, {
       to: 'http://localhost',
       samplesLocation: './node_modules/@openstapps/core/test/resources',
-    }).should.be.rejectedWith(ApiError);
+    }).should.eventually.include('');
   });
 
   it('should fail to index directory without data', async function () {

packages/api-cli/turbo.json

@@ -0,0 +1,9 @@
+{
+  "extends": ["//"],
+  "pipeline": {
+    "deploy": {
+      "dependsOn": ["@openstapps/api-cli#build"],
+      "outputs": [".deploy/api-cli"]
+    }
+  }
+}

packages/api-plugin/package.json

@@ -28,7 +28,7 @@
     "format:fix": "prettier --write . --ignore-path ../../.gitignore",
     "lint": "eslint --ext .ts src/",
     "lint:fix": "eslint --fix --ext .ts src/",
-    "test": "c8 mocha"
+    "test": "c8 mocha --exit"
   },
   "dependencies": {
     "@openstapps/api": "workspace:*",
@@ -58,6 +58,7 @@
     "chai-spies": "1.0.0",
     "conventional-changelog-cli": "2.2.2",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",

packages/api/package.json

@@ -51,6 +51,7 @@
     "conventional-changelog-cli": "2.2.2",
     "date-fns": "2.30.0",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "traverse": "0.6.7",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",

packages/collection-utils/package.json

@@ -26,6 +26,7 @@
     "c8": "7.14.0",
     "chai": "4.3.7",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",
     "typedoc": "0.24.8",

packages/core-tools/package.json

@@ -79,6 +79,7 @@
     "c8": "7.14.0",
     "chai": "4.3.7",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",

packages/core/package.json

@@ -67,6 +67,7 @@
     "chai": "4.3.7",
     "conditional-type-checks": "1.0.6",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "source-map-support": "0.5.21",
     "surge": "0.23.1",
     "ts-node": "10.9.1",

packages/easy-ast/package.json

@@ -37,6 +37,7 @@
     "c8": "7.14.0",
     "chai": "4.3.7",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",
     "typedoc": "0.24.8"

packages/es-mapping-generator/package.json

@@ -43,6 +43,7 @@
     "c8": "7.14.0",
     "chai": "4.3.7",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "nock": "13.3.1",
     "rimraf": "5.0.0",
     "ts-node": "10.9.1"

packages/gitlab-api/.gitignore

@@ -1,91 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
-.env
-
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# next.js build output
-.next
-
-# nuxt.js build output
-.nuxt
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-#DynamoDB Local files
-.dynamodb/
-
-########## end of https://github.com/github/gitignore/blob/master/Node.gitignore
-
-# ignore ide files
-.idea
-.vscode
-
-# ignore lib
-lib
-
-# ignore docs
-docs

packages/logger/package.json

@@ -46,6 +46,7 @@
     "chai-as-promised": "7.1.1",
     "chai-spies": "1.0.0",
     "mocha": "10.2.0",
+    "mocha-junit-reporter": "2.2.0",
     "ts-node": "10.9.1",
     "tsup": "6.7.0",
     "typedoc": "0.24.8",