refactor: split api into api, api-cli & api-plugin

packages/api-cli/src/app.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2018-2022 Open StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCThingType} from '@openstapps/core';
+import {Logger} from '@openstapps/logger';
+import {Command} from 'commander';
+import {readFileSync} from 'fs';
+import path from 'path';
+import {URL} from 'url';
+import waitOn from 'wait-on';
+import {HttpClient} from '@openstapps/api';
+import {copy} from './copy.js';
+// eslint-disable-next-line unicorn/prevent-abbreviations
+import {e2eRun} from './e2e.js';
+
+process.on('unhandledRejection', async error => {
+  await Logger.error('unhandledRejection', error);
+});
+
+// eslint-disable-next-line unicorn/prefer-module
+const packageJson = JSON.parse(readFileSync(path.join(__dirname, '..', 'package.json')).toString());
+
+const client = new HttpClient();
+const commander = new Command();
+const helpAndExit = (help: string) => {
+  // eslint-disable-next-line no-console
+  console.log(help);
+  process.exit(-1);
+};
+
+commander
+  .command('e2e <to>')
+  .version(packageJson.version)
+  .description(
+    'Run in end to end test mode. Indexing and afterwards retrieving all test files from @openstapp/core to the backend',
+  )
+  .option(
+    '-s --samples [path]',
+    'Path to @openstapp/core test files',
+    './node_modules/@openstapps/core/test/resources/indexable',
+  )
+  .option('-w --waiton [resource]', 'wait-on resource parameter see "www.npmjs.com/wait-on"')
+  // eslint-disable-next-line unicorn/prevent-abbreviations
+  .action(async (to, e2eCommand) => {
+    let toURL = '';
+    // validate url
+    try {
+      toURL = new URL(to).toString();
+    } catch (error) {
+      await Logger.error('expected parameter <to> to be valid url', error);
+      helpAndExit(e2eCommand.helpInformation());
+    }
+
+    try {
+      if (typeof e2eCommand.waiton === 'string') {
+        Logger.info(`Waiting for availibilty of resource: ${e2eCommand.waiton}`);
+        await waitOn({
+          resources: [e2eCommand.waiton],
+          timeout: 300_000,
+        });
+        Logger.info(`Resource became available`);
+      }
+      await e2eRun(client, {to: toURL, samplesLocation: e2eCommand.samples});
+      Logger.ok('Done');
+    } catch (error) {
+      await Logger.error(error);
+    }
+  });
+
+commander
+  .command('copy <type> <from> <to> <batchSize>')
+  .version(packageJson.version)
+  .description('Copy data from one instance to another')
+  .option(
+    '-s, --bulkSource <bulkSource>',
+    'The source identifier for the bulk to use with the target instance [copy]',
+    'copy',
+  )
+  // TODO: remove
+  .option('-a, --appVersion <version>', 'The App version to use [unset by default]')
+  .allowUnknownOption(false)
+  .action(async (type, from, to, batchSize, copyCommand) => {
+    // validate type
+    if (typeof type !== 'string') {
+      await Logger.error('expected parameter "type" to be of type: string');
+      copyCommand.help();
+      helpAndExit(copyCommand.helpInformation());
+    }
+
+    let fromURL = '';
+    let toURL = '';
+
+    // validate urls
+    try {
+      fromURL = new URL(from).toString();
+      toURL = new URL(to).toString();
+    } catch (error) {
+      await Logger.error('expected parameters "from" and "to" to be valid urls', error);
+      helpAndExit(copyCommand.helpInformation());
+    }
+
+    // validate batchSize
+    if (Number.isNaN(Number.parseInt(batchSize, 10))) {
+      await Logger.error('expected parameter "batchSize" to be of type: number');
+      helpAndExit(copyCommand.helpInformation());
+    }
+
+    Logger.info(`Copying ${type} objects from ${fromURL} to ${toURL}`);
+
+    copy(client, {
+      batchSize: Number.parseInt(batchSize, 10),
+      from: fromURL,
+      source: copyCommand.bulkSource,
+      to: toURL,
+      type: type as SCThingType,
+      version: copyCommand.appVersion,
+    }).then(
+      () => {
+        Logger.ok('Done');
+      },
+      error => {
+        throw error;
+      },
+    );
+  });
+
+commander.parse(process.argv);

packages/api-cli/src/copy.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2018-2022 Open StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCSearchRequest, SCThingType} from '@openstapps/core';
+import {Bar} from 'cli-progress';
+import {Client, ConnectorClient, OutOfRangeError, HttpClientInterface} from '@openstapps/api';
+
+/**
+ * Options to set up copying data from one backend to another
+ */
+export interface CopyOptions {
+  /**
+   * Batch size to copy at once
+   */
+  batchSize: number;
+
+  /**
+   * URL of the backend to copy from
+   */
+  from: string;
+
+  /**
+   * Source identifier
+   */
+  source: string;
+
+  /**
+   * URL of the backend to copy to
+   */
+  to: string;
+
+  /**
+   * StAppsCore type to copy
+   */
+  type: SCThingType;
+
+  /**
+   * StApps version identifier to copy data for
+   */
+  version: string;
+}
+
+/**
+ * Copy data for a StAppsCore type from one backend to another
+ * @param client HTTP client
+ * @param options Map of options
+ */
+export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
+  const apiIn = new Client(client, options.from, options.version);
+  const apiOut = new ConnectorClient(client, options.to);
+
+  // open a bulk
+  const bulk = await apiOut.bulk(options.type, options.source);
+
+  let searchRequest: SCSearchRequest = {
+    filter: {
+      arguments: {
+        field: 'type',
+        value: options.type,
+      },
+      type: 'value',
+    },
+    size: 0,
+  };
+
+  let searchResponse = await apiIn.search(searchRequest);
+
+  searchRequest.size = options.batchSize;
+
+  const progressBar = new Bar({});
+  progressBar.start(searchResponse.pagination.total, 0);
+
+  let outOfRange = false;
+
+  do {
+    try {
+      ({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
+      const things = searchResponse.data;
+
+      await Promise.all(
+        Array.from({length: ConnectorClient.ITEM_CONCURRENT_LIMIT}).map(async () => {
+          for (let item = things.shift(); item !== undefined; item = things.shift()) {
+            progressBar.increment(1);
+            await bulk.add(item);
+          }
+        }),
+      );
+    } catch (error) {
+      if (error instanceof OutOfRangeError) {
+        outOfRange = true;
+      } else {
+        progressBar.stop();
+        throw error;
+      }
+    }
+  } while (!outOfRange);
+
+  await bulk.done();
+
+  progressBar.stop();
+}

packages/api-cli/src/e2e.ts

@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2019-2022 Open StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+/* eslint-disable unicorn/prevent-abbreviations */
+
+import {SCSearchRequest, SCThings, SCThingType} from '@openstapps/core';
+import {Logger} from '@openstapps/logger';
+import {deepStrictEqual} from 'assert';
+import {readdir, readFile} from 'fs';
+import path from 'path';
+import {promisify} from 'util';
+import {ConnectorClient, HttpClientInterface} from '@openstapps/api';
+
+const localItemMap: Map<string, SCThings> = new Map();
+const remoteItemMap: Map<string, SCThings> = new Map();
+
+/**
+ * Options to set up indexing core test files to backend
+ */
+export interface E2EOptions {
+  /**
+   * File path of the directory containing core test files
+   */
+  samplesLocation: string;
+
+  /**
+   * URL of the backend to index to
+   */
+  to: string;
+}
+
+/**
+ * Function that can be used for integration tests.
+ * Adds all the SCThings that getItemsFromSamples() returns to the backend.
+ * Afterwards retrieves the items from backend and checks for differences with original ones.
+ */
+export async function e2eRun(client: HttpClientInterface, options: E2EOptions): Promise<void> {
+  localItemMap.clear();
+  remoteItemMap.clear();
+
+  const api = new ConnectorClient(client, options.to);
+  try {
+    await indexSamples(api, options);
+    Logger.info(`All samples have been indexed via the backend`);
+
+    await retrieveItems(api);
+    Logger.info(`All samples have been retrieved from the backend`);
+    compareItems();
+  } catch (error) {
+    throw error;
+  }
+}
+
+/**
+ * Retieves all samples previously index using the api
+ */
+async function retrieveItems(api: ConnectorClient): Promise<void> {
+  const singleItemSearchRequest: SCSearchRequest = {
+    filter: {
+      arguments: {
+        field: 'uid',
+        value: 'replace-me',
+      },
+      type: 'value',
+    },
+  };
+  for (const uid of localItemMap.keys()) {
+    singleItemSearchRequest.filter!.arguments.value = uid;
+    const searchResonse = await api.search(singleItemSearchRequest);
+    if (searchResonse.data.length !== 1) {
+      throw new Error(
+        `Search for single SCThing with uid: ${uid} returned ${searchResonse.data.length} results`,
+      );
+    }
+    remoteItemMap.set(uid, searchResonse.data[0]);
+  }
+}
+
+/**
+ * Compares all samples (local and remote) with the same uid and throws if they're not deep equal
+ */
+function compareItems() {
+  for (const localThing of localItemMap.values()) {
+    /* istanbul ignore next  retrieveItems will throw before*/
+    if (!remoteItemMap.has(localThing.uid)) {
+      throw new Error(`Did not retrieve expected SCThing with uid: ${localThing.uid}`);
+    }
+    const remoteThing = remoteItemMap.get(localThing.uid);
+    deepStrictEqual(remoteThing, localThing, `Unexpected difference between original and retrieved sample`);
+  }
+  Logger.info(
+    `All samples retrieved from the backend are the same (deep equal) as the original ones submitted`,
+  );
+}
+/**
+ * Function to add all the SCThings that getItemsFromSamples() returns to the backend
+ */
+async function indexSamples(api: ConnectorClient, options: E2EOptions): Promise<void> {
+  try {
+    const items = await getItemsFromSamples(options.samplesLocation);
+
+    if (items.length === 0) {
+      throw new Error('Could not index samples. None were retrieved from the file system.');
+    }
+
+    // sort items by type
+    const itemMap: Map<SCThingType, SCThings[]> = new Map();
+    for (const item of items) {
+      if (!itemMap.has(item.type)) {
+        itemMap.set(item.type, []);
+      }
+      const itemsOfSameType = itemMap.get(item.type) as SCThings[];
+      itemsOfSameType.push(item);
+      itemMap.set(item.type, itemsOfSameType);
+      localItemMap.set(item.uid, item);
+    }
+    // add items depending on their type property with one type per bulk
+    for (const type of itemMap.keys()) {
+      await api.index(itemMap.get(type) as SCThings[], 'stapps-core-sample-data');
+    }
+  } catch (error) {
+    throw error;
+  }
+}
+
+/**
+ * Get all SCThings from the predefined core test json files
+ * @param samplesDirectory Filepath to the directory containing to the core test json files
+ * @returns an Array of all the SCThings specified for test usage
+ */
+export async function getItemsFromSamples<T extends SCThings>(samplesDirectory: string): Promise<T[]> {
+  const readDirPromised = promisify(readdir);
+  const readFilePromised = promisify(readFile);
+
+  const things: T[] = [];
+  try {
+    const fileNames = await readDirPromised(samplesDirectory);
+    for (const fileName of fileNames) {
+      const filePath = path.join(samplesDirectory, fileName);
+      if (filePath.endsWith('.json')) {
+        const fileContent = await readFilePromised(filePath, {encoding: 'utf8'});
+        const schemaObject = JSON.parse(fileContent);
+        if (schemaObject.errorNames.length === 0 && typeof schemaObject.instance.type === 'string') {
+          things.push(schemaObject.instance);
+        }
+      }
+    }
+  } catch (error) {
+    throw error;
+  }
+
+  return things;
+}