refactor: split api into api, api-cli & api-plugin

packages/api-cli/src/copy.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2018-2022 Open StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCSearchRequest, SCThingType} from '@openstapps/core';
+import {Bar} from 'cli-progress';
+import {Client, ConnectorClient, OutOfRangeError, HttpClientInterface} from '@openstapps/api';
+
+/**
+ * Options to set up copying data from one backend to another
+ */
+export interface CopyOptions {
+  /**
+   * Batch size to copy at once
+   */
+  batchSize: number;
+
+  /**
+   * URL of the backend to copy from
+   */
+  from: string;
+
+  /**
+   * Source identifier
+   */
+  source: string;
+
+  /**
+   * URL of the backend to copy to
+   */
+  to: string;
+
+  /**
+   * StAppsCore type to copy
+   */
+  type: SCThingType;
+
+  /**
+   * StApps version identifier to copy data for
+   */
+  version: string;
+}
+
+/**
+ * Copy data for a StAppsCore type from one backend to another
+ * @param client HTTP client
+ * @param options Map of options
+ */
+export async function copy(client: HttpClientInterface, options: CopyOptions): Promise<void> {
+  const apiIn = new Client(client, options.from, options.version);
+  const apiOut = new ConnectorClient(client, options.to);
+
+  // open a bulk
+  const bulk = await apiOut.bulk(options.type, options.source);
+
+  let searchRequest: SCSearchRequest = {
+    filter: {
+      arguments: {
+        field: 'type',
+        value: options.type,
+      },
+      type: 'value',
+    },
+    size: 0,
+  };
+
+  let searchResponse = await apiIn.search(searchRequest);
+
+  searchRequest.size = options.batchSize;
+
+  const progressBar = new Bar({});
+  progressBar.start(searchResponse.pagination.total, 0);
+
+  let outOfRange = false;
+
+  do {
+    try {
+      ({searchRequest, searchResponse} = await apiIn.searchNext(searchRequest, searchResponse));
+      const things = searchResponse.data;
+
+      await Promise.all(
+        Array.from({length: ConnectorClient.ITEM_CONCURRENT_LIMIT}).map(async () => {
+          for (let item = things.shift(); item !== undefined; item = things.shift()) {
+            progressBar.increment(1);
+            await bulk.add(item);
+          }
+        }),
+      );
+    } catch (error) {
+      if (error instanceof OutOfRangeError) {
+        outOfRange = true;
+      } else {
+        progressBar.stop();
+        throw error;
+      }
+    }
+  } while (!outOfRange);
+
+  await bulk.done();
+
+  progressBar.stop();
+}