refactor: provide common functions and abstraction

src/Connector.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCLicensePlate, SCThingOriginType, SCThingRemoteOrigin, SCThings} from '@openstapps/core';
+import {createUUID} from './common';
+
+/**
+ * Provides abstracted methods for the connector execution process 
+ * 
+ * By extending this class connector-developers only need to implement load and transform of the data
+ * Pushing the data to the backend will be handled automatically
+ *
+ * @typeparam T Any serializable type
+ */
+export abstract class Connector<T extends SCThings> {
+  /**
+   * License plate of the school
+   */
+  protected licensePlate: SCLicensePlate;
+  /**
+   * Name of the connector
+   */
+  public origin: string;
+
+  /**
+   * Abstract constructor for a connector 
+   * 
+   * @param licensePlate License plate of the school
+   * @param origin Name of the connector
+   */
+  constructor(licensePlate: SCLicensePlate, origin: string) {
+    this.licensePlate = licensePlate;
+    this.origin = origin;
+  }
+
+  /**
+   * Will fetch items from systems
+   *
+   * Implementation according to your schools requirements
+   */
+  protected abstract async fetchItems(): Promise<T[]>;
+
+  /**
+   * Creates a remote origin with the current date-time
+   */
+  createRemoteOrigin(): SCThingRemoteOrigin {
+    return {
+      indexed: new Date().toISOString(),
+      name: this.origin,
+      type: SCThingOriginType.Remote,
+    };
+  }
+
+  /**
+   * Fetches items and generates missing uids
+   */
+  async getItems(): Promise<T[]> {
+    const importedItems = await this.fetchItems();
+
+    for (const item of importedItems) {
+      if (item.uid.length === 0) {
+        item.uid = createUUID(item, this.licensePlate);
+      }
+    }
+
+    return importedItems;
+  }
+}

src/MinimalConnector.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2018, 2019 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {SCLicensePlate, SCMessage, SCThingRemoteOrigin, SCThingType} from '@openstapps/core';
+import {createUUID} from './common';
+import {Connector} from './Connector';
+
+/**
+ * Example connector
+ */
+export class MinimalConnector extends Connector<SCMessage> {
+  // for quick access to the type
+  private type: SCThingType.Message = SCThingType.Message;
+
+  /**
+   * Constructor for the MinimalConnector
+   * 
+   * @param licensePlate License plate of the school
+   * @param origin Name of the connector
+   */
+  constructor(licensePlate: SCLicensePlate, origin: string) {
+    super(licensePlate, origin);
+  }
+
+  /**
+   * Use or override the `createRemoteOrigin` method to customize the remote origin
+   */
+  private createCustomRemoteOrigin(): SCThingRemoteOrigin {
+    const customRemoteOrigin = this.createRemoteOrigin();
+    // may add a maintainer or other attributes here
+    customRemoteOrigin.url = 'http://your.backend.url';
+    return customRemoteOrigin;
+  }
+
+  /**
+   * Mock-up data
+   */
+  protected async fetchItems(): Promise<SCMessage[]> {
+    const importedItems: SCMessage[] = [
+      {
+        audiences: ['students', 'employees'],
+        description: 'Some description 1',
+        message: 'Some message 1',
+        name: 'Some name 1',
+        origin: this.createCustomRemoteOrigin(),
+        type: this.type,
+        uid: createUUID({id: 'message_1'}, this.licensePlate),
+      },
+      {
+        audiences: ['students', 'employees'],
+        description: 'Some description 2',
+        message: 'Some message 2',
+        name: 'Some name 2',
+        origin: this.createCustomRemoteOrigin(),
+        type: this.type,
+        uid: '', // see Connetor.getItems()
+      },
+      {
+        audiences: ['students', 'employees'],
+        description: 'Some description 3',
+        message: 'Some message 3',
+        name: 'Some name 3',
+        origin: this.createCustomRemoteOrigin(),
+        type: this.type,
+        uid: createUUID({id: 'message_3'}, this.licensePlate),
+      },
+    ];
+    return importedItems;
+  }
+}

src/cli.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2019 StApps
+ * Copyright (C) 2018, 2019 StApps
  * This program is free software: you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation, version 3.
@@ -12,58 +12,49 @@
  * You should have received a copy of the GNU General Public License along with
  * this program. If not, see <https://www.gnu.org/licenses/>.
  */
-import {Bulk} from '@openstapps/api/lib/bulk';
-import {ConnectorClient as Client} from '@openstapps/api/lib/connectorClient';
-import {HttpClient} from '@openstapps/api/lib/httpClient';
-import {SCBulkAddResponse, SCMessage, SCThingType} from '@openstapps/core';
 import {Logger} from '@openstapps/logger';
-import * as promiseLimit from 'promise-limit';
-import {MinimalConnector} from '.';
+import * as commander from 'commander';
+import {readFileSync} from 'fs';
+import {join} from 'path';
+import {executeConnector, isValidSCNamespace} from './common';
+import {MinimalConnector} from './MinimalConnector';
 
-const api = new Client(new HttpClient(), 'http://localhost:3000');
-const logger = new Logger();
+const connectorVersion = JSON.parse(
+  readFileSync(join(__dirname, '..', 'package.json')).toString(),
+).version;
 
-async function runConnector() {
-
-  const connector = new MinimalConnector();
-  const items = await connector.getItems();
-
-  if (items.length === 0) {
-    throw new Error('No items fetched.');
-  }
-
-  let bulk: Bulk<SCMessage>;
-
-  try {
-    bulk = await api.bulk<SCMessage>(SCThingType.Message, 'minimal-connector');
-  } catch (err) {
-    logger.error('Couldn\'t open bulk.');
-    throw err;
-  }
-
-  // create a concurrency limit
-  const limit = promiseLimit<SCBulkAddResponse>(5);
-
-  try {
-    // index all items with our concurrency limit
-    await Promise.all(items.map((item) => {
-      return limit(() => bulk.add(item));
-    }));
-  } catch (err) {
-    logger.error('Error while indexing items.');
-    throw err;
-  }
-
-  try {
-    await bulk.done();
-  } catch (err) {
-    logger.error('Error while closing bulk');
-    throw err;
-  }
-}
-
-runConnector().then(() => {
-  logger.log('Done.');
-}, (err) => {
-  throw err;
+process.on('unhandledRejection', (error) => {
+  throw error;
 });
+
+/**
+ * Uses arguments to paramtrize the connector execution
+ */
+commander
+  .version(connectorVersion)
+  .command('run')
+  .option('-b <backend>', 'URL of the StApps backend deployment', 'http://localhost:3000')
+  .option('-o <origin>', 'Origin, where the data comes from. Typically the name of the connector', 'minimal-connector')
+  .option('-l <licensePlate>', 'The license plate of your school. Must be matched to a SCNamespace', 'f-u')
+  .action(async (backend: string, origin: string, licensePlate: string) => {
+      if (backend.length === 0) {
+        throw new Error('Param "backend" needs to have a length greater zero.');
+      }
+      const originRegex = /^[a-z\-\_0-9]*$/;
+      if (!originRegex.test(origin)) {
+        throw new Error('Origin name can only consist of lowercase letters from a-z, "-", "_" and integer numbers.');
+      }
+
+      if (!isValidSCNamespace(licensePlate)) {
+        throw new Error('Not a valid license plate. Please register a namespace with a unique-id in "core"');
+      }
+
+      // TODO for connector-developers: set your connector here
+      const connector = new MinimalConnector(licensePlate, origin);
+
+      executeConnector(backend, connector);
+      Logger.ok('Done');
+    },
+  );
+
+commander.parse(process.argv);

src/common.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2019 StApps
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+import {ConnectorClient} from '@openstapps/api/lib/connectorClient';
+import {HttpClient} from '@openstapps/api/lib/httpClient';
+import {
+  SCLicensePlate,
+  SCNamespaces,
+  SCThings,
+} from '@openstapps/core';
+import {Connector} from './Connector';
+
+/**
+ * Checks if the input is a valid SCNamespace
+ *
+ * @param input Name of the potential SCNamespace
+ */
+export function isValidSCNamespace(input: string): input is SCLicensePlate {
+  return Object.keys(SCNamespaces).indexOf(input) > 0;
+}
+
+/**
+ * Creates a uuid from a JSON stringified item identifier
+ * 
+ * You may create custom itemIdentifier-Interfaces to generate UIDs consistently
+ *
+ * @param itemIdentifier Identifying representation of the item 
+ * @param licensePlate License plate of the school
+ */
+export function createUUID(itemIdentifier: any, licensePlate: SCLicensePlate): string {
+  return ConnectorClient.makeUUID(JSON.stringify(itemIdentifier), licensePlate);
+}
+
+/**
+ * Fetches items specified by the connector and pushs them to the backend,
+ * by overwriting the bulk indexed with the `origin`.
+ *
+ * @param backend URL of the StApps backend eployment
+ * @param connector Connector to be executed
+ */
+export async function executeConnector<T extends SCThings>(
+  backend: string,
+  connector: Connector<T>,
+) {
+  const items: T[] = await connector.getItems();
+  const client: ConnectorClient = new ConnectorClient(
+    new HttpClient(),
+    backend,
+  );
+  try {
+    await client.index<T>(items, connector.origin);
+  } catch (err) {
+    throw err;
+  }
+}

src/index.ts

@@ -1,84 +0,0 @@
-/*
- * Copyright (C) 2018-2019 StApps
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, version 3.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <https://www.gnu.org/licenses/>.
- */
-import {ConnectorClient as Client} from '@openstapps/api/lib/connectorClient';
-import {SCMessage, SCThingOriginType, SCThingType} from '@openstapps/core';
-
-export class MinimalConnector {
-  // Data is mocked inside of getItems()
-  private items: SCMessage[] = [];
-
-  /**
-   * Provides "fetched" items (e.g. messages)
-   * 
-   * @returns {Promise<SCMessage[]>}
-   * @memberof MinimalConnector
-   */
-  async getItems(): Promise<SCMessage[]> {
-    // reset items
-    this.items.length = 0;
-
-    // sample items (messages) "fetched" from some source
-    const importedItems: SCMessage[] = [
-      {
-        audiences: ['students', 'employees'],
-        description: 'Some description 1' ,
-        message: 'Some message 1',
-        name: 'Some name 1',
-        origin: {
-          indexed: (new Date()).toISOString(),
-          name: 'minimal connector',
-          type: SCThingOriginType.Remote,
-        },
-        type: SCThingType.Message,
-        uid: '',
-      },
-      {
-        audiences: ['students', 'employees'],
-        description: 'Some description 2',
-        message: 'Some message 2',
-        name: 'Some name 2',
-        origin: {
-          indexed: (new Date()).toISOString(),
-          name: 'minimal connector',
-          type: SCThingOriginType.Remote,
-        },
-        type: SCThingType.Message,
-        uid: '',
-      },
-      {
-        audiences: ['students', 'employees'],
-        description: 'Some description 3',
-        message: 'Some message 3',
-        name: 'Some name 3',
-        origin: {
-          indexed: (new Date()).toISOString(),
-          name: 'minimal connector',
-          type: SCThingOriginType.Remote,
-        },
-        type: SCThingType.Message,
-        uid: '',
-      },
-    ];
-
-    // create a universally unique identifier for each item
-    for (const item of importedItems) {
-      // each uid is generated by a string and the namespace id of your university
-      item.uid = Client.makeUUID(JSON.stringify(item), 'f-u');
-      this.items.push(item);
-    }
-
-    return this.items;
-  }
-}