fix: pipeline

This commit is contained in:
2023-05-31 15:30:25 +02:00
parent 68400f2480
commit 45444d9373
31 changed files with 74 additions and 120 deletions

View File

@@ -15,24 +15,29 @@
import {Logger} from '@openstapps/logger';
import {Command} from 'commander';
import {readFileSync} from 'fs';
import {join} from 'path';
import path from 'path';
import {executeConnector, isValidSCNamespace} from './common.js';
import {MinimalConnector} from './minimal-connector.js';
import * as url from 'url';
process.on('unhandledRejection', error => {
throw error;
});
const connectorVersion = JSON.parse(readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
const dirname = path.dirname(url.fileURLToPath(import.meta.url));
const connectorVersion = JSON.parse(
readFileSync(path.join(dirname, '..', 'package.json')).toString(),
).version;
const commander = new Command();
/**
* Uses arguments to paramtrize the connector execution
* Uses arguments to parameterize the connector execution
*
* backendURL - URL of the StApps backend deployment e.g. http://localhost:3000
* origin - Origin, where the data comes from. Typically the name of the connector e.g. minimal-connector
* licensePlate - The license plate of your school. Must be matched to a SCNamespace e.g. f-u
* origin - Origin, where the data comes from.
* Typically, the name of the connector e.g., minimal-connector licensePlate - The license plate of your school.
* Must be matched to a SCNamespace e.g., f-u
*/
commander
.command('run <backendURL> <origin> <licensePlate>')
@@ -41,7 +46,7 @@ commander
if (backendURL.length === 0) {
throw new Error('Param "backend" needs to have a length greater zero.');
}
const originRegex = /^[a-z\-\_0-9]+$/;
const originRegex = /^[a-z\-_0-9]+$/;
if (!originRegex.test(origin)) {
throw new Error(
'Origin name can only consist of lowercase letters from a-z, "-", "_" and integer numbers.',

View File

@@ -12,8 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ConnectorClient} from '@openstapps/api/lib/connector-client.js';
import {HttpClient} from '@openstapps/api/lib/http-client.js';
import {ConnectorClient, HttpClient} from '@openstapps/api';
import {SCLicensePlate, SCNamespaces, SCThings} from '@openstapps/core';
import {Connector} from './connector.js';
@@ -48,9 +47,6 @@ export function createUUID(itemIdentifier: unknown, licensePlate: SCLicensePlate
export async function executeConnector<T extends SCThings>(backend: string, connector: Connector<T>) {
const items: T[] = await connector.getItems();
const client: ConnectorClient = new ConnectorClient(new HttpClient(), backend);
try {
await client.index<T>(items, connector.origin);
} catch (err) {
throw err;
}
// this might throw an error!
await client.index<T>(items, connector.origin);
}

View File

@@ -18,16 +18,17 @@ import {createUUID} from './common.js';
/**
* Provides abstracted methods for the connector execution process
*
* By extending this class connector-developers only need to implement load and transform of the data
* By extending this class connector-developers only need to implement the load and transform of the data
* Pushing the data to the backend will be handled automatically
*
* @typeparam T Any serializable type
* @template T Any serializable type
*/
export abstract class Connector<T extends SCThings> {
/**
* License plate of the school
*/
protected licensePlate: SCLicensePlate;
/**
* Name of the connector
*/

View File

@@ -50,7 +50,7 @@ export class MinimalConnector extends Connector<SCMessage> {
* Mock-up data
*/
protected async fetchItems(): Promise<SCMessage[]> {
const importedItems: SCMessage[] = [
return [
{
audiences: ['students', 'employees'],
categories: [],
@@ -82,7 +82,5 @@ export class MinimalConnector extends Connector<SCMessage> {
uid: createUUID({id: 'message_3'}, this.licensePlate),
},
];
return importedItems;
}
}