fix: pipeline

This commit is contained in:
2023-05-31 15:30:25 +02:00
parent 68400f2480
commit 45444d9373
31 changed files with 74 additions and 120 deletions

3
.gitignore vendored
View File

@@ -12,6 +12,9 @@ pids
*.pid.lock
.DS_Store
.angular/
www/
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov

View File

@@ -3,7 +3,7 @@
/**
* This is the database configuration for the technical university of berlin
*
* @type {import('@openstapps/logger').RecursivePartial<import('../../src/storage/elasticsearch/types/elasticsearch.js').ElasticsearchConfigFile>}
* @type {import('@openstapps/logger').RecursivePartial<import('../../src/storage/elasticsearch/types/elasticsearch-config.js').ElasticsearchConfigFile>}
*/
const config = {
internal: {

View File

@@ -11,7 +11,7 @@
*
* To get more information about the meaning of specific fields, please use your IDE to read the TSDoc documentation.
*
* @type {import('../../src/storage/elasticsearch/types/elasticsearch.js')}
* @type {import('../../src/storage/elasticsearch/types/elasticsearch-config.js')}
*/
const config = {
internal: {

View File

@@ -1,5 +1,6 @@
declare global {
namespace NodeJS {
// eslint-disable-next-line unicorn/prevent-abbreviations
interface ProcessEnv {
NODE_APP_INSTANCE: 'default' | 'b-tu' | 'f-u' | 'fb-fh' | 'ks-ug' | string | undefined;
NODE_CONFIG_ENV: 'elasticsearch' | string | undefined;
@@ -12,3 +13,5 @@ declare global {
}
}
}
export {};

View File

@@ -18,7 +18,7 @@ import {
AggregationsAggregate,
AggregationsFiltersAggregate,
AggregationsMultiTermsBucket,
} from '@elastic/elasticsearch/lib/api/types';
} from '@elastic/elasticsearch/lib/api/types.js';
import {SCFacet, SCThingType} from '@openstapps/core';
/**

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types.js';
import {SCBackendConfigurationSearchBoostingType} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util.js';

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslFunctionScoreContainer} from '@elastic/elasticsearch/lib/api/types.js';
import {SCBackendConfigurationSearchBoostingContext, SCSearchContext} from '@openstapps/core';
import {buildFunctionsForBoostingTypes} from './boost-functions.js';

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types.js';
import {SCSearchFilter} from '@openstapps/core';
import {buildBooleanFilter} from './filters/boolean.js';
import {buildAvailabilityFilter} from './filters/availability.js';
@@ -29,19 +29,26 @@ import {buildValueFilter} from './filters/value.js';
*/
export function buildFilter(filter: SCSearchFilter): QueryDslQueryContainer {
switch (filter.type) {
case 'value':
case 'value': {
return buildValueFilter(filter);
case 'availability':
}
case 'availability': {
return buildAvailabilityFilter(filter);
case 'distance':
}
case 'distance': {
return buildDistanceFilter(filter);
case 'boolean':
}
case 'boolean': {
return buildBooleanFilter(filter);
case 'numeric range':
}
case 'numeric range': {
return buildNumericRangeFilter(filter);
case 'date range':
}
case 'date range': {
return buildDateRangeFilter(filter);
case 'geo':
}
case 'geo': {
return buildGeoFilter(filter);
}
}
}

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslBoolQuery} from '@elastic/elasticsearch/lib/api/types.js';
import {SCSearchBooleanFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util.js';
import {buildFilter} from '../filter.js';

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslDateRangeQuery} from '@elastic/elasticsearch/lib/api/types.js';
import {SCSearchDateRangeFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util.js';

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslGeoDistanceQuery} from '@elastic/elasticsearch/lib/api/types.js';
import {SCSearchDistanceFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util.js';

View File

@@ -12,8 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslNumberRangeQuery} from '@elastic/elasticsearch/lib/api/types.js';
import {SCSearchNumericRangeFilter} from '@openstapps/core';
import {QueryDslSpecificQueryContainer} from '../../types/util.js';

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
import {SCDistanceSort} from '@openstapps/core';
/**

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
import {SCDucetSort} from '@openstapps/core';
/**

View File

@@ -12,7 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {SortOptions} from '@elastic/elasticsearch/lib/api/types';
import {SortOptions} from '@elastic/elasticsearch/lib/api/types.js';
import {SCPriceSort, SCSportCoursePriceGroup, SCThingsField} from '@openstapps/core';
/**

View File

@@ -12,8 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types';
import {QueryDslQueryContainer} from '@elastic/elasticsearch/lib/api/types.js';
export type QueryDslSpecificQueryContainer<T extends keyof QueryDslQueryContainer> = Required<
Pick<QueryDslQueryContainer, T>

View File

@@ -31,7 +31,7 @@ import {buildQuery} from '../../../src/storage/elasticsearch/query/query.js';
import {buildSort} from '../../../src/storage/elasticsearch/query/sort.js';
import {ElasticsearchConfig} from '../../../src/storage/elasticsearch/types/elasticsearch-config.js';
import {QueryDslSpecificQueryContainer} from '../../../src/storage/elasticsearch/types/util.js';
import {SortCombinations} from '@elastic/elasticsearch/lib/api/types.js';
import {QueryDslRangeQuery, SortCombinations} from '@elastic/elasticsearch/lib/api/types.js';
import {backendConfig} from '../../../src/config.js';
describe('Query', function () {
@@ -363,8 +363,10 @@ describe('Query', function () {
const filter = buildFilter(rawFilter) as QueryDslSpecificQueryContainer<'range'>;
expect(filter).to.deep.equal(expectedFilter);
for (const bound of ['g', 'l']) {
const inclusiveExists = typeof filter.range.price[`${bound}t`] !== 'undefined';
const exclusiveExists = typeof filter.range.price[`${bound}te`] !== 'undefined';
const inclusiveExists =
typeof filter.range.price?.[`${bound}t` as keyof QueryDslRangeQuery] !== 'undefined';
const exclusiveExists =
typeof filter.range.price?.[`${bound}te` as keyof QueryDslRangeQuery] !== 'undefined';
// only one should exist at the same time
expect(inclusiveExists && exclusiveExists).to.be.false;

View File

@@ -15,24 +15,29 @@
import {Logger} from '@openstapps/logger';
import {Command} from 'commander';
import {readFileSync} from 'fs';
import {join} from 'path';
import path from 'path';
import {executeConnector, isValidSCNamespace} from './common.js';
import {MinimalConnector} from './minimal-connector.js';
import * as url from 'url';
process.on('unhandledRejection', error => {
throw error;
});
const connectorVersion = JSON.parse(readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
const dirname = path.dirname(url.fileURLToPath(import.meta.url));
const connectorVersion = JSON.parse(
readFileSync(path.join(dirname, '..', 'package.json')).toString(),
).version;
const commander = new Command();
/**
* Uses arguments to paramtrize the connector execution
* Uses arguments to parameterize the connector execution
*
* backendURL - URL of the StApps backend deployment e.g. http://localhost:3000
* origin - Origin, where the data comes from. Typically the name of the connector e.g. minimal-connector
* licensePlate - The license plate of your school. Must be matched to a SCNamespace e.g. f-u
* origin - Origin, where the data comes from.
* Typically, the name of the connector e.g., minimal-connector licensePlate - The license plate of your school.
* Must be matched to a SCNamespace e.g., f-u
*/
commander
.command('run <backendURL> <origin> <licensePlate>')
@@ -41,7 +46,7 @@ commander
if (backendURL.length === 0) {
throw new Error('Param "backend" needs to have a length greater zero.');
}
const originRegex = /^[a-z\-\_0-9]+$/;
const originRegex = /^[a-z\-_0-9]+$/;
if (!originRegex.test(origin)) {
throw new Error(
'Origin name can only consist of lowercase letters from a-z, "-", "_" and integer numbers.',

View File

@@ -12,8 +12,7 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {ConnectorClient} from '@openstapps/api/lib/connector-client.js';
import {HttpClient} from '@openstapps/api/lib/http-client.js';
import {ConnectorClient, HttpClient} from '@openstapps/api';
import {SCLicensePlate, SCNamespaces, SCThings} from '@openstapps/core';
import {Connector} from './connector.js';
@@ -48,9 +47,6 @@ export function createUUID(itemIdentifier: unknown, licensePlate: SCLicensePlate
export async function executeConnector<T extends SCThings>(backend: string, connector: Connector<T>) {
const items: T[] = await connector.getItems();
const client: ConnectorClient = new ConnectorClient(new HttpClient(), backend);
try {
// this might throw an error!
await client.index<T>(items, connector.origin);
} catch (err) {
throw err;
}
}

View File

@@ -18,16 +18,17 @@ import {createUUID} from './common.js';
/**
* Provides abstracted methods for the connector execution process
*
* By extending this class connector-developers only need to implement load and transform of the data
* By extending this class connector-developers only need to implement the load and transform of the data
* Pushing the data to the backend will be handled automatically
*
* @typeparam T Any serializable type
* @template T Any serializable type
*/
export abstract class Connector<T extends SCThings> {
/**
* License plate of the school
*/
protected licensePlate: SCLicensePlate;
/**
* Name of the connector
*/

View File

@@ -50,7 +50,7 @@ export class MinimalConnector extends Connector<SCMessage> {
* Mock-up data
*/
protected async fetchItems(): Promise<SCMessage[]> {
const importedItems: SCMessage[] = [
return [
{
audiences: ['students', 'employees'],
categories: [],
@@ -82,7 +82,5 @@ export class MinimalConnector extends Connector<SCMessage> {
uid: createUUID({id: 'message_3'}, this.licensePlate),
},
];
return importedItems;
}
}

View File

@@ -13,7 +13,6 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
import {HttpClient, PluginClient} from '@openstapps/api';
import {Converter} from '@openstapps/core-tools';
import {Logger} from '@openstapps/logger';

View File

@@ -32,13 +32,7 @@ export class MinimalPlugin extends Plugin {
* @param numbers the list of numbers
*/
private static sum(numbers: number[]): number {
let out = 0;
for (const num of numbers) {
out += num;
}
return out;
return numbers.reduce((a, b) => a + b);
}
/**
@@ -47,13 +41,13 @@ export class MinimalPlugin extends Plugin {
* For this example the whole purpose of the plugin is to receive a list of numbers and return the sum of them.
* TODO: remove the body of the function and replace with your own logic
*
* @param req the express request object
* @param res the express response object
* @param request the express request object
* @param response the express response object
*/
// tslint:disable-next-line:prefer-function-over-method
protected async onRouteInvoke(req: express.Request, res: express.Response): Promise<void> {
protected async onRouteInvoke(request: express.Request, response: express.Response): Promise<void> {
// get the body from the request as a SCMinimalRequest for static type-safety
const requestBody = req.body as SCMinimalRequest;
const requestBody = request.body as SCMinimalRequest;
// create out response body
const responseBody: SCMinimalResponse = {
@@ -61,6 +55,6 @@ export class MinimalPlugin extends Plugin {
};
// send the response
res.json(responseBody);
response.json(responseBody);
}
}

View File

@@ -1,10 +1,5 @@
<<<<<<<< HEAD:packages/es-mapping-generator/test/mapping-model/aggregations/src/types.ts
/*
* Copyright (C) 2020 StApps
========
/*!
* Copyright (C) 2023 StApps
>>>>>>>> app/develop:frontend/app/src/app/modules/about/about-licenses.scss
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
@@ -17,18 +12,6 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
<<<<<<<< HEAD:packages/es-mapping-generator/test/mapping-model/aggregations/src/types.ts
export enum ThingType {
AggArray = 'agg array',
AggGlobal = 'agg global',
AggGlobalNested = 'agg global nested',
AggNested = 'agg nested',
AggInherited = 'agg inherited',
AggInheritedGlobal = 'agg inherited global',
AggInheritedOverwritten = 'agg inherited overwritten',
}
========
ion-content > div {
height: 100%;
}
@@ -52,4 +35,3 @@ cdk-virtual-scroll-viewport {
vertical-align: text-top;
height: 14px;
}
>>>>>>>> app/develop:frontend/app/src/app/modules/about/about-licenses.scss

View File

@@ -12,35 +12,9 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
<<<<<<<< HEAD:frontend/app/src/app/modules/data/list/data-list-item.scss
:host {
display: block;
}
ion-item::part(native) {
height: 100%;
}
.ion-text-wrap ::ng-deep ion-label {
white-space: normal !important;
display: -webkit-box;
-webkit-box-orient: vertical;
-webkit-line-clamp: 3;
}
ion-item {
--border-color: transparent;
@include border-radius-in-parallax(var(--border-radius-default));
overflow: hidden;
--inner-padding-end: 0;
--padding-start: var(--spacing-sm);
margin: var(--spacing-sm);
========
simple-swiper {
--swiper-slide-width: 256px;
}
>>>>>>>> app/develop:frontend/app/src/app/modules/dashboard/sections/news-section/news-section.component.scss
.more-news {
font-size: var(--font-size-xl);

View File

@@ -12,9 +12,6 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
<<<<<<<< HEAD:frontend/app/src/app/modules/settings/item/settings-item.scss
========
ion-list {
background: none;
}
@@ -22,4 +19,3 @@ ion-list {
skeleton-list {
height: 100%;
}
>>>>>>>> app/develop:frontend/app/src/app/modules/data/list/data-list.scss

View File

@@ -1,10 +1,5 @@
<<<<<<<< HEAD:examples/minimal-plugin/src/common.ts
/*
* Copyright (C) 2019 StApps
========
/*!
* Copyright (C) 2023 StApps
>>>>>>>> app/develop:frontend/app/src/app/modules/settings/item/settings-item.scss
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, version 3.
@@ -17,8 +12,3 @@
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <https://www.gnu.org/licenses/>.
*/
<<<<<<<< HEAD:examples/minimal-plugin/src/common.ts
// TODO: place commonly used values here
========
>>>>>>>> app/develop:frontend/app/src/app/modules/settings/item/settings-item.scss

View File

@@ -2,17 +2,17 @@
"name": "@openstapps/openstapps",
"version": "2.1.0",
"scripts": {
"build": "dotenv -c -- turbo run build",
"build": "dotenv -c -- turbo run build --filter=!@openstapps/app",
"dev": "dotenv -c -- turbo run dev",
"docs": "typedoc",
"format": "dotenv -c -- turbo run format",
"format:fix": "dotenv -c -- turbo run format:fix",
"lint": "dotenv -c -- turbo run lint",
"lint:fix": "dotenv -c -- turbo run lint:fix",
"lint": "dotenv -c -- turbo run lint --filter=!@openstapps/app",
"lint:fix": "dotenv -c -- turbo run lint:fix --filter=!@openstapps/app",
"publish-packages": "dotenv -c -- turbo run build format lint test && changeset version && changeset publish",
"syncpack": "syncpack list-mismatches && syncpack lint-semver-ranges",
"syncpack:fix": "syncpack format && syncpack fix-mismatches",
"test": "dotenv -c -- turbo run test && node coverage.mjs"
"test": "dotenv -c -- turbo run test --filter=!@openstapps/app && node coverage.mjs"
},
"devDependencies": {
"@changesets/cli": "2.26.1",

View File

@@ -1,3 +1,9 @@
{
"extends": "@openstapps"
"extends": "@openstapps",
"rules": {
"no-console": "off",
"unicorn/no-negated-condition": "off",
"unicorn/no-null": "off",
"jsdoc/check-tag-names": "off"
}
}