fix: make facets work again

package-lock.json

@@ -219,9 +219,9 @@
       }
     },
     "@openstapps/core": {
-      "version": "0.26.0",
+      "version": "0.28.0",
-      "resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.26.0.tgz",
+      "resolved": "https://registry.npmjs.org/@openstapps/core/-/core-0.28.0.tgz",
-      "integrity": "sha512-r8mAplHPY7gS8EsuQv8NHfqR4TZ2ptEouMPjtvx2L7I0g0+YgnYO9ZP0QQGHmEeLYoJ01XdDkxOAasXsa2KGBw==",
+      "integrity": "sha512-VwL0ngs2o1xEsgNdne/XipYQimidrtfxT/DemVf28SMbGGjXDDS6NO8er4nMVV9C1uKm6SnKwWlzhKQF2OJjYg==",
       "requires": {
         "@types/geojson": "1.0.6",
         "@types/json-patch": "0.0.30",
@@ -241,9 +241,9 @@
       }
     },
     "@openstapps/core-tools": {
-      "version": "0.8.0",
+      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/@openstapps/core-tools/-/core-tools-0.8.0.tgz",
+      "resolved": "https://registry.npmjs.org/@openstapps/core-tools/-/core-tools-0.9.0.tgz",
-      "integrity": "sha512-2HaMQ3cxuhyvWRUPxED3/XOJilPq6A5nBVXzthgxpxeu5Wl8D/zD1Y7He3BIfDGgu+Pp+aDFkqvNKKe/3Hrn8Q==",
+      "integrity": "sha512-ltkQVc3ykGsqnPUop+lwp1ctlAlvJWt9L7FZ+3q+6Eepvjiqu/nZJM5N11qDIptOfjB0yXY0ovdTqJFQ+fc0uQ==",
       "requires": {
         "@krlwlfrt/async-pool": "0.1.0",
         "@openstapps/logger": "0.3.1",
@@ -293,11 +293,6 @@
           "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.8.tgz",
           "integrity": "sha512-I4+DbJEhLEg4/vIy/2gkWDvXBOOtPKV9EnLhYjMoqxcRW+TTZtUftkHktz/a8suoD5mUL7m6ReLrkPvSsCQQmw=="
         },
-        "flatted": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
-          "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg=="
-        },
         "glob": {
           "version": "7.1.4",
           "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",

package.json

@@ -25,12 +25,12 @@
     "preversion": "npm run prepublishOnly",
     "push": "git push && git push origin \"v$npm_package_version\"",
     "start": "NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true node ./lib/cli.js",
-    "test": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true nyc mocha --require ts-node/register --ui mocha-typescript --exit 'test/**/*.ts'",
+    "test": "env NODE_CONFIG_ENV=elasticsearch ALLOW_NO_TRANSPORT=true ES_FORCE_MAPPING_UPDATE=true nyc mocha --require ts-node/register --exit 'test/**/*.spec.ts'",
     "tslint": "tslint -p tsconfig.json -c tslint.json 'src/**/*.ts'"
   },
   "dependencies": {
-    "@openstapps/core": "0.26.0",
+    "@openstapps/core": "0.28.0",
-    "@openstapps/core-tools": "0.8.0",
+    "@openstapps/core-tools": "0.9.0",
     "@openstapps/logger": "0.4.0",
     "@types/node": "10.14.12",
     "commander": "2.20.0",

src/app.ts

@@ -22,7 +22,7 @@ import {
 import {Logger} from '@openstapps/logger';
 import * as config from 'config';
 import * as cors from 'cors';
-import * as express from 'express';
+import {Express} from 'express';
 import * as morgan from 'morgan';
 import {join} from 'path';
 import {configFile, isTestEnvironment, mailer, plugins, validator} from './common';
@@ -40,15 +40,10 @@ import {BulkStorage} from './storage/bulk-storage';
 import {DatabaseConstructor} from './storage/database';
 import {Elasticsearch} from './storage/elasticsearch/elasticsearch';
 
-/**
- * Created express application
- */
-export const app = express();
-
 /**
  * Configure the backend
  */
-export async function configureApp() {
+export async function configureApp(app: Express) {
   // request loggers have to be the first middleware to be set in express
   app.use(morgan('dev'));
 

src/cli.ts

@@ -14,8 +14,11 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 import {Logger} from '@openstapps/logger';
+import * as express from 'express';
 import * as http from 'http';
-import {app, configureApp} from './app';
+import {configureApp} from './app';
+
+const app = express();
 
 /**
  * Get port from environment and store in Express.
@@ -95,7 +98,7 @@ function onListening() {
   Logger.ok(`Listening on ${bind}`);
 }
 
-configureApp()
+configureApp(app)
   .then(() => {
     Logger.ok('Sucessfully configured express server');
   })

src/storage/elasticsearch/aggregations.ts

@@ -13,13 +13,15 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {SCBackendAggregationConfiguration, SCFacet, SCThingType} from '@openstapps/core';
+import {SCBackendAggregationConfiguration, SCFacet} from '@openstapps/core';
-import {AggregationSchema} from './common';
+import {
-
+  AggregationResponse,
-/**
+  AggregationSchema,
- * Provide information on which type (or on all) an aggregation happens
+  ESNestedAggregation,
- */
+  isBucketAggregation,
-export type aggregationType = SCThingType | '@all';
+  isESNestedAggregation,
+  isESTermsFilter, isNestedAggregation,
+} from './common';
 
 /**
  * Builds the aggregation
@@ -29,51 +31,40 @@ export function buildAggregations(aggsConfig: SCBackendAggregationConfiguration[
 
   const result: AggregationSchema = {};
 
-  aggsConfig.forEach((aggregation) => {
+  for (const aggregation of aggsConfig) {
+    if (typeof aggregation.onlyOnTypes !== 'undefined') {
+      for (const type of aggregation.onlyOnTypes) {
+        if (typeof result[type] === 'undefined') {
+          result[type] = {
+            aggs: {},
+            filter: {
+              type: {
+                value: type,
+              },
+            },
+          };
+        }
 
-    result[aggregation.fieldName] = {
+        (result[type] as ESNestedAggregation).aggs[aggregation.fieldName] = {
-      terms: {
+          terms: {
-        field: `${aggregation.fieldName}.raw`,
+            field: `${aggregation.fieldName}.keyword`,
-        size: 1000,
+            size: 1000,
-      },
+          },
-    };
+        };
-  });
+      }
+    } else {
+      result[aggregation.fieldName] = {
+        terms: {
+          field: `${aggregation.fieldName}.keyword`,
+          size: 1000,
+        },
+      };
+    }
+  }
 
   return result;
 }
 
-/**
- * An elasticsearch aggregation bucket
- */
-interface Bucket {
-  /**
-   * Number of documents in the agregation bucket
-   */
-  doc_count: number;
-
-  /**
-   * Text representing the documents in the bucket
-   */
-  key: string;
-}
-
-/**
- * An elasticsearch aggregation response
- */
-interface AggregationResponse {
-  [field: string]: {
-    /**
-     * Buckets in an aggregation
-     */
-    buckets: Bucket[];
-
-    /**
-     * Number of documents in an aggregation
-     */
-    doc_count?: number;
-  };
-}
-
 /**
  * Parses elasticsearch aggregations (response from es) to facets for the app
  * @param aggregationSchema - aggregation-schema for elasticsearch
@@ -85,22 +76,52 @@ export function parseAggregations(
 
   const facets: SCFacet[] = [];
 
-  const aggregationNames = Object.keys(aggregations);
+  // get all names of the types an aggregation is on
+  for (const typeName in aggregationSchema) {
+    if (aggregationSchema.hasOwnProperty(typeName) && aggregations.hasOwnProperty(typeName)) {
+      // the type object from the schema
+      const type = aggregationSchema[typeName];
+      // the "real" type object from the response
+      const realType = aggregations[typeName];
 
-  aggregationNames.forEach((aggregationName) => {
+      // both conditions must apply, else we have an error somewhere
-    const buckets = aggregations[aggregationName].buckets;
+      if (isESNestedAggregation(type) && isNestedAggregation(realType)) {
-    const facet: SCFacet = {
+        for (const fieldName in type.aggs) {
-      buckets: buckets.map((bucket) => {
+          if (type.aggs.hasOwnProperty(fieldName) && realType.hasOwnProperty(fieldName)) {
-        return {
+            // the field object from the schema
-          count: bucket.doc_count,
+            const field = type.aggs[fieldName];
-          key: bucket.key,
+            // the "real" field object from the response
-        };
+            const realField = realType[fieldName];
-      }),
-      field: `${aggregationSchema[aggregationName].terms.field}.raw`,
-    };
 
-    facets.push(facet);
+            // this should always be true in theory...
-  });
+            if (isESTermsFilter(field) && isBucketAggregation(realField)) {
+              facets.push({
+                buckets: realField.buckets.map((bucket) => {
+                  return {
+                    count: bucket.doc_count,
+                    key: bucket.key,
+                  };
+                }),
+                field: fieldName,
+                onlyOnType: type.filter.type.value,
+              });
+            }
+          }
+        }
+        // the last part here means that it is a bucket aggregation
+      } else if (isESTermsFilter(type) && !isNestedAggregation(realType)) {
+        facets.push({
+          buckets: realType.buckets.map((bucket) => {
+            return {
+              count: bucket.doc_count,
+              key: bucket.key,
+            };
+          }),
+          field: typeName,
+        });
+      }
+    }
+  }
 
   return facets;
 }

src/storage/elasticsearch/common.ts

@@ -17,12 +17,83 @@ import {SCThingType} from '@openstapps/core';
 import {SCThing} from '@openstapps/core';
 import {NameList} from 'elasticsearch';
 
+/**
+ * An elasticsearch aggregation bucket
+ */
+interface Bucket {
+  /**
+   * Number of documents in the agregation bucket
+   */
+  doc_count: number;
+
+  /**
+   * Text representing the documents in the bucket
+   */
+  key: string;
+}
+
+/**
+ * An elasticsearch aggregation response
+ */
+export interface AggregationResponse {
+  /**
+   * The individual aggregations
+   */
+  [field: string]: BucketAggregation | NestedAggregation;
+}
+
+/**
+ * An elasticsearch bucket aggregation
+ */
+export interface BucketAggregation {
+  /**
+   * Buckets in an aggregation
+   */
+  buckets: Bucket[];
+
+  /**
+   * Number of documents in an aggregation
+   */
+  doc_count?: number;
+}
+
+/**
+ * Checks if the type is a BucketAggregation
+ * @param agg the type to check
+ */
+export function isBucketAggregation(agg: BucketAggregation | number): agg is BucketAggregation {
+  return typeof agg !== 'number';
+}
+
+/**
+ * An aggregation that contains more aggregations nested inside
+ */
+export interface NestedAggregation {
+  /**
+   * Number of documents in an aggregation
+   */
+  doc_count: number;
+
+  /**
+   * Any nested responses
+   */
+  [name: string]: BucketAggregation | number;
+}
+
+/**
+ * Checks if the type is a NestedAggregation
+ * @param agg the type to check
+ */
+export function isNestedAggregation(agg: BucketAggregation | NestedAggregation): agg is NestedAggregation {
+  return typeof (agg as BucketAggregation).buckets === 'undefined';
+}
+
 /**
  * An elasticsearch bucket aggregation
  * @see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-aggregations-bucket.html
  */
 export interface AggregationSchema {
-  [aggregationName: string]: ESTermsFilter;
+  [aggregationName: string]: ESTermsFilter | ESNestedAggregation;
 }
 
 /**
@@ -224,6 +295,46 @@ export interface ESTermsFilter {
   };
 }
 
+/**
+ * Checks if the parameter is of type ESTermsFilter
+ * @param agg the value to check
+ */
+export function isESTermsFilter(agg: ESTermsFilter | ESNestedAggregation): agg is ESTermsFilter {
+  return typeof (agg as ESTermsFilter).terms !== 'undefined';
+}
+
+/**
+ * For nested aggregations
+ */
+export interface ESNestedAggregation {
+  /**
+   * Possible nested Aggregations
+   */
+  aggs: AggregationSchema;
+  /**
+   * Possible filter for types
+   */
+  filter: {
+    /**
+     * The type of the object to find
+     */
+    type: {
+      /**
+       * The name of the type
+       */
+      value: SCThingType;
+    };
+  };
+}
+
+/**
+ * Checks if the parameter is of type ESTermsFilter
+ * @param agg the value to check
+ */
+export function isESNestedAggregation(agg: ESTermsFilter | ESNestedAggregation): agg is ESNestedAggregation {
+  return typeof (agg as ESNestedAggregation).aggs !== 'undefined';
+}
+
 /**
  * An elasticsearch type filter
  */

src/storage/elasticsearch/templating.ts

@@ -27,7 +27,7 @@ const templatePath = resolve(dirPath, `template_${coreVersion}.json`);
 const errorPath = resolve(dirPath, `failed_template_${coreVersion}.json`);
 const errorReportPath = resolve(dirPath, `error_report_${coreVersion}.txt`);
 
-const ignoredTags = ['minlength', 'pattern', 'see'];
+const ignoredTags = ['minlength', 'pattern', 'see', 'tjs-format']; // TODO: put this into config
 
 /**
  * Check if the correct template exists

test/app.spec.disabled.ts

@@ -13,26 +13,31 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+// tslint:disable
 import * as supertest from 'supertest';
 import * as chaiAsPromised from 'chai-as-promised';
-import {timeout, slow} from 'mocha-typescript';
+import {timeout, slow, test, suite} from 'mocha-typescript';
 import {should, use} from 'chai';
-import {configureApp, app} from '../src/app';
+import {configureApp} from '../src/app';
 import {registerAddRequest, registerRemoveRequest} from './routes/plugin-register-route.spec';
 import {plugins} from '../src/common';
 import {SCPluginRemove} from '@openstapps/core';
 import * as nock from 'nock';
 import * as got from 'got';
 import * as sinon from 'sinon';
+import {Logger} from '@openstapps/logger';
+import * as express from 'express';
 
 should();
 use(chaiAsPromised);
 let appTest: supertest.SuperTest<supertest.Test>;
 // configures the backend and creates supertest
-const prepareTestApp = async () => {
+async function prepareTestApp() {
-  await configureApp();
+  const app = express();
+  await configureApp(app);
+  Logger.ok('App Configured');
   appTest = supertest(app);
-};
+}
 
 /**
  * Tests plugin registration routes
@@ -47,6 +52,7 @@ export class AppPluginRegisterSpec {
   async after() {
     // remove plugins
     plugins.clear();
+
   }
 
   @test
@@ -119,7 +125,7 @@ export class AppPluginRegisterSpec {
 /**
  * Tests functioning of already registered plugins
  */
-@suite(timeout(10000), slow(5000))
+@suite(timeout(50000), slow(5000))
 export class AppPluginSpec {
   static async before() {
     if (typeof appTest === 'undefined') {

test/routes/plugin-register-route.spec.ts

@@ -13,6 +13,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+// tslint:disable
 import {
   SCPluginAdd,
   SCPluginAlreadyRegisteredErrorResponse,
@@ -20,7 +21,7 @@ import {
   SCNotFoundErrorResponse,
 } from '@openstapps/core';
 import {should, use} from 'chai';
-import {slow, timeout} from 'mocha-typescript';
+import {slow, timeout, test, suite} from 'mocha-typescript';
 import * as chaiAsPromised from 'chai-as-promised';
 import {plugins} from '../../src/common';
 import {pluginRegisterHandler} from '../../src/routes/plugin-register-route';

test/routes/virtual-plugin-route.spec.ts

@@ -13,9 +13,10 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+// tslint:disable
 import {should, use, expect} from 'chai';
 import * as chaiAsPromised from 'chai-as-promised';
-import {slow, timeout} from 'mocha-typescript';
+import {slow, timeout, test, suite} from 'mocha-typescript';
 import {SCPluginMetaData, SCInternalServerErrorResponse, SCValidationErrorResponse} from '@openstapps/core';
 import {virtualPluginRoute} from '../../src/routes/virtual-plugin-route';
 import {mockReq} from 'sinon-express-mock'