feat: migrate to protomaps and maplibre

backend/backend/src/storage/database.ts

@@ -13,15 +13,34 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
-import {SCConfigFile, SCSearchQuery, SCSearchResponse, SCThings, SCUuid} from '@openstapps/core';
+import {
+  SCConfigFile,
+  SCPlace,
+  SCPlaceWithoutReferences,
+  SCSearchQuery,
+  SCSearchResponse,
+  SCThingWithCategoriesWithoutReferences,
+  SCThings,
+  SCUuid,
+} from '@openstapps/core';
 import {MailQueue} from '../notification/mail-queue.js';
 import {Bulk} from './bulk-storage.js';
+import {FeatureCollection, Point, Polygon} from 'geojson';
 
 /**
  * Creates an instance of a database
  */
 export type DatabaseConstructor = new (config: SCConfigFile, mailQueue?: MailQueue) => Database;
 
+export type SupplementaryGeoJSON = FeatureCollection<Point | Polygon, SupplementaryGeoJSONThing>;
+export type SupplementaryGeoJSONThing = Pick<
+  Extract<SCThings, SCPlace>,
+  Exclude<
+    keyof SCPlaceWithoutReferences | keyof SCThingWithCategoriesWithoutReferences<never, never>,
+    'geo' | 'origin' | 'translations'
+  >
+>;
+
 /**
  * Defines what one database class needs to have defined
  */
@@ -82,4 +101,9 @@ export interface Database {
    * @param params Parameters which form a search query to search the backend data
    */
   search(parameters: SCSearchQuery): Promise<SCSearchResponse>;
+
+  /**
+   * Get geo info for display on a map
+   */
+  geo(): Promise<SupplementaryGeoJSON>;
 }

backend/backend/src/storage/elasticsearch/elasticsearch.ts

@@ -26,7 +26,7 @@ import {Logger} from '@openstapps/logger';
 import moment from 'moment';
 import {MailQueue} from '../../notification/mail-queue.js';
 import {Bulk} from '../bulk-storage.js';
-import {Database} from '../database.js';
+import {Database, SupplementaryGeoJSON, SupplementaryGeoJSONThing} from '../database.js';
 import {parseAggregations} from './aggregations.js';
 import * as Monitoring from './monitoring.js';
 import {buildQuery} from './query/query.js';
@@ -46,6 +46,7 @@ import {
 } from './util/index.js';
 import {noUndefined} from './util/no-undefined.js';
 import {retryCatch, RetryOptions} from './util/retry.js';
+import {Feature, Point, Polygon} from 'geojson';
 
 /**
  * A database interface for elasticsearch
@@ -405,4 +406,49 @@ export class Elasticsearch implements Database {
       },
     };
   }
+
+  async geo(): Promise<SupplementaryGeoJSON> {
+    const searchResponse = await this.client.search<Extract<SCThings, {geo: unknown}>>({
+      body: {
+        query: {
+          exists: {
+            field: 'geo',
+          },
+        },
+      },
+      from: 0,
+      allow_no_indices: true,
+      index: ACTIVE_INDICES_ALIAS,
+      size: 1,
+    });
+
+    return {
+      type: 'FeatureCollection',
+      features: searchResponse.hits.hits
+        .map(thing => {
+          return thing._source?.geo
+            ? ({
+                id: Number(thing._source.identifiers?.['OSM']) || undefined,
+                type: 'Feature',
+                geometry: thing._source.geo.polygon ?? thing._source.geo.point,
+                properties: {
+                  name: thing._source.name,
+                  sameAs: thing._source.sameAs,
+                  image: thing._source.image,
+                  alternateNames: thing._source.alternateNames,
+                  description: thing._source.description,
+                  identifiers: thing._source.identifiers,
+                  categories: thing._source.categories,
+                  categorySpecificValues: thing._source.categorySpecificValues,
+                  openingHours: thing._source.openingHours,
+                  address: thing._source.address,
+                  uid: thing._source.uid,
+                  type: thing._source.type,
+                },
+              } satisfies Feature<Polygon | Point, SupplementaryGeoJSONThing>)
+            : undefined;
+        })
+        .filter(noUndefined),
+    };
+  }
 }

backend/backend/src/storage/elasticsearch/query/filters/geo.ts

@@ -19,14 +19,29 @@ import {QueryDslSpecificQueryContainer} from '../../types/util.js';
  * Converts a geo filter to elasticsearch syntax
  * @param filter A search filter for the retrieval of the data
  */
-export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'geo_shape'> {
+export function buildGeoFilter(filter: SCGeoFilter): QueryDslSpecificQueryContainer<'bool'> {
   return {
-    geo_shape: {
-      ignore_unmapped: true,
-      [`${filter.arguments.field}.polygon`]: {
-        shape: filter.arguments.shape,
-        relation: filter.arguments.spatialRelation,
-      },
+    bool: {
+      should: [
+        {
+          geo_shape: {
+            ignore_unmapped: true,
+            [`${filter.arguments.field}.polygon`]: {
+              shape: filter.arguments.shape,
+              relation: filter.arguments.spatialRelation,
+            },
+          },
+        } satisfies QueryDslSpecificQueryContainer<'geo_shape'>,
+        {
+          geo_shape: {
+            ignore_unmapped: true,
+            [`${filter.arguments.field}.point`]: {
+              shape: filter.arguments.shape,
+              relation: filter.arguments.spatialRelation,
+            },
+          },
+        } satisfies QueryDslSpecificQueryContainer<'geo_shape'>,
+      ],
     },
   };
 }

backend/backend/test/common.ts

@@ -22,7 +22,7 @@ import http from 'http';
 import {MailQueue} from '../src/notification/mail-queue.js';
 import {Bulk, BulkStorage} from '../src/storage/bulk-storage.js';
 import getPort from 'get-port';
-import {Database} from '../src/storage/database.js';
+import {Database, SupplementaryGeoJSON} from '../src/storage/database.js';
 import {v4} from 'uuid';
 import {backendConfig} from '../src/config.js';
 import {getIndexUID} from '../src/storage/elasticsearch/util/index.js';
@@ -58,7 +58,6 @@ export async function startApp(): Promise<Express> {
  * An elasticsearch mock
  */
 export class ElasticsearchMock implements Database {
-  // @ts-expect-error never read
   private bulk: Bulk | undefined;
 
   private storageMock = new Map<string, SCThings>();
@@ -67,6 +66,10 @@ export class ElasticsearchMock implements Database {
     // Nothing to do here
   }
 
+  geo(): Promise<SupplementaryGeoJSON> {
+    throw new Error('Method not implemented.');
+  }
+
   bulkCreated(bulk: Bulk): Promise<void> {
     this.bulk = bulk;
     return Promise.resolve(undefined);

backend/backend/test/storage/elasticsearch/query.spec.ts

@@ -479,18 +479,39 @@ describe('Query', function () {
     it('should build geo filter for shapes and points', function () {
       const filter = buildFilter(searchFilters.geoPoint);
       const expectedFilter = {
-        geo_shape: {
-          'geo.polygon': {
-            relation: undefined,
-            shape: {
-              type: 'envelope',
-              coordinates: [
-                [50.123, 8.123],
-                [50.123, 8.123],
-              ],
+        bool: {
+          should: [
+            {
+              geo_shape: {
+                'geo.polygon': {
+                  relation: undefined,
+                  shape: {
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                    type: 'envelope',
+                  },
+                },
+                'ignore_unmapped': true,
+              },
             },
-          },
-          'ignore_unmapped': true,
+            {
+              geo_shape: {
+                'geo.point': {
+                  relation: undefined,
+                  shape: {
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                    type: 'envelope',
+                  },
+                },
+                'ignore_unmapped': true,
+              },
+            },
+          ],
         },
       };
 
@@ -500,18 +521,39 @@ describe('Query', function () {
     it('should build geo filter for shapes only', function () {
       const filter = buildFilter(searchFilters.geoShape);
       const expectedFilter = {
-        geo_shape: {
-          'geo.polygon': {
-            relation: 'contains',
-            shape: {
-              type: 'envelope',
-              coordinates: [
-                [50.123, 8.123],
-                [50.123, 8.123],
-              ],
+        bool: {
+          should: [
+            {
+              geo_shape: {
+                'geo.polygon': {
+                  relation: 'contains',
+                  shape: {
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                    type: 'envelope',
+                  },
+                },
+                'ignore_unmapped': true,
+              },
             },
-          },
-          'ignore_unmapped': true,
+            {
+              geo_shape: {
+                'geo.point': {
+                  relation: 'contains',
+                  shape: {
+                    coordinates: [
+                      [50.123, 8.123],
+                      [50.123, 8.123],
+                    ],
+                    type: 'envelope',
+                  },
+                },
+                'ignore_unmapped': true,
+              },
+            },
+          ],
         },
       };