fix: search uses inactive indices

test/storage/elasticsearch/elasticsearch.spec.ts

@@ -37,8 +37,13 @@ import {Logger} from '@openstapps/logger';
 import {SMTP} from '@openstapps/logger/lib/smtp';
 import {expect, use} from 'chai';
 import chaiAsPromised from 'chai-as-promised';
+import {beforeEach} from 'mocha';
 import mockedEnv from 'mocked-env';
-import {ALL_INDICES_QUERY, parseIndexName} from '../../../src/storage/elasticsearch/util';
+import {
+  ACTIVE_INDICES_ALIAS,
+  INACTIVE_INDICES_ALIAS,
+  parseIndexName,
+} from '../../../src/storage/elasticsearch/util';
 import * as queryModule from '../../../src/storage/elasticsearch/query/query';
 import * as sortModule from '../../../src/storage/elasticsearch/query/sort';
 import sinon, {SinonStub} from 'sinon';
@@ -287,7 +292,6 @@ describe('Elasticsearch', function () {
       let deleteStub: SinonStub;
       let refreshStub: SinonStub;
       let updateAliasesStub: SinonStub;
-      let existsStub: SinonStub;
       const oldIndex = 'stapps_footype_foosource_oldindex';
 
       beforeEach(function () {
@@ -297,7 +301,6 @@ describe('Elasticsearch', function () {
         sandbox.stub(Indices.prototype, 'putTemplate').resolves({} as any);
         createStub = sandbox.stub(Indices.prototype, 'create').resolves({} as any);
         deleteStub = sandbox.stub(Indices.prototype, 'delete').resolves({} as any);
-        existsStub = sandbox.stub(Indices.prototype, 'exists').resolves({} as any);
         refreshStub = sandbox.stub(Indices.prototype, 'refresh').resolves({} as any);
         updateAliasesStub = sandbox.stub(Indices.prototype, 'updateAliases').resolves({} as any);
         es = new Elasticsearch(configFile);
@@ -329,7 +332,7 @@ describe('Elasticsearch', function () {
           await es.bulkCreated(bulk);
 
           expect(putTemplateStub.called).to.be.true;
-          expect(createStub.calledWith({index})).to.be.true;
+          expect(createStub.calledWith({index, aliases: {[INACTIVE_INDICES_ALIAS]: {}}})).to.be.true;
         });
       });
 
@@ -341,6 +344,7 @@ describe('Elasticsearch', function () {
         it('should cleanup index in case of the expired bulk for bulk whose index is not in use', async function () {
           sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
 
+          await es.init();
           await es.bulkExpired({...bulk, state: 'in progress'});
 
           expect(deleteStub.called).to.be.true;
@@ -349,6 +353,7 @@ describe('Elasticsearch', function () {
         it('should not cleanup index in case of the expired bulk for bulk whose index is in use', async function () {
           sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
 
+          await es.init();
           await es.bulkExpired({...bulk, state: 'done'});
 
           expect(deleteStub.called).to.be.false;
@@ -368,44 +373,44 @@ describe('Elasticsearch', function () {
           return expect(es.bulkUpdated(bulk)).to.be.rejectedWith('Index');
         });
 
-        it("should create templates if index doesn't exist", async function () {
+        it("should refuse to finalize bulk if index doesn't exist", async function () {
           await es.init();
-          existsStub.resolves(false);
-          const putTemplateSpy = sandbox.spy(templating, 'putTemplate');
-          await es.bulkUpdated(bulk);
+          refreshStub.throws();
+          await expect(es.bulkUpdated(bulk)).to.eventually.throw;
 
-          expect(createStub.called).to.be.true;
-          expect(putTemplateSpy.called).to.be.true;
+          expect(
+            refreshStub.calledWith({
+              index: getThingIndexName(bulk.type, bulk.source, bulk),
+              allow_no_indices: false,
+            }),
+          ).to.be.true;
         });
 
         it('should create a new index', async function () {
           const index = getIndex();
           const expectedRefreshActions = [
             {
-              add: {index: index, alias: SCThingType.Book},
+              add: {index: index, alias: ACTIVE_INDICES_ALIAS},
             },
             {
-              remove: {index: oldIndex, alias: SCThingType.Book},
+              remove: {index: index, alias: INACTIVE_INDICES_ALIAS},
+            },
+            {
+              remove_index: {index: oldIndex},
             },
           ];
           sandbox.stub(utilModule, 'getThingIndexName').returns(index);
-          sandbox.stub(es, 'aliasMap').value({
-            [SCThingType.Book]: {
-              [bulk.source]: oldIndex,
-            },
-          });
           sandbox.stub(templating, 'putTemplate');
           await es.init();
 
           await es.bulkUpdated(bulk);
 
-          expect(refreshStub.calledWith({index})).to.be.true;
+          expect(refreshStub.calledWith({index, allow_no_indices: false})).to.be.true;
           expect(
             updateAliasesStub.calledWith({
               actions: expectedRefreshActions,
             }),
           ).to.be.true;
-          expect(deleteStub.called).to.be.true;
         });
       });
     });
@@ -449,6 +454,10 @@ describe('Elasticsearch', function () {
         es = new Elasticsearch(configFile);
       });
 
+      beforeEach(function () {
+        sandbox.stub(Indices.prototype, 'getAlias').resolves({} as any);
+      });
+
       afterEach(function () {
         sandbox.restore();
       });
@@ -459,33 +468,20 @@ describe('Elasticsearch', function () {
         const object: SearchHit<SCMessage> = {
           _id: '',
           _index: oldIndex,
-          _score: 0,
           _source: message as SCMessage,
         };
-        sandbox.stub(es.client, 'search').resolves(searchResponse(object));
+        sandbox.stub(es.client, 'search').resolves(searchResponse<SCMessage>(object));
         sandbox.stub(utilModule, 'getThingIndexName').returns(index);
 
-        return expect(es.post(object._source!, bulk)).to.rejectedWith('exist');
-      });
-
-      it('should not reject if the object already exists but in an index which will be rolled over', async function () {
-        const object: SearchHit<SCMessage> = {
-          _id: '',
-          _index: getIndex(),
-          _score: 0,
-          _source: message as SCMessage,
-        };
-        sandbox.stub(es.client, 'search').resolves(searchResponse(object));
-        // return index name with different generated UID (see getIndex method)
-        sandbox.stub(utilModule, 'getThingIndexName').returns(getIndex());
-
-        return expect(es.post(object._source!, bulk)).to.not.rejectedWith('exist');
+        await es.init();
+        return expect(es.post(object._source!, bulk)).to.be.rejectedWith('UID conflict');
       });
 
       it('should reject if there is an object creation error on the elasticsearch side', async function () {
         sandbox.stub(es.client, 'search').resolves(searchResponse());
         sandbox.stub(es.client, 'create').resolves({result: 'not_found'} as CreateResponse);
 
+        await es.init();
         return expect(es.post(message as SCMessage, bulk)).to.rejectedWith('creation');
       });
 
@@ -498,6 +494,7 @@ describe('Elasticsearch', function () {
           return Promise.resolve({result: 'created'});
         });
 
+        await es.init();
         await es.post(message as SCMessage, bulk);
 
         expect(createStub.called).to.be.true;
@@ -684,7 +681,7 @@ describe('Elasticsearch', function () {
           query: fakeResponse,
           sort: fakeBuildSortResponse,
           from: parameters.from,
-          index: ALL_INDICES_QUERY,
+          index: ACTIVE_INDICES_ALIAS,
           size: parameters.size,
         });
       });