Cleanup, add tests

application/src/main/java/uk/gov/hmcts/ccd/definition/store/SecurityConfiguration.java

@@ -19,6 +19,7 @@
 import org.springframework.security.web.SecurityFilterChain;
 
 import uk.gov.hmcts.ccd.definition.store.elastic.endpoint.ElasticsearchIndexController;
+import uk.gov.hmcts.ccd.definition.store.elastic.endpoint.ReindexTaskController;
 import uk.gov.hmcts.ccd.definition.store.excel.endpoint.ImportController;
 import uk.gov.hmcts.ccd.definition.store.security.JwtGrantedAuthoritiesConverter;
 import uk.gov.hmcts.ccd.definition.store.security.filters.ExceptionHandlingFilter;
@@ -76,7 +77,8 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
             .formLogin(fl -> fl.disable())
             .logout(lg -> lg.disable())
             .authorizeHttpRequests(ar -> 
-                ar.requestMatchers(ImportController.URI_IMPORT, ElasticsearchIndexController.ELASTIC_INDEX_URI)
+                ar.requestMatchers(ImportController.URI_IMPORT, ElasticsearchIndexController.ELASTIC_INDEX_URI,
+                        ReindexTaskController.REINDEX_TASKS_URI)
                 .hasAuthority("ccd-import")
                 .anyRequest()
                 .authenticated()

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/AsynchronousElasticDefinitionImportListener.java

@@ -10,7 +10,7 @@
 import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties;
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler;
 import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
-import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexTaskService;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexDBService;
 import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
 
 @Service
@@ -22,9 +22,10 @@ public AsynchronousElasticDefinitionImportListener(CcdElasticSearchProperties co
                                                        CaseMappingGenerator mappingGenerator,
                                                        ObjectFactory<HighLevelCCDElasticClient> clientFactory,
                                                        ElasticsearchErrorHandler elasticsearchErrorHandler,
-                                                       ReindexService reindexService,
-                                                       ReindexTaskService reindexTaskService) {
-        super(config, mappingGenerator, clientFactory, elasticsearchErrorHandler, reindexService, reindexTaskService);
+                                                       ElasticReindexService elasticReindexService,
+                                                       ReindexDBService reindexDBService) {
+        super(config, mappingGenerator, clientFactory, elasticsearchErrorHandler, elasticReindexService,
+            reindexDBService);
     }
 
     @Async

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java

@@ -9,7 +9,7 @@
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.ElasticSearchInitialisationException;
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler;
 import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
-import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexTaskService;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexDBService;
 import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity;
 
@@ -29,21 +29,22 @@ public abstract class ElasticDefinitionImportListener {
 
     private final ElasticsearchErrorHandler elasticsearchErrorHandler;
 
-    private final ReindexService reindexService;
+    private final ElasticReindexService elasticReindexService;
 
-    private final ReindexTaskService reindexTaskService;
+    private final ReindexDBService reindexDBService;
 
 
     public ElasticDefinitionImportListener(CcdElasticSearchProperties config, CaseMappingGenerator mappingGenerator,
                                            ObjectFactory<HighLevelCCDElasticClient> clientFactory,
-                                           ElasticsearchErrorHandler elasticsearchErrorHandler, ReindexService reindexService,
-                                           ReindexTaskService reindexTaskService) {
+                                           ElasticsearchErrorHandler elasticsearchErrorHandler,
+                                           ElasticReindexService elasticReindexService,
+                                           ReindexDBService reindexDBService) {
         this.config = config;
         this.mappingGenerator = mappingGenerator;
         this.clientFactory = clientFactory;
         this.elasticsearchErrorHandler = elasticsearchErrorHandler;
-        this.reindexService = reindexService;
-        this.reindexTaskService = reindexTaskService;
+        this.elasticReindexService = elasticReindexService;
+        this.reindexDBService = reindexDBService;
     }
 
     public abstract void onDefinitionImported(DefinitionImportedEvent event) throws IOException;
@@ -56,9 +57,6 @@ public ElasticDefinitionImportListener(CcdElasticSearchProperties config, CaseMa
     @Transactional
     public void initialiseElasticSearch(DefinitionImportedEvent event) {
         List<CaseTypeEntity> caseTypes = event.getContent();
-        boolean reindex = event.isReindex();
-        boolean reindexStarted = false;
-        String newIndexName = null;
         String caseMapping = null;
         CaseTypeEntity currentCaseType = null;
 
@@ -71,8 +69,8 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) {
                     String actualIndexName = baseIndexName + FIRST_INDEX_SUFFIX;
                     elasticClient.createIndex(actualIndexName, baseIndexName);
                 }
-                if (reindex) {
-                    reindexService.asyncReindex(event, baseIndexName, caseType, reindexStarted);
+                if (event.isReindex()) {
+                    elasticReindexService.asyncReindex(event, baseIndexName, caseType);
                 } else {
                     caseMapping = mappingGenerator.generateMapping(caseType);
                     log.debug("case mapping: {}", caseMapping);
@@ -81,15 +79,9 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) {
             }
         } catch (ElasticsearchStatusException exc) {
             logMapping(caseMapping);
-            if (reindex && !reindexStarted) {
-                reindexTaskService.updateEntity(newIndexName, exc);
-            }
             throw elasticsearchErrorHandler.createException(exc, currentCaseType);
         } catch (Exception exc) {
             logMapping(caseMapping);
-            if (reindex && !reindexStarted) {
-                reindexTaskService.updateEntity(newIndexName, exc);
-            }
             throw new ElasticSearchInitialisationException(exc);
         }
     }

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticReindexService.java

@@ -0,0 +1,143 @@
+package uk.gov.hmcts.ccd.definition.store.elastic;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.GetAliasesResponse;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient;
+import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexDBService;
+import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
+import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity;
+import uk.gov.hmcts.ccd.definition.store.repository.entity.ReindexEntity;
+
+import java.io.IOException;
+import java.util.concurrent.CompletionException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+@Service
+public class ElasticReindexService {
+
+    private final CaseMappingGenerator mappingGenerator;
+
+    private final ObjectFactory<HighLevelCCDElasticClient> clientFactory;
+
+    private final ReindexDBService reindexDBService;
+
+    public ElasticReindexService(CaseMappingGenerator mappingGenerator,
+                                 ObjectFactory<HighLevelCCDElasticClient> clientFactory,
+                                 ReindexDBService reindexDBService) {
+        this.mappingGenerator = mappingGenerator;
+        this.clientFactory = clientFactory;
+        this.reindexDBService = reindexDBService;
+    }
+
+    @Async("reindexExecutor")
+    public void asyncReindex(DefinitionImportedEvent event,
+                             String baseIndexName,
+                             CaseTypeEntity caseType) throws IOException {
+        HighLevelCCDElasticClient elasticClient = clientFactory.getObject();
+        GetAliasesResponse aliasResponse = elasticClient.getAlias(baseIndexName);
+        String indexName = aliasResponse.getAliases().keySet().iterator().next();
+        String newIndexName = incrementIndexNumber(indexName);
+        boolean reindexStarted = false;
+
+        //prepare for db
+        ReindexEntity reindexEntity = reindexDBService.saveEntity(event.isReindex(),
+            event.isDeleteOldIndex(), caseType, newIndexName);
+
+        try {
+            //create new index with generated mapping and incremented case type name (no alias update yet)
+            String caseMapping = mappingGenerator.generateMapping(caseType);
+            log.debug("case mapping: {}", caseMapping);
+            elasticClient.setIndexReadOnly(baseIndexName, true);
+            elasticClient.createIndexAndMapping(newIndexName, caseMapping);
+
+            //initiate reindexing
+            reindexStarted = true;
+            handleReindexing(elasticClient, baseIndexName, indexName, newIndexName, event.isDeleteOldIndex());
+            //dummy value for phase 1
+            event.setTaskId("taskID");
+            log.debug("reindexing successful for case type: {}", caseType.getReference());
+            log.debug("task id returned from the import: {}", event.getTaskId());
+        } catch (Exception e) {
+            if (!reindexStarted) {
+                reindexDBService.updateEntity(newIndexName, e);
+            }
+            throw new CompletionException(e);
+        }
+    }
+
+    private void handleReindexing(HighLevelCCDElasticClient elasticClient, String baseIndexName,
+                                  String oldIndexName, String newIndexName,
+                                  boolean deleteOldIndex) {
+        elasticClient.reindexData(oldIndexName, newIndexName, new ActionListener<>() {
+            @Override
+            public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
+                try (elasticClient; HighLevelCCDElasticClient highLevelCCDElasticClient = clientFactory.getObject()) {
+                    //if success set writable and update alias to new index
+                    log.debug("updating alias from {} to {}", oldIndexName, newIndexName);
+                    highLevelCCDElasticClient.setIndexReadOnly(baseIndexName, false);
+                    highLevelCCDElasticClient.updateAlias(baseIndexName, oldIndexName, newIndexName);
+                    if (deleteOldIndex) {
+                        log.debug("deleting old index {}", oldIndexName);
+                        highLevelCCDElasticClient.removeIndex(oldIndexName);
+                    }
+                    //set success status and end time for db
+                    reindexDBService.updateEntity(newIndexName, bulkByScrollResponse.toString());
+                    log.info("saved reindex entity"
+                             + " metadata for case type {} to DB", baseIndexName);
+                } catch (IOException e) {
+                    log.error("failed to clean up after reindexing success", e);
+                    reindexDBService.updateEntity(newIndexName, e);
+                    throw new CompletionException(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception ex) {
+                try (elasticClient; HighLevelCCDElasticClient highLevelCCDElasticClient = clientFactory.getObject()) {
+                    //set failure status, end time and ex for db
+                    reindexDBService.updateEntity(newIndexName, ex);
+
+                    //if failed delete new index, set old index writable
+                    log.debug("reindexing failed", ex);
+                    highLevelCCDElasticClient.removeIndex(newIndexName);
+                    log.debug("{} deleted", newIndexName);
+                    highLevelCCDElasticClient.setIndexReadOnly(oldIndexName, false);
+                    log.debug("{} set to writable", oldIndexName);
+                } catch (IOException e) {
+                    log.error("failed to clean up after reindexing failure", e);
+                    reindexDBService.updateEntity(newIndexName, e);
+                    throw new CompletionException(e);
+                }
+                throw new CompletionException(ex);
+            }
+        });
+    }
+
+    String incrementIndexNumber(String indexName) {
+        Pattern pattern = Pattern.compile("(.+_cases-)(\\d+)$");
+        Matcher matcher = pattern.matcher(indexName);
+
+        if (!matcher.matches() || matcher.groupCount() < 2) {
+            throw new IllegalArgumentException("invalid index name format: " + indexName);
+        }
+
+        String prefix = matcher.group(1);
+        String numberStr = matcher.group(2);
+
+        int incremented = Integer.parseInt(numberStr) + 1;
+        String formattedNumber = StringUtils.leftPad(String.valueOf(incremented), numberStr.length(), '0');
+
+        String incrementedIndexName = prefix + formattedNumber;
+        return incrementedIndexName;
+    }
+
+}

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/SynchronousElasticDefinitionImportListener.java

@@ -10,7 +10,7 @@
 import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties;
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler;
 import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
-import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexTaskService;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexDBService;
 import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
 
 @Service
@@ -23,9 +23,10 @@ public SynchronousElasticDefinitionImportListener(CcdElasticSearchProperties con
                                                       CaseMappingGenerator mappingGenerator,
                                                       ObjectFactory<HighLevelCCDElasticClient> clientFactory,
                                                       ElasticsearchErrorHandler elasticsearchErrorHandler,
-                                                      ReindexService reindexService,
-                                                      ReindexTaskService reindexTaskService) {
-        super(config, mappingGenerator, clientFactory, elasticsearchErrorHandler, reindexService, reindexTaskService);
+                                                      ElasticReindexService elasticReindexService,
+                                                      ReindexDBService reindexDBService) {
+        super(config, mappingGenerator, clientFactory, elasticsearchErrorHandler, elasticReindexService,
+            reindexDBService);
     }
 
     @EventListener

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/endpoint/ReindexTaskController.java

@@ -10,7 +10,7 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
-import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexTaskService;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexDBService;
 import uk.gov.hmcts.ccd.definition.store.repository.model.ReindexTask;
 
 import java.util.List;
@@ -22,11 +22,11 @@ public class ReindexTaskController {
 
     public static final String REINDEX_TASKS_URI = "/elastic-support/reindex/tasks";
 
-    private final ReindexTaskService reindexTaskService;
+    private final ReindexDBService reindexDBService;
 
     @Autowired
-    public ReindexTaskController(ReindexTaskService reindexTaskService) {
-        this.reindexTaskService = reindexTaskService;
+    public ReindexTaskController(ReindexDBService reindexDBService) {
+        this.reindexDBService = reindexDBService;
     }
 
     @GetMapping
@@ -40,7 +40,7 @@ public ReindexTaskController(ReindexTaskService reindexTaskService) {
     public ResponseEntity<List<ReindexTask>> getReindexTasksByCaseType(
         @RequestParam(value = "caseType", required = false) String caseType
     ) {
-        List<ReindexTask> response = reindexTaskService.getTasksByCaseType(caseType);
+        List<ReindexTask> response = reindexDBService.getTasksByCaseType(caseType);
         return ResponseEntity.ok(response);
     }
 }

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/service/ReindexTaskService.java → elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/service/ReindexDBService.java

@@ -6,7 +6,7 @@
 
 import java.util.List;
 
-public interface ReindexTaskService {
+public interface ReindexDBService {
     List<ReindexTask> getAll();
 
     List<ReindexTask> getTasksByCaseType(String caseType);

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/service/ReindexTaskServiceImpl.java → elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/service/ReindexDBServiceImpl.java

@@ -18,13 +18,13 @@
 
 @Service
 @Slf4j
-public class ReindexTaskServiceImpl implements ReindexTaskService {
+public class ReindexDBServiceImpl implements ReindexDBService {
 
     private final ReindexRepository reindexRepository;
     private final EntityToResponseDTOMapper mapper;
 
     @Autowired
-    public ReindexTaskServiceImpl(ReindexRepository reindexRepository, EntityToResponseDTOMapper mapper) {
+    public ReindexDBServiceImpl(ReindexRepository reindexRepository, EntityToResponseDTOMapper mapper) {
         this.reindexRepository = reindexRepository;
         this.mapper = mapper;
     }