CME-239 - Add GET reindexing tasks

application/src/main/java/uk/gov/hmcts/ccd/definition/store/SecurityConfiguration.java

@@ -19,6 +19,7 @@
 import org.springframework.security.web.SecurityFilterChain;
 
 import uk.gov.hmcts.ccd.definition.store.elastic.endpoint.ElasticsearchIndexController;
+import uk.gov.hmcts.ccd.definition.store.elastic.endpoint.ReindexTaskController;
 import uk.gov.hmcts.ccd.definition.store.excel.endpoint.ImportController;
 import uk.gov.hmcts.ccd.definition.store.security.JwtGrantedAuthoritiesConverter;
 import uk.gov.hmcts.ccd.definition.store.security.filters.ExceptionHandlingFilter;
@@ -76,8 +77,9 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
             .csrf(csrf -> csrf.disable()) // NOSONAR - CSRF is disabled as per security requirements
             .formLogin(fl -> fl.disable())
             .logout(lg -> lg.disable())
-            .authorizeHttpRequests(ar ->
-                ar.requestMatchers(ImportController.URI_IMPORT, ElasticsearchIndexController.ELASTIC_INDEX_URI)
+            .authorizeHttpRequests(ar -> 
+                ar.requestMatchers(ImportController.URI_IMPORT, ElasticsearchIndexController.ELASTIC_INDEX_URI,
+                        ReindexTaskController.REINDEX_TASKS_URI)
                 .hasAuthority("ccd-import")
                 .anyRequest()
                 .authenticated()

build.gradle

@@ -130,7 +130,6 @@ allprojects {
         testImplementation group: 'org.apache.groovy', name: 'groovy-xml', version: groovyVersion
         testImplementation group: 'org.apache.groovy', name: 'groovy-json', version: groovyVersion
         testImplementation group: 'com.github.hmcts', name: 'fortify-client', version: '1.4.10', classifier: 'all'
-
     }
 
 }

domain/src/main/java/uk/gov/hmcts/ccd/definition/store/domain/service/EntityToResponseDTOMapper.java

@@ -32,6 +32,7 @@
 import uk.gov.hmcts.ccd.definition.store.repository.entity.FieldTypeListItemEntity;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.JurisdictionEntity;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.JurisdictionUiConfigEntity;
+import uk.gov.hmcts.ccd.definition.store.repository.entity.ReindexEntity;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.RoleToAccessProfilesEntity;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.SearchAliasFieldEntity;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.SearchCasesResultFieldEntity;
@@ -65,6 +66,7 @@
 import uk.gov.hmcts.ccd.definition.store.repository.model.FixedListItem;
 import uk.gov.hmcts.ccd.definition.store.repository.model.Jurisdiction;
 import uk.gov.hmcts.ccd.definition.store.repository.model.JurisdictionUiConfig;
+import uk.gov.hmcts.ccd.definition.store.repository.model.ReindexTask;
 import uk.gov.hmcts.ccd.definition.store.repository.model.RoleAssignment;
 import uk.gov.hmcts.ccd.definition.store.repository.model.RoleToAccessProfiles;
 import uk.gov.hmcts.ccd.definition.store.repository.model.SearchAliasField;
@@ -334,6 +336,8 @@ static List<CaseEventFieldComplex> map(List<? extends EventComplexTypeEntity> ev
     @Mapping(source = "caseType.jurisdiction.name", target = "jurisdictionName")
     AccessTypeField map(AccessTypeEntity accessTypeEntity);
 
+    ReindexTask map(ReindexEntity reindexEntity);
+
     @Mapping(source = "caseType.reference", target = "id")
     @Mapping(source = "roleToAccessProfilesEntity.roleName", target = "name")
     RoleAssignment roleToAccessProfilesEntityToRoleAssignment(RoleToAccessProfilesEntity roleToAccessProfilesEntity);

elastic-search-support/build.gradle

@@ -15,6 +15,7 @@ bootJar {
 
 dependencies {
 
+    implementation group:  'org.mapstruct', name: 'mapstruct', version: '1.5.5.Final'
     implementation project(':domain')
     implementation project(':repository')
     // need these for command line to pass in the gradle version,
@@ -25,6 +26,8 @@ dependencies {
     testImplementation group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.3'
     testImplementation group: 'org.testcontainers', name: 'elasticsearch', version: testcontainersVersion
     testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: testcontainersVersion
+    testImplementation 'net.ttddyy:datasource-proxy:1.8'
+    testImplementation(project(":application"))
 }
 
 pitest {

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/AsynchronousElasticDefinitionImportListener.java

@@ -10,6 +10,7 @@
 import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties;
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler;
 import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexService;
 import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
 
 @Service

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java

@@ -9,6 +9,7 @@
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.ElasticSearchInitialisationException;
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler;
 import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexService;
 import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
 import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity;
 
@@ -30,6 +31,7 @@ public abstract class ElasticDefinitionImportListener {
 
     private final ReindexService reindexService;
 
+
     public ElasticDefinitionImportListener(CcdElasticSearchProperties config, CaseMappingGenerator mappingGenerator,
                                            ObjectFactory<HighLevelCCDElasticClient> clientFactory,
                                            ElasticsearchErrorHandler elasticsearchErrorHandler,
@@ -51,8 +53,6 @@ public ElasticDefinitionImportListener(CcdElasticSearchProperties config, CaseMa
     @Transactional
     public void initialiseElasticSearch(DefinitionImportedEvent event) {
         List<CaseTypeEntity> caseTypes = event.getContent();
-        boolean reindex = event.isReindex();
-
         String caseMapping = null;
         CaseTypeEntity currentCaseType = null;
 
@@ -65,7 +65,7 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) {
                     String actualIndexName = baseIndexName + FIRST_INDEX_SUFFIX;
                     elasticClient.createIndex(actualIndexName, baseIndexName);
                 }
-                if (reindex) {
+                if (event.isReindex()) {
                     reindexService.asyncReindex(event, baseIndexName, caseType);
                 } else {
                     caseMapping = mappingGenerator.generateMapping(caseType);

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ReindexHelper.java

@@ -64,7 +64,7 @@ public String reindexIndex(String sourceIndex,
                         TaskInfo taskInfo = taskResponse.get().getTaskInfo();
                         if (!shouldWaitForMissingInfo(taskInfo, taskId, pollIntervalMs)) {
                             if (taskResponse.get().isCompleted()) {
-                                if (handleCompletion(taskInfo, listener, destIndex)) {
+                                if (handleCompletion(taskInfo, listener)) {
                                     break;
                                 }
                             } else {
@@ -111,22 +111,27 @@ private boolean shouldWaitForMissingInfo(TaskInfo taskInfo,
     }
 
     private boolean handleCompletion(TaskInfo taskInfo,
-                                     ReindexListener listener,
-                                     String destIndex) throws IOException {
+                                     ReindexListener listener) throws IOException {
         Object statusObj = taskInfo.getStatus();
         if (statusObj == null) {
-            listener.onFailure(new RuntimeException("Reindex process completed with unknown status"));
+            listener.onFailure(new RuntimeException(
+                String.format("Reindex process completed with unknown status (taskId=%s)", taskInfo.getTaskId())
+                ));
             return true;
         }
 
         JsonNode statusJson = toStatusJson(statusObj);
         if (hasFailures(statusJson)) {
-            listener.onFailure(new RuntimeException("Reindex process failed: " + statusJson.path("failures")));
+            String failures = statusJson.path("failures").toString();
+            listener.onFailure(new RuntimeException(
+                String.format("Reindex process failed (taskId=%s): %s", taskInfo.getTaskId(), failures)
+            ));
             return true;
         }
 
-        logProgress(destIndex, statusJson);
-        listener.onSuccess();
+        String response = mapper.writeValueAsString(taskInfo);
+        listener.onSuccess(response);
+        log.info("Reindex task completed successfully (taskId={}): {}", taskInfo.getTaskId(), response);
         return true;
     }
 
@@ -139,15 +144,6 @@ private boolean hasFailures(JsonNode statusJson) {
         return statusJson.has("failures") && !statusJson.path("failures").isEmpty();
     }
 
-    private void logProgress(String destIndex, JsonNode statusJson) {
-        int total = statusJson.path("total").asInt(0);
-        int created = statusJson.path("created").asInt(0);
-        int updated = statusJson.path("updated").asInt(0);
-        int batches = statusJson.path("batches").asInt(0);
-        log.info("Progress for index {}: total={}, created={}, updated={}, batches={}",
-            destIndex, total, created, updated, batches);
-    }
-
     public Executor asyncExecutor() {
         ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
         executor.setCorePoolSize(10);

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/SynchronousElasticDefinitionImportListener.java

@@ -10,6 +10,7 @@
 import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties;
 import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler;
 import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexService;
 import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
 
 @Service

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/endpoint/ReindexTaskController.java

@@ -0,0 +1,46 @@
+package uk.gov.hmcts.ccd.definition.store.elastic.endpoint;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import uk.gov.hmcts.ccd.definition.store.elastic.service.ReindexService;
+import uk.gov.hmcts.ccd.definition.store.repository.model.ReindexTask;
+
+import java.util.List;
+
+@RestController
+@RequestMapping(ReindexTaskController.REINDEX_TASKS_URI)
+@Api(value = ReindexTaskController.REINDEX_TASKS_URI)
+public class ReindexTaskController {
+
+    public static final String REINDEX_TASKS_URI = "/elastic-support/reindex/tasks";
+
+    private final ReindexService reindexService;
+
+    @Autowired
+    public ReindexTaskController(ReindexService reindexService) {
+        this.reindexService = reindexService;
+    }
+
+    @GetMapping
+    @ApiOperation(value = "Get all reindex tasks, optionally by case type",
+        response = ReindexTask.class,
+        responseContainer = "List")
+    @ApiResponses(value = {
+        @ApiResponse(code = 200, message = "Successfully retrieved reindex tasks"),
+        @ApiResponse(code = 500, message = "Internal Server Error")
+    })
+    public ResponseEntity<List<ReindexTask>> getReindexTasksByCaseType(
+        @RequestParam(value = "caseType", required = false) String caseType
+    ) {
+        List<ReindexTask> response = reindexService.getTasksByCaseType(caseType);
+        return ResponseEntity.ok(response);
+    }
+}

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/listener/ReindexListener.java

@@ -2,7 +2,7 @@
 
 public interface ReindexListener {
 
-    void onSuccess();
+    void onSuccess(String reindexResponse);
 
     void onFailure(Exception ex);
 }

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/service/ReindexService.java

@@ -0,0 +1,26 @@
+package uk.gov.hmcts.ccd.definition.store.elastic.service;
+
+import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent;
+import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity;
+import uk.gov.hmcts.ccd.definition.store.repository.entity.ReindexEntity;
+import uk.gov.hmcts.ccd.definition.store.repository.model.ReindexTask;
+
+import java.io.IOException;
+import java.util.List;
+
+public interface ReindexService {
+    void asyncReindex(DefinitionImportedEvent event, String baseIndexName, CaseTypeEntity caseType) throws IOException;
+
+    String incrementIndexNumber(String indexName);
+
+    List<ReindexTask> getAll();
+
+    List<ReindexTask> getTasksByCaseType(String caseType);
+
+    ReindexEntity saveEntity(Boolean deleteOldIndex, CaseTypeEntity caseType,
+                             String newIndexName);
+
+    void updateEntity(String newIndexName, String response);
+
+    void updateEntity(String newIndexName, Exception exception);
+}