CCD-5849: elastic search upgrade

Jenkinsfile_CNP

@@ -10,6 +10,9 @@ def app = "definition-store-api"
 
 def branchesToSync = ['demo', 'ithc', 'perftest', 'develop']
 
+// Variables to switch pipeline logic and wiring per type of build
+def definitionStoreDevelopPr = "PR-1574" // This doesn't change frequently, but when it does, only change this value.
+
 def secrets = [
     'ccd-${env}': [
         secret('ccd-caseworker-autotest-email', 'CCD_CASEWORKER_AUTOTEST_EMAIL'),

README.md

@@ -8,7 +8,7 @@
 [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/d3b02d95faf6419ca6fbb15b2e712b8b)](https://www.codacy.com/app/adr1ancho/ccd-definition-store-api?utm_source=github.com&utm_medium=referral&utm_content=hmcts/ccd-definition-store-api&utm_campaign=Badge_Coverage)
 [![Known Vulnerabilities](https://snyk.io/test/github/hmcts/ccd-definition-store-api/badge.svg)](https://snyk.io/test/github/hmcts/ccd-definition-store-api)
 [![HitCount](http://hits.dwyl.io/hmcts/ccd-definition-store-api.svg)](#ccd-definition-store-api)
-
+  
 Validation and persistence of definitions for field types, jurisdictions, case types and associated display elements.
 
 ## Overview

application/build.gradle

@@ -30,6 +30,7 @@ dependencies {
 
     testImplementation project(":app-insights").sourceSets.main.output
     testImplementation project(":commons").sourceSets.main.output
+    testImplementation("org.json:json:20250517")
 }
 
 rootProject.tasks.named("distTar") {

application/src/test/java/uk/gov/hmcts/net/ccd/definition/store/excel/SpreadSheetImportTest.java

@@ -1,5 +1,19 @@
 package uk.gov.hmcts.net.ccd.definition.store.excel;
 
+import com.github.tomakehurst.wiremock.client.WireMock;
+import org.apache.http.HttpStatus;
+import org.hamcrest.Matcher;
+import org.junit.jupiter.api.Test;
+import org.skyscreamer.jsonassert.JSONAssert;
+import org.skyscreamer.jsonassert.JSONCompareMode;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.web.servlet.MvcResult;
+import org.springframework.test.web.servlet.ResultMatcher;
+import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
+import org.springframework.transaction.annotation.Transactional;
 import uk.gov.hmcts.ccd.definition.store.excel.client.translation.DictionaryRequest;
 import uk.gov.hmcts.ccd.definition.store.excel.client.translation.Translation;
 import uk.gov.hmcts.ccd.definition.store.repository.SecurityClassification;
@@ -17,22 +31,6 @@
 import java.util.Map;
 import java.util.regex.Pattern;
 
-import com.github.tomakehurst.wiremock.client.WireMock;
-import org.apache.http.HttpStatus;
-import org.hamcrest.Matcher;
-import org.json.JSONException;
-import org.junit.jupiter.api.Test;
-import org.skyscreamer.jsonassert.JSONAssert;
-import org.skyscreamer.jsonassert.JSONCompareMode;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.mock.web.MockMultipartFile;
-import org.springframework.test.context.TestPropertySource;
-import org.springframework.test.web.servlet.MvcResult;
-import org.springframework.test.web.servlet.ResultMatcher;
-import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
-import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
-import org.springframework.transaction.annotation.Transactional;
-
 import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
@@ -131,7 +129,7 @@ public void importValidDefinitionFile_TranslationService_return4XX() throws Exce
             stubForPutDictionaryReturns4XX(getDictionaryRequest());
 
             MvcResult mvcResult = mockMvc.perform(MockMvcRequestBuilders.multipart(IMPORT_URL)
-            
+
                 .file(file)
                 .header(AUTHORIZATION, "Bearer testUser"))
                 .andReturn();
@@ -348,12 +346,12 @@ public static Matcher<Map<String, Object>> hasColumn(String key, Object value) {
         return hasColumn(is(key), is(value));
     }
 
-    private void assertBody(String contentAsString) throws IOException, URISyntaxException, JSONException {
+    private void assertBody(String contentAsString) throws IOException, URISyntaxException {
         assertBody(contentAsString, RESPONSE_JSON_V55);
     }
 
     private void assertBody(String contentAsString, String fileName)
-        throws IOException, URISyntaxException, JSONException {
+        throws IOException, URISyntaxException {
 
         String expected = readFileToString(new File(getClass().getClassLoader()
             .getResource(fileName)

build.gradle

@@ -31,14 +31,14 @@ tasks.register("printSuppressionFile") {
 }
 
 ext {
-    set('elasticsearch.version', '8.16.2')
     set('spring-framework.version', '6.2.1')
     set('spring-security.version', '6.4.2')
     set('springCloudVersion', '2024.0.0')
     set('jackson.version', '2.16.0')
     set('snakeyaml.version', '2.3')
     set('log4j2.version', '2.24.3')
 
+    elasticsearchVersion = '9.1.2'
     junitJupiterVersion = '5.11.4'
     junitVintageVersion = '5.11.4'
     springDocVersion = '2.8.14'
@@ -91,13 +91,34 @@ allprojects {
         mockitoAgent
         codacy
     }
+    configurations.configureEach {
+        exclude group: 'com.vaadin.external.google', module: 'android-json'
+    }
+
+    configurations.all {
+        resolutionStrategy {
+            eachDependency { DependencyResolveDetails details ->
+                if (details.requested.group == 'org.elasticsearch') {
+                    details.useVersion elasticsearchVersion
+                }
+
+                if (details.requested.group == 'co.elastic.clients') {
+                    details.useVersion elasticsearchVersion
+                }
+
+                if (details.requested.group == 'org.elasticsearch.client') {
+                    details.useVersion elasticsearchVersion
+                }
+            }
+        }
+    }
 
     apply plugin: 'java'
 
     // Global constraints
     dependencies {
 
-        // start::CVE Vulnerability dependency overrides 
+        // start::CVE Vulnerability dependency overrides
 
         implementation group: 'commons-fileupload', name: 'commons-fileupload', version: '1.6.0' // spring-cloud-starter-openfeign
         implementation group: 'commons-io', name: 'commons-io', version: '2.20.0' // spring-cloud-starter-openfeign
@@ -130,7 +151,7 @@ allprojects {
         testImplementation group: 'org.apache.groovy', name: 'groovy-xml', version: groovyVersion
         testImplementation group: 'org.apache.groovy', name: 'groovy-json', version: groovyVersion
         testImplementation group: 'com.github.hmcts', name: 'fortify-client', version: '1.4.10', classifier: 'all'
-
+        testImplementation("org.json:json:20250517")
     }
 
 }
@@ -195,7 +216,7 @@ subprojects { subproject ->
 
     dependencies {
 
-        // start::CVE Vulnerability dependency overrides 
+        // start::CVE Vulnerability dependency overrides
 
         implementation group: 'com.nimbusds', name: 'nimbus-jose-jwt', version: '10.5' // spring-boot-starter-oauth2-client
         implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.20.0' // azure-storage
@@ -243,8 +264,9 @@ subprojects { subproject ->
         implementation group: 'org.apache.poi', name: 'poi', version: apachePoiVersion
         implementation group: 'org.apache.poi', name: 'poi-ooxml', version: apachePoiVersion
         implementation group: 'org.apache.poi', name: 'poi-scratchpad', version: apachePoiVersion
-        implementation group: 'org.elasticsearch', name: 'elasticsearch', version: elasticSearchVersion
-        implementation group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: elasticSearchVersion
+        implementation group: 'org.elasticsearch', name: 'elasticsearch', version: elasticsearchVersion
+        implementation group: 'co.elastic.clients', name: 'elasticsearch-java', version: elasticsearchVersion
+        implementation group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: elasticsearchVersion
         implementation group: 'org.flywaydb', name: 'flyway-core', version: flywayVersion
         implementation group: 'org.flywaydb', name: 'flyway-database-postgresql', version: flywayVersion
         implementation group: 'org.jooq', name: 'jool-java-8', version: '0.9.15'
@@ -525,4 +547,4 @@ void loadEnvSecrets(String env) {
             project.file("./.${env}-remote-env").write(new String(os.toString().replace('\n', '').decodeBase64(), java.nio.charset.StandardCharsets.UTF_8))
         }
     }
-}
+}

charts/ccd-definition-store-api/values.preview.template.yaml

@@ -23,10 +23,13 @@ java:
 
     # enable whenever required and provide host url to match with corresponding data-store-api
     ELASTIC_SEARCH_ENABLED: true
-    ELASTIC_SEARCH_HOST: "{{ .Release.Name }}-es-master"
+
+    ELASTIC_SEARCH_HOST: ccd-data-store-api-pr-2620-es-master
+
     USER_PROFILE_HOST: http://${SERVICE_NAME}-ccd-user-profile-api
     TS_TRANSLATION_SERVICE_HOST: http://${SERVICE_NAME}-translation-service
 
+
     # ccd-test app-insights key - remove once testing is done
     APPINSIGHTS_INSTRUMENTATIONKEY: 2dcb834e-768e-4429-9050-ab15af959995
 
@@ -109,7 +112,7 @@ ccd:
           secretRef: "{{ .Values.global.postgresSecret }}"
           key: PASSWORD
           disabled: true
-          
+
 
 elastic:
   enabled: true

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java

@@ -1,11 +1,11 @@
 package uk.gov.hmcts.ccd.definition.store.elastic;
 
+import co.elastic.clients.elasticsearch.core.ReindexResponse;
+import co.elastic.clients.elasticsearch.indices.GetAliasResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.GetAliasesResponse;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.springframework.beans.factory.ObjectFactory;
 import org.springframework.transaction.annotation.Transactional;
 import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient;
@@ -60,7 +60,8 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) {
         String caseMapping = null;
         CaseTypeEntity currentCaseType = null;
 
-        try (HighLevelCCDElasticClient elasticClient = clientFactory.getObject()) {
+        HighLevelCCDElasticClient elasticClient = clientFactory.getObject();
+        try {
             for (CaseTypeEntity caseType : caseTypes) {
                 currentCaseType = caseType;
                 String baseIndexName = baseIndexName(caseType);
@@ -71,8 +72,8 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) {
                 }
                 if (reindex) {
                     //get current alias index
-                    GetAliasesResponse aliasResponse = elasticClient.getAlias(baseIndexName);
-                    String caseTypeName = aliasResponse.getAliases().keySet().iterator().next();
+                    GetAliasResponse aliasResponse = elasticClient.getAlias(baseIndexName);
+                    String caseTypeName = aliasResponse.aliases().keySet().iterator().next();
 
                     //create new index with generated mapping and incremented case type name (no alias update yet)
                     caseMapping = mappingGenerator.generateMapping(caseType);
@@ -109,7 +110,7 @@ private void handleReindexing(String baseIndexName,
         HighLevelCCDElasticClient elasticClient = clientFactory.getObject();
         elasticClient.reindexData(oldIndex, newIndex, new ActionListener<>() {
             @Override
-            public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
+            public void onResponse(ReindexResponse reindexResponse) {
                 try (elasticClient; HighLevelCCDElasticClient asyncElasticClient = clientFactory.getObject()) {
                     //if success set writable and update alias to new index
                     log.info("updating alias from {} to {}", oldIndex, newIndex);

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticGlobalSearchListener.java

@@ -47,10 +47,6 @@ public void initialiseElasticSearchForGlobalSearch() {
 
         } catch (IOException | ElasticsearchStatusException e) {
             throw new ElasticSearchInitialisationException(e);
-        } finally {
-            if (elasticClient != null) {
-                elasticClient.close();
-            }
         }
     }
 }

elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/ElasticsearchClientFactory.java

@@ -0,0 +1,40 @@
+package uk.gov.hmcts.ccd.definition.store.elastic.client;
+
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.json.JsonData;
+import co.elastic.clients.json.jackson.JacksonJsonpMapper;
+import co.elastic.clients.transport.rest_client.RestClientTransport;
+import org.elasticsearch.client.RestClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class ElasticsearchClientFactory {
+
+    private static final Logger log = LoggerFactory.getLogger(ElasticsearchClientFactory.class);
+    private final Supplier<RestClient> restClientSupplier;
+    private final JacksonJsonpMapper mapper;
+
+    public ElasticsearchClientFactory(Supplier<RestClient> restClientSupplier, JacksonJsonpMapper mapper) {
+        this.restClientSupplier = restClientSupplier;
+        this.mapper = mapper;
+    }
+
+    public ElasticsearchClient createClient() {
+        RestClient restClient = restClientSupplier.get();
+        RestClientTransport transport = new RestClientTransport(restClient, mapper);
+        ElasticsearchClient client = new ElasticsearchClient(transport);
+        try {
+            client.cluster().putSettings(s ->
+                s.persistent(Map.of("action.destructive_requires_name", JsonData.of(false)))
+            );
+        } catch (IOException e) {
+            log.error(e.getMessage(), e);
+        }
+
+        return client;
+    }
+}