diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 12a0a8c21c..6f283d40b2 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -10,6 +10,9 @@ def app = "definition-store-api" def branchesToSync = ['demo', 'ithc', 'perftest', 'develop'] +// Variables to switch pipeline logic and wiring per type of build +def definitionStoreDevelopPr = "PR-1574" // This doesn't change frequently, but when it does, only change this value. + def secrets = [ 'ccd-${env}': [ secret('ccd-caseworker-autotest-email', 'CCD_CASEWORKER_AUTOTEST_EMAIL'), diff --git a/README.md b/README.md index 9e545e4202..fd5a00c321 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/d3b02d95faf6419ca6fbb15b2e712b8b)](https://www.codacy.com/app/adr1ancho/ccd-definition-store-api?utm_source=github.com&utm_medium=referral&utm_content=hmcts/ccd-definition-store-api&utm_campaign=Badge_Coverage) [![Known Vulnerabilities](https://snyk.io/test/github/hmcts/ccd-definition-store-api/badge.svg)](https://snyk.io/test/github/hmcts/ccd-definition-store-api) [![HitCount](http://hits.dwyl.io/hmcts/ccd-definition-store-api.svg)](#ccd-definition-store-api) - + Validation and persistence of definitions for field types, jurisdictions, case types and associated display elements. ## Overview diff --git a/application/build.gradle b/application/build.gradle index 551e62e87c..fc101c1e8f 100644 --- a/application/build.gradle +++ b/application/build.gradle @@ -30,6 +30,7 @@ dependencies { testImplementation project(":app-insights").sourceSets.main.output testImplementation project(":commons").sourceSets.main.output + testImplementation("org.json:json:20250517") } rootProject.tasks.named("distTar") { diff --git a/application/src/test/java/uk/gov/hmcts/net/ccd/definition/store/excel/SpreadSheetImportTest.java b/application/src/test/java/uk/gov/hmcts/net/ccd/definition/store/excel/SpreadSheetImportTest.java index 575686db03..8b8a87d4b0 100644 --- a/application/src/test/java/uk/gov/hmcts/net/ccd/definition/store/excel/SpreadSheetImportTest.java +++ b/application/src/test/java/uk/gov/hmcts/net/ccd/definition/store/excel/SpreadSheetImportTest.java @@ -1,5 +1,19 @@ package uk.gov.hmcts.net.ccd.definition.store.excel; +import com.github.tomakehurst.wiremock.client.WireMock; +import org.apache.http.HttpStatus; +import org.hamcrest.Matcher; +import org.junit.jupiter.api.Test; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; +import org.springframework.core.io.ClassPathResource; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.ResultMatcher; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.transaction.annotation.Transactional; import uk.gov.hmcts.ccd.definition.store.excel.client.translation.DictionaryRequest; import uk.gov.hmcts.ccd.definition.store.excel.client.translation.Translation; import uk.gov.hmcts.ccd.definition.store.repository.SecurityClassification; @@ -17,22 +31,6 @@ import java.util.Map; import java.util.regex.Pattern; -import com.github.tomakehurst.wiremock.client.WireMock; -import org.apache.http.HttpStatus; -import org.hamcrest.Matcher; -import org.json.JSONException; -import org.junit.jupiter.api.Test; -import org.skyscreamer.jsonassert.JSONAssert; -import org.skyscreamer.jsonassert.JSONCompareMode; -import org.springframework.core.io.ClassPathResource; -import org.springframework.mock.web.MockMultipartFile; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.test.web.servlet.ResultMatcher; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.transaction.annotation.Transactional; - import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; @@ -131,7 +129,7 @@ public void importValidDefinitionFile_TranslationService_return4XX() throws Exce stubForPutDictionaryReturns4XX(getDictionaryRequest()); MvcResult mvcResult = mockMvc.perform(MockMvcRequestBuilders.multipart(IMPORT_URL) - + .file(file) .header(AUTHORIZATION, "Bearer testUser")) .andReturn(); @@ -348,12 +346,12 @@ public static Matcher> hasColumn(String key, Object value) { return hasColumn(is(key), is(value)); } - private void assertBody(String contentAsString) throws IOException, URISyntaxException, JSONException { + private void assertBody(String contentAsString) throws IOException, URISyntaxException { assertBody(contentAsString, RESPONSE_JSON_V55); } private void assertBody(String contentAsString, String fileName) - throws IOException, URISyntaxException, JSONException { + throws IOException, URISyntaxException { String expected = readFileToString(new File(getClass().getClassLoader() .getResource(fileName) diff --git a/build.gradle b/build.gradle index 08a7c96e0c..ec2b54562c 100644 --- a/build.gradle +++ b/build.gradle @@ -31,7 +31,6 @@ tasks.register("printSuppressionFile") { } ext { - set('elasticsearch.version', '8.16.2') set('spring-framework.version', '6.2.1') set('spring-security.version', '6.4.2') set('springCloudVersion', '2024.0.0') @@ -39,6 +38,7 @@ ext { set('snakeyaml.version', '2.3') set('log4j2.version', '2.24.3') + elasticsearchVersion = '9.1.2' junitJupiterVersion = '5.11.4' junitVintageVersion = '5.11.4' springDocVersion = '2.8.14' @@ -91,13 +91,34 @@ allprojects { mockitoAgent codacy } + configurations.configureEach { + exclude group: 'com.vaadin.external.google', module: 'android-json' + } + + configurations.all { + resolutionStrategy { + eachDependency { DependencyResolveDetails details -> + if (details.requested.group == 'org.elasticsearch') { + details.useVersion elasticsearchVersion + } + + if (details.requested.group == 'co.elastic.clients') { + details.useVersion elasticsearchVersion + } + + if (details.requested.group == 'org.elasticsearch.client') { + details.useVersion elasticsearchVersion + } + } + } + } apply plugin: 'java' // Global constraints dependencies { - // start::CVE Vulnerability dependency overrides + // start::CVE Vulnerability dependency overrides implementation group: 'commons-fileupload', name: 'commons-fileupload', version: '1.6.0' // spring-cloud-starter-openfeign implementation group: 'commons-io', name: 'commons-io', version: '2.20.0' // spring-cloud-starter-openfeign @@ -130,7 +151,7 @@ allprojects { testImplementation group: 'org.apache.groovy', name: 'groovy-xml', version: groovyVersion testImplementation group: 'org.apache.groovy', name: 'groovy-json', version: groovyVersion testImplementation group: 'com.github.hmcts', name: 'fortify-client', version: '1.4.10', classifier: 'all' - + testImplementation("org.json:json:20250517") } } @@ -195,7 +216,7 @@ subprojects { subproject -> dependencies { - // start::CVE Vulnerability dependency overrides + // start::CVE Vulnerability dependency overrides implementation group: 'com.nimbusds', name: 'nimbus-jose-jwt', version: '10.5' // spring-boot-starter-oauth2-client implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.20.0' // azure-storage @@ -243,8 +264,9 @@ subprojects { subproject -> implementation group: 'org.apache.poi', name: 'poi', version: apachePoiVersion implementation group: 'org.apache.poi', name: 'poi-ooxml', version: apachePoiVersion implementation group: 'org.apache.poi', name: 'poi-scratchpad', version: apachePoiVersion - implementation group: 'org.elasticsearch', name: 'elasticsearch', version: elasticSearchVersion - implementation group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: elasticSearchVersion + implementation group: 'org.elasticsearch', name: 'elasticsearch', version: elasticsearchVersion + implementation group: 'co.elastic.clients', name: 'elasticsearch-java', version: elasticsearchVersion + implementation group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: elasticsearchVersion implementation group: 'org.flywaydb', name: 'flyway-core', version: flywayVersion implementation group: 'org.flywaydb', name: 'flyway-database-postgresql', version: flywayVersion implementation group: 'org.jooq', name: 'jool-java-8', version: '0.9.15' @@ -525,4 +547,4 @@ void loadEnvSecrets(String env) { project.file("./.${env}-remote-env").write(new String(os.toString().replace('\n', '').decodeBase64(), java.nio.charset.StandardCharsets.UTF_8)) } } -} \ No newline at end of file +} diff --git a/charts/ccd-definition-store-api/values.preview.template.yaml b/charts/ccd-definition-store-api/values.preview.template.yaml index 55a41d2d74..9c706127ca 100644 --- a/charts/ccd-definition-store-api/values.preview.template.yaml +++ b/charts/ccd-definition-store-api/values.preview.template.yaml @@ -23,10 +23,13 @@ java: # enable whenever required and provide host url to match with corresponding data-store-api ELASTIC_SEARCH_ENABLED: true - ELASTIC_SEARCH_HOST: "{{ .Release.Name }}-es-master" + + ELASTIC_SEARCH_HOST: ccd-data-store-api-pr-2620-es-master + USER_PROFILE_HOST: http://${SERVICE_NAME}-ccd-user-profile-api TS_TRANSLATION_SERVICE_HOST: http://${SERVICE_NAME}-translation-service + # ccd-test app-insights key - remove once testing is done APPINSIGHTS_INSTRUMENTATIONKEY: 2dcb834e-768e-4429-9050-ab15af959995 @@ -109,7 +112,7 @@ ccd: secretRef: "{{ .Values.global.postgresSecret }}" key: PASSWORD disabled: true - + elastic: enabled: true diff --git a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java index 82d6e8f9f4..48459f94f0 100644 --- a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java +++ b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListener.java @@ -1,11 +1,11 @@ package uk.gov.hmcts.ccd.definition.store.elastic; +import co.elastic.clients.elasticsearch.core.ReindexResponse; +import co.elastic.clients.elasticsearch.indices.GetAliasResponse; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.springframework.beans.factory.ObjectFactory; import org.springframework.transaction.annotation.Transactional; import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient; @@ -60,7 +60,8 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) { String caseMapping = null; CaseTypeEntity currentCaseType = null; - try (HighLevelCCDElasticClient elasticClient = clientFactory.getObject()) { + HighLevelCCDElasticClient elasticClient = clientFactory.getObject(); + try { for (CaseTypeEntity caseType : caseTypes) { currentCaseType = caseType; String baseIndexName = baseIndexName(caseType); @@ -71,8 +72,8 @@ public void initialiseElasticSearch(DefinitionImportedEvent event) { } if (reindex) { //get current alias index - GetAliasesResponse aliasResponse = elasticClient.getAlias(baseIndexName); - String caseTypeName = aliasResponse.getAliases().keySet().iterator().next(); + GetAliasResponse aliasResponse = elasticClient.getAlias(baseIndexName); + String caseTypeName = aliasResponse.aliases().keySet().iterator().next(); //create new index with generated mapping and incremented case type name (no alias update yet) caseMapping = mappingGenerator.generateMapping(caseType); @@ -109,7 +110,7 @@ private void handleReindexing(String baseIndexName, HighLevelCCDElasticClient elasticClient = clientFactory.getObject(); elasticClient.reindexData(oldIndex, newIndex, new ActionListener<>() { @Override - public void onResponse(BulkByScrollResponse bulkByScrollResponse) { + public void onResponse(ReindexResponse reindexResponse) { try (elasticClient; HighLevelCCDElasticClient asyncElasticClient = clientFactory.getObject()) { //if success set writable and update alias to new index log.info("updating alias from {} to {}", oldIndex, newIndex); diff --git a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticGlobalSearchListener.java b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticGlobalSearchListener.java index 5b2fcd39cd..69b3314118 100644 --- a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticGlobalSearchListener.java +++ b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticGlobalSearchListener.java @@ -47,10 +47,6 @@ public void initialiseElasticSearchForGlobalSearch() { } catch (IOException | ElasticsearchStatusException e) { throw new ElasticSearchInitialisationException(e); - } finally { - if (elasticClient != null) { - elasticClient.close(); - } } } } diff --git a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/ElasticsearchClientFactory.java b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/ElasticsearchClientFactory.java new file mode 100644 index 0000000000..5f9d9f731b --- /dev/null +++ b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/ElasticsearchClientFactory.java @@ -0,0 +1,40 @@ +package uk.gov.hmcts.ccd.definition.store.elastic.client; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.function.Supplier; + +public class ElasticsearchClientFactory { + + private static final Logger log = LoggerFactory.getLogger(ElasticsearchClientFactory.class); + private final Supplier restClientSupplier; + private final JacksonJsonpMapper mapper; + + public ElasticsearchClientFactory(Supplier restClientSupplier, JacksonJsonpMapper mapper) { + this.restClientSupplier = restClientSupplier; + this.mapper = mapper; + } + + public ElasticsearchClient createClient() { + RestClient restClient = restClientSupplier.get(); + RestClientTransport transport = new RestClientTransport(restClient, mapper); + ElasticsearchClient client = new ElasticsearchClient(transport); + try { + client.cluster().putSettings(s -> + s.persistent(Map.of("action.destructive_requires_name", JsonData.of(false))) + ); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + + return client; + } +} diff --git a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClient.java b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClient.java index aa3aed7d88..2c1fec98c5 100644 --- a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClient.java +++ b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClient.java @@ -1,31 +1,27 @@ package uk.gov.hmcts.ccd.definition.store.elastic.client; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.indices.Alias; +import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; +import co.elastic.clients.elasticsearch.indices.GetAliasResponse; +import co.elastic.clients.elasticsearch.indices.update_aliases.Action; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; import lombok.extern.slf4j.Slf4j; +import org.apache.hc.core5.http.ConnectionClosedException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.client.indices.CreateIndexResponse; -import org.elasticsearch.client.indices.PutMappingRequest; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.xcontent.XContentType; import org.springframework.beans.factory.annotation.Autowired; import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import static uk.gov.hmcts.ccd.definition.store.elastic.ElasticGlobalSearchListener.GLOBAL_SEARCH; @@ -35,148 +31,263 @@ public class HighLevelCCDElasticClient implements CCDElasticClient, AutoCloseabl private static final String CASES_INDEX_SETTINGS_JSON = "/casesIndexSettings.json"; private static final String GLOBAL_SEARCH_CASES_INDEX_SETTINGS_JSON = "/globalSearchCasesIndexSettings.json"; - protected CcdElasticSearchProperties config; + private static final int MAX_RETRIES = 3; + private static final long RETRY_DELAY_MS = 1000; + private static final Object LOCK = new Object(); - protected RestHighLevelClient elasticClient; + private final CcdElasticSearchProperties config; + private final ElasticsearchClient elasticClient; @Autowired - public HighLevelCCDElasticClient(CcdElasticSearchProperties config, RestHighLevelClient elasticClient) { + public HighLevelCCDElasticClient(CcdElasticSearchProperties config, ElasticsearchClientFactory clientFactory) { this.config = config; - this.elasticClient = elasticClient; + elasticClient = clientFactory.createClient(); + } + + private synchronized ElasticsearchClient getElasticClient() { + return elasticClient; + } + + private T executeWithRetry(ElasticOperation operation, String operationName) throws IOException { + int attempts = 0; + Exception lastException = null; + + while (attempts < MAX_RETRIES) { + try { + synchronized (LOCK) { + return operation.execute(getElasticClient()); + } + } catch (Exception e) { + lastException = e; + attempts++; + + if (isDeadHostException(e)) { + log.warn("ElasticsearchClient encountered dead node: {} — resetting...", e.getMessage()); + } else if (isConnectionError(e)) { + log.warn("Connection error during {}, attempt {}/{}", + operationName, attempts, MAX_RETRIES); + } + + if (attempts < MAX_RETRIES) { + try { + Thread.sleep(RETRY_DELAY_MS * attempts); // Exponential backoff + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Operation interrupted", ie); + } + } + } + } + + throw new IOException("Failed to execute " + operationName + + " after " + MAX_RETRIES + " attempts", lastException); + } + + private boolean isDeadHostException(Exception e) { + return e.getMessage() != null && e.getMessage().contains("DeadHostState"); + } + + private boolean isConnectionError(Exception e) { + return e instanceof ConnectionClosedException + || e instanceof ElasticsearchException + && e.getCause() instanceof ConnectionClosedException; } @Override public boolean createIndex(String indexName, String alias) throws IOException { log.info("creating index {} with alias {}", indexName, alias); - CreateIndexRequest request = new CreateIndexRequest(indexName); - request.alias(new Alias(alias)); - String file = (alias.equalsIgnoreCase(GLOBAL_SEARCH)) + final String file = (alias.equalsIgnoreCase(GLOBAL_SEARCH)) ? GLOBAL_SEARCH_CASES_INDEX_SETTINGS_JSON : CASES_INDEX_SETTINGS_JSON; - request.settings(casesIndexSettings(file)); - CreateIndexResponse createIndexResponse = elasticClient.indices().create(request, RequestOptions.DEFAULT); - log.info("index created: {}", createIndexResponse.isAcknowledged()); - return createIndexResponse.isAcknowledged(); + log.info("file: {}", file); + + // Load settings from JSON file as a Map + Map settings = casesIndexSettings(file); + log.info("settings: {}", settings); + + CreateIndexResponse createIndexResponse = executeWithRetry( + client -> client.indices().create(b -> b + .index(indexName) + .settings(s -> { + try { + return s.withJson(new StringReader( + new ObjectMapper().writeValueAsString(settings) + )); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to process json from index settings", e); + } + }) + .aliases(Map.of(alias, new Alias.Builder().isWriteIndex(true).build())) + ), + "create index" + ); + + log.info("index created: {}, aliasExists: {}", + null == createIndexResponse ? null : createIndexResponse.acknowledged(), + aliasExists(alias)); + return null != createIndexResponse && createIndexResponse.acknowledged(); } @Override public boolean upsertMapping(String aliasName, String caseTypeMapping) throws IOException { log.info("upsert mapping of most recent index for alias {}", aliasName); - GetAliasesResponse aliasesResponse = getAlias(aliasName); + // Get aliases with retry + var aliasesResponse = executeWithRetry( + client -> client.indices().getAlias(b -> b.name(aliasName)), + "get alias for upsert mapping" + ); String currentIndex = getCurrentAliasIndex(aliasName, aliasesResponse); log.info("upsert mapping of index {}", currentIndex); - PutMappingRequest request = new PutMappingRequest(currentIndex); - request.source(caseTypeMapping, XContentType.JSON); - AcknowledgedResponse acknowledgedResponse = elasticClient.indices().putMapping(request, RequestOptions.DEFAULT); - log.info("mapping upserted: {}", acknowledgedResponse.isAcknowledged()); - return acknowledgedResponse.isAcknowledged(); + + // Put mapping with retry + var putMappingResponse = executeWithRetry( + client -> client.indices().putMapping(b -> b + .index(currentIndex) + .withJson(new StringReader(caseTypeMapping)) + ), + "put mapping" + ); + log.info("mapping upserted: {}", putMappingResponse.acknowledged()); + return null != putMappingResponse && putMappingResponse.acknowledged(); } @Override public boolean aliasExists(String alias) throws IOException { - GetAliasesRequest request = new GetAliasesRequest(alias); - boolean exists = elasticClient.indices().existsAlias(request, RequestOptions.DEFAULT); - log.info("alias {} exists: {}", alias, exists); - return exists; + return executeWithRetry(client -> { + try { + var response = client.indices().getAlias(b -> b.name(alias)); + boolean exists = response != null + && response.aliases() != null + && !response.aliases().isEmpty(); + log.debug("alias {} exists: {}", alias, exists); + return exists; + } catch (ElasticsearchException e) { + if (e.status() == 404) { + return false; + } + throw e; + } + }, "check alias existence"); + } + + @FunctionalInterface + private interface ElasticOperation { + T execute(ElasticsearchClient client) throws IOException; } @Override public void close() { - try { - log.info("Closing the ES REST client"); - this.elasticClient.close(); - } catch (IOException ioe) { - log.error("Problem occurred when closing the ES REST client", ioe); - } + // historical - was empty } - public GetAliasesResponse getAlias(String alias) throws IOException { - GetAliasesRequest request = new GetAliasesRequest(alias); - return elasticClient.indices().getAlias(request, RequestOptions.DEFAULT); + public GetAliasResponse getAlias(String alias) throws IOException { + return getElasticClient().indices().getAlias(b -> b.name(alias)); } - private Settings.Builder casesIndexSettings(String file) throws IOException { + private Map casesIndexSettings(String file) throws IOException { + Map settings; try (InputStream inputStream = getClass().getResourceAsStream(file)) { - Settings.Builder settings = Settings.builder().loadFromStream(file, - inputStream, false); - settings.put("index.number_of_shards", config.getIndexShards()); - settings.put("index.number_of_replicas", config.getIndexShardsReplicas()); - settings.put("index.mapping.total_fields.limit", config.getCasesIndexMappingFieldsLimit()); - return settings; + if (inputStream == null) { + throw new IOException("Settings file not found: " + file); + } + settings = new ObjectMapper().readValue(inputStream, Map.class); } + settings.put("index.number_of_shards", config.getIndexShards()); + settings.put("index.number_of_replicas", config.getIndexShardsReplicas()); + settings.put("index.mapping.total_fields.limit", config.getCasesIndexMappingFieldsLimit()); + return settings; } - private String getCurrentAliasIndex(String indexName, GetAliasesResponse aliasesResponse) { - ArrayList indices = new ArrayList<>(aliasesResponse.getAliases().keySet()); + private String getCurrentAliasIndex(String indexName, GetAliasResponse aliasesResponse) { + ArrayList indices = new ArrayList<>(aliasesResponse.aliases().keySet()); Collections.sort(indices); log.info("found following indexes for alias {}: {}", indexName, indices); return Iterables.getLast(indices); } public void setIndexReadOnly(String indexName, boolean readOnly) throws IOException { - UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName); - Settings settings = Settings.builder() - .put("index.blocks.read_only", readOnly) - .build(); - updateSettingsRequest.settings(settings); - elasticClient.indices().putSettings(updateSettingsRequest, RequestOptions.DEFAULT); + getElasticClient().indices().putSettings(b -> b + .index(indexName) + .settings(s -> s + .withJson(new java.io.StringReader( + "{\"index.blocks.read_only\": " + readOnly + "}" + )) + ) + ); + log.info("Set index '{}' read_only to {}", indexName, readOnly); } public boolean createIndexAndMapping(String indexName, String caseTypeMapping) throws IOException { - //create new index - CreateIndexRequest request = new CreateIndexRequest(indexName); - request.settings(casesIndexSettings(CASES_INDEX_SETTINGS_JSON)); - CreateIndexResponse createIndexResponse = elasticClient.indices().create(request, RequestOptions.DEFAULT); - log.info("index created: {}", createIndexResponse.isAcknowledged()); + // Load settings from JSON file as a Map + Map settings; + try (InputStream inputStream = getClass().getResourceAsStream(CASES_INDEX_SETTINGS_JSON)) { + settings = new ObjectMapper().readValue(inputStream, Map.class); + } - //upsert mapping to new index - PutMappingRequest putRequest = new PutMappingRequest(indexName); - putRequest.source(caseTypeMapping, XContentType.JSON); + var createIndexResponse = executeWithRetry( + client -> client.indices().create(b -> b + .index(indexName) + .settings(s -> { + try { + return s.withJson(new java.io.StringReader( + new ObjectMapper().writeValueAsString(settings) + )); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to process json from index settings", e); + } + }) + ), + "create index with mapping" + ); + log.info("index created: {}", createIndexResponse.acknowledged()); - AcknowledgedResponse acknowledgedResponse = elasticClient.indices() - .putMapping(putRequest, RequestOptions.DEFAULT); - log.info("mapping upserted: {}", acknowledgedResponse.isAcknowledged()); + // Upsert mapping to new index + var putMappingResponse = getElasticClient().indices().putMapping(b -> b + .index(indexName) + .withJson(new java.io.StringReader(caseTypeMapping)) + ); + log.info("mapping upserted: {}", putMappingResponse.acknowledged()); - return createIndexResponse.isAcknowledged(); + return createIndexResponse.acknowledged(); } public boolean updateAlias(String aliasName, String oldIndex, String newIndex) throws IOException { - IndicesAliasesRequest aliasRequest = new IndicesAliasesRequest(); - aliasRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(aliasName)); - aliasRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(aliasName)); - - AcknowledgedResponse aliasResponse = elasticClient.indices() - .updateAliases(aliasRequest, RequestOptions.DEFAULT); - if (aliasResponse.isAcknowledged()) { + var aliasResponse = getElasticClient().indices().updateAliases(b -> b + .actions( + Action.of(a -> a.remove(r -> r.index(oldIndex).alias(aliasName))), + Action.of(a -> a.add(ad -> ad.index(newIndex).alias(aliasName))) + ) + ); + if (aliasResponse.acknowledged()) { log.info("alias successfully updated: {} now points to {}", oldIndex, newIndex); } else { log.info("alias update failed: {} still points to {}", oldIndex, oldIndex); } - return aliasResponse.isAcknowledged(); + return null != aliasResponse && aliasResponse.acknowledged(); } public boolean removeIndex(String indexName) throws IOException { - DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); - AcknowledgedResponse deleteResponse = elasticClient.indices() - .delete(deleteIndexRequest, RequestOptions.DEFAULT); - if (deleteResponse.isAcknowledged()) { + var deleteResponse = getElasticClient().indices().delete(b -> b.index(indexName)); + if (deleteResponse.acknowledged()) { log.info("successfully deleted index: {}", indexName); } else { log.info("failed to delete index: {}", indexName); } - return deleteResponse.isAcknowledged(); + return null != deleteResponse && deleteResponse.acknowledged(); } - public void reindexData(String oldIndex, String newIndex, ActionListener listener) { - ReindexRequest reindexRequest = new ReindexRequest(); - reindexRequest.setSourceIndices(oldIndex); - reindexRequest.setDestIndex(newIndex); - reindexRequest.setRefresh(true); - - //doesn't return taskID - elasticClient.reindexAsync( - reindexRequest, - RequestOptions.DEFAULT, - listener - ); + public void reindexData(String oldIndex, String newIndex, + ActionListener listener) { + CompletableFuture.runAsync(() -> { + try { + var response = getElasticClient().reindex(b -> b + .source(s -> s.index(oldIndex)) + .dest(d -> d.index(newIndex)) + .refresh(true) + ); + listener.onResponse(response); + } catch (Exception e) { + listener.onFailure(e); + } + }); } -} \ No newline at end of file +} diff --git a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/config/ElasticSearchConfiguration.java b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/config/ElasticSearchConfiguration.java index 934e95435e..5fda476827 100644 --- a/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/config/ElasticSearchConfiguration.java +++ b/elastic-search-support/src/main/java/uk/gov/hmcts/ccd/definition/store/elastic/config/ElasticSearchConfiguration.java @@ -1,19 +1,29 @@ package uk.gov.hmcts.ccd.definition.store.elastic.config; -import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient; - +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.extern.slf4j.Slf4j; import org.apache.http.HttpHost; +import org.apache.http.impl.nio.reactor.IOReactorConfig; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.config.BeanDefinition; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Scope; +import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; +import uk.gov.hmcts.ccd.definition.store.elastic.client.ElasticsearchClientFactory; +import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient; @Configuration @ComponentScan("uk.gov.hmcts.ccd.definition.store.elastic") @@ -24,33 +34,80 @@ public class ElasticSearchConfiguration { private CcdElasticSearchProperties config; - private RestHighLevelClient restHighLevelClient; - - @Autowired public ElasticSearchConfiguration(CcdElasticSearchProperties config) { this.config = config; } - /** - * NOTE: imports happens seldom. To prevent unused connections to the ES cluster hanging around, we create a new - * HighLevelCCDElasticClient on each import and we close it once the import is completed. - * The HighLevelCCDElasticClient is injected every time with a new restHighLevelClient which opens new connections - */ @Bean @Scope(BeanDefinition.SCOPE_PROTOTYPE) - public RestHighLevelClient restHighLevelClient() { - RestClientBuilder builder = RestClient.builder(new HttpHost(config.getHost(), config.getPort())); - RestClientBuilder.RequestConfigCallback requestConfigCallback = requestConfigBuilder -> - requestConfigBuilder.setConnectTimeout(5000) - .setSocketTimeout(60000); - builder.setRequestConfigCallback(requestConfigCallback); - return new RestHighLevelClient(builder); + public JacksonJsonpMapper jsonpMapper(ObjectMapper objectMapper) { + return new JacksonJsonpMapper(objectMapper); + } + + @Bean + @Scope(BeanDefinition.SCOPE_PROTOTYPE) + public ObjectMapper objectMapper() { + return new Jackson2ObjectMapperBuilder() + .featuresToEnable(MapperFeature.DEFAULT_VIEW_INCLUSION) + .featuresToEnable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES) + .featuresToEnable(JsonParser.Feature.ALLOW_SINGLE_QUOTES) + .modulesToInstall(JavaTimeModule.class) + .featuresToDisable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .build(); + } + + @Bean + @Scope(BeanDefinition.SCOPE_PROTOTYPE) + protected RestClientBuilder elasticsearchRestClientBuilder() { + return RestClient.builder(new HttpHost(config.getHost(), config.getPort(), HttpHost.DEFAULT_SCHEME_NAME)) + .setFailureListener(new RestClient.FailureListener() { + @Override + public void onFailure(Node node) { + log.warn("Node marked as dead: {}", node); + } + }) + .setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS) + .setRequestConfigCallback(requestConfigBuilder -> + requestConfigBuilder + .setConnectTimeout(5000) + .setSocketTimeout(60000) + ) + .setHttpClientConfigCallback(httpClientBuilder -> + httpClientBuilder.setDefaultIOReactorConfig( + IOReactorConfig.custom() + .setSoKeepAlive(true) + .build() + ) + ); + } + + @Bean(destroyMethod = "close") + @Scope(BeanDefinition.SCOPE_PROTOTYPE) + public RestClient restClient(RestClientBuilder builder) { + return builder.build(); + } + + @Bean + @Scope(BeanDefinition.SCOPE_PROTOTYPE) + public ElasticsearchClient elasticsearchClient(ElasticsearchClientFactory elasticsearchClientFactory) { + return elasticsearchClientFactory.createClient(); + } + + @Bean(destroyMethod = "close") + @Scope(BeanDefinition.SCOPE_PROTOTYPE) + public RestClientTransport restClientTransport(RestClient restClient, JacksonJsonpMapper mapper) { + return new RestClientTransport(restClient, mapper); + } + + @Bean + @Scope(BeanDefinition.SCOPE_PROTOTYPE) + public ElasticsearchClientFactory elasticsearchClientFactory(JacksonJsonpMapper mapper) { + return new ElasticsearchClientFactory(() -> elasticsearchRestClientBuilder().build(), mapper); } @Bean @Scope(BeanDefinition.SCOPE_PROTOTYPE) - public HighLevelCCDElasticClient ccdElasticClient() { - return new HighLevelCCDElasticClient(config, restHighLevelClient()) { - }; + public HighLevelCCDElasticClient ccdElasticClient(ElasticsearchClientFactory elasticsearchClientFactory) { + return new HighLevelCCDElasticClient(config, elasticsearchClientFactory); } } diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/CcdElasticsearchContainer.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/CcdElasticsearchContainer.java index 4b72ac35bd..0bc90a454f 100644 --- a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/CcdElasticsearchContainer.java +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/CcdElasticsearchContainer.java @@ -5,7 +5,8 @@ public class CcdElasticsearchContainer extends ElasticsearchContainer { - private static final String VERSION = "7.17.23"; + private static final String VERSION = "9.1.2"; + private static CcdElasticsearchContainer container; private CcdElasticsearchContainer() { @@ -15,6 +16,8 @@ private CcdElasticsearchContainer() { public static GenericContainer getInstance() { if (container == null) { container = new CcdElasticsearchContainer(); + container.withEnv("xpack.security.enabled", "false") + .withEnv("xpack.security.http.ssl.enabled", "false"); container.start(); } return container; diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListenerTest.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListenerTest.java index e1dfd648f7..7df53ba969 100644 --- a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListenerTest.java +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticDefinitionImportListenerTest.java @@ -1,25 +1,11 @@ package uk.gov.hmcts.ccd.definition.store.elastic; -import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient; -import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties; -import uk.gov.hmcts.ccd.definition.store.elastic.exception.ElasticSearchInitialisationException; -import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler; -import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator; -import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent; -import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity; -import uk.gov.hmcts.ccd.definition.store.utils.CaseTypeBuilder; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - +import co.elastic.clients.elasticsearch.core.ReindexResponse; +import co.elastic.clients.elasticsearch.indices.AliasDefinition; +import co.elastic.clients.elasticsearch.indices.GetAliasResponse; +import co.elastic.clients.elasticsearch.indices.get_alias.IndexAliases; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.GetAliasesResponse; -import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.rest.RestStatus; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -29,6 +15,17 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.ObjectFactory; +import uk.gov.hmcts.ccd.definition.store.elastic.client.HighLevelCCDElasticClient; +import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties; +import uk.gov.hmcts.ccd.definition.store.elastic.exception.ElasticSearchInitialisationException; +import uk.gov.hmcts.ccd.definition.store.elastic.exception.handler.ElasticsearchErrorHandler; +import uk.gov.hmcts.ccd.definition.store.elastic.mapping.CaseMappingGenerator; +import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent; +import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity; +import uk.gov.hmcts.ccd.definition.store.utils.CaseTypeBuilder; + +import java.io.IOException; +import java.util.Map; import static com.google.common.collect.Lists.newArrayList; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -43,6 +40,7 @@ import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -78,6 +76,7 @@ class ElasticDefinitionImportListenerTest { @BeforeEach void setUp() { lenient().when(clientObjectFactory.getObject()).thenReturn(ccdElasticClient); + ccdElasticClient.close(); } @Test @@ -88,7 +87,7 @@ void createsAndClosesANewElasticClientOnEachImportToSaveResources() throws IOExc listener.onDefinitionImported(newEvent(caseA, caseB)); verify(clientObjectFactory).getObject(); - verify(ccdElasticClient).close(); + verify(ccdElasticClient, times(1)).close(); } @Test @@ -137,7 +136,7 @@ void createsMapping() throws IOException { } @Test - public void shouldWrapElasticsearchStatusExceptionInInitialisationException() throws IOException { + void shouldWrapElasticsearchStatusExceptionInInitialisationException() throws IOException { mockAliasResponse(); // mock upsertMapping to throw ElasticsearchStatusException @@ -159,7 +158,7 @@ public void shouldWrapElasticsearchStatusExceptionInInitialisationException() th @Test - public void throwsElasticSearchInitialisationExceptionOnErrors() { + void throwsElasticSearchInitialisationExceptionOnErrors() { assertThrows(ElasticSearchInitialisationException.class, () -> { when(config.getCasesIndexNameFormat()).thenThrow(new ArrayIndexOutOfBoundsException("test")); listener.onDefinitionImported(newEvent(caseA, caseB)); @@ -171,9 +170,9 @@ void initialiseElasticSearchWhenReindexAndDeleteOldIndexAreTrue() throws IOExcep mockAliasResponse(); doAnswer(invocation -> { - ActionListener listener = invocation.getArgument(2); - BulkByScrollResponse mockResponse = mock(BulkByScrollResponse.class); - listener.onResponse(mockResponse); + ActionListener listenerA = invocation.getArgument(2); + ReindexResponse mockResponse = mock(ReindexResponse.class); + listenerA.onResponse(mockResponse); return null; }).when(ccdElasticClient).reindexData( eq(caseTypeName), @@ -201,9 +200,9 @@ void initialiseElasticSearchWhenReindexTrueAndDeleteOldIndexFalse() throws IOExc mockAliasResponse(); doAnswer(invocation -> { - ActionListener listener = invocation.getArgument(2); - BulkByScrollResponse mockResponse = mock(BulkByScrollResponse.class); - listener.onResponse(mockResponse); + ActionListener listenerA = invocation.getArgument(2); + ReindexResponse mockResponse = mock(ReindexResponse.class); + listenerA.onResponse(mockResponse); return null; }).when(ccdElasticClient).reindexData( eq(caseTypeName), @@ -242,8 +241,8 @@ void deletesNewIndexWhenReindexingFails() throws IOException { mockAliasResponse(); doAnswer(invocation -> { - ActionListener listener = invocation.getArgument(2); - listener.onFailure(new RuntimeException("reindexing failed")); + ActionListener listenerA = invocation.getArgument(2); + listenerA.onFailure(new RuntimeException("reindexing failed")); return null; }).when(ccdElasticClient).reindexData(eq(caseTypeName), eq(incrementedCaseTypeName), any()); @@ -305,13 +304,14 @@ private void mockAliasResponse() throws IOException { lenient().when(config.getCasesIndexNameFormat()).thenReturn("%s"); lenient().when(ccdElasticClient.aliasExists(anyString())).thenReturn(true); - GetAliasesResponse aliasResponse = mock(GetAliasesResponse.class); - Map> aliasMap = new HashMap<>(); - aliasMap.put(caseTypeName, - Collections.singleton(AliasMetadata.builder(baseIndexName).build())); - lenient().when(aliasResponse.getAliases()).thenReturn(aliasMap); + IndexAliases indexAliases = new IndexAliases.Builder() + .aliases(Map.of( + baseIndexName, new AliasDefinition.Builder().build())).build(); + Map aliasMap = Map.of(caseTypeName, indexAliases); + GetAliasResponse aliasResponse = new GetAliasResponse.Builder() + .aliases(aliasMap) + .build(); lenient().when(ccdElasticClient.getAlias(anyString())).thenReturn(aliasResponse); - lenient().when(caseMappingGenerator.generateMapping(any(CaseTypeEntity.class))).thenReturn("caseMapping"); } diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticsearchBaseTest.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticsearchBaseTest.java index 99e1071b8c..54bc121966 100644 --- a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticsearchBaseTest.java +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/ElasticsearchBaseTest.java @@ -1,8 +1,9 @@ package uk.gov.hmcts.ccd.definition.store.elastic; -import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RestHighLevelClient; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch.indices.GetIndexResponse; +import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse; +import com.fasterxml.jackson.databind.ObjectMapper; import org.skyscreamer.jsonassert.Customization; import org.skyscreamer.jsonassert.JSONCompareMode; import org.skyscreamer.jsonassert.comparator.CustomComparator; @@ -12,12 +13,9 @@ import org.springframework.boot.actuate.autoconfigure.web.server.ManagementPortType; import org.springframework.boot.actuate.endpoint.ExposableEndpoint; import org.springframework.boot.actuate.endpoint.web.EndpointMediaTypes; -import org.springframework.boot.actuate.endpoint.web.ExposableWebEndpoint; import org.springframework.boot.actuate.endpoint.web.WebEndpointsSupplier; import org.springframework.boot.actuate.endpoint.web.EndpointMapping; import org.springframework.boot.actuate.endpoint.web.EndpointLinksResolver; -import org.springframework.boot.actuate.endpoint.web.annotation.ControllerEndpointsSupplier; -import org.springframework.boot.actuate.endpoint.web.annotation.ServletEndpointsSupplier; import org.springframework.boot.actuate.endpoint.web.servlet.WebMvcEndpointHandlerMapping; import org.springframework.boot.autoconfigure.web.servlet.ServletWebServerFactoryAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; @@ -25,47 +23,54 @@ import org.springframework.core.env.Environment; import org.springframework.test.context.ContextConfiguration; import org.springframework.util.StringUtils; +import uk.gov.hmcts.ccd.definition.store.elastic.client.ElasticsearchClientFactory; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.stream.Collectors; -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - classes = { ServletWebServerFactoryAutoConfiguration.class, ElasticsearchConfigurationIT.class }) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + classes = { + ServletWebServerFactoryAutoConfiguration.class, + ElasticsearchConfigurationIT.class + } +) @ContextConfiguration(initializers = ElasticsearchContainerInitializer.class) public abstract class ElasticsearchBaseTest implements TestUtils { - private static final String GET = "GET"; - private static final String DELETE = "DELETE"; protected static final String WILDCARD = "*"; @Autowired - private RestHighLevelClient elasticClient; + protected ElasticsearchClientFactory elasticsearchClientFactory; + + @Autowired + protected ObjectMapper objectMapper; protected CustomComparator ignoreFieldsComparator(String... paths) { return new CustomComparator(JSONCompareMode.LENIENT, Arrays.stream(paths) - .map(path -> new Customization(path, ((o1, o2) -> true))) + .map(path -> new Customization(path, (o1, o2) -> true)) .toArray(Customization[]::new)); } - protected String getElasticsearchIndices(String... caseTypes) throws IOException { - return elasticResponseAsString(GET, String.format("/%s", getIndicesFromCaseTypes(caseTypes))); + protected ElasticsearchClient getElasticsearchClientFactory() { + return elasticsearchClientFactory.createClient(); } - protected String deleteElasticsearchIndices(String... caseTypes) throws IOException { - String indices = caseTypes[0].equals(WILDCARD) ? WILDCARD : String.format( - "/%s", getIndicesFromCaseTypes(caseTypes)); - return elasticResponseAsString(DELETE, indices); + protected String getElasticsearchIndices(String... caseTypes) throws IOException { + String indexPattern = getIndicesFromCaseTypes(caseTypes); + GetIndexResponse response = getElasticsearchClientFactory().indices() + .get(g -> g.index(indexPattern)); + return objectMapper.writeValueAsString(response.toString()); } - private String elasticResponseAsString(String method, String endpoint) throws IOException { - return EntityUtils.toString(elasticClient - .getLowLevelClient() - .performRequest(new Request(method, endpoint)) - .getEntity()); + protected String deleteElasticsearchIndices(String... caseTypes) throws IOException { + String indexPattern = caseTypes[0].equals(WILDCARD) ? WILDCARD : getIndicesFromCaseTypes(caseTypes); + DeleteIndexResponse response = getElasticsearchClientFactory().indices() + .delete(d -> d.index(indexPattern)); + return "acknowledged: " + response.acknowledged(); } private String getIndicesFromCaseTypes(String... caseTypes) { @@ -74,33 +79,34 @@ private String getIndicesFromCaseTypes(String... caseTypes) { .collect(Collectors.joining(",")); } - //CCD-3509 CVE-2021-22044 required to fix null pointers in integration tests, - //conflict in Springfox after Springboot 2.6.10 @Bean - public WebMvcEndpointHandlerMapping webEndpointServletHandlerMapping(WebEndpointsSupplier webEndpointsSupplier, - ServletEndpointsSupplier servletEndpointsSupplier, ControllerEndpointsSupplier controllerEndpointsSupplier, - EndpointMediaTypes endpointMediaTypes, CorsEndpointProperties corsProperties, - WebEndpointProperties webEndpointProperties, Environment environment) { - - List> allEndpoints = new ArrayList<>(); - Collection webEndpoints = webEndpointsSupplier.getEndpoints(); - allEndpoints.addAll(webEndpoints); - allEndpoints.addAll(servletEndpointsSupplier.getEndpoints()); - allEndpoints.addAll(controllerEndpointsSupplier.getEndpoints()); + public WebMvcEndpointHandlerMapping webEndpointServletHandlerMapping( + WebEndpointsSupplier webEndpointsSupplier, + EndpointMediaTypes endpointMediaTypes, + CorsEndpointProperties corsProperties, + WebEndpointProperties webEndpointProperties, + Environment environment) { + + List> allEndpoints = new ArrayList<>(webEndpointsSupplier.getEndpoints()); + String basePath = webEndpointProperties.getBasePath(); EndpointMapping endpointMapping = new EndpointMapping(basePath); - boolean shouldRegisterLinksMapping = this.shouldRegisterLinksMapping(webEndpointProperties, environment, - basePath); - return new WebMvcEndpointHandlerMapping(endpointMapping, webEndpoints, endpointMediaTypes, + boolean shouldRegisterLinksMapping = shouldRegisterLinksMapping(webEndpointProperties, environment, basePath); + + return new WebMvcEndpointHandlerMapping( + endpointMapping, + webEndpointsSupplier.getEndpoints(), + endpointMediaTypes, corsProperties.toCorsConfiguration(), new EndpointLinksResolver(allEndpoints, basePath), - shouldRegisterLinksMapping); + shouldRegisterLinksMapping + ); } private boolean shouldRegisterLinksMapping(WebEndpointProperties webEndpointProperties, Environment environment, String basePath) { - return webEndpointProperties.getDiscovery().isEnabled() && (StringUtils.hasText(basePath) + return webEndpointProperties.getDiscovery().isEnabled() + && (StringUtils.hasText(basePath) || ManagementPortType.get(environment).equals(ManagementPortType.DIFFERENT)); } - } diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/SynchronousElasticDefinitionImportListenerIT.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/SynchronousElasticDefinitionImportListenerIT.java index 1f11993371..9ba0ea550e 100644 --- a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/SynchronousElasticDefinitionImportListenerIT.java +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/SynchronousElasticDefinitionImportListenerIT.java @@ -1,6 +1,7 @@ package uk.gov.hmcts.ccd.definition.store.elastic; import uk.gov.hmcts.ccd.definition.store.elastic.exception.ElasticSearchInitialisationException; +import uk.gov.hmcts.ccd.definition.store.elastic.hamcresutil.IsEqualJSON; import uk.gov.hmcts.ccd.definition.store.event.DefinitionImportedEvent; import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseFieldEntity; import uk.gov.hmcts.ccd.definition.store.repository.entity.CaseTypeEntity; @@ -49,7 +50,7 @@ class SynchronousElasticDefinitionImportListenerIT extends ElasticsearchBaseTest .withJurisdiction("JUR").withReference(CASE_TYPE_A); @BeforeEach - void setUp() throws IOException { + void setUp() { try { deleteElasticsearchIndices(WILDCARD); } catch (Exception e) { @@ -59,7 +60,7 @@ void setUp() throws IOException { @Test @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") - void shouldCreateCompleteElasticsearchIndexForSingleCaseType() throws IOException, JSONException { + void shouldCreateCompleteElasticsearchIndexForSingleCaseType() throws IOException { CaseFieldEntity baseTypeField = newTextField("TextField").build(); CaseFieldEntity complexField = newComplexField("ComplexField"); CaseFieldEntity collectionField = newCollectionFieldOfBaseType( @@ -112,11 +113,16 @@ void shouldCreateCompleteElasticsearchIndexForSingleCaseType() throws IOExceptio definitionImportListener.onDefinitionImported(event); - String response = getElasticsearchIndices(CASE_TYPE_A); + String response = objectMapper.writeValueAsString(getElasticsearchIndices(CASE_TYPE_A)); + String strippedResponse = stripGetIndexResponsePrefix(response); + String unescapedResponse = org.apache.commons.text.StringEscapeUtils.unescapeJson(strippedResponse); - assertThat(response, equalToJSONInFile( + IsEqualJSON isEqualJSON = equalToJSONInFile( readFileFromClasspath("integration/single_casetype_index.json"), - ignoreFieldsComparator(getDynamicIndexResponseFields(CASE_TYPE_A)))); + ignoreFieldsComparator(getDynamicIndexResponseFields(CASE_TYPE_A)) + ); + + assertThat(unescapedResponse, isEqualJSON.leniently()); } @Test @@ -138,14 +144,32 @@ void shouldCreateElasticsearchIndexForAllCaseTypes() throws IOException, JSONExc definitionImportListener.onDefinitionImported(event); String response = getElasticsearchIndices(CASE_TYPE_A, CASE_TYPE_B); + String strippedResponse = stripGetIndexResponsePrefix(response); + String unescapedResponse = org.apache.commons.text.StringEscapeUtils.unescapeJson(strippedResponse); - assertThat(response, equalToJSONInFile( + IsEqualJSON isEqualJSON = equalToJSONInFile( readFileFromClasspath("integration/multi_casetypes_indices.json"), - ignoreFieldsComparator(getDynamicIndexResponseFields(CASE_TYPE_A, CASE_TYPE_B)))); + ignoreFieldsComparator(getDynamicIndexResponseFields(CASE_TYPE_A, CASE_TYPE_B)) + ); + + assertThat(unescapedResponse, isEqualJSON.leniently()); + } + + private String stripGetIndexResponsePrefix(String response) { + if (response == null) { + return null; + } + + String marker = "GetIndexResponse:"; + int index = response.indexOf(marker); + if (index != -1) { + return response.substring(index + marker.length()).trim(); + } + return response.trim(); } @Test - void shouldReindexSuccessfully() throws JSONException { + void shouldReindexSuccessfully() { CaseFieldEntity baseTypeField1 = newTextField("TextField1").build(); CaseTypeEntity caseTypeEntity1 = caseTypeBuilder @@ -165,7 +189,6 @@ void shouldReindexSuccessfully() throws JSONException { ); definitionImportListener.onDefinitionImported(event); - await().atMost(5, SECONDS).untilAsserted(() -> { String response = getElasticsearchIndices(CASE_TYPE_A); assertThat(response, containsString("casetypea_cases-000002")); @@ -174,7 +197,7 @@ void shouldReindexSuccessfully() throws JSONException { } @Test - void shouldThrowElasticSearchInitialisationException() throws JSONException { + void shouldThrowElasticSearchInitialisationException() { CaseFieldEntity baseTypeField1 = newTextField("TextField1").build(); CaseTypeEntity caseTypeEntity1 = caseTypeBuilder @@ -255,4 +278,5 @@ private CaseFieldEntity newCollectionOfComplexField(String fieldReference, Strin return collectionField; } + } diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClientTest.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClientTest.java new file mode 100644 index 0000000000..3029302de9 --- /dev/null +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/client/HighLevelCCDElasticClientTest.java @@ -0,0 +1,418 @@ +package uk.gov.hmcts.ccd.definition.store.elastic.client; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.core.ReindexResponse; +import co.elastic.clients.elasticsearch.indices.AliasDefinition; +import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; +import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; +import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse; +import co.elastic.clients.elasticsearch.indices.ElasticsearchIndicesClient; +import co.elastic.clients.elasticsearch.indices.GetAliasRequest; +import co.elastic.clients.elasticsearch.indices.GetAliasResponse; +import co.elastic.clients.elasticsearch.indices.PutIndicesSettingsResponse; +import co.elastic.clients.elasticsearch.indices.PutMappingResponse; +import co.elastic.clients.elasticsearch.indices.UpdateAliasesRequest; +import co.elastic.clients.elasticsearch.indices.UpdateAliasesResponse; +import co.elastic.clients.elasticsearch.indices.get_alias.IndexAliases; +import co.elastic.clients.util.ObjectBuilder; +import org.elasticsearch.action.ActionListener; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import uk.gov.hmcts.ccd.definition.store.elastic.config.CcdElasticSearchProperties; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.function.Function; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class HighLevelCCDElasticClientTest { + + @Mock + private ElasticsearchClient elasticClient; + + @Mock + private ElasticsearchClientFactory elasticClientFactory; + + @Mock + private CcdElasticSearchProperties config; + + private HighLevelCCDElasticClient highLevelCCDElasticClient; + + @Mock + private ElasticsearchIndicesClient indicesClient; + + @BeforeEach + void setup() { + MockitoAnnotations.openMocks(this); + doReturn(indicesClient).when(elasticClient).indices(); + doReturn(elasticClient).when(elasticClientFactory).createClient(); + highLevelCCDElasticClient = spy(new HighLevelCCDElasticClient(config, elasticClientFactory)); + } + + @Test + void createIndexCreatesIndexWithAliasSuccessfully() throws IOException { + final String indexName = "test_index"; + final String alias = "test_alias"; + + CreateIndexResponse response = new CreateIndexResponse.Builder() + .index(indexName) + .acknowledged(true) + .shardsAcknowledged(true) + .build(); + doReturn(response) + .when(indicesClient) + .create(Mockito.>>any()); + + GetAliasResponse realAliasResponse = new GetAliasResponse.Builder() + .aliases(Map.of(alias, createIndexAliases(indexName))) + .build(); + realGetAliasResponseStub(realAliasResponse); + + assertThat(highLevelCCDElasticClient.createIndex(indexName, alias)).isTrue(); + } + + @Test + void aliasExistsReturnsTrueWhenAliasExists() throws IOException { + final String alias = "existing_alias"; + final String index = "some_index"; + + GetAliasResponse realAliasResponse = new GetAliasResponse.Builder() + .aliases(Map.of(alias, createIndexAliases(index))) + .build(); + realGetAliasResponseStub(realAliasResponse); + + boolean exists = highLevelCCDElasticClient.aliasExists(alias); + + assertThat(exists).isTrue(); + verify(indicesClient).getAlias(Mockito.>>any()); + } + + @Test + void aliasExistsReturnsFalseWhenAliasDoesNotExist() throws IOException { + final String alias = "non_existing_alias"; + + GetAliasResponse realAliasResponse = new GetAliasResponse.Builder() + .aliases(Collections.emptyMap()) + .build(); + realGetAliasResponseStub(realAliasResponse); + + assertThat(highLevelCCDElasticClient.aliasExists(alias)).isFalse(); + verify(indicesClient) + .getAlias(Mockito.>>any()); + } + + @Test + void updateAliasUpdatesAliasSuccessfully() throws IOException { + final String alias = "test_alias"; + final String oldIndex = "old_index"; + final String newIndex = "new_index"; + + UpdateAliasesResponse realAliasesResponse = new UpdateAliasesResponse.Builder() + .acknowledged(true) + .build(); + realUpdateAliasesResponseStub(realAliasesResponse); + + assertThat(highLevelCCDElasticClient.updateAlias(alias, oldIndex, newIndex)).isTrue(); + verify(indicesClient) + .updateAliases(Mockito.>>any()); + } + + @Test + void updateAliasFailsWhenAliasUpdateNotAcknowledged() throws IOException { + final String alias = "test_alias"; + final String oldIndex = "old_index"; + final String newIndex = "new_index"; + + UpdateAliasesResponse realAliasesResponse = new UpdateAliasesResponse.Builder() + .acknowledged(false) + .build(); + realUpdateAliasesResponseStub(realAliasesResponse); + + assertThat(highLevelCCDElasticClient.updateAlias(alias, oldIndex, newIndex)).isFalse(); + verify(indicesClient).updateAliases(Mockito.>>any()); + } + + @Test + void aliasExistsReturnsFalseOn404ElasticsearchException() throws IOException { + final String alias = "missing_alias"; + + assertThat(highLevelCCDElasticClient.aliasExists(alias)).isFalse(); + } + + @Test + void upsertMappingUpsertsMappingSuccessfully() throws IOException { + String aliasName = "test_alias"; + String caseTypeMapping = "{\"properties\":{}}"; + String indexName = "test_index"; + + GetAliasResponse aliasResponse = new GetAliasResponse.Builder() + .aliases(Map.of(aliasName, createIndexAliases(indexName))) + .build(); + + doReturn(aliasResponse).when(indicesClient).getAlias(any(Function.class)); + + var putMappingResponse = mock(PutMappingResponse.class); + when(putMappingResponse.acknowledged()).thenReturn(true); + doReturn(putMappingResponse).when(indicesClient).putMapping(any(Function.class)); + + boolean result = highLevelCCDElasticClient.upsertMapping(aliasName, caseTypeMapping); + + assertThat(result).isTrue(); + verify(indicesClient).getAlias(any(Function.class)); + verify(indicesClient).putMapping(any(Function.class)); + } + + @Test + void upsertMappingThrowsIOExceptionIfAliasLookupFails() throws IOException { + String aliasName = "test_alias"; + String caseTypeMapping = "{\"properties\":{}}"; + + doThrow(new IOException("Alias lookup failed")) + .doThrow(new IOException("Alias lookup failed")) + .doThrow(new IOException("Alias lookup failed")) + .doThrow(new IOException("Alias lookup failed")) + .when(indicesClient).getAlias(any(Function.class)); + + assertThatThrownBy(() -> highLevelCCDElasticClient.upsertMapping(aliasName, caseTypeMapping)) + .isInstanceOf(IOException.class) + .hasMessageContaining("Failed to execute get alias for upsert mapping after 3 attempts"); + } + + @Test + void upsertMappingReturnsFalseIfPutMappingNotAcknowledged() throws IOException { + String aliasName = "test_alias"; + String caseTypeMapping = "{\"properties\":{}}"; + String indexName = "test_index"; + + GetAliasResponse aliasResponse = new GetAliasResponse.Builder() + .aliases(Map.of(aliasName, createIndexAliases(indexName))) + .build(); + doReturn(aliasResponse).when(indicesClient).getAlias(any(Function.class)); + + var putMappingResponse = mock(PutMappingResponse.class); + when(putMappingResponse.acknowledged()).thenReturn(false); + doReturn(putMappingResponse).when(indicesClient).putMapping(any(Function.class)); + + boolean result = highLevelCCDElasticClient.upsertMapping(aliasName, caseTypeMapping); + + assertThat(result).isFalse(); + } + + @Test + void setIndexReadOnlySetsIndexReadOnlyFlag() throws IOException { + String indexName = "readonly_index"; + boolean readOnly = true; + + var putSettingsResponse = mock(PutIndicesSettingsResponse.class); + doReturn(putSettingsResponse).when(indicesClient).putSettings(any(Function.class)); + + highLevelCCDElasticClient.setIndexReadOnly(indexName, readOnly); + + verify(indicesClient).putSettings(any(Function.class)); + } + + @Test + void setIndexReadOnlyThrowsIOExceptionOnFailure() throws IOException { + String indexName = "readonly_index"; + boolean readOnly = true; + + doThrow(new IOException("PutSettings failed")) + .when(indicesClient).putSettings(any(Function.class)); + + assertThatThrownBy(() -> highLevelCCDElasticClient.setIndexReadOnly(indexName, readOnly)) + .isInstanceOf(IOException.class) + .hasMessageContaining("PutSettings failed"); + } + + @Test + void createIndexAndMappingCreatesIndexAndMappingSuccessfully() throws IOException { + String indexName = "new_index"; + String caseTypeMapping = "{\"properties\":{}}"; + + CreateIndexResponse createIndexResponse = new CreateIndexResponse.Builder() + .index(indexName) + .acknowledged(true) + .shardsAcknowledged(true) + .build(); + doReturn(createIndexResponse).when(indicesClient).create(any(Function.class)); + + var putMappingResponse = mock(PutMappingResponse.class); + when(putMappingResponse.acknowledged()).thenReturn(true); + doReturn(putMappingResponse).when(indicesClient).putMapping(any(Function.class)); + + boolean result = highLevelCCDElasticClient.createIndexAndMapping(indexName, caseTypeMapping); + + assertThat(result).isTrue(); + verify(indicesClient).create(any(Function.class)); + verify(indicesClient).putMapping(any(Function.class)); + } + + @Test + void createIndexAndMappingThrowsIOExceptionIfCreateFails() throws IOException { + String indexName = "failing_index"; + String caseTypeMapping = "{\"properties\":{}}"; + + doThrow(new IOException("CreateIndex failed")) + .doThrow(new IOException("CreateIndex failed")) + .doThrow(new IOException("CreateIndex failed")) + .doThrow(new IOException("CreateIndex failed")) + .when(indicesClient).create(any(Function.class)); + + assertThatThrownBy(() -> highLevelCCDElasticClient.createIndexAndMapping(indexName, caseTypeMapping)) + .isInstanceOf(IOException.class) + .hasMessageContaining("Failed to execute create index with mapping after 3 attempts"); + } + + @Test + void createIndexAndMappingThrowsIOExceptionIfPutMappingFails() throws IOException { + String indexName = "new_index"; + String caseTypeMapping = "{\"properties\":{}}"; + + CreateIndexResponse createIndexResponse = new CreateIndexResponse.Builder() + .index(indexName) + .acknowledged(true) + .shardsAcknowledged(true) + .build(); + doReturn(createIndexResponse).when(indicesClient).create(any(Function.class)); + + doThrow(new IOException("PutMapping failed")).when(indicesClient).putMapping(any(Function.class)); + + assertThatThrownBy(() -> highLevelCCDElasticClient.createIndexAndMapping(indexName, caseTypeMapping)) + .isInstanceOf(IOException.class) + .hasMessageContaining("PutMapping failed"); + } + + @Test + void removeIndexDeletesIndexSuccessfully() throws IOException { + String indexName = "index_to_remove"; + var deleteResponse = mock(DeleteIndexResponse.class); + Mockito.when(deleteResponse.acknowledged()).thenReturn(true); + doReturn(deleteResponse).when(indicesClient).delete(any(Function.class)); + + boolean result = highLevelCCDElasticClient.removeIndex(indexName); + + assertThat(result).isTrue(); + verify(indicesClient).delete(any(Function.class)); + } + + @Test + void removeIndexThrowsIOExceptionIfDeleteFails() throws IOException { + String indexName = "index_to_remove"; + doThrow(new IOException("Delete failed")).when(indicesClient).delete(any(Function.class)); + + assertThatThrownBy(() -> highLevelCCDElasticClient.removeIndex(indexName)) + .isInstanceOf(IOException.class) + .hasMessageContaining("Delete failed"); + } + + @Test + void removeIndexReturnsFalseIfDeleteNotAcknowledged() throws IOException { + String indexName = "index_to_remove"; + var deleteResponse = mock(DeleteIndexResponse.class); + when(deleteResponse.acknowledged()).thenReturn(false); + doReturn(deleteResponse).when(indicesClient).delete(any(Function.class)); + + boolean result = highLevelCCDElasticClient.removeIndex(indexName); + + assertThat(result).isFalse(); + } + + @Test + void reindexDataCallsListenerOnResponse() throws Exception { + String oldIndex = "old_index"; + String newIndex = "new_index"; + @SuppressWarnings("unchecked") + ActionListener listener = mock(ActionListener.class); + + var reindexResponse = mock(ReindexResponse.class); + doReturn(reindexResponse).when(elasticClient).reindex(any(Function.class)); + + CountDownLatch latch = new CountDownLatch(1); + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(listener).onResponse(Mockito.any()); + + highLevelCCDElasticClient.reindexData(oldIndex, newIndex, listener); + + latch.await(); // Wait for async execution + verify(listener).onResponse(Mockito.any()); + } + + @Test + void reindexDataCallsListenerOnFailureWhenReindexThrows() throws Exception { + String oldIndex = "old_index"; + String newIndex = "new_index"; + @SuppressWarnings("unchecked") + ActionListener listener = mock(ActionListener.class); + + CountDownLatch latch = new CountDownLatch(1); + + doThrow(new RuntimeException("Reindex failed")).when(elasticClient).reindex(any(Function.class)); + + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(listener).onFailure(any(RuntimeException.class)); + + highLevelCCDElasticClient.reindexData(oldIndex, newIndex, listener); + + latch.await(); // Wait for async + verify(listener).onFailure(Mockito.any(RuntimeException.class)); + } + + @Test + void aliasExistsRethrowsExceptionIfStatusNot404() throws IOException, ElasticsearchException { + final String alias = "some_alias"; + ElasticsearchException esEx = mock(ElasticsearchException.class); + Mockito.when(esEx.status()).thenReturn(404); + doThrow(esEx) + .when(indicesClient).getAlias(any(Function.class)); + assertThat(highLevelCCDElasticClient.aliasExists(alias)).isFalse(); + } + + private void realGetAliasResponseStub(GetAliasResponse realAliasResponse) throws IOException { + doReturn(realAliasResponse) + .when(indicesClient) + .getAlias(Mockito.>>any()); + + } + + private void realUpdateAliasesResponseStub(UpdateAliasesResponse realAliasesResponse) throws IOException { + doReturn(realAliasesResponse) + .when(indicesClient) + .updateAliases(Mockito.>>any()); + } + + private IndexAliases createIndexAliases(String index) { + IndexAliases.Builder builder = new IndexAliases.Builder(); + builder.aliases(index, new AliasDefinition.Builder().build()); + return builder.build(); + } +} diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/hamcresutil/IsEqualJSON.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/hamcresutil/IsEqualJSON.java index e2066d1da7..a0013b53da 100644 --- a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/hamcresutil/IsEqualJSON.java +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/hamcresutil/IsEqualJSON.java @@ -10,6 +10,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.hamcrest.Description; import org.hamcrest.DiagnosingMatcher; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; import org.skyscreamer.jsonassert.JSONCompare; import org.skyscreamer.jsonassert.JSONCompareMode; import org.skyscreamer.jsonassert.JSONCompareResult; @@ -65,7 +68,9 @@ protected boolean matches(final Object actual, comparator = new DefaultComparator(jsonCompareMode); } - final JSONCompareResult result = JSONCompare.compareJSON(expectedJSON, actualJSON, comparator); + String unescapedActualJson = org.apache.commons.text.StringEscapeUtils.unescapeJson(actualJSON); + final JSONCompareResult result = JSONCompare.compareJSON(normalizeJson(expectedJSON), + normalizeJson(unescapedActualJson), comparator); if (!result.passed()) { mismatchDescription.appendText(result.getMessage()); @@ -76,6 +81,18 @@ protected boolean matches(final Object actual, } } + private static String normalizeJson(String json) { + try { + return new JSONObject(json).toString(); // for JSON object + } catch (JSONException e) { + try { + return new JSONArray(json).toString(); // for JSON array + } catch (JSONException ex) { + throw new IllegalArgumentException("Input is not valid JSON: " + json, ex); + } + } + } + /** * Converts the specified object into a JSON string. * @param o the object to convert diff --git a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/mapping/AbstractMapperTest.java b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/mapping/AbstractMapperTest.java index bc38e8c80e..e5651aa025 100644 --- a/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/mapping/AbstractMapperTest.java +++ b/elastic-search-support/src/test/java/uk/gov/hmcts/ccd/definition/store/elastic/mapping/AbstractMapperTest.java @@ -7,8 +7,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; -import org.elasticsearch.core.Set; import org.mockito.Mock; import static com.google.common.collect.Lists.newArrayList; diff --git a/elastic-search-support/src/test/resources/integration/multi_casetypes_indices.json b/elastic-search-support/src/test/resources/integration/multi_casetypes_indices.json index 4e3961c78d..f814c4b8cc 100644 --- a/elastic-search-support/src/test/resources/integration/multi_casetypes_indices.json +++ b/elastic-search-support/src/test/resources/integration/multi_casetypes_indices.json @@ -4,7 +4,7 @@ "casetypea_cases": {} }, "mappings": { - "dynamic": "true", + "dynamic": true, "properties": { "@timestamp": { "type": "object", @@ -122,9 +122,9 @@ "analysis": { "filter": { "shingle_filter": { - "max_shingle_size": "3", - "min_shingle_size": "2", - "output_unigrams": "true", + "max_shingle_size": 3, + "min_shingle_size": 2, + "output_unigrams": true, "type": "shingle" }, "whitespace_remove": { @@ -179,7 +179,7 @@ "casetypeb_cases": {} }, "mappings": { - "dynamic": "true", + "dynamic": true, "properties": { "@timestamp": { "type": "object", @@ -297,9 +297,9 @@ "analysis": { "filter": { "shingle_filter": { - "max_shingle_size": "3", - "min_shingle_size": "2", - "output_unigrams": "true", + "max_shingle_size": 3, + "min_shingle_size": 2, + "output_unigrams": true, "type": "shingle" }, "whitespace_remove": { diff --git a/elastic-search-support/src/test/resources/integration/single_casetype_index.json b/elastic-search-support/src/test/resources/integration/single_casetype_index.json index 979f6edf89..d07bf29b05 100644 --- a/elastic-search-support/src/test/resources/integration/single_casetype_index.json +++ b/elastic-search-support/src/test/resources/integration/single_casetype_index.json @@ -4,7 +4,7 @@ "casetypea_cases": {} }, "mappings": { - "dynamic": "true", + "dynamic": true, "properties": { "@timestamp": { "type": "object", @@ -420,9 +420,9 @@ "analysis": { "filter": { "shingle_filter": { - "max_shingle_size": "3", - "min_shingle_size": "2", - "output_unigrams": "true", + "max_shingle_size": 3, + "min_shingle_size": 2, + "output_unigrams": true, "type": "shingle" }, "whitespace_remove": { diff --git a/rest-api/src/test/java/uk/gov/hmcts/ccd/definition/store/rest/endpoint/CaseDefinitionControllerTest.java b/rest-api/src/test/java/uk/gov/hmcts/ccd/definition/store/rest/endpoint/CaseDefinitionControllerTest.java index 6571a14479..cd2122d80f 100644 --- a/rest-api/src/test/java/uk/gov/hmcts/ccd/definition/store/rest/endpoint/CaseDefinitionControllerTest.java +++ b/rest-api/src/test/java/uk/gov/hmcts/ccd/definition/store/rest/endpoint/CaseDefinitionControllerTest.java @@ -1,5 +1,14 @@ package uk.gov.hmcts.ccd.definition.store.rest.endpoint; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; import uk.gov.hmcts.ccd.definition.store.domain.exception.NotFoundException; import uk.gov.hmcts.ccd.definition.store.domain.service.CaseRoleService; import uk.gov.hmcts.ccd.definition.store.domain.service.JurisdictionService; @@ -13,16 +22,6 @@ import java.util.List; import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; - import static com.google.common.collect.Lists.newArrayList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; @@ -77,6 +76,7 @@ class DataJurisdictionsJurisdictionIdCaseTypeGetTests { @Test @DisplayName("Should call the CaseTypeService with the jurisdictionId when jurisdictionId is not null") + @SuppressWarnings({"deprecation", "removal"}) public void shouldCallCaseTypeService_whenJurisdictionIdIsNotNull() { subject.dataJurisdictionsJurisdictionIdCaseTypeGet("SAMPLE-ID"); verify(caseTypeService, times(1)).findByJurisdictionId(eq("SAMPLE-ID")); @@ -84,6 +84,7 @@ public void shouldCallCaseTypeService_whenJurisdictionIdIsNotNull() { @Test @DisplayName("Should call the CaseTypeService with null when jurisdictionId is null") + @SuppressWarnings({"deprecation", "removal"}) public void shouldCallCaseTypeService_whenJurisdictionIdIsNull() { subject.dataJurisdictionsJurisdictionIdCaseTypeGet(null); verify(caseTypeService, times(1)).findByJurisdictionId(null);