From c0184d91f7c09ab971efbdf68e885ce4c8e6598d Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 9 Jun 2021 21:37:18 -0700 Subject: [PATCH 01/14] Merge MCE and MAE with GMS --- gms/war/build.gradle | 2 ++ .../entity/ebean/EbeanEntityService.java | 26 ++++++++++++++++++- metadata-jobs/mae-consumer-job/build.gradle | 11 +------- .../kafka/DataHubUsageEventsProcessor.java | 8 +++--- .../kafka/MaeConsumerApplication.java | 15 ----------- .../kafka/config/IndexBuildersConfig.java | 10 ------- .../{KafkaConfig.java => MaeKafkaConfig.java} | 2 +- .../ElasticsearchConnectorFactory.java | 2 +- .../kafka/hydrator/ChartHydrator.java | 3 ++- .../kafka/hydrator/CorpUserHydrator.java | 3 ++- .../kafka/hydrator/DashboardHydrator.java | 3 ++- .../kafka/hydrator/DataFlowHydrator.java | 3 ++- .../kafka/hydrator/DataJobHydrator.java | 3 ++- .../kafka/hydrator/HydratorFactory.java | 3 ++- .../DataHubUsageEventTransformer.java | 3 ++- metadata-jobs/mce-consumer-job/build.gradle | 13 ++-------- .../kafka/MceConsumerApplication.java | 15 ----------- .../kafka/MetadataChangeEventsProcessor.java | 2 +- .../{KafkaConfig.java => MceKafkaConfig.java} | 5 ++-- 19 files changed, 54 insertions(+), 78 deletions(-) delete mode 100644 metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java rename metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/{KafkaConfig.java => MaeKafkaConfig.java} (99%) delete mode 100644 metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java rename metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/{KafkaConfig.java => MceKafkaConfig.java} (98%) diff --git a/gms/war/build.gradle b/gms/war/build.gradle index 4ad1448db3a392..d6304c9bc9d5be 100644 --- a/gms/war/build.gradle +++ b/gms/war/build.gradle @@ -4,6 +4,8 @@ ext.apiProject = project(':gms:api') dependencies { runtime project(':gms:factories') + runtime project(':metadata-jobs:mce-consumer-job') + runtime project(':metadata-jobs:mae-consumer-job') runtime externalDependency.h2 runtime externalDependency.logbackClassic diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index ce7e246148140b..e2c76e6e145699 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -24,6 +24,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.linkedin.metadata.PegasusUtils.*; @@ -35,9 +37,9 @@ public class EbeanEntityService extends EntityService { private static final int DEFAULT_MAX_TRANSACTION_RETRY = 3; + private static final Logger LOG = LoggerFactory.getLogger("EbeanEntityService"); private final EbeanAspectDao _entityDao; - private Boolean _alwaysEmitAuditEvent = false; public EbeanEntityService( @@ -51,6 +53,9 @@ public EbeanEntityService( @Override @Nonnull public Map> getLatestAspects(@Nonnull final Set urns, @Nonnull final Set aspectNames) { + + LOG.info(String.format("Getting latest aspects. urns: %s, aspectNames: %s.", urns, aspectNames)); + // Create DB keys final Set dbKeys = urns.stream() .map(urn -> { @@ -74,11 +79,13 @@ public Map> getLatestAspects(@Nonnull final Set u // Add "key" aspects for each urn. TODO: Replace this with a materialized key aspect. urnToAspects.keySet().forEach(key -> { + LOG.info(String.format("Adding key aspect. urn: %s, aspectNames: %s.", key, aspectNames)); final RecordTemplate keyAspect = buildKeyAspect(key); urnToAspects.get(key).add(keyAspect); }); _entityDao.batchGet(dbKeys).forEach((key, aspectEntry) -> { + LOG.info(String.format("Successfully fetched aspect. urn: %s, aspect: %s.", key.getUrn(), key.getAspect())); final Urn urn = toUrn(key.getUrn()); final String aspectName = key.getAspect(); final RecordTemplate aspectRecord = toAspectRecord(urn, aspectName, aspectEntry.getMetadata()); @@ -145,20 +152,27 @@ private RecordTemplate ingestAspect( @Nonnull final AuditStamp auditStamp, final int maxTransactionRetry) { + LOG.info(String.format("Ingesting aspect with urn: %s, name: %s", urn, aspectName)); + final AddAspectResult result = _entityDao.runInTransactionWithRetry(() -> { // 1. Fetch the latest existing version of the aspect. final EbeanAspectV2 latest = _entityDao.getLatestAspect(urn.toString(), aspectName); + LOG.info(String.format("Fetched latest aspect to ingest with urn: %s, name: %s", urn, aspectName)); + // 2. Compare the latest existing and new. final RecordTemplate oldValue = latest == null ? null : toAspectRecord(urn, aspectName, latest.getMetadata()); final RecordTemplate newValue = updateLambda.apply(Optional.ofNullable(oldValue)); // 3. Skip updating if there is no difference between existing and new. if (oldValue != null && DataTemplateUtil.areEqual(oldValue, newValue)) { + LOG.info(String.format("Aspect has not changed between previous and new versions. Skipping ingest. urn: %s, name: %s", urn, aspectName)); return new AddAspectResult(urn, oldValue, oldValue); } + LOG.info(String.format("Found differing aspects. Ingesting new aspect version. urn: %s, name: %s", urn, aspectName)); + // 4. Save the newValue as the latest version _entityDao.saveLatestAspect( urn.toString(), @@ -182,7 +196,10 @@ private RecordTemplate ingestAspect( // 5. Produce MAE after a successful update if (oldValue != newValue || _alwaysEmitAuditEvent) { + LOG.info(String.format("Producing MAE for ingested aspect. urn: %s, name: %s", urn, aspectName)); produceMetadataAuditEvent(urn, oldValue, newValue); + } else { + LOG.info(String.format("Skipping MAE for ingested aspect. urn: %s, name: %s", urn, aspectName)); } return newValue; @@ -217,12 +234,16 @@ private RecordTemplate updateAspect( @Nonnull final boolean emitMae, final int maxTransactionRetry) { + LOG.info(String.format("Updating aspect with urn: %s, name: %s", urn, aspectName)); + final AddAspectResult result = _entityDao.runInTransactionWithRetry(() -> { final EbeanAspectV2 oldAspect = _entityDao.getAspect(urn.toString(), aspectName, version); final RecordTemplate oldValue = oldAspect == null ? null : toAspectRecord(urn, aspectName, oldAspect.getMetadata()); + LOG.info(String.format("Fetched old aspect to update with urn: %s, name: %s. Saving new version.", urn, aspectName)); + _entityDao.saveAspect( urn.toString(), aspectName, @@ -242,7 +263,10 @@ private RecordTemplate updateAspect( final RecordTemplate newValue = result.getNewValue(); if (emitMae) { + LOG.info(String.format("Producing MAE for updated aspect. %s, name: %s.", urn, aspectName)); produceMetadataAuditEvent(urn, oldValue, newValue); + } else { + LOG.info(String.format("Skipping MAE for updated aspect. %s, name: %s.", urn, aspectName)); } return newValue; diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle index a52a468511a62b..69fe765ef40ba4 100644 --- a/metadata-jobs/mae-consumer-job/build.gradle +++ b/metadata-jobs/mae-consumer-job/build.gradle @@ -1,5 +1,4 @@ plugins { - id 'org.springframework.boot' id 'java' } @@ -31,10 +30,6 @@ dependencies { compile externalDependency.kafkaAvroSerde compile externalDependency.neo4jJavaDriver - compile (externalDependency.springBootStarterWeb) { - exclude module: "spring-boot-starter-tomcat" - } - compile externalDependency.springBootStarterJetty compile externalDependency.springKafka compile externalDependency.springActuator @@ -59,8 +54,4 @@ compileJava.dependsOn avroSchemaSources clean { project.delete("src/main/resources/avro") -} - -bootJar { - mainClassName = 'com.linkedin.metadata.kafka.MaeConsumerApplication' -} +} \ No newline at end of file diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java index 43cd68802830d5..9857527f7a25a7 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java @@ -11,7 +11,6 @@ import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.stereotype.Component; @@ -19,7 +18,6 @@ @Slf4j @Component -@ConditionalOnProperty(value = "DATAHUB_ANALYTICS_ENABLED", havingValue = "true", matchIfMissing = true) @EnableKafka public class DataHubUsageEventsProcessor { @@ -27,8 +25,10 @@ public class DataHubUsageEventsProcessor { private final DataHubUsageEventTransformer dataHubUsageEventTransformer; private final String indexName; - public DataHubUsageEventsProcessor(ElasticsearchConnector elasticSearchConnector, - DataHubUsageEventTransformer dataHubUsageEventTransformer, IndexConvention indexConvention) { + public DataHubUsageEventsProcessor( + ElasticsearchConnector elasticSearchConnector, + DataHubUsageEventTransformer dataHubUsageEventTransformer, + IndexConvention indexConvention) { this.elasticSearchConnector = elasticSearchConnector; this.dataHubUsageEventTransformer = dataHubUsageEventTransformer; this.indexName = indexConvention.getIndexName("datahub_usage_event"); diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java deleted file mode 100644 index 389f40477ad13c..00000000000000 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.linkedin.metadata.kafka; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration; - -@SuppressWarnings("checkstyle:HideUtilityClassConstructor") -@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}) -public class MaeConsumerApplication { - - public static void main(String[] args) { - Class[] primarySources = {MaeConsumerApplication.class, MaeConsumerConfig.class}; - SpringApplication.run(primarySources, args); - } -} diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java index 73f138da446e13..92b4d624fb71c5 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java @@ -15,8 +15,6 @@ import com.linkedin.metadata.builders.search.GlossaryTermInfoIndexBuilder; import com.linkedin.metadata.builders.search.GlossaryNodeInfoIndexBuilder; import com.linkedin.metadata.restli.DefaultRestliClientFactory; -import com.linkedin.metadata.utils.elasticsearch.IndexConvention; -import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; import com.linkedin.restli.client.Client; import java.util.HashSet; import java.util.Set; @@ -71,12 +69,4 @@ public Set> indexBuilders(@Nonnull Cl public Client restliClient() { return DefaultRestliClientFactory.getRestLiClient(gmsHost, gmsPort); } - - /** - * Convention for naming search indices - */ - @Bean - public IndexConvention indexConvention() { - return new IndexConventionImpl(indexPrefix); - } } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java similarity index 99% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java rename to metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java index 97fbe48653baed..b054f4086aaa5e 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java @@ -20,7 +20,7 @@ @Slf4j @Configuration -public class KafkaConfig { +public class MaeKafkaConfig { @Value("${KAFKA_BOOTSTRAP_SERVER:http://localhost:9092}") private String kafkaBootstrapServer; @Value("${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081}") diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java index bd53263d1ae57a..74eb559e8eeba5 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java @@ -21,7 +21,7 @@ public class ElasticsearchConnectorFactory { @Bean(name = "elasticsearchConnector") @Nonnull - public ElasticsearchConnector createInstance(RestHighLevelClient elasticSearchRestHighLevelClient) { + public ElasticsearchConnector createInstance(@Nonnull RestHighLevelClient elasticSearchRestHighLevelClient) { return new ElasticsearchConnector(elasticSearchRestHighLevelClient, bulkRequestsLimit, bulkFlushPeriod); } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java index 7ac89c804468b9..689445c6199927 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java @@ -9,6 +9,7 @@ import com.linkedin.restli.client.Client; import java.net.URISyntaxException; import java.util.Optional; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,7 +21,7 @@ public class ChartHydrator implements Hydrator { private static final String DASHBOARD_TOOL = "dashboardTool"; private static final String TITLE = "title"; - public ChartHydrator(Client restliClient) { + public ChartHydrator(@Nonnull Client restliClient) { _restliClient = restliClient; _remoteDAO = new RestliRemoteDAO<>(ChartSnapshot.class, ChartAspect.class, _restliClient); } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java index 8741dc36d844f4..fc1ab67b2ed591 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java @@ -9,6 +9,7 @@ import com.linkedin.restli.client.Client; import java.net.URISyntaxException; import java.util.Optional; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,7 +21,7 @@ public class CorpUserHydrator implements Hydrator { private static final String USER_NAME = "username"; private static final String NAME = "name"; - public CorpUserHydrator(Client restliClient) { + public CorpUserHydrator(@Nonnull Client restliClient) { _restliClient = restliClient; _remoteDAO = new RestliRemoteDAO<>(CorpUserSnapshot.class, CorpUserAspect.class, _restliClient); } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java index ca7e4f2a1b0791..c2be3043911640 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java @@ -9,6 +9,7 @@ import com.linkedin.restli.client.Client; import java.net.URISyntaxException; import java.util.Optional; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,7 +21,7 @@ public class DashboardHydrator implements Hydrator { private static final String DASHBOARD_TOOL = "dashboardTool"; private static final String TITLE = "title"; - public DashboardHydrator(Client restliClient) { + public DashboardHydrator(@Nonnull Client restliClient) { _restliClient = restliClient; _remoteDAO = new RestliRemoteDAO<>(DashboardSnapshot.class, DashboardAspect.class, _restliClient); } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java index ce6def8ce4c0ed..69761b11392620 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java @@ -9,6 +9,7 @@ import com.linkedin.restli.client.Client; import java.net.URISyntaxException; import java.util.Optional; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,7 +21,7 @@ public class DataFlowHydrator implements Hydrator { private static final String ORCHESTRATOR = "orchestrator"; private static final String NAME = "name"; - public DataFlowHydrator(Client restliClient) { + public DataFlowHydrator(@Nonnull Client restliClient) { _restliClient = restliClient; _remoteDAO = new RestliRemoteDAO<>(DataFlowSnapshot.class, DataFlowAspect.class, _restliClient); } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java index ea4eae333ad038..e1b40e571cce45 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java @@ -9,6 +9,7 @@ import com.linkedin.restli.client.Client; import java.net.URISyntaxException; import java.util.Optional; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -20,7 +21,7 @@ public class DataJobHydrator implements Hydrator { private static final String ORCHESTRATOR = "orchestrator"; private static final String NAME = "name"; - public DataJobHydrator(Client restliClient) { + public DataJobHydrator(@Nonnull Client restliClient) { _restliClient = restliClient; _remoteDAO = new RestliRemoteDAO<>(DataJobSnapshot.class, DataJobAspect.class, _restliClient); } diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java index 3e171fc41f09a8..e9e1defaf3a0ed 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import javax.annotation.Nonnull; public class HydratorFactory { @@ -14,7 +15,7 @@ public class HydratorFactory { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public HydratorFactory(Client restliClient) { + public HydratorFactory(@Nonnull Client restliClient) { _restliClient = restliClient; _hydratorMap = new HashMap<>(); _hydratorMap.put(EntityType.DATASET, new DatasetHydrator()); diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java index 83e959187ea7de..c6ea3fa8341622 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java @@ -9,6 +9,7 @@ import com.linkedin.metadata.kafka.hydrator.HydratorFactory; import java.util.Optional; import java.util.Set; +import javax.annotation.Nonnull; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -36,7 +37,7 @@ public static class TransformedDocument { String document; } - public DataHubUsageEventTransformer(HydratorFactory hydratorFactory) { + public DataHubUsageEventTransformer(@Nonnull HydratorFactory hydratorFactory) { this.hydratorFactory = hydratorFactory; } diff --git a/metadata-jobs/mce-consumer-job/build.gradle b/metadata-jobs/mce-consumer-job/build.gradle index c72b1cea0f7f72..a895437841f531 100644 --- a/metadata-jobs/mce-consumer-job/build.gradle +++ b/metadata-jobs/mce-consumer-job/build.gradle @@ -1,5 +1,4 @@ plugins { - id 'org.springframework.boot' id 'java' } @@ -26,12 +25,8 @@ dependencies { compile spec.product.pegasus.restliCommon compile externalDependency.elasticSearchRest compile externalDependency.kafkaAvroSerde - compile (externalDependency.springBootStarterWeb) { - exclude module: "spring-boot-starter-tomcat" - } - compile externalDependency.springBootStarterJetty - compile externalDependency.springKafka + compile externalDependency.springKafka compile externalDependency.springActuator compileOnly externalDependency.lombok @@ -54,8 +49,4 @@ compileJava.dependsOn avroSchemaSources clean { project.delete("src/main/resources/avro") -} - -bootJar { - mainClassName = 'com.linkedin.metadata.kafka.MceConsumerApplication' -} +} \ No newline at end of file diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java deleted file mode 100644 index 9d6bc19801374c..00000000000000 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.linkedin.metadata.kafka; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration; - - -@SuppressWarnings("checkstyle:HideUtilityClassConstructor") -@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}) -public class MceConsumerApplication { - - public static void main(String[] args) { - SpringApplication.run(MceConsumerApplication.class, args); - } -} diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index a1403f9748ffe1..3f00c1be0d70ea 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -40,7 +40,7 @@ public MetadataChangeEventsProcessor( } @KafkaListener(id = "${KAFKA_CONSUMER_GROUP_ID:mce-consumer-job-client}", topics = "${KAFKA_MCE_TOPIC_NAME:" - + Topics.METADATA_CHANGE_EVENT + "}") + + Topics.METADATA_CHANGE_EVENT + "}", containerFactory = "mceKafkaContainerFactory") public void consume(final ConsumerRecord consumerRecord) { final GenericRecord record = consumerRecord.value(); log.debug("Record ", record); diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java similarity index 98% rename from metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java rename to metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java index 98e9e236a55279..15d264dc9695f5 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/KafkaConfig.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java @@ -24,13 +24,14 @@ @Slf4j @Configuration -public class KafkaConfig { +public class MceKafkaConfig { + @Value("${KAFKA_BOOTSTRAP_SERVER:http://localhost:9092}") private String kafkaBootstrapServers; @Value("${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081}") private String kafkaSchemaRegistryUrl; - @Bean + @Bean(name = "mceKafkaContainerFactory") public KafkaListenerContainerFactory kafkaListenerContainerFactory(KafkaProperties properties) { KafkaProperties.Consumer consumerProps = properties.getConsumer(); From 5621deb0450ce55f2ee4458f3ef8af745c766eff Mon Sep 17 00:00:00 2001 From: John Joyce Date: Thu, 10 Jun 2021 09:46:49 -0700 Subject: [PATCH 02/14] Merging mae-consumer and mce-consumer into GMS --- .github/workflows/docker-mae-consumer.yml | 73 ------- .github/workflows/docker-mce-consumer.yml | 73 ------- .../charts/datahub-mae-consumer/.helmignore | 23 --- .../charts/datahub-mae-consumer/Chart.yaml | 21 --- .../charts/datahub-mae-consumer/README.md | 59 ------ .../datahub-mae-consumer/templates/NOTES.txt | 21 --- .../templates/_helpers.tpl | 63 ------- .../templates/config-jmx-exporter.yaml | 17 -- .../templates/deployment.yaml | 156 --------------- .../templates/service.yaml | 15 -- .../templates/serviceaccount.yaml | 12 -- .../charts/datahub-mae-consumer/values.yaml | 178 ------------------ .../charts/datahub-mce-consumer/.helmignore | 23 --- .../charts/datahub-mce-consumer/Chart.yaml | 21 --- .../charts/datahub-mce-consumer/README.md | 52 ----- .../datahub-mce-consumer/templates/NOTES.txt | 21 --- .../templates/_helpers.tpl | 63 ------- .../templates/config-jmx-exporter.yaml | 17 -- .../templates/deployment.yaml | 122 ------------ .../templates/serviceaccount.yaml | 12 -- .../charts/datahub-mce-consumer/values.yaml | 164 ---------------- docker/datahub-mae-consumer/Dockerfile | 42 ----- docker/datahub-mae-consumer/README.md | 5 - docker/datahub-mae-consumer/env/docker.env | 29 --- docker/datahub-mae-consumer/start.sh | 25 --- docker/datahub-mce-consumer/Dockerfile | 42 ----- docker/datahub-mce-consumer/README.md | 5 - docker/datahub-mce-consumer/env/docker.env | 15 -- docker/datahub-mce-consumer/start.sh | 6 - metadata-jobs/mae-consumer-job/README.md | 13 +- metadata-jobs/mce-consumer-job/README.md | 22 +-- 31 files changed, 2 insertions(+), 1408 deletions(-) delete mode 100644 .github/workflows/docker-mae-consumer.yml delete mode 100644 .github/workflows/docker-mce-consumer.yml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml delete mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml delete mode 100644 docker/datahub-mae-consumer/Dockerfile delete mode 100644 docker/datahub-mae-consumer/README.md delete mode 100644 docker/datahub-mae-consumer/env/docker.env delete mode 100755 docker/datahub-mae-consumer/start.sh delete mode 100644 docker/datahub-mce-consumer/Dockerfile delete mode 100644 docker/datahub-mce-consumer/README.md delete mode 100644 docker/datahub-mce-consumer/env/docker.env delete mode 100755 docker/datahub-mce-consumer/start.sh diff --git a/.github/workflows/docker-mae-consumer.yml b/.github/workflows/docker-mae-consumer.yml deleted file mode 100644 index 1b0748742d141e..00000000000000 --- a/.github/workflows/docker-mae-consumer.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: datahub-mae-consumer docker -on: - push: - branches: - - master - paths-ignore: - - 'docs/**' - - '**.md' - pull_request: - branches: - - master - paths: - - 'docker/**' - - '.github/workflows/docker-mae-consumer.yml' - paths_ignore: - - '**.md' - - '**.env' - release: - types: [published, edited] - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - publish: ${{ steps.publish.outputs.publish }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Compute Tag - id: tag - run: | - echo "GITHUB_REF: $GITHUB_REF" - SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") - TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g') - echo "tag=$TAG" - echo "::set-output name=tag::$TAG" - - name: Check whether publishing enabled - id: publish - env: - ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }} - run: | - echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}" - echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}" - push_to_registries: - name: Build and Push Docker Image to Docker Hub - runs-on: ubuntu-latest - needs: setup - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - # list of Docker images to use as base name for tags - images: | - linkedin/datahub-mae-consumer - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Login to DockerHub - if: ${{ needs.setup.outputs.publish == 'true' }} - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push image - uses: docker/build-push-action@v2 - with: - file: ./docker/datahub-mae-consumer/Dockerfile - tags: ${{ steps.docker_meta.outputs.tags }} - push: ${{ needs.setup.outputs.publish == 'true' }} diff --git a/.github/workflows/docker-mce-consumer.yml b/.github/workflows/docker-mce-consumer.yml deleted file mode 100644 index 5c69ce5453a652..00000000000000 --- a/.github/workflows/docker-mce-consumer.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: datahub-mce-consumer docker -on: - push: - branches: - - master - paths-ignore: - - 'docs/**' - - '**.md' - pull_request: - branches: - - master - paths: - - 'docker/**' - - '.github/workflows/docker-mce-consumer.yml' - paths_ignore: - - '**.md' - - '**.env' - release: - types: [published, edited] - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.tag.outputs.tag }} - publish: ${{ steps.publish.outputs.publish }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Compute Tag - id: tag - run: | - echo "GITHUB_REF: $GITHUB_REF" - SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") - TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g') - echo "tag=$TAG" - echo "::set-output name=tag::$TAG" - - name: Check whether publishing enabled - id: publish - env: - ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }} - run: | - echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}" - echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}" - push_to_registries: - name: Build and Push Docker Image to Docker Hub - runs-on: ubuntu-latest - needs: setup - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - # list of Docker images to use as base name for tags - images: | - linkedin/datahub-mce-consumer - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Login to DockerHub - if: ${{ needs.setup.outputs.publish == 'true' }} - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push image - uses: docker/build-push-action@v2 - with: - file: ./docker/datahub-mce-consumer/Dockerfile - tags: ${{ steps.docker_meta.outputs.tags }} - push: ${{ needs.setup.outputs.publish == 'true' }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore deleted file mode 100644 index 0e8a0eb36f4ca2..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml deleted file mode 100644 index 4e5116696dc366..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: datahub-mae-consumer -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -version: 0.2.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -appVersion: 0.3.1 diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md deleted file mode 100644 index 29696289f30d37..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md +++ /dev/null @@ -1,59 +0,0 @@ -datahub-mae-consumer -==================== -A Helm chart for datahub-mae-consumer - -Current chart version is `0.2.0` - -## Chart Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | object | `{}` | | -| exporters.jmx.enabled | boolean | false | | -| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | -| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | -| fullnameOverride | string | `"datahub-mae-consumer"` | | -| global.datahub_analytics_enabled | boolean | true | | -| global.elasticsearch.host | string | `"elasticsearch"` | | -| global.elasticsearch.port | string | `"9200"` | | -| global.kafka.bootstrap.server | string | `"broker:9092"` | | -| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` | | -| global.neo4j.host | string | `"neo4j:7474"` | | -| global.neo4j.uri | string | `"bolt://neo4j"` | | -| global.neo4j.username | string | `"neo4j"` | | -| global.neo4j.password.secretRef | string | `"neo4j-secrets"` | | -| global.neo4j.password.secretKey | string | `"neo4j-password"` | | -| global.hostAliases[0].hostnames[0] | string | `"broker"` | | -| global.hostAliases[0].hostnames[1] | string | `"mysql"` | | -| global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` | | -| global.hostAliases[0].hostnames[3] | string | `"neo4j"` | | -| global.hostAliases[0].ip | string | `"192.168.0.104"` | | -| image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"linkedin/datahub-mae-consumer"` | | -| image.tag | string | `"v0.8.1"` | | -| imagePullSecrets | list | `[]` | | -| ingress.annotations | object | `{}` | | -| ingress.enabled | bool | `false` | | -| ingress.hosts[0].host | string | `"chart-example.local"` | | -| ingress.hosts[0].paths | list | `[]` | | -| ingress.tls | list | `[]` | | -| livenessProbe.initialDelaySeconds | int | `60` | | -| livenessProbe.periodSeconds | int | `30` | | -| livenessProbe.failureThreshold | int | `8` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| podAnnotations | object | `{}` | | -| podSecurityContext | object | `{}` | | -| readinessProbe.initialDelaySeconds | int | `60` | | -| readinessProbe.periodSeconds | int | `30` | | -| readinessProbe.failureThreshold | int | `8` | | -| replicaCount | int | `1` | | -| resources | object | `{}` | | -| securityContext | object | `{}` | | -| service.port | int | `80` | | -| service.type | string | `"ClusterIP"` | | -| serviceAccount.annotations | object | `{}` | | -| serviceAccount.create | bool | `true` | | -| serviceAccount.name | string | `nil` | | -| tolerations | list | `[]` | | diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt deleted file mode 100644 index 6f2f355847e9d0..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt +++ /dev/null @@ -1,21 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "datahub-mae-consumer.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "datahub-mae-consumer.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "datahub-mae-consumer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "datahub-mae-consumer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 -{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl deleted file mode 100644 index e677e252a6270c..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "datahub-mae-consumer.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datahub-mae-consumer.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datahub-mae-consumer.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "datahub-mae-consumer.labels" -}} -helm.sh/chart: {{ include "datahub-mae-consumer.chart" . }} -{{ include "datahub-mae-consumer.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - -{{/* -Selector labels -*/}} -{{- define "datahub-mae-consumer.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datahub-mae-consumer.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datahub-mae-consumer.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "datahub-mae-consumer.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml deleted file mode 100644 index 72431b713e836e..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.exporters.jmx.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter - labels: - {{- include "datahub-mae-consumer.labels" . | nindent 4 }} -data: - config.yml: |- - hostPort: localhost:{{ .Values.env.JMXPORT }} - lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }} - lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }} - rules: -{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }} - ssl: false - startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }} -{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml deleted file mode 100644 index 158c051bb8b1e6..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml +++ /dev/null @@ -1,156 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datahub-mae-consumer.fullname" . }} - labels: - {{- include "datahub-mae-consumer.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "datahub-mae-consumer.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "datahub-mae-consumer.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.global.hostAliases }} - hostAliases: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datahub-mae-consumer.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - volumes: - {{- with .Values.global.credentialsAndCertsSecrets }} - - name: datahub-certs-dir - secret: - defaultMode: 0444 - secretName: {{ .name }} - {{- end }} - {{- if .Values.exporters.jmx.enabled }} - - name: config-jmx-exporter - configMap: - name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter - {{- end }} - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - initContainers: - {{- with .Values.extraInitContainers }} - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 9091 - protocol: TCP - livenessProbe: - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - readinessProbe: - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - env: - - name: GMS_HOST - value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} - - name: GMS_PORT - value: "{{ .Values.global.datahub.gms.port }}" - - name: KAFKA_BOOTSTRAP_SERVER - value: "{{ .Values.global.kafka.bootstrap.server }}" - - name: KAFKA_SCHEMAREGISTRY_URL - value: "{{ .Values.global.kafka.schemaregistry.url }}" - - name: ELASTICSEARCH_HOST - value: "{{ .Values.global.elasticsearch.host }}" - - name: ELASTICSEARCH_PORT - value: "{{ .Values.global.elasticsearch.port }}" - {{- with .Values.global.elasticsearch.useSSL }} - - name: ELASTICSEARCH_USE_SSL - value: {{ . | quote }} - {{- end }} - {{- with .Values.global.elasticsearch.auth }} - - name: ELASTICSEARCH_USERNAME - value: {{ .username }} - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .password.secretRef }}" - key: "{{ .password.secretKey }}" - {{- end }} - - name: NEO4J_HOST - value: "{{ .Values.global.neo4j.host }}" - - name: NEO4J_URI - value: "{{ .Values.global.neo4j.uri }}" - - name: NEO4J_USERNAME - value: "{{ .Values.global.neo4j.username }}" - - name: NEO4J_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Values.global.neo4j.password.secretRef }}" - key: "{{ .Values.global.neo4j.password.secretKey }}" - - name: DATAHUB_ANALYTICS_ENABLED - value: "{{ .Values.global.datahub_analytics_enabled }}" - {{- if .Values.global.springKafkaConfigurationOverrides }} - {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }} - - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }} - value: {{ $configValue }} - {{- end }} - {{- end }} - {{- if .Values.global.credentialsAndCertsSecrets }} - {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }} - - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }} - valueFrom: - secretKeyRef: - name: {{ $.Values.global.credentialsAndCertsSecrets.name }} - key: {{ $envVarValue }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvs }} - {{- toYaml . | nindent 12 }} - {{- end }} - volumeMounts: - {{- with .Values.global.credentialsAndCertsSecrets }} - - name: datahub-certs-dir - mountPath: {{ .path | default "/mnt/certs" }} - {{- end }} - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - - {{- include "datahub-jmxexporter.container" . }} - - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml deleted file mode 100644 index 4a2178ff52e86a..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ printf "%s-%s" .Release.Name "datahub-mae-consumer" }} - labels: - {{- include "datahub-mae-consumer.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.global.datahub.mae_consumer.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "datahub-mae-consumer.selectorLabels" . | nindent 4 }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml deleted file mode 100644 index 14e6c5f6ced0ae..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datahub-mae-consumer.serviceAccountName" . }} - labels: - {{- include "datahub-mae-consumer.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml deleted file mode 100644 index 3da74b68321730..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# Default values for datahub-mae-consumer. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: linkedin/datahub-mae-consumer - pullPolicy: Always - tag: v0.8.1 - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: - -podAnnotations: {} - # co.elastic.logs/enabled: "true" - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: [] - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -# Extra environment variables -# This will be appended to the current 'env:' key. You can use any of the kubernetes env -# syntax here -extraEnvs: [] - # - name: MY_ENVIRONMENT_VAR - # value: the_value_goes_here - -extraVolumes: [] - # - name: extras - # emptyDir: {} - -extraVolumeMounts: [] - # - name: extras - # mountPath: /usr/share/extras - # readOnly: true - -extraInitContainers: [] - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -env: - JMXPORT: 1099 - -exporters: - jmx: - enabled: false -# image: -# repository: bitnami/jmx-exporter -# tag: 0.15.0 -# pullPolicy: IfNotPresent -# config: -# lowercaseOutputName: true -# rules: -# - pattern: ".*" -# startDelaySeconds: 30 -# env: {} -# resources: {} -# path: /metrics -# ports: -# jmxxp: -# containerPort: 5556 -# protocol: TCP -# livenessProbe: -# httpGet: -# path: /metrics -# port: jmxxp -# initialDelaySeconds: 30 -# periodSeconds: 15 -# timeoutSeconds: 60 -# failureThreshold: 8 -# successThreshold: 1 -# readinessProbe: -# httpGet: -# path: /metrics -# port: jmxxp -# initialDelaySeconds: 30 -# periodSeconds: 15 -# timeoutSeconds: 60 -# failureThreshold: 8 -# successThreshold: 1 -# serviceMonitor: -# interval: 30s -# scrapeTimeout: 30s -# scheme: http - -livenessProbe: - initialDelaySeconds: 60 - periodSeconds: 30 - failureThreshold: 8 - -readinessProbe: - initialDelaySeconds: 60 - periodSeconds: 30 - failureThreshold: 8 - -global: - datahub_analytics_enabled: true - - elasticsearch: - host: "elasticsearch" - port: "9200" - - kafka: - bootstrap: - server: "broker:9092" - schemaregistry: - url: "http://schema-registry:8081" - - neo4j: - host: "neo4j:7474" - uri: "bolt://neo4j" - username: "neo4j" - password: - secretRef: "neo4j-secrets" - secretKey: "neo4j-password" - - datahub: - mae_consumer: - port: "9091" - - hostAliases: - - ip: "192.168.0.104" - hostnames: - - "broker" - - "mysql" - - "elasticsearch" - - "neo4j" \ No newline at end of file diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore deleted file mode 100644 index 0e8a0eb36f4ca2..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml deleted file mode 100644 index bd00d74aa261ae..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: datahub-mce-consumer -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -version: 0.2.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -appVersion: 0.3.1 diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md deleted file mode 100644 index 7cae56b3df6e88..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md +++ /dev/null @@ -1,52 +0,0 @@ -datahub-mce-consumer -==================== -A Helm chart for datahub-mce-consumer - -Current chart version is `0.2.0` - -## Chart Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| affinity | object | `{}` | | -| exporters.jmx.enabled | boolean | false | | -| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | -| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | -| fullnameOverride | string | `""` | | -| global.kafka.bootstrap.server | string | `"broker:9092"` | | -| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` | | -| global.datahub.gms.port | string | `"8080"` | | -| global.hostAliases[0].hostnames[0] | string | `"broker"` | | -| global.hostAliases[0].hostnames[1] | string | `"mysql"` | | -| global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` | | -| global.hostAliases[0].hostnames[3] | string | `"neo4j"` | | -| global.hostAliases[0].ip | string | `"192.168.0.104"` | | -| image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"linkedin/datahub-mce-consumer"` | | -| image.tag | string | `"v0.8.1"` | | -| imagePullSecrets | list | `[]` | | -| ingress.annotations | object | `{}` | | -| ingress.enabled | bool | `false` | | -| ingress.hosts[0].host | string | `"chart-example.local"` | | -| ingress.hosts[0].paths | list | `[]` | | -| ingress.tls | list | `[]` | | -| livenessProbe.initialDelaySeconds | int | `60` | | -| livenessProbe.periodSeconds | int | `30` | | -| livenessProbe.failureThreshold | int | `4` | | -| nameOverride | string | `""` | | -| nodeSelector | object | `{}` | | -| podAnnotations | object | `{}` | | -| podSecurityContext | object | `{}` | | -| readinessProbe.initialDelaySeconds | int | `60` | | -| readinessProbe.periodSeconds | int | `30` | | -| readinessProbe.failureThreshold | int | `4` | | -| replicaCount | int | `1` | | -| resources | object | `{}` | | -| securityContext | object | `{}` | | -| service.port | int | `80` | | -| service.type | string | `"ClusterIP"` | | -| serviceAccount.annotations | object | `{}` | | -| serviceAccount.create | bool | `true` | | -| serviceAccount.name | string | `nil` | | -| tolerations | list | `[]` | | diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt deleted file mode 100644 index 8124ab7da3da8d..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt +++ /dev/null @@ -1,21 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "datahub-mce-consumer.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "datahub-mce-consumer.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "datahub-mce-consumer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "datahub-mce-consumer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 -{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl deleted file mode 100644 index f7c47a79c45e34..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "datahub-mce-consumer.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datahub-mce-consumer.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datahub-mce-consumer.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "datahub-mce-consumer.labels" -}} -helm.sh/chart: {{ include "datahub-mce-consumer.chart" . }} -{{ include "datahub-mce-consumer.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - -{{/* -Selector labels -*/}} -{{- define "datahub-mce-consumer.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datahub-mce-consumer.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datahub-mce-consumer.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "datahub-mce-consumer.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml deleted file mode 100644 index 7d1f05c346a143..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.exporters.jmx.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter - labels: - {{- include "datahub-mce-consumer.labels" . | nindent 4 }} -data: - config.yml: |- - hostPort: localhost:{{ .Values.env.JMXPORT }} - lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }} - lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }} - rules: -{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }} - ssl: false - startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }} -{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml deleted file mode 100644 index 0c83919dd26d09..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datahub-mce-consumer.fullname" . }} - labels: - {{- include "datahub-mce-consumer.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "datahub-mce-consumer.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "datahub-mce-consumer.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.global.hostAliases }} - hostAliases: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datahub-mce-consumer.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - volumes: - {{- if .Values.global.credentialsAndCertsSecrets }} - - name: datahub-certs-dir - secret: - defaultMode: 0444 - secretName: {{ .Values.global.credentialsAndCertsSecrets.name }} - {{- end }} - {{- if .Values.exporters.jmx.enabled }} - - name: config-jmx-exporter - configMap: - name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter - {{- end }} - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - initContainers: - {{- with .Values.extraInitContainers }} - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - livenessProbe: - httpGet: - path: /actuator/health - port: 9090 - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - readinessProbe: - httpGet: - path: /actuator/health - port: 9090 - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - env: - - name: KAFKA_BOOTSTRAP_SERVER - value: "{{ .Values.global.kafka.bootstrap.server }}" - - name: KAFKA_SCHEMAREGISTRY_URL - value: "{{ .Values.global.kafka.schemaregistry.url }}" - - name: GMS_HOST - value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} - - name: GMS_PORT - value: "{{ .Values.global.datahub.gms.port }}" - {{- if .Values.global.springKafkaConfigurationOverrides }} - {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }} - - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }} - value: {{ $configValue }} - {{- end }} - {{- end }} - {{- if .Values.global.credentialsAndCertsSecrets }} - {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }} - - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }} - valueFrom: - secretKeyRef: - name: {{ $.Values.global.credentialsAndCertsSecrets.name }} - key: {{ $envVarValue }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvs }} - {{- toYaml . | nindent 12 }} - {{- end }} - volumeMounts: - {{- with .Values.global.credentialsAndCertsSecrets }} - - name: datahub-certs-dir - mountPath: {{ .path | default "/mnt/certs" }} - {{- end }} - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - - {{- include "datahub-jmxexporter.container" . }} - - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml deleted file mode 100644 index d8570643f3bf61..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datahub-mce-consumer.serviceAccountName" . }} - labels: - {{- include "datahub-mce-consumer.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml deleted file mode 100644 index cdcf61ad52f4cc..00000000000000 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml +++ /dev/null @@ -1,164 +0,0 @@ -# Default values for datahub-mce-consumer. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: linkedin/datahub-mce-consumer - pullPolicy: Always - tag: v0.8.1 - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: - -podAnnotations: {} - # co.elastic.logs/enabled: "true" - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: [] - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -# Extra environment variables -# This will be appended to the current 'env:' key. You can use any of the kubernetes env -# syntax here -extraEnvs: [] - # - name: MY_ENVIRONMENT_VAR - # value: the_value_goes_here - -extraVolumes: [] - # - name: extras - # emptyDir: {} - -extraVolumeMounts: [] - # - name: extras - # mountPath: /usr/share/extras - # readOnly: true - -extraInitContainers: [] - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -env: - JMXPORT: 1099 - -exporters: - jmx: - enabled: false -# image: -# repository: bitnami/jmx-exporter -# tag: 0.15.0 -# pullPolicy: IfNotPresent -# config: -# lowercaseOutputName: true -# rules: -# - pattern: ".*" -# startDelaySeconds: 30 -# env: {} -# resources: {} -# path: /metrics -# ports: -# jmxxp: -# containerPort: 5556 -# protocol: TCP -# livenessProbe: -# httpGet: -# path: /metrics -# port: jmxxp -# initialDelaySeconds: 30 -# periodSeconds: 15 -# timeoutSeconds: 60 -# failureThreshold: 8 -# successThreshold: 1 -# readinessProbe: -# httpGet: -# path: /metrics -# port: jmxxp -# initialDelaySeconds: 30 -# periodSeconds: 15 -# timeoutSeconds: 60 -# failureThreshold: 8 -# successThreshold: 1 -# serviceMonitor: -# interval: 30s -# scrapeTimeout: 30s -# scheme: http - -livenessProbe: - initialDelaySeconds: 60 - periodSeconds: 30 - failureThreshold: 4 - -readinessProbe: - initialDelaySeconds: 60 - periodSeconds: 30 - failureThreshold: 4 - -global: - kafka: - bootstrap: - server: "broker:9092" - schemaregistry: - url: "http://schema-registry:8081" - - datahub: - gms: - port: "8080" - - hostAliases: - - ip: "192.168.0.104" - hostnames: - - "broker" - - "mysql" - - "elasticsearch" - - "neo4j" \ No newline at end of file diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile deleted file mode 100644 index 3d40e1581a8353..00000000000000 --- a/docker/datahub-mae-consumer/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Defining environment -ARG APP_ENV=prod - -FROM openjdk:8-jre-alpine as base -ENV DOCKERIZE_VERSION v0.6.1 -RUN apk --no-cache add curl tar \ - && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv - -# Workaround alpine issue with /lib64 not being in the ld library path -# https://gitlab.alpinelinux.org/alpine/aports/-/issues/10140 -ENV LD_LIBRARY_PATH=/lib64 - -# Add glibc compat layer into alpine linux, needed by java-snappy if kafka topics are compressed with snappy -RUN apk add libc6-compat - -FROM openjdk:8 as prod-build -COPY . datahub-src -RUN cd datahub-src && ./gradlew :metadata-jobs:mae-consumer-job:build -RUN cd datahub-src && cp metadata-jobs/mae-consumer-job/build/libs/mae-consumer-job.jar ../mae-consumer-job.jar - -FROM base as prod-install -COPY --from=prod-build /mae-consumer-job.jar /datahub/datahub-mae-consumer/bin/ -COPY --from=prod-build /datahub-src/docker/datahub-mae-consumer/start.sh /datahub/datahub-mae-consumer/scripts/ -RUN chmod +x /datahub/datahub-mae-consumer/scripts/start.sh - -FROM base as dev-install -# Dummy stage for development. Assumes code is built on your machine and mounted to this image. -# See this excellent thread https://github.com/docker/cli/issues/1134 - -FROM ${APP_ENV}-install as final - -RUN addgroup -S datahub && adduser -S datahub -G datahub -USER datahub - -ENV JMX_OPTS="" -ENV JAVA_OPTS="" - -EXPOSE 9090 - -HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:9091/actuator/health || exit 1 - -CMD /datahub/datahub-mae-consumer/scripts/start.sh \ No newline at end of file diff --git a/docker/datahub-mae-consumer/README.md b/docker/datahub-mae-consumer/README.md deleted file mode 100644 index e95e8754405246..00000000000000 --- a/docker/datahub-mae-consumer/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# DataHub MetadataAuditEvent (MAE) Consumer Docker Image -[![datahub-mae-consumer docker](https://github.com/linkedin/datahub/workflows/datahub-mae-consumer%20docker/badge.svg)](https://github.com/linkedin/datahub/actions?query=workflow%3A%22datahub-mae-consumer+docker%22) - -Refer to [DataHub MAE Consumer Job](../../metadata-jobs/mae-consumer-job) to have a quick understanding of the architecture and -responsibility of this service for the DataHub. diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env deleted file mode 100644 index 290e4df515257a..00000000000000 --- a/docker/datahub-mae-consumer/env/docker.env +++ /dev/null @@ -1,29 +0,0 @@ -KAFKA_BOOTSTRAP_SERVER=broker:29092 -KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 -ELASTICSEARCH_HOST=elasticsearch -ELASTICSEARCH_PORT=9200 -NEO4J_HOST=http://neo4j:7474 -NEO4J_URI=bolt://neo4j -NEO4J_USERNAME=neo4j -NEO4J_PASSWORD=datahub -GMS_HOST=datahub-gms -GMS_PORT=8080 - -# Uncomment to disable persistence of client-side analytics events -# DATAHUB_ANALYTICS_ENABLED=false - -# Uncomment to configure topic names -# Make sure these names are consistent across the whole deployment -# KAFKA_TOPIC_NAME=MetadataAuditEvent_v4 -# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 - -# Uncomment and set these to support SSL connection to Elasticsearch -# ELASTICSEARCH_USE_SSL= -# ELASTICSEARCH_SSL_PROTOCOL= -# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL= -# ELASTICSEARCH_SSL_TRUSTSTORE_FILE= -# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE= -# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD= -# ELASTICSEARCH_SSL_KEYSTORE_FILE= -# ELASTICSEARCH_SSL_KEYSTORE_TYPE= -# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= \ No newline at end of file diff --git a/docker/datahub-mae-consumer/start.sh b/docker/datahub-mae-consumer/start.sh deleted file mode 100755 index 8acea00366a4b4..00000000000000 --- a/docker/datahub-mae-consumer/start.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -# Add default URI (http) scheme if needed -if ! echo $NEO4J_HOST | grep -q "://" ; then - NEO4J_HOST="http://$NEO4J_HOST" -fi - -if [[ -z $ELASTICSEARCH_USERNAME ]]; then - ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST -else - ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD@$ELASTICSEARCH_HOST -fi - -if [[ $ELASTICSEARCH_USE_SSL == true ]]; then - ELASTICSEARCH_PROTOCOL=https -else - ELASTICSEARCH_PROTOCOL=http -fi - -dockerize \ - -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ - -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT \ - -wait $NEO4J_HOST \ - -timeout 240s \ - java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar \ No newline at end of file diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile deleted file mode 100644 index e6f9affa58023a..00000000000000 --- a/docker/datahub-mce-consumer/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Defining environment -ARG APP_ENV=prod - -FROM openjdk:8-jre-alpine as base -ENV DOCKERIZE_VERSION v0.6.1 -RUN apk --no-cache add curl tar \ - && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv - -# Workaround alpine issue with /lib64 not being in the ld library path -# https://gitlab.alpinelinux.org/alpine/aports/-/issues/10140 -ENV LD_LIBRARY_PATH=/lib64 - -# Add glibc compat layer into alpine linux, needed by java-snappy if kafka topics are compressed with snappy -RUN apk add libc6-compat - -FROM openjdk:8 as prod-build -COPY . datahub-src -RUN cd datahub-src && ./gradlew :metadata-jobs:mce-consumer-job:build -RUN cd datahub-src && cp metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar ../mce-consumer-job.jar - -FROM base as prod-install -COPY --from=prod-build /mce-consumer-job.jar /datahub/datahub-mce-consumer/bin/ -COPY --from=prod-build /datahub-src/docker/datahub-mce-consumer/start.sh /datahub/datahub-mce-consumer/scripts/ -RUN chmod +x /datahub/datahub-mce-consumer/scripts/start.sh - -FROM base as dev-install -# Dummy stage for development. Assumes code is built on your machine and mounted to this image. -# See this excellent thread https://github.com/docker/cli/issues/1134 - -FROM ${APP_ENV}-install as final - -RUN addgroup -S datahub && adduser -S datahub -G datahub -USER datahub - -ENV JMX_OPTS="" -ENV JAVA_OPTS="" - -EXPOSE 9090 - -HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:9090/actuator/health || exit 1 - -CMD /datahub/datahub-mce-consumer/scripts/start.sh \ No newline at end of file diff --git a/docker/datahub-mce-consumer/README.md b/docker/datahub-mce-consumer/README.md deleted file mode 100644 index e4a05fe85110be..00000000000000 --- a/docker/datahub-mce-consumer/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# DataHub MetadataChangeEvent (MCE) Consumer Docker Image -[![datahub-mce-consumer docker](https://github.com/linkedin/datahub/workflows/datahub-mce-consumer%20docker/badge.svg)](https://github.com/linkedin/datahub/actions?query=workflow%3A%22datahub-mce-consumer+docker%22) - -Refer to [DataHub MCE Consumer Job](../../metadata-jobs/mce-consumer-job) to have a quick understanding of the architecture and -responsibility of this service for the DataHub. diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env deleted file mode 100644 index c89f317a921140..00000000000000 --- a/docker/datahub-mce-consumer/env/docker.env +++ /dev/null @@ -1,15 +0,0 @@ -KAFKA_BOOTSTRAP_SERVER=broker:29092 -KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 -GMS_HOST=datahub-gms -GMS_PORT=8080 - -# Uncomment to configure kafka topic names -# Make sure these names are consistent across the whole deployment -# KAFKA_MCE_TOPIC_NAME=MetadataChangeEvent_v4 -# KAFKA_FMCE_TOPIC_NAME=FailedMetadataChangeEvent_v4 - -# Uncomment and set these to support SSL connection to GMS -# NOTE: Currently GMS itself does not offer SSL support, these settings are intended for when there is a proxy in front -# of GMS that handles SSL, such as an EC2 Load Balancer. -#GMS_USE_SSL=true -#GMS_SSL_PROTOCOL= \ No newline at end of file diff --git a/docker/datahub-mce-consumer/start.sh b/docker/datahub-mce-consumer/start.sh deleted file mode 100755 index 6bbfeeee08867c..00000000000000 --- a/docker/datahub-mce-consumer/start.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -dockerize \ - -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ - -timeout 240s \ - java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar \ No newline at end of file diff --git a/metadata-jobs/mae-consumer-job/README.md b/metadata-jobs/mae-consumer-job/README.md index 08bf7a2bde5ced..664e288a16d8c2 100644 --- a/metadata-jobs/mae-consumer-job/README.md +++ b/metadata-jobs/mae-consumer-job/README.md @@ -22,18 +22,7 @@ However, if you only want to build `MAE Consumer Job` specifically: Before starting `MAE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup), [Elasticsearch](../../docker/elasticsearch), and [Neo4j](../../docker/neo4j) Docker containers are up and running. -## Start via Docker image -Quickest way to try out `MAE Consumer Job` is running the [Docker image](../../docker/datahub-mae-consumer). - -## Start via command line -If you do modify things and want to try it out quickly without building the Docker image, you can also run -the application directly from command line after a successful [build](#build): -``` -./gradlew :metadata-jobs:mae-consumer-job:bootRun -``` - -## Endpoints -Spring boot actuator has been enabled for MAE Application. +## Endpoints TODO: Reevaluate these `healthcheck`, `metrics` and `info` web endpoints are enabled by default. `healthcheck` - http://localhost:9091/actuator/health diff --git a/metadata-jobs/mce-consumer-job/README.md b/metadata-jobs/mce-consumer-job/README.md index 0aefe70d75bad2..4ca44f76336098 100644 --- a/metadata-jobs/mce-consumer-job/README.md +++ b/metadata-jobs/mce-consumer-job/README.md @@ -22,29 +22,9 @@ However, if you only want to build `MCE Consumer Job` specifically: ## Dependencies Before starting `MCE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup) -and [DataHub GMS](../../docker/datahub-gms) Docker containers are up and running. - -## Start via Docker image -Quickest way to try out `MCE Consumer Job` is running the [Docker image](../../docker/datahub-mce-consumer). - -## Start via command line -If you do modify things and want to try it out quickly without building the Docker image, you can also run -the application directly from command line after a successful [build](#build): -``` -./gradlew :metadata-jobs:mce-consumer-job:bootRun -``` - -## Debugging - -To debug with an IDE (i.e. IntelliJ), run the `bootRun` task with the `--debug-jvm` flag. This will launch the app and -listen on port 5005 for a remote debugger. - -``` -./gradlew :metadata-jobs:mce-consumer-job:bootRun --debug-jvm -``` +Docker containers are up and running. MCE Consumer Job runs as part of the `datahub-gms` container. ## Endpoints -Spring boot actuator has been enabled for MCE Application. `healthcheck`, `metrics` and `info` web endpoints are enabled by default. `healthcheck` - http://localhost:9090/actuator/health From cb743fe6a64b04b7fe054c94b098e984e8bd9723 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Thu, 10 Jun 2021 09:50:55 -0700 Subject: [PATCH 03/14] Removing mae-consumer mce-consumer from compose files --- docker/docker-compose.dev.yml | 22 ---------------------- docker/docker-compose.yml | 29 ----------------------------- 2 files changed, 51 deletions(-) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index dc82594fa38e16..fe6997ba904a9b 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,25 +50,3 @@ services: APP_ENV: dev volumes: - ../datahub-frontend/build/stage/datahub-frontend:/datahub-frontend - - datahub-mae-consumer: - image: linkedin/datahub-mae-consumer:debug - build: - context: datahub-mae-consumer - dockerfile: Dockerfile - args: - APP_ENV: dev - volumes: - - ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh - - ../metadata-jobs/mae-consumer-job/build/libs/:/datahub/datahub-mae-consumer/bin/ - - datahub-mce-consumer: - image: linkedin/datahub-mce-consumer:debug - build: - context: datahub-mce-consumer - dockerfile: Dockerfile - args: - APP_ENV: dev - volumes: - - ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh - - ../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5e38f20eda7fa3..74ab7845c7ca77 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -160,35 +160,6 @@ services: depends_on: - datahub-gms - datahub-mae-consumer: - build: - context: ../ - dockerfile: docker/datahub-mae-consumer/Dockerfile - image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest} - env_file: datahub-mae-consumer/env/docker.env - hostname: datahub-mae-consumer - container_name: datahub-mae-consumer - ports: - - "9091:9091" - depends_on: - - kafka-setup - - elasticsearch-setup - - neo4j - - datahub-mce-consumer: - build: - context: ../ - dockerfile: docker/datahub-mce-consumer/Dockerfile - image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest} - env_file: datahub-mce-consumer/env/docker.env - hostname: datahub-mce-consumer - container_name: datahub-mce-consumer - ports: - - "9090:9090" - depends_on: - - kafka-setup - - datahub-gms - networks: default: name: datahub_network From 08f40a3f7aee4b0759cc90e6944dacb87b1a1224 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Thu, 10 Jun 2021 09:51:21 -0700 Subject: [PATCH 04/14] Updating docs --- docker/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 5bcb5ca4310ab1..948bb7d6aa38b1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -25,8 +25,6 @@ DataHub Docker Images: * [linkedin/datahub-gms](https://cloud.docker.com/repository/docker/linkedin/datahub-gms/) * [linkedin/datahub-frontend-react](https://cloud.docker.com/repository/docker/linkedin/datahub-frontend-react/) -* [linkedin/datahub-mae-consumer](https://cloud.docker.com/repository/docker/linkedin/datahub-mae-consumer/) -* [linkedin/datahub-mce-consumer](https://cloud.docker.com/repository/docker/linkedin/datahub-mce-consumer/) Dependencies: * [Kafka, Zookeeper, and Schema Registry](kafka-setup) From 2f8b272ac0cc82e62595947535ddc18243101b5d Mon Sep 17 00:00:00 2001 From: John Joyce Date: Thu, 10 Jun 2021 18:26:19 -0700 Subject: [PATCH 05/14] Addressing comments etc --- .github/workflows/docker-mae-consumer.yml | 73 ++++++++ .github/workflows/docker-mce-consumer.yml | 73 ++++++++ .../charts/datahub-mae-consumer/.helmignore | 23 +++ .../charts/datahub-mae-consumer/Chart.yaml | 21 +++ .../charts/datahub-mae-consumer/README.md | 59 ++++++ .../datahub-mae-consumer/templates/NOTES.txt | 21 +++ .../templates/_helpers.tpl | 63 +++++++ .../templates/config-jmx-exporter.yaml | 17 ++ .../templates/deployment.yaml | 154 +++++++++++++++ .../templates/serviceaccount.yaml | 12 ++ .../charts/datahub-mae-consumer/values.yaml | 175 ++++++++++++++++++ .../charts/datahub-mce-consumer/.helmignore | 23 +++ .../charts/datahub-mce-consumer/Chart.yaml | 21 +++ .../charts/datahub-mce-consumer/README.md | 52 ++++++ .../datahub-mce-consumer/templates/NOTES.txt | 21 +++ .../templates/_helpers.tpl | 63 +++++++ .../templates/config-jmx-exporter.yaml | 17 ++ .../templates/deployment.yaml | 122 ++++++++++++ .../templates/serviceaccount.yaml | 12 ++ .../charts/datahub-mce-consumer/values.yaml | 164 ++++++++++++++++ .../datahub/quickstart-values.yaml | 4 +- datahub-kubernetes/datahub/values.yaml | 4 +- docker/datahub-gms/env/docker.env | 2 + docker/datahub-gms/env/docker.mariadb.env | 1 + docker/datahub-gms/env/docker.postgres.env | 1 + docker/datahub-mae-consumer/Dockerfile | 42 +++++ docker/datahub-mae-consumer/README.md | 5 + docker/datahub-mae-consumer/env/docker.env | 29 +++ docker/datahub-mae-consumer/start.sh | 25 +++ docker/datahub-mce-consumer/Dockerfile | 42 +++++ docker/datahub-mce-consumer/README.md | 5 + docker/datahub-mce-consumer/env/docker.env | 15 ++ docker/datahub-mce-consumer/start.sh | 6 + .../entity/ebean/EbeanEntityService.java | 52 ++---- metadata-jobs/mae-consumer-job/build.gradle | 54 +----- .../kafka/MaeConsumerApplication.java | 15 ++ .../.gitignore | 0 .../README.md | 0 metadata-jobs/mae-consumer/build.gradle | 57 ++++++ .../kafka/DataHubUsageEventsProcessor.java | 0 .../metadata/kafka/MaeConsumerConfig.java | 0 .../kafka/MetadataAuditEventsProcessor.java | 0 .../kafka/config/ElasticSearchConfig.java | 0 .../kafka/config/HydratorFactoryConfig.java | 0 .../kafka/config/IndexBuildersConfig.java | 0 .../metadata/kafka/config/MaeKafkaConfig.java | 0 ...MetadataAuditEventsProcessorCondition.java | 18 ++ .../kafka/elasticsearch/ElasticEvent.java | 0 .../elasticsearch/ElasticsearchConnector.java | 0 .../ElasticsearchConnectorFactory.java | 0 .../kafka/elasticsearch/JsonElasticEvent.java | 0 .../kafka/elasticsearch/MCEElasticEvent.java | 0 .../kafka/hydrator/ChartHydrator.java | 0 .../kafka/hydrator/CorpUserHydrator.java | 0 .../kafka/hydrator/DashboardHydrator.java | 0 .../kafka/hydrator/DataFlowHydrator.java | 0 .../kafka/hydrator/DataJobHydrator.java | 0 .../kafka/hydrator/DatasetHydrator.java | 0 .../metadata/kafka/hydrator/EntityType.java | 0 .../metadata/kafka/hydrator/Hydrator.java | 0 .../kafka/hydrator/HydratorFactory.java | 0 .../DataHubUsageEventConstants.java | 0 .../DataHubUsageEventTransformer.java | 0 .../transformer/DataHubUsageEventType.java | 0 .../src/main/resources/application.properties | 0 .../src/main/resources/logback.xml | 0 metadata-jobs/mce-consumer-job/build.gradle | 51 +---- .../kafka/MceConsumerApplication.java | 15 ++ .../.gitignore | 0 .../README.md | 0 metadata-jobs/mce-consumer/build.gradle | 52 ++++++ .../kafka/MetadataChangeEventsProcessor.java | 3 + .../kafka/config/EntityClientConfig.java | 0 .../metadata/kafka/config/MceKafkaConfig.java | 0 ...etadataChangeEventsProcessorCondition.java | 18 ++ .../src/main/resources/application.properties | 0 .../src/main/resources/logback.xml | 0 settings.gradle | 2 + 78 files changed, 1569 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/docker-mae-consumer.yml create mode 100644 .github/workflows/docker-mce-consumer.yml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml create mode 100644 datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml create mode 100644 docker/datahub-mae-consumer/Dockerfile create mode 100644 docker/datahub-mae-consumer/README.md create mode 100644 docker/datahub-mae-consumer/env/docker.env create mode 100755 docker/datahub-mae-consumer/start.sh create mode 100644 docker/datahub-mce-consumer/Dockerfile create mode 100644 docker/datahub-mce-consumer/README.md create mode 100644 docker/datahub-mce-consumer/env/docker.env create mode 100755 docker/datahub-mce-consumer/start.sh create mode 100644 metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java rename metadata-jobs/{mae-consumer-job => mae-consumer}/.gitignore (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/README.md (100%) create mode 100644 metadata-jobs/mae-consumer/build.gradle rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/MaeConsumerConfig.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/config/ElasticSearchConfig.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/config/HydratorFactoryConfig.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java (100%) create mode 100644 metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/DatasetHydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityType.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/Hydrator.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventConstants.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventType.java (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/resources/application.properties (100%) rename metadata-jobs/{mae-consumer-job => mae-consumer}/src/main/resources/logback.xml (100%) create mode 100644 metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java rename metadata-jobs/{mce-consumer-job => mce-consumer}/.gitignore (100%) rename metadata-jobs/{mce-consumer-job => mce-consumer}/README.md (100%) create mode 100644 metadata-jobs/mce-consumer/build.gradle rename metadata-jobs/{mce-consumer-job => mce-consumer}/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java (94%) rename metadata-jobs/{mce-consumer-job => mce-consumer}/src/main/java/com/linkedin/metadata/kafka/config/EntityClientConfig.java (100%) rename metadata-jobs/{mce-consumer-job => mce-consumer}/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java (100%) create mode 100644 metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java rename metadata-jobs/{mce-consumer-job => mce-consumer}/src/main/resources/application.properties (100%) rename metadata-jobs/{mce-consumer-job => mce-consumer}/src/main/resources/logback.xml (100%) diff --git a/.github/workflows/docker-mae-consumer.yml b/.github/workflows/docker-mae-consumer.yml new file mode 100644 index 00000000000000..3c98cb55f94c09 --- /dev/null +++ b/.github/workflows/docker-mae-consumer.yml @@ -0,0 +1,73 @@ +name: datahub-mae-consumer docker +on: + push: + branches: + - master + paths-ignore: + - 'docs/**' + - '**.md' + pull_request: + branches: + - master + paths: + - 'docker/**' + - '.github/workflows/docker-mae-consumer.yml' + paths_ignore: + - '**.md' + - '**.env' + release: + types: [published, edited] + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + publish: ${{ steps.publish.outputs.publish }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Compute Tag + id: tag + run: | + echo "GITHUB_REF: $GITHUB_REF" + SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") + TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g') + echo "tag=$TAG" + echo "::set-output name=tag::$TAG" + - name: Check whether publishing enabled + id: publish + env: + ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}" + echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}" + push_to_registries: + name: Build and Push Docker Image to Docker Hub + runs-on: ubuntu-latest + if: ${{ needs.setup.outputs.publish == 'true' }} + needs: setup + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + # list of Docker images to use as base name for tags + images: | + linkedin/datahub-mae-consumer + # add git short SHA as Docker tag + tag-custom: ${{ needs.setup.outputs.tag }} + tag-custom-only: true + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and Push image + uses: docker/build-push-action@v2 + with: + file: ./docker/datahub-mae-consumer/Dockerfile + tags: ${{ steps.docker_meta.outputs.tags }} + push: ${{ needs.setup.outputs.publish == 'true' }} diff --git a/.github/workflows/docker-mce-consumer.yml b/.github/workflows/docker-mce-consumer.yml new file mode 100644 index 00000000000000..e1034b1069546e --- /dev/null +++ b/.github/workflows/docker-mce-consumer.yml @@ -0,0 +1,73 @@ +name: datahub-mce-consumer docker +on: + push: + branches: + - master + paths-ignore: + - 'docs/**' + - '**.md' + pull_request: + branches: + - master + paths: + - 'docker/**' + - '.github/workflows/docker-mce-consumer.yml' + paths_ignore: + - '**.md' + - '**.env' + release: + types: [published, edited] + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + publish: ${{ steps.publish.outputs.publish }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Compute Tag + id: tag + run: | + echo "GITHUB_REF: $GITHUB_REF" + SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") + TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g') + echo "tag=$TAG" + echo "::set-output name=tag::$TAG" + - name: Check whether publishing enabled + id: publish + env: + ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}" + echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}" + push_to_registries: + name: Build and Push Docker Image to Docker Hub + runs-on: ubuntu-latest + if: ${{ needs.setup.outputs.publish == 'true' }} + needs: setup + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + # list of Docker images to use as base name for tags + images: | + linkedin/datahub-mce-consumer + # add git short SHA as Docker tag + tag-custom: ${{ needs.setup.outputs.tag }} + tag-custom-only: true + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and Push image + uses: docker/build-push-action@v2 + with: + file: ./docker/datahub-mce-consumer/Dockerfile + tags: ${{ steps.docker_meta.outputs.tags }} + push: ${{ needs.setup.outputs.publish == 'true' }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore new file mode 100644 index 00000000000000..0e8a0eb36f4ca2 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml new file mode 100644 index 00000000000000..4e5116696dc366 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: datahub-mae-consumer +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.2.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 0.3.1 diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md new file mode 100644 index 00000000000000..13d48dc2d12c96 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md @@ -0,0 +1,59 @@ +datahub-mae-consumer +==================== +A Helm chart for datahub-mae-consumer + +Current chart version is `0.2.0` + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| exporters.jmx.enabled | boolean | false | | +| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | +| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | +| fullnameOverride | string | `"datahub-mae-consumer"` | | +| global.datahub_analytics_enabled | boolean | true | | +| global.elasticsearch.host | string | `"elasticsearch"` | | +| global.elasticsearch.port | string | `"9200"` | | +| global.kafka.bootstrap.server | string | `"broker:9092"` | | +| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` | | +| global.neo4j.host | string | `"neo4j:7474"` | | +| global.neo4j.uri | string | `"bolt://neo4j"` | | +| global.neo4j.username | string | `"neo4j"` | | +| global.neo4j.password.secretRef | string | `"neo4j-secrets"` | | +| global.neo4j.password.secretKey | string | `"neo4j-password"` | | +| global.hostAliases[0].hostnames[0] | string | `"broker"` | | +| global.hostAliases[0].hostnames[1] | string | `"mysql"` | | +| global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` | | +| global.hostAliases[0].hostnames[3] | string | `"neo4j"` | | +| global.hostAliases[0].ip | string | `"192.168.0.104"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"linkedin/datahub-mae-consumer"` | | +| image.tag | string | `"latest"` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths | list | `[]` | | +| ingress.tls | list | `[]` | | +| livenessProbe.initialDelaySeconds | int | `60` | | +| livenessProbe.periodSeconds | int | `30` | | +| livenessProbe.failureThreshold | int | `8` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| readinessProbe.initialDelaySeconds | int | `60` | | +| readinessProbe.periodSeconds | int | `30` | | +| readinessProbe.failureThreshold | int | `8` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `nil` | | +| tolerations | list | `[]` | | diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt new file mode 100644 index 00000000000000..6f2f355847e9d0 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "datahub-mae-consumer.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "datahub-mae-consumer.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "datahub-mae-consumer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "datahub-mae-consumer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl new file mode 100644 index 00000000000000..e677e252a6270c --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "datahub-mae-consumer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "datahub-mae-consumer.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "datahub-mae-consumer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "datahub-mae-consumer.labels" -}} +helm.sh/chart: {{ include "datahub-mae-consumer.chart" . }} +{{ include "datahub-mae-consumer.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "datahub-mae-consumer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "datahub-mae-consumer.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "datahub-mae-consumer.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "datahub-mae-consumer.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml new file mode 100644 index 00000000000000..72431b713e836e --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/config-jmx-exporter.yaml @@ -0,0 +1,17 @@ +{{- if .Values.exporters.jmx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter + labels: + {{- include "datahub-mae-consumer.labels" . | nindent 4 }} +data: + config.yml: |- + hostPort: localhost:{{ .Values.env.JMXPORT }} + lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }} + lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }} + rules: +{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }} + ssl: false + startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }} +{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml new file mode 100644 index 00000000000000..adb550b6f27de0 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml @@ -0,0 +1,154 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datahub-mae-consumer.fullname" . }} + labels: + {{- include "datahub-mae-consumer.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "datahub-mae-consumer.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "datahub-mae-consumer.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.global.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datahub-mae-consumer.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- with .Values.global.credentialsAndCertsSecrets }} + - name: datahub-certs-dir + secret: + defaultMode: 0444 + secretName: {{ .name }} + {{- end }} + {{- if .Values.exporters.jmx.enabled }} + - name: config-jmx-exporter + configMap: + name: {{ include "datahub-mae-consumer.fullname" . }}-config-jmx-exporter + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + httpGet: + path: /actuator/health + port: 9091 + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + readinessProbe: + httpGet: + path: /actuator/health + port: 9091 + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + env: + - name: GMS_HOST + value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} + - name: GMS_PORT + value: "{{ .Values.global.datahub.gms.port }}" + - name: KAFKA_BOOTSTRAP_SERVER + value: "{{ .Values.global.kafka.bootstrap.server }}" + - name: KAFKA_SCHEMAREGISTRY_URL + value: "{{ .Values.global.kafka.schemaregistry.url }}" + - name: ELASTICSEARCH_HOST + value: "{{ .Values.global.elasticsearch.host }}" + - name: ELASTICSEARCH_PORT + value: "{{ .Values.global.elasticsearch.port }}" + {{- with .Values.global.elasticsearch.useSSL }} + - name: ELASTICSEARCH_USE_SSL + value: {{ . | quote }} + {{- end }} + {{- with .Values.global.elasticsearch.auth }} + - name: ELASTICSEARCH_USERNAME + value: {{ .username }} + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ .password.secretRef }}" + key: "{{ .password.secretKey }}" + {{- end }} + - name: NEO4J_HOST + value: "{{ .Values.global.neo4j.host }}" + - name: NEO4J_URI + value: "{{ .Values.global.neo4j.uri }}" + - name: NEO4J_USERNAME + value: "{{ .Values.global.neo4j.username }}" + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Values.global.neo4j.password.secretRef }}" + key: "{{ .Values.global.neo4j.password.secretKey }}" + - name: DATAHUB_ANALYTICS_ENABLED + value: "{{ .Values.global.datahub_analytics_enabled }}" + - name: GMS_MODE + value: "{{ .Values.global.datahub_gms_mode }}" + {{- if .Values.global.springKafkaConfigurationOverrides }} + {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }} + - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }} + value: {{ $configValue }} + {{- end }} + {{- end }} + {{- if .Values.global.credentialsAndCertsSecrets }} + {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }} + - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }} + valueFrom: + secretKeyRef: + name: {{ $.Values.global.credentialsAndCertsSecrets.name }} + key: {{ $envVarValue }} + {{- end }} + {{- end }} + {{- with .Values.extraEnvs }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- with .Values.global.credentialsAndCertsSecrets }} + - name: datahub-certs-dir + mountPath: {{ .path | default "/mnt/certs" }} + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + + {{- include "datahub-jmxexporter.container" . }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml new file mode 100644 index 00000000000000..14e6c5f6ced0ae --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datahub-mae-consumer.serviceAccountName" . }} + labels: + {{- include "datahub-mae-consumer.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml new file mode 100644 index 00000000000000..093d25ff4f5eba --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml @@ -0,0 +1,175 @@ +# Default values for datahub-mae-consumer. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: linkedin/datahub-mae-consumer + pullPolicy: Always + tag: latest + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +podAnnotations: {} + # co.elastic.logs/enabled: "true" + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +# Extra environment variables +# This will be appended to the current 'env:' key. You can use any of the kubernetes env +# syntax here +extraEnvs: [] + # - name: MY_ENVIRONMENT_VAR + # value: the_value_goes_here + +extraVolumes: [] + # - name: extras + # emptyDir: {} + +extraVolumeMounts: [] + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true + +extraInitContainers: [] + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +env: + JMXPORT: 1099 + +exporters: + jmx: + enabled: false +# image: +# repository: bitnami/jmx-exporter +# tag: 0.15.0 +# pullPolicy: IfNotPresent +# config: +# lowercaseOutputName: true +# rules: +# - pattern: ".*" +# startDelaySeconds: 30 +# env: {} +# resources: {} +# path: /metrics +# ports: +# jmxxp: +# containerPort: 5556 +# protocol: TCP +# livenessProbe: +# httpGet: +# path: /metrics +# port: jmxxp +# initialDelaySeconds: 30 +# periodSeconds: 15 +# timeoutSeconds: 60 +# failureThreshold: 8 +# successThreshold: 1 +# readinessProbe: +# httpGet: +# path: /metrics +# port: jmxxp +# initialDelaySeconds: 30 +# periodSeconds: 15 +# timeoutSeconds: 60 +# failureThreshold: 8 +# successThreshold: 1 +# serviceMonitor: +# interval: 30s +# scrapeTimeout: 30s +# scheme: http + +livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 8 + +readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 8 + +global: + datahub_analytics_enabled: true + datahub_gms_mode: "SINGLE_NODE" + + elasticsearch: + host: "elasticsearch" + port: "9200" + + kafka: + bootstrap: + server: "broker:9092" + schemaregistry: + url: "http://schema-registry:8081" + + neo4j: + host: "neo4j:7474" + uri: "bolt://neo4j" + username: "neo4j" + password: + secretRef: "neo4j-secrets" + secretKey: "neo4j-password" + + hostAliases: + - ip: "192.168.0.104" + hostnames: + - "broker" + - "mysql" + - "elasticsearch" + - "neo4j" \ No newline at end of file diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore new file mode 100644 index 00000000000000..0e8a0eb36f4ca2 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml new file mode 100644 index 00000000000000..bd00d74aa261ae --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: datahub-mce-consumer +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.2.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 0.3.1 diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md new file mode 100644 index 00000000000000..693e5b640fd669 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md @@ -0,0 +1,52 @@ +datahub-mce-consumer +==================== +A Helm chart for datahub-mce-consumer + +Current chart version is `0.2.0` + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| exporters.jmx.enabled | boolean | false | | +| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" | +| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" | +| fullnameOverride | string | `""` | | +| global.kafka.bootstrap.server | string | `"broker:9092"` | | +| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` | | +| global.datahub.gms.port | string | `"8080"` | | +| global.hostAliases[0].hostnames[0] | string | `"broker"` | | +| global.hostAliases[0].hostnames[1] | string | `"mysql"` | | +| global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` | | +| global.hostAliases[0].hostnames[3] | string | `"neo4j"` | | +| global.hostAliases[0].ip | string | `"192.168.0.104"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"linkedin/datahub-mce-consumer"` | | +| image.tag | string | `"latest"` | | +| imagePullSecrets | list | `[]` | | +| ingress.annotations | object | `{}` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths | list | `[]` | | +| ingress.tls | list | `[]` | | +| livenessProbe.initialDelaySeconds | int | `60` | | +| livenessProbe.periodSeconds | int | `30` | | +| livenessProbe.failureThreshold | int | `4` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podSecurityContext | object | `{}` | | +| readinessProbe.initialDelaySeconds | int | `60` | | +| readinessProbe.periodSeconds | int | `30` | | +| readinessProbe.failureThreshold | int | `4` | | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| securityContext | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `nil` | | +| tolerations | list | `[]` | | diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt new file mode 100644 index 00000000000000..8124ab7da3da8d --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "datahub-mce-consumer.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "datahub-mce-consumer.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "datahub-mce-consumer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "datahub-mce-consumer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl new file mode 100644 index 00000000000000..f7c47a79c45e34 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "datahub-mce-consumer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "datahub-mce-consumer.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "datahub-mce-consumer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "datahub-mce-consumer.labels" -}} +helm.sh/chart: {{ include "datahub-mce-consumer.chart" . }} +{{ include "datahub-mce-consumer.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "datahub-mce-consumer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "datahub-mce-consumer.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "datahub-mce-consumer.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "datahub-mce-consumer.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml new file mode 100644 index 00000000000000..7d1f05c346a143 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/config-jmx-exporter.yaml @@ -0,0 +1,17 @@ +{{- if .Values.exporters.jmx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter + labels: + {{- include "datahub-mce-consumer.labels" . | nindent 4 }} +data: + config.yml: |- + hostPort: localhost:{{ .Values.env.JMXPORT }} + lowercaseOutputName: {{ .Values.exporters.jmx.config.lowercaseOutputName }} + lowercaseOutputLabelNames: {{ .Values.exporters.jmx.config.lowercaseOutputLabelNames }} + rules: +{{ .Values.exporters.jmx.config.rules | toYaml | indent 6 }} + ssl: false + startDelaySeconds: {{ .Values.exporters.jmx.config.startDelaySeconds }} +{{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml new file mode 100644 index 00000000000000..0c83919dd26d09 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml @@ -0,0 +1,122 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datahub-mce-consumer.fullname" . }} + labels: + {{- include "datahub-mce-consumer.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "datahub-mce-consumer.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "datahub-mce-consumer.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.global.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datahub-mce-consumer.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.global.credentialsAndCertsSecrets }} + - name: datahub-certs-dir + secret: + defaultMode: 0444 + secretName: {{ .Values.global.credentialsAndCertsSecrets.name }} + {{- end }} + {{- if .Values.exporters.jmx.enabled }} + - name: config-jmx-exporter + configMap: + name: {{ include "datahub-mce-consumer.fullname" . }}-config-jmx-exporter + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + httpGet: + path: /actuator/health + port: 9090 + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + readinessProbe: + httpGet: + path: /actuator/health + port: 9090 + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + env: + - name: KAFKA_BOOTSTRAP_SERVER + value: "{{ .Values.global.kafka.bootstrap.server }}" + - name: KAFKA_SCHEMAREGISTRY_URL + value: "{{ .Values.global.kafka.schemaregistry.url }}" + - name: GMS_HOST + value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} + - name: GMS_PORT + value: "{{ .Values.global.datahub.gms.port }}" + {{- if .Values.global.springKafkaConfigurationOverrides }} + {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }} + - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }} + value: {{ $configValue }} + {{- end }} + {{- end }} + {{- if .Values.global.credentialsAndCertsSecrets }} + {{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }} + - name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }} + valueFrom: + secretKeyRef: + name: {{ $.Values.global.credentialsAndCertsSecrets.name }} + key: {{ $envVarValue }} + {{- end }} + {{- end }} + {{- with .Values.extraEnvs }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- with .Values.global.credentialsAndCertsSecrets }} + - name: datahub-certs-dir + mountPath: {{ .path | default "/mnt/certs" }} + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + + {{- include "datahub-jmxexporter.container" . }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml new file mode 100644 index 00000000000000..d8570643f3bf61 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datahub-mce-consumer.serviceAccountName" . }} + labels: + {{- include "datahub-mce-consumer.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml new file mode 100644 index 00000000000000..b7734141bf4a76 --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml @@ -0,0 +1,164 @@ +# Default values for datahub-mce-consumer. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: linkedin/datahub-mce-consumer + pullPolicy: Always + tag: latest + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +podAnnotations: {} + # co.elastic.logs/enabled: "true" + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +# Extra environment variables +# This will be appended to the current 'env:' key. You can use any of the kubernetes env +# syntax here +extraEnvs: [] + # - name: MY_ENVIRONMENT_VAR + # value: the_value_goes_here + +extraVolumes: [] + # - name: extras + # emptyDir: {} + +extraVolumeMounts: [] + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true + +extraInitContainers: [] + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +env: + JMXPORT: 1099 + +exporters: + jmx: + enabled: false +# image: +# repository: bitnami/jmx-exporter +# tag: 0.15.0 +# pullPolicy: IfNotPresent +# config: +# lowercaseOutputName: true +# rules: +# - pattern: ".*" +# startDelaySeconds: 30 +# env: {} +# resources: {} +# path: /metrics +# ports: +# jmxxp: +# containerPort: 5556 +# protocol: TCP +# livenessProbe: +# httpGet: +# path: /metrics +# port: jmxxp +# initialDelaySeconds: 30 +# periodSeconds: 15 +# timeoutSeconds: 60 +# failureThreshold: 8 +# successThreshold: 1 +# readinessProbe: +# httpGet: +# path: /metrics +# port: jmxxp +# initialDelaySeconds: 30 +# periodSeconds: 15 +# timeoutSeconds: 60 +# failureThreshold: 8 +# successThreshold: 1 +# serviceMonitor: +# interval: 30s +# scrapeTimeout: 30s +# scheme: http + +livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 4 + +readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 4 + +global: + kafka: + bootstrap: + server: "broker:9092" + schemaregistry: + url: "http://schema-registry:8081" + + datahub: + gms: + port: "8080" + + hostAliases: + - ip: "192.168.0.104" + hostnames: + - "broker" + - "mysql" + - "elasticsearch" + - "neo4j" \ No newline at end of file diff --git a/datahub-kubernetes/datahub/quickstart-values.yaml b/datahub-kubernetes/datahub/quickstart-values.yaml index 130ad93deb828a..4eee21e2225be7 100644 --- a/datahub-kubernetes/datahub/quickstart-values.yaml +++ b/datahub-kubernetes/datahub/quickstart-values.yaml @@ -16,13 +16,13 @@ datahub-frontend: enabled: false datahub-mae-consumer: - enabled: true + enabled: false image: repository: linkedin/datahub-mae-consumer tag: "v0.8.1" datahub-mce-consumer: - enabled: true + enabled: false image: repository: linkedin/datahub-mce-consumer tag: "v0.8.1" diff --git a/datahub-kubernetes/datahub/values.yaml b/datahub-kubernetes/datahub/values.yaml index be3592fff32041..41deac12370b65 100644 --- a/datahub-kubernetes/datahub/values.yaml +++ b/datahub-kubernetes/datahub/values.yaml @@ -16,13 +16,13 @@ datahub-frontend: enabled: false datahub-mae-consumer: - enabled: true + enabled: false image: repository: linkedin/datahub-mae-consumer tag: "v0.8.1" datahub-mce-consumer: - enabled: true + enabled: false image: repository: linkedin/datahub-mce-consumer tag: "v0.8.1" diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index af97ac2307c89a..682e94740fb792 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -13,6 +13,8 @@ NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub +GMS_MODE=SINGLE_NODE + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 diff --git a/docker/datahub-gms/env/docker.mariadb.env b/docker/datahub-gms/env/docker.mariadb.env index 52f80765962a70..e0329c2a6b1903 100644 --- a/docker/datahub-gms/env/docker.mariadb.env +++ b/docker/datahub-gms/env/docker.mariadb.env @@ -11,3 +11,4 @@ NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub +GMS_MODE=SINGLE_NODE diff --git a/docker/datahub-gms/env/docker.postgres.env b/docker/datahub-gms/env/docker.postgres.env index b92fd5b9e5f5b7..7dceaf8f8566bc 100644 --- a/docker/datahub-gms/env/docker.postgres.env +++ b/docker/datahub-gms/env/docker.postgres.env @@ -11,3 +11,4 @@ NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub +GMS_MODE=SINGLE_NODE diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile new file mode 100644 index 00000000000000..3d40e1581a8353 --- /dev/null +++ b/docker/datahub-mae-consumer/Dockerfile @@ -0,0 +1,42 @@ +# Defining environment +ARG APP_ENV=prod + +FROM openjdk:8-jre-alpine as base +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk --no-cache add curl tar \ + && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv + +# Workaround alpine issue with /lib64 not being in the ld library path +# https://gitlab.alpinelinux.org/alpine/aports/-/issues/10140 +ENV LD_LIBRARY_PATH=/lib64 + +# Add glibc compat layer into alpine linux, needed by java-snappy if kafka topics are compressed with snappy +RUN apk add libc6-compat + +FROM openjdk:8 as prod-build +COPY . datahub-src +RUN cd datahub-src && ./gradlew :metadata-jobs:mae-consumer-job:build +RUN cd datahub-src && cp metadata-jobs/mae-consumer-job/build/libs/mae-consumer-job.jar ../mae-consumer-job.jar + +FROM base as prod-install +COPY --from=prod-build /mae-consumer-job.jar /datahub/datahub-mae-consumer/bin/ +COPY --from=prod-build /datahub-src/docker/datahub-mae-consumer/start.sh /datahub/datahub-mae-consumer/scripts/ +RUN chmod +x /datahub/datahub-mae-consumer/scripts/start.sh + +FROM base as dev-install +# Dummy stage for development. Assumes code is built on your machine and mounted to this image. +# See this excellent thread https://github.com/docker/cli/issues/1134 + +FROM ${APP_ENV}-install as final + +RUN addgroup -S datahub && adduser -S datahub -G datahub +USER datahub + +ENV JMX_OPTS="" +ENV JAVA_OPTS="" + +EXPOSE 9090 + +HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:9091/actuator/health || exit 1 + +CMD /datahub/datahub-mae-consumer/scripts/start.sh \ No newline at end of file diff --git a/docker/datahub-mae-consumer/README.md b/docker/datahub-mae-consumer/README.md new file mode 100644 index 00000000000000..e95e8754405246 --- /dev/null +++ b/docker/datahub-mae-consumer/README.md @@ -0,0 +1,5 @@ +# DataHub MetadataAuditEvent (MAE) Consumer Docker Image +[![datahub-mae-consumer docker](https://github.com/linkedin/datahub/workflows/datahub-mae-consumer%20docker/badge.svg)](https://github.com/linkedin/datahub/actions?query=workflow%3A%22datahub-mae-consumer+docker%22) + +Refer to [DataHub MAE Consumer Job](../../metadata-jobs/mae-consumer-job) to have a quick understanding of the architecture and +responsibility of this service for the DataHub. diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env new file mode 100644 index 00000000000000..290e4df515257a --- /dev/null +++ b/docker/datahub-mae-consumer/env/docker.env @@ -0,0 +1,29 @@ +KAFKA_BOOTSTRAP_SERVER=broker:29092 +KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 +ELASTICSEARCH_HOST=elasticsearch +ELASTICSEARCH_PORT=9200 +NEO4J_HOST=http://neo4j:7474 +NEO4J_URI=bolt://neo4j +NEO4J_USERNAME=neo4j +NEO4J_PASSWORD=datahub +GMS_HOST=datahub-gms +GMS_PORT=8080 + +# Uncomment to disable persistence of client-side analytics events +# DATAHUB_ANALYTICS_ENABLED=false + +# Uncomment to configure topic names +# Make sure these names are consistent across the whole deployment +# KAFKA_TOPIC_NAME=MetadataAuditEvent_v4 +# DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1 + +# Uncomment and set these to support SSL connection to Elasticsearch +# ELASTICSEARCH_USE_SSL= +# ELASTICSEARCH_SSL_PROTOCOL= +# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL= +# ELASTICSEARCH_SSL_TRUSTSTORE_FILE= +# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE= +# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD= +# ELASTICSEARCH_SSL_KEYSTORE_FILE= +# ELASTICSEARCH_SSL_KEYSTORE_TYPE= +# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= \ No newline at end of file diff --git a/docker/datahub-mae-consumer/start.sh b/docker/datahub-mae-consumer/start.sh new file mode 100755 index 00000000000000..8acea00366a4b4 --- /dev/null +++ b/docker/datahub-mae-consumer/start.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# Add default URI (http) scheme if needed +if ! echo $NEO4J_HOST | grep -q "://" ; then + NEO4J_HOST="http://$NEO4J_HOST" +fi + +if [[ -z $ELASTICSEARCH_USERNAME ]]; then + ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_HOST +else + ELASTICSEARCH_HOST_URL=$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD@$ELASTICSEARCH_HOST +fi + +if [[ $ELASTICSEARCH_USE_SSL == true ]]; then + ELASTICSEARCH_PROTOCOL=https +else + ELASTICSEARCH_PROTOCOL=http +fi + +dockerize \ + -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ + -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST_URL:$ELASTICSEARCH_PORT \ + -wait $NEO4J_HOST \ + -timeout 240s \ + java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar \ No newline at end of file diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile new file mode 100644 index 00000000000000..e6f9affa58023a --- /dev/null +++ b/docker/datahub-mce-consumer/Dockerfile @@ -0,0 +1,42 @@ +# Defining environment +ARG APP_ENV=prod + +FROM openjdk:8-jre-alpine as base +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk --no-cache add curl tar \ + && curl -L https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar -C /usr/local/bin -xzv + +# Workaround alpine issue with /lib64 not being in the ld library path +# https://gitlab.alpinelinux.org/alpine/aports/-/issues/10140 +ENV LD_LIBRARY_PATH=/lib64 + +# Add glibc compat layer into alpine linux, needed by java-snappy if kafka topics are compressed with snappy +RUN apk add libc6-compat + +FROM openjdk:8 as prod-build +COPY . datahub-src +RUN cd datahub-src && ./gradlew :metadata-jobs:mce-consumer-job:build +RUN cd datahub-src && cp metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar ../mce-consumer-job.jar + +FROM base as prod-install +COPY --from=prod-build /mce-consumer-job.jar /datahub/datahub-mce-consumer/bin/ +COPY --from=prod-build /datahub-src/docker/datahub-mce-consumer/start.sh /datahub/datahub-mce-consumer/scripts/ +RUN chmod +x /datahub/datahub-mce-consumer/scripts/start.sh + +FROM base as dev-install +# Dummy stage for development. Assumes code is built on your machine and mounted to this image. +# See this excellent thread https://github.com/docker/cli/issues/1134 + +FROM ${APP_ENV}-install as final + +RUN addgroup -S datahub && adduser -S datahub -G datahub +USER datahub + +ENV JMX_OPTS="" +ENV JAVA_OPTS="" + +EXPOSE 9090 + +HEALTHCHECK --start-period=2m --retries=4 CMD curl --fail http://localhost:9090/actuator/health || exit 1 + +CMD /datahub/datahub-mce-consumer/scripts/start.sh \ No newline at end of file diff --git a/docker/datahub-mce-consumer/README.md b/docker/datahub-mce-consumer/README.md new file mode 100644 index 00000000000000..e4a05fe85110be --- /dev/null +++ b/docker/datahub-mce-consumer/README.md @@ -0,0 +1,5 @@ +# DataHub MetadataChangeEvent (MCE) Consumer Docker Image +[![datahub-mce-consumer docker](https://github.com/linkedin/datahub/workflows/datahub-mce-consumer%20docker/badge.svg)](https://github.com/linkedin/datahub/actions?query=workflow%3A%22datahub-mce-consumer+docker%22) + +Refer to [DataHub MCE Consumer Job](../../metadata-jobs/mce-consumer-job) to have a quick understanding of the architecture and +responsibility of this service for the DataHub. diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env new file mode 100644 index 00000000000000..c89f317a921140 --- /dev/null +++ b/docker/datahub-mce-consumer/env/docker.env @@ -0,0 +1,15 @@ +KAFKA_BOOTSTRAP_SERVER=broker:29092 +KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 +GMS_HOST=datahub-gms +GMS_PORT=8080 + +# Uncomment to configure kafka topic names +# Make sure these names are consistent across the whole deployment +# KAFKA_MCE_TOPIC_NAME=MetadataChangeEvent_v4 +# KAFKA_FMCE_TOPIC_NAME=FailedMetadataChangeEvent_v4 + +# Uncomment and set these to support SSL connection to GMS +# NOTE: Currently GMS itself does not offer SSL support, these settings are intended for when there is a proxy in front +# of GMS that handles SSL, such as an EC2 Load Balancer. +#GMS_USE_SSL=true +#GMS_SSL_PROTOCOL= \ No newline at end of file diff --git a/docker/datahub-mce-consumer/start.sh b/docker/datahub-mce-consumer/start.sh new file mode 100755 index 00000000000000..6bbfeeee08867c --- /dev/null +++ b/docker/datahub-mce-consumer/start.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +dockerize \ + -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') \ + -timeout 240s \ + java $JAVA_OPTS $JMX_OPTS -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index e2c76e6e145699..7b405f69a48abd 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -24,8 +24,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Value; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static com.linkedin.metadata.PegasusUtils.*; @@ -37,9 +35,9 @@ public class EbeanEntityService extends EntityService { private static final int DEFAULT_MAX_TRANSACTION_RETRY = 3; - private static final Logger LOG = LoggerFactory.getLogger("EbeanEntityService"); private final EbeanAspectDao _entityDao; + private Boolean _alwaysEmitAuditEvent = false; public EbeanEntityService( @@ -53,39 +51,22 @@ public EbeanEntityService( @Override @Nonnull public Map> getLatestAspects(@Nonnull final Set urns, @Nonnull final Set aspectNames) { - - LOG.info(String.format("Getting latest aspects. urns: %s, aspectNames: %s.", urns, aspectNames)); - // Create DB keys final Set dbKeys = urns.stream() .map(urn -> { final Set aspectsToFetch = aspectNames.isEmpty() - ? getEntityAspectNames(urn) - : aspectNames; + ? getEntityAspectNames(urn) + : aspectNames; return aspectsToFetch.stream() - .map(aspectName -> new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, LATEST_ASPECT_VERSION)) - .collect(Collectors.toList()); + .map(aspectName -> new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, LATEST_ASPECT_VERSION)) + .collect(Collectors.toList()); }) .flatMap(List::stream) .collect(Collectors.toSet()); // Fetch from db and populate urn -> aspect map. final Map> urnToAspects = new HashMap<>(); - - // Each urn should have some result, regardless of whether aspects are found in the DB. - for (Urn urn: urns) { - urnToAspects.putIfAbsent(urn, new ArrayList<>()); - } - - // Add "key" aspects for each urn. TODO: Replace this with a materialized key aspect. - urnToAspects.keySet().forEach(key -> { - LOG.info(String.format("Adding key aspect. urn: %s, aspectNames: %s.", key, aspectNames)); - final RecordTemplate keyAspect = buildKeyAspect(key); - urnToAspects.get(key).add(keyAspect); - }); - _entityDao.batchGet(dbKeys).forEach((key, aspectEntry) -> { - LOG.info(String.format("Successfully fetched aspect. urn: %s, aspect: %s.", key.getUrn(), key.getAspect())); final Urn urn = toUrn(key.getUrn()); final String aspectName = key.getAspect(); final RecordTemplate aspectRecord = toAspectRecord(urn, aspectName, aspectEntry.getMetadata()); @@ -93,6 +74,12 @@ public Map> getLatestAspects(@Nonnull final Set u urnToAspects.get(urn).add(aspectRecord); }); + // Add "key" aspects to any non null keys. + urnToAspects.keySet().forEach(key -> { + final RecordTemplate keyAspect = buildKeyAspect(key); + urnToAspects.get(key).add(keyAspect); + }); + return urnToAspects; } @@ -152,27 +139,20 @@ private RecordTemplate ingestAspect( @Nonnull final AuditStamp auditStamp, final int maxTransactionRetry) { - LOG.info(String.format("Ingesting aspect with urn: %s, name: %s", urn, aspectName)); - final AddAspectResult result = _entityDao.runInTransactionWithRetry(() -> { // 1. Fetch the latest existing version of the aspect. final EbeanAspectV2 latest = _entityDao.getLatestAspect(urn.toString(), aspectName); - LOG.info(String.format("Fetched latest aspect to ingest with urn: %s, name: %s", urn, aspectName)); - // 2. Compare the latest existing and new. final RecordTemplate oldValue = latest == null ? null : toAspectRecord(urn, aspectName, latest.getMetadata()); final RecordTemplate newValue = updateLambda.apply(Optional.ofNullable(oldValue)); // 3. Skip updating if there is no difference between existing and new. if (oldValue != null && DataTemplateUtil.areEqual(oldValue, newValue)) { - LOG.info(String.format("Aspect has not changed between previous and new versions. Skipping ingest. urn: %s, name: %s", urn, aspectName)); return new AddAspectResult(urn, oldValue, oldValue); } - LOG.info(String.format("Found differing aspects. Ingesting new aspect version. urn: %s, name: %s", urn, aspectName)); - // 4. Save the newValue as the latest version _entityDao.saveLatestAspect( urn.toString(), @@ -196,10 +176,7 @@ private RecordTemplate ingestAspect( // 5. Produce MAE after a successful update if (oldValue != newValue || _alwaysEmitAuditEvent) { - LOG.info(String.format("Producing MAE for ingested aspect. urn: %s, name: %s", urn, aspectName)); produceMetadataAuditEvent(urn, oldValue, newValue); - } else { - LOG.info(String.format("Skipping MAE for ingested aspect. urn: %s, name: %s", urn, aspectName)); } return newValue; @@ -234,16 +211,12 @@ private RecordTemplate updateAspect( @Nonnull final boolean emitMae, final int maxTransactionRetry) { - LOG.info(String.format("Updating aspect with urn: %s, name: %s", urn, aspectName)); - final AddAspectResult result = _entityDao.runInTransactionWithRetry(() -> { final EbeanAspectV2 oldAspect = _entityDao.getAspect(urn.toString(), aspectName, version); final RecordTemplate oldValue = oldAspect == null ? null : toAspectRecord(urn, aspectName, oldAspect.getMetadata()); - LOG.info(String.format("Fetched old aspect to update with urn: %s, name: %s. Saving new version.", urn, aspectName)); - _entityDao.saveAspect( urn.toString(), aspectName, @@ -263,10 +236,7 @@ private RecordTemplate updateAspect( final RecordTemplate newValue = result.getNewValue(); if (emitMae) { - LOG.info(String.format("Producing MAE for updated aspect. %s, name: %s.", urn, aspectName)); produceMetadataAuditEvent(urn, oldValue, newValue); - } else { - LOG.info(String.format("Skipping MAE for updated aspect. %s, name: %s.", urn, aspectName)); } return newValue; diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle index 69fe765ef40ba4..4e7f4221218311 100644 --- a/metadata-jobs/mae-consumer-job/build.gradle +++ b/metadata-jobs/mae-consumer-job/build.gradle @@ -1,57 +1,17 @@ plugins { + id 'org.springframework.boot' id 'java' } -apply plugin: 'pegasus' - -configurations { - avro -} - dependencies { - avro project(path: ':metadata-models', configuration: 'avroSchema') - - compile project(':li-utils') - compile (project(':gms:factories')) { - exclude group: 'org.neo4j.test' + compile project(':metadata-jobs:mae-consumer') + compile(externalDependency.springBootStarterWeb) { + exclude module: "spring-boot-starter-tomcat" } - compile project(':metadata-utils') - compile project(":entity-registry") - compile project(':metadata-builders') - compile project(':metadata-dao-impl:restli-dao') - compile project(':metadata-events:mxe-schemas') - compile project(':metadata-events:mxe-avro-1.7') - compile project(':metadata-events:mxe-registration') - compile project(':metadata-events:mxe-utils-avro-1.7') - - compile externalDependency.elasticSearchRest - compile externalDependency.gmaDaoApi - compile externalDependency.gmaNeo4jDao - compile externalDependency.kafkaAvroSerde - compile externalDependency.neo4jJavaDriver - + compile externalDependency.springBootStarterJetty compile externalDependency.springKafka - compile externalDependency.springActuator - - compileOnly externalDependency.lombok - - annotationProcessor externalDependency.lombok - - runtime externalDependency.logbackClassic } -task avroSchemaSources(type: Copy) { - dependsOn configurations.avro - - from { // use of closure defers evaluation until execution time - configurations.avro.collect { zipTree(it) } - } - into("src/main/resources/") - include("avro/com/linkedin/mxe/") -} - -compileJava.dependsOn avroSchemaSources - -clean { - project.delete("src/main/resources/avro") +bootJar { + mainClassName = 'com.linkedin.metadata.kafka.MaeConsumerApplication' } \ No newline at end of file diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java new file mode 100644 index 00000000000000..1a0635c9e2f51c --- /dev/null +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java @@ -0,0 +1,15 @@ +package com.linkedin.metadata.kafka; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration; + +@SuppressWarnings("checkstyle:HideUtilityClassConstructor") +@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}) +public class MaeConsumerApplication { + + public static void main(String[] args) { + Class[] primarySources = {MaeConsumerApplication.class, com.linkedin.metadata.kafka.MaeConsumerConfig.class}; + SpringApplication.run(primarySources, args); + } +} \ No newline at end of file diff --git a/metadata-jobs/mae-consumer-job/.gitignore b/metadata-jobs/mae-consumer/.gitignore similarity index 100% rename from metadata-jobs/mae-consumer-job/.gitignore rename to metadata-jobs/mae-consumer/.gitignore diff --git a/metadata-jobs/mae-consumer-job/README.md b/metadata-jobs/mae-consumer/README.md similarity index 100% rename from metadata-jobs/mae-consumer-job/README.md rename to metadata-jobs/mae-consumer/README.md diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle new file mode 100644 index 00000000000000..69fe765ef40ba4 --- /dev/null +++ b/metadata-jobs/mae-consumer/build.gradle @@ -0,0 +1,57 @@ +plugins { + id 'java' +} + +apply plugin: 'pegasus' + +configurations { + avro +} + +dependencies { + avro project(path: ':metadata-models', configuration: 'avroSchema') + + compile project(':li-utils') + compile (project(':gms:factories')) { + exclude group: 'org.neo4j.test' + } + compile project(':metadata-utils') + compile project(":entity-registry") + compile project(':metadata-builders') + compile project(':metadata-dao-impl:restli-dao') + compile project(':metadata-events:mxe-schemas') + compile project(':metadata-events:mxe-avro-1.7') + compile project(':metadata-events:mxe-registration') + compile project(':metadata-events:mxe-utils-avro-1.7') + + compile externalDependency.elasticSearchRest + compile externalDependency.gmaDaoApi + compile externalDependency.gmaNeo4jDao + compile externalDependency.kafkaAvroSerde + compile externalDependency.neo4jJavaDriver + + compile externalDependency.springKafka + compile externalDependency.springActuator + + compileOnly externalDependency.lombok + + annotationProcessor externalDependency.lombok + + runtime externalDependency.logbackClassic +} + +task avroSchemaSources(type: Copy) { + dependsOn configurations.avro + + from { // use of closure defers evaluation until execution time + configurations.avro.collect { zipTree(it) } + } + into("src/main/resources/") + include("avro/com/linkedin/mxe/") +} + +compileJava.dependsOn avroSchemaSources + +clean { + project.delete("src/main/resources/avro") +} \ No newline at end of file diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerConfig.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MaeConsumerConfig.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerConfig.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MaeConsumerConfig.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/ElasticSearchConfig.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/ElasticSearchConfig.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/ElasticSearchConfig.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/ElasticSearchConfig.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/HydratorFactoryConfig.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/HydratorFactoryConfig.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/HydratorFactoryConfig.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/HydratorFactoryConfig.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/IndexBuildersConfig.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MaeKafkaConfig.java diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java new file mode 100644 index 00000000000000..59de044548fe18 --- /dev/null +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java @@ -0,0 +1,18 @@ +package com.linkedin.metadata.kafka.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + + +public class MetadataAuditEventsProcessorCondition implements Condition { + @Override + public boolean matches( + ConditionContext context, + AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + return env != null + && "SINGLE_NODE".equals(env.getProperty("GMS_MODE")); + } +} diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticEvent.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnector.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/ElasticsearchConnectorFactory.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/JsonElasticEvent.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/elasticsearch/MCEElasticEvent.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/ChartHydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/CorpUserHydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DashboardHydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DataFlowHydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DataJobHydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DatasetHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DatasetHydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/DatasetHydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/DatasetHydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityType.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityType.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityType.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityType.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/Hydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/Hydrator.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/Hydrator.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/Hydrator.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/HydratorFactory.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventConstants.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventConstants.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventConstants.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventConstants.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventTransformer.java diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventType.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventType.java similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventType.java rename to metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/transformer/DataHubUsageEventType.java diff --git a/metadata-jobs/mae-consumer-job/src/main/resources/application.properties b/metadata-jobs/mae-consumer/src/main/resources/application.properties similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/resources/application.properties rename to metadata-jobs/mae-consumer/src/main/resources/application.properties diff --git a/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml b/metadata-jobs/mae-consumer/src/main/resources/logback.xml similarity index 100% rename from metadata-jobs/mae-consumer-job/src/main/resources/logback.xml rename to metadata-jobs/mae-consumer/src/main/resources/logback.xml diff --git a/metadata-jobs/mce-consumer-job/build.gradle b/metadata-jobs/mce-consumer-job/build.gradle index a895437841f531..bd0b4b192ca6c8 100644 --- a/metadata-jobs/mce-consumer-job/build.gradle +++ b/metadata-jobs/mce-consumer-job/build.gradle @@ -1,52 +1,17 @@ plugins { + id 'org.springframework.boot' id 'java' } -apply plugin: 'pegasus' - -configurations { - avro -} - dependencies { - avro project(path: ':metadata-models', configuration: 'avroSchema') - - compile project(':li-utils') - compile project(':metadata-utils') - compile project(':metadata-builders') - compile project(':metadata-events:mxe-schemas') - compile project(':metadata-events:mxe-avro-1.7') - compile project(':metadata-events:mxe-registration') - compile project(':metadata-events:mxe-utils-avro-1.7') - compile project(':metadata-dao-impl:restli-dao') - compile project(':metadata-io') - compile project(':gms:client') - compile spec.product.pegasus.restliClient - compile spec.product.pegasus.restliCommon - compile externalDependency.elasticSearchRest - compile externalDependency.kafkaAvroSerde - - compile externalDependency.springKafka - compile externalDependency.springActuator - - compileOnly externalDependency.lombok - - annotationProcessor externalDependency.lombok - -} - -task avroSchemaSources(type: Copy) { - dependsOn configurations.avro - - from { // use of closure defers evaluation until execution time - configurations.avro.collect { zipTree(it) } + compile project(':metadata-jobs:mce-consumer') + compile(externalDependency.springBootStarterWeb) { + exclude module: "spring-boot-starter-tomcat" } - into("src/main/resources/") - include("avro/com/linkedin/mxe/") + compile externalDependency.springBootStarterJetty + compile externalDependency.springKafka } -compileJava.dependsOn avroSchemaSources - -clean { - project.delete("src/main/resources/avro") +bootJar { + mainClassName = 'com.linkedin.metadata.kafka.MceConsumerApplication' } \ No newline at end of file diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java new file mode 100644 index 00000000000000..b308b5dba8b5c8 --- /dev/null +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java @@ -0,0 +1,15 @@ +package com.linkedin.metadata.kafka; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration; + + +@SuppressWarnings("checkstyle:HideUtilityClassConstructor") +@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}) +public class MceConsumerApplication { + + public static void main(String[] args) { + SpringApplication.run(MceConsumerApplication.class, args); + } +} \ No newline at end of file diff --git a/metadata-jobs/mce-consumer-job/.gitignore b/metadata-jobs/mce-consumer/.gitignore similarity index 100% rename from metadata-jobs/mce-consumer-job/.gitignore rename to metadata-jobs/mce-consumer/.gitignore diff --git a/metadata-jobs/mce-consumer-job/README.md b/metadata-jobs/mce-consumer/README.md similarity index 100% rename from metadata-jobs/mce-consumer-job/README.md rename to metadata-jobs/mce-consumer/README.md diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle new file mode 100644 index 00000000000000..a895437841f531 --- /dev/null +++ b/metadata-jobs/mce-consumer/build.gradle @@ -0,0 +1,52 @@ +plugins { + id 'java' +} + +apply plugin: 'pegasus' + +configurations { + avro +} + +dependencies { + avro project(path: ':metadata-models', configuration: 'avroSchema') + + compile project(':li-utils') + compile project(':metadata-utils') + compile project(':metadata-builders') + compile project(':metadata-events:mxe-schemas') + compile project(':metadata-events:mxe-avro-1.7') + compile project(':metadata-events:mxe-registration') + compile project(':metadata-events:mxe-utils-avro-1.7') + compile project(':metadata-dao-impl:restli-dao') + compile project(':metadata-io') + compile project(':gms:client') + compile spec.product.pegasus.restliClient + compile spec.product.pegasus.restliCommon + compile externalDependency.elasticSearchRest + compile externalDependency.kafkaAvroSerde + + compile externalDependency.springKafka + compile externalDependency.springActuator + + compileOnly externalDependency.lombok + + annotationProcessor externalDependency.lombok + +} + +task avroSchemaSources(type: Copy) { + dependsOn configurations.avro + + from { // use of closure defers evaluation until execution time + configurations.avro.collect { zipTree(it) } + } + into("src/main/resources/") + include("avro/com/linkedin/mxe/") +} + +compileJava.dependsOn avroSchemaSources + +clean { + project.delete("src/main/resources/avro") +} \ No newline at end of file diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java similarity index 94% rename from metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java rename to metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index 3f00c1be0d70ea..826141c928a7b7 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -3,6 +3,7 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; import com.linkedin.metadata.EventUtils; +import com.linkedin.metadata.kafka.config.MetadataChangeEventsProcessorCondition; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.mxe.FailedMetadataChangeEvent; import com.linkedin.mxe.MetadataChangeEvent; @@ -15,6 +16,7 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Conditional; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.KafkaTemplate; @@ -24,6 +26,7 @@ @Slf4j @Component @EnableKafka +@Conditional(MetadataChangeEventsProcessorCondition.class) public class MetadataChangeEventsProcessor { private EntityClient entityClient; diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/EntityClientConfig.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/EntityClientConfig.java similarity index 100% rename from metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/EntityClientConfig.java rename to metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/EntityClientConfig.java diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java similarity index 100% rename from metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java rename to metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MceKafkaConfig.java diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java new file mode 100644 index 00000000000000..a6e5bd34aa2693 --- /dev/null +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java @@ -0,0 +1,18 @@ +package com.linkedin.metadata.kafka.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + + +public class MetadataChangeEventsProcessorCondition implements Condition { + @Override + public boolean matches( + ConditionContext context, + AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + return env != null + && "SINGLE_NODE".equals(env.getProperty("GMS_MODE")); + } +} diff --git a/metadata-jobs/mce-consumer-job/src/main/resources/application.properties b/metadata-jobs/mce-consumer/src/main/resources/application.properties similarity index 100% rename from metadata-jobs/mce-consumer-job/src/main/resources/application.properties rename to metadata-jobs/mce-consumer/src/main/resources/application.properties diff --git a/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml b/metadata-jobs/mce-consumer/src/main/resources/logback.xml similarity index 100% rename from metadata-jobs/mce-consumer-job/src/main/resources/logback.xml rename to metadata-jobs/mce-consumer/src/main/resources/logback.xml diff --git a/settings.gradle b/settings.gradle index c2c0686e00901d..832b65d925b1c9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -21,6 +21,8 @@ include 'metadata-ingestion' include 'metadata-ingestion-examples:common' include 'metadata-ingestion-examples:kafka-etl' include 'metadata-ingestion-examples:mce-cli' +include 'metadata-jobs:mae-consumer' +include 'metadata-jobs:mce-consumer' include 'metadata-jobs:mae-consumer-job' include 'metadata-jobs:mce-consumer-job' include 'metadata-models' From 20a1075bc560b141625b0b66156c2d52ed9fce48 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 10:53:20 -0700 Subject: [PATCH 06/14] Fixing charts & more --- datahub-kubernetes/datahub/Chart.yaml | 4 +-- .../datahub-gms/templates/deployment.yaml | 6 ++++ .../templates/deployment.yaml | 4 +-- .../charts/datahub-mae-consumer/values.yaml | 2 +- .../templates/deployment.yaml | 2 ++ .../charts/datahub-mce-consumer/values.yaml | 3 ++ datahub-kubernetes/datahub/values.yaml | 2 ++ docker/datahub-gms/env/docker.env | 3 +- docker/datahub-gms/env/docker.mariadb.env | 3 +- docker/datahub-gms/env/docker.postgres.env | 4 ++- docker/datahub-mae-consumer/env/docker.env | 1 + docker/datahub-mce-consumer/env/docker.env | 1 + docker/docker-compose.consumers.dev.yml | 23 ++++++++++++++ docker/docker-compose.consumers.yml | 31 +++++++++++++++++++ gms/war/build.gradle | 4 +-- .../README.md | 0 ...MetadataAuditEventsProcessorCondition.java | 2 +- .../README.md | 0 ...etadataChangeEventsProcessorCondition.java | 2 +- 19 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 docker/docker-compose.consumers.dev.yml create mode 100644 docker/docker-compose.consumers.yml rename metadata-jobs/{mae-consumer => mae-consumer-job}/README.md (100%) rename metadata-jobs/{mce-consumer => mce-consumer-job}/README.md (100%) diff --git a/datahub-kubernetes/datahub/Chart.yaml b/datahub-kubernetes/datahub/Chart.yaml index 71589e532fe2df..f2527266654cec 100644 --- a/datahub-kubernetes/datahub/Chart.yaml +++ b/datahub-kubernetes/datahub/Chart.yaml @@ -20,11 +20,11 @@ dependencies: - name: datahub-mae-consumer version: 0.2.1 repository: file://./charts/datahub-mae-consumer - condition: datahub-mae-consumer.enabled + condition: global.datahub_standalone_consumers_enabled - name: datahub-mce-consumer version: 0.2.1 repository: file://./charts/datahub-mce-consumer - condition: datahub-mce-consumer.enabled + condition: global.datahub_standalone_consumers_enabled - name: datahub-ingestion-cron version: 0.2.1 repository: file://./charts/datahub-ingestion-cron diff --git a/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml index 2389325ae12153..ac3608e13f1b7b 100644 --- a/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml @@ -73,6 +73,12 @@ spec: periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} env: + {{- if ne .Values.global.datahub_standalone_consumers_enabled true }} + - name: MCE_CONSUMER_ENABLED + value: "true" + - name: MAE_CONSUMER_ENABLED + value: "true" + {{- end }} - name: EBEAN_DATASOURCE_USERNAME value: "{{ .Values.global.sql.datasource.username }}" - name: EBEAN_DATASOURCE_PASSWORD diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml index adb550b6f27de0..008f75e32f5519 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml @@ -69,6 +69,8 @@ spec: periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} env: + - name: MAE_CONSUMER_ENABLED + value: "true" - name: GMS_HOST value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} - name: GMS_PORT @@ -107,8 +109,6 @@ spec: key: "{{ .Values.global.neo4j.password.secretKey }}" - name: DATAHUB_ANALYTICS_ENABLED value: "{{ .Values.global.datahub_analytics_enabled }}" - - name: GMS_MODE - value: "{{ .Values.global.datahub_gms_mode }}" {{- if .Values.global.springKafkaConfigurationOverrides }} {{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }} - name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml index 093d25ff4f5eba..16fbd734708428 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml @@ -146,7 +146,7 @@ readinessProbe: global: datahub_analytics_enabled: true - datahub_gms_mode: "SINGLE_NODE" + datahub_consumer_jobs_enabled: true elasticsearch: host: "elasticsearch" diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml index 0c83919dd26d09..650668e6fbfb98 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml @@ -69,6 +69,8 @@ spec: periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} env: + - name: MCE_CONSUMER_ENABLED + value: "true" - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" - name: KAFKA_SCHEMAREGISTRY_URL diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml index b7734141bf4a76..b98ecfbc25898e 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml @@ -145,6 +145,9 @@ readinessProbe: failureThreshold: 4 global: + + datahub_consumer_jobs_enabled: true + kafka: bootstrap: server: "broker:9092" diff --git a/datahub-kubernetes/datahub/values.yaml b/datahub-kubernetes/datahub/values.yaml index 41deac12370b65..266429249795e8 100644 --- a/datahub-kubernetes/datahub/values.yaml +++ b/datahub-kubernetes/datahub/values.yaml @@ -58,7 +58,9 @@ datahubUpgrade: tag: "v0.8.1" global: + datahub_analytics_enabled: true + datahub_standalone_consumers_enabled: false elasticsearch: host: "elasticsearch-master" diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index 682e94740fb792..55c348d1da4765 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -13,7 +13,8 @@ NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub -GMS_MODE=SINGLE_NODE +MAE_CONSUMER_ENABLED=true +MCE_CONSUMER_ENABLED=true # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment diff --git a/docker/datahub-gms/env/docker.mariadb.env b/docker/datahub-gms/env/docker.mariadb.env index e0329c2a6b1903..5e75abd87238e2 100644 --- a/docker/datahub-gms/env/docker.mariadb.env +++ b/docker/datahub-gms/env/docker.mariadb.env @@ -11,4 +11,5 @@ NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub -GMS_MODE=SINGLE_NODE +MAE_CONSUMER_ENABLED=true +MCE_CONSUMER_ENABLED=true diff --git a/docker/datahub-gms/env/docker.postgres.env b/docker/datahub-gms/env/docker.postgres.env index 7dceaf8f8566bc..efb60a4f55b0ed 100644 --- a/docker/datahub-gms/env/docker.postgres.env +++ b/docker/datahub-gms/env/docker.postgres.env @@ -11,4 +11,6 @@ NEO4J_HOST=http://neo4j:7474 NEO4J_URI=bolt://neo4j NEO4J_USERNAME=neo4j NEO4J_PASSWORD=datahub -GMS_MODE=SINGLE_NODE +MAE_CONSUMER_ENABLED=true +MCE_CONSUMER_ENABLED=true + diff --git a/docker/datahub-mae-consumer/env/docker.env b/docker/datahub-mae-consumer/env/docker.env index 290e4df515257a..e7f192901ea0e3 100644 --- a/docker/datahub-mae-consumer/env/docker.env +++ b/docker/datahub-mae-consumer/env/docker.env @@ -1,3 +1,4 @@ +MAE_CONSUMER_ENABLED=true KAFKA_BOOTSTRAP_SERVER=broker:29092 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 ELASTICSEARCH_HOST=elasticsearch diff --git a/docker/datahub-mce-consumer/env/docker.env b/docker/datahub-mce-consumer/env/docker.env index c89f317a921140..ec47943a4ab95e 100644 --- a/docker/datahub-mce-consumer/env/docker.env +++ b/docker/datahub-mce-consumer/env/docker.env @@ -1,3 +1,4 @@ +MCE_CONSUMER_ENABLED=true KAFKA_BOOTSTRAP_SERVER=broker:29092 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 GMS_HOST=datahub-gms diff --git a/docker/docker-compose.consumers.dev.yml b/docker/docker-compose.consumers.dev.yml new file mode 100644 index 00000000000000..4d354fda97a13d --- /dev/null +++ b/docker/docker-compose.consumers.dev.yml @@ -0,0 +1,23 @@ +version: '3.8' +services: + datahub-mae-consumer: + image: linkedin/datahub-mae-consumer:debug + build: + context: datahub-mae-consumer + dockerfile: Dockerfile + args: + APP_ENV: dev + volumes: + - ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh + - ../metadata-jobs/mae-consumer-job/build/libs/:/datahub/datahub-mae-consumer/bin/ + + datahub-mce-consumer: + image: linkedin/datahub-mce-consumer:debug + build: + context: datahub-mce-consumer + dockerfile: Dockerfile + args: + APP_ENV: dev + volumes: + - ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh + - ../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin \ No newline at end of file diff --git a/docker/docker-compose.consumers.yml b/docker/docker-compose.consumers.yml new file mode 100644 index 00000000000000..3dc39521fd05e0 --- /dev/null +++ b/docker/docker-compose.consumers.yml @@ -0,0 +1,31 @@ +# Service definitions for standalone Kafka consumer containers. +version: '3.8' +services: + datahub-mae-consumer: + build: + context: ../ + dockerfile: docker/datahub-mae-consumer/Dockerfile + image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest} + env_file: datahub-mae-consumer/env/docker.env + hostname: datahub-mae-consumer + container_name: datahub-mae-consumer + ports: + - "9091:9091" + depends_on: + - kafka-setup + - elasticsearch-setup + - neo4j + + datahub-mce-consumer: + build: + context: ../ + dockerfile: docker/datahub-mce-consumer/Dockerfile + image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest} + env_file: datahub-mce-consumer/env/docker.env + hostname: datahub-mce-consumer + container_name: datahub-mce-consumer + ports: + - "9090:9090" + depends_on: + - kafka-setup + - datahub-gms \ No newline at end of file diff --git a/gms/war/build.gradle b/gms/war/build.gradle index d6304c9bc9d5be..7df5e426f6b963 100644 --- a/gms/war/build.gradle +++ b/gms/war/build.gradle @@ -4,8 +4,8 @@ ext.apiProject = project(':gms:api') dependencies { runtime project(':gms:factories') - runtime project(':metadata-jobs:mce-consumer-job') - runtime project(':metadata-jobs:mae-consumer-job') + runtime project(':metadata-jobs:mce-consumer') + runtime project(':metadata-jobs:mae-consumer') runtime externalDependency.h2 runtime externalDependency.logbackClassic diff --git a/metadata-jobs/mae-consumer/README.md b/metadata-jobs/mae-consumer-job/README.md similarity index 100% rename from metadata-jobs/mae-consumer/README.md rename to metadata-jobs/mae-consumer-job/README.md diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java index 59de044548fe18..dbb4483a1c7126 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java @@ -13,6 +13,6 @@ public boolean matches( AnnotatedTypeMetadata metadata) { Environment env = context.getEnvironment(); return env != null - && "SINGLE_NODE".equals(env.getProperty("GMS_MODE")); + && "true".equals(env.getProperty("MAE_CONSUMER_ENABLED")); } } diff --git a/metadata-jobs/mce-consumer/README.md b/metadata-jobs/mce-consumer-job/README.md similarity index 100% rename from metadata-jobs/mce-consumer/README.md rename to metadata-jobs/mce-consumer-job/README.md diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java index a6e5bd34aa2693..6551afb549fcd8 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java @@ -13,6 +13,6 @@ public boolean matches( AnnotatedTypeMetadata metadata) { Environment env = context.getEnvironment(); return env != null - && "SINGLE_NODE".equals(env.getProperty("GMS_MODE")); + && "true".equals(env.getProperty("MCE_CONSUMER_ENABLED")); } } From 1954d187e00fbd6c37180b73ff242bcd80b9d945 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 12:38:22 -0700 Subject: [PATCH 07/14] Fixing issues --- .../datahub-gms/templates/deployment.yaml | 2 ++ .../datahub/charts/datahub-gms/values.yaml | 2 ++ docker/datahub-gms/env/docker.env | 3 +++ .../entity/ebean/EbeanEntityService.java | 26 ++++++++++++------- .../kafka/DataHubUsageEventsProcessor.java | 3 +++ .../kafka/MetadataAuditEventsProcessor.java | 3 +++ .../DataHubUsageEventsProcessorCondition.java | 19 ++++++++++++++ ...MetadataAuditEventsProcessorCondition.java | 3 +-- .../kafka/MetadataChangeEventsProcessor.java | 2 +- ...etadataChangeEventsProcessorCondition.java | 3 +-- 10 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java diff --git a/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml index ac3608e13f1b7b..69219181e50cfb 100644 --- a/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml @@ -79,6 +79,8 @@ spec: - name: MAE_CONSUMER_ENABLED value: "true" {{- end }} + - name: DATAHUB_ANALYTICS_ENABLED + value: "{{ .Values.global.datahub_analytics_enabled }}" - name: EBEAN_DATASOURCE_USERNAME value: "{{ .Values.global.sql.datasource.username }}" - name: EBEAN_DATASOURCE_PASSWORD diff --git a/datahub-kubernetes/datahub/charts/datahub-gms/values.yaml b/datahub-kubernetes/datahub/charts/datahub-gms/values.yaml index 4480e91987fa00..b5e07b5ad0c3d8 100644 --- a/datahub-kubernetes/datahub/charts/datahub-gms/values.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-gms/values.yaml @@ -146,6 +146,8 @@ readinessProbe: #This section is useful if we are installing this chart separately for testing # helm install datahub-gms datahub-gms/ global: + datahub_analytics_enabled: true + elasticsearch: host: "elasticsearch" port: "9200" diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index 55c348d1da4765..b56337df5e3dee 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -16,6 +16,9 @@ NEO4J_PASSWORD=datahub MAE_CONSUMER_ENABLED=true MCE_CONSUMER_ENABLED=true +# Uncomment to disable persistence of client-side analytics events +# DATAHUB_ANALYTICS_ENABLED=false + # Uncomment to configure kafka topic names # Make sure these names are consistent across the whole deployment # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index 7b405f69a48abd..ce7e246148140b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -55,17 +55,29 @@ public Map> getLatestAspects(@Nonnull final Set u final Set dbKeys = urns.stream() .map(urn -> { final Set aspectsToFetch = aspectNames.isEmpty() - ? getEntityAspectNames(urn) - : aspectNames; + ? getEntityAspectNames(urn) + : aspectNames; return aspectsToFetch.stream() - .map(aspectName -> new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, LATEST_ASPECT_VERSION)) - .collect(Collectors.toList()); + .map(aspectName -> new EbeanAspectV2.PrimaryKey(urn.toString(), aspectName, LATEST_ASPECT_VERSION)) + .collect(Collectors.toList()); }) .flatMap(List::stream) .collect(Collectors.toSet()); // Fetch from db and populate urn -> aspect map. final Map> urnToAspects = new HashMap<>(); + + // Each urn should have some result, regardless of whether aspects are found in the DB. + for (Urn urn: urns) { + urnToAspects.putIfAbsent(urn, new ArrayList<>()); + } + + // Add "key" aspects for each urn. TODO: Replace this with a materialized key aspect. + urnToAspects.keySet().forEach(key -> { + final RecordTemplate keyAspect = buildKeyAspect(key); + urnToAspects.get(key).add(keyAspect); + }); + _entityDao.batchGet(dbKeys).forEach((key, aspectEntry) -> { final Urn urn = toUrn(key.getUrn()); final String aspectName = key.getAspect(); @@ -74,12 +86,6 @@ public Map> getLatestAspects(@Nonnull final Set u urnToAspects.get(urn).add(aspectRecord); }); - // Add "key" aspects to any non null keys. - urnToAspects.keySet().forEach(key -> { - final RecordTemplate keyAspect = buildKeyAspect(key); - urnToAspects.get(key).add(keyAspect); - }); - return urnToAspects; } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java index 9857527f7a25a7..cdb4ecc7450ccd 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/DataHubUsageEventsProcessor.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.kafka; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.kafka.config.DataHubUsageEventsProcessorCondition; import com.linkedin.metadata.kafka.elasticsearch.ElasticsearchConnector; import com.linkedin.metadata.kafka.elasticsearch.JsonElasticEvent; import com.linkedin.metadata.kafka.transformer.DataHubUsageEventTransformer; @@ -11,6 +12,7 @@ import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.context.annotation.Conditional; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.stereotype.Component; @@ -19,6 +21,7 @@ @Slf4j @Component @EnableKafka +@Conditional(DataHubUsageEventsProcessorCondition.class) public class DataHubUsageEventsProcessor { private final ElasticsearchConnector elasticSearchConnector; diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java index 3cce2a2155ab89..d03a5da1376892 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataAuditEventsProcessor.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.extractor.FieldExtractor; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.kafka.config.MetadataAuditEventsProcessorCondition; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.RelationshipFieldSpec; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; @@ -34,6 +35,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Import; import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.annotation.KafkaListener; @@ -44,6 +46,7 @@ @Slf4j @Component +@Conditional(MetadataAuditEventsProcessorCondition.class) @Import({GraphServiceFactory.class, SearchServiceFactory.class}) @EnableKafka public class MetadataAuditEventsProcessor { diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java new file mode 100644 index 00000000000000..e61547edfac0d2 --- /dev/null +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java @@ -0,0 +1,19 @@ +package com.linkedin.metadata.kafka.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + + +public class DataHubUsageEventsProcessorCondition implements Condition { + @Override + public boolean matches( + ConditionContext context, + AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + return "true".equals(env.getProperty("MAE_CONSUMER_ENABLED")) && ( + env.getProperty("DATAHUB_ANALYTICS_ENABLED") == null || + "true".equals(env.getProperty("DATAHUB_ANALYTICS_ENABLED"))); + } +} \ No newline at end of file diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java index dbb4483a1c7126..4dc7598ceaacbc 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataAuditEventsProcessorCondition.java @@ -12,7 +12,6 @@ public boolean matches( ConditionContext context, AnnotatedTypeMetadata metadata) { Environment env = context.getEnvironment(); - return env != null - && "true".equals(env.getProperty("MAE_CONSUMER_ENABLED")); + return "true".equals(env.getProperty("MAE_CONSUMER_ENABLED")); } } diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index 826141c928a7b7..90e68e6eb34ff8 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -25,8 +25,8 @@ @Slf4j @Component -@EnableKafka @Conditional(MetadataChangeEventsProcessorCondition.class) +@EnableKafka public class MetadataChangeEventsProcessor { private EntityClient entityClient; diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java index 6551afb549fcd8..9739a40f5df656 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeEventsProcessorCondition.java @@ -12,7 +12,6 @@ public boolean matches( ConditionContext context, AnnotatedTypeMetadata metadata) { Environment env = context.getEnvironment(); - return env != null - && "true".equals(env.getProperty("MCE_CONSUMER_ENABLED")); + return "true".equals(env.getProperty("MCE_CONSUMER_ENABLED")); } } From fea09ccf3ecff35472ed687e67a33ddb4cacf245 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 12:47:58 -0700 Subject: [PATCH 08/14] Fixing mis-applied changes --- .github/workflows/docker-mae-consumer.yml | 2 +- .github/workflows/docker-mce-consumer.yml | 2 +- .../charts/datahub-mae-consumer/README.md | 2 +- .../templates/deployment.yaml | 8 +++++-- .../templates/service.yaml | 15 +++++++++++++ .../charts/datahub-mae-consumer/values.yaml | 7 ++++-- .../charts/datahub-mce-consumer/README.md | 2 +- .../charts/datahub-mce-consumer/values.yaml | 5 +---- .../datahub/quickstart-values.yaml | 12 ---------- datahub-kubernetes/datahub/values.yaml | 4 ++-- metadata-jobs/mae-consumer-job/README.md | 13 ++++++++++- metadata-jobs/mce-consumer-job/README.md | 22 ++++++++++++++++++- 12 files changed, 66 insertions(+), 28 deletions(-) create mode 100644 datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml diff --git a/.github/workflows/docker-mae-consumer.yml b/.github/workflows/docker-mae-consumer.yml index 3c98cb55f94c09..1b0748742d141e 100644 --- a/.github/workflows/docker-mae-consumer.yml +++ b/.github/workflows/docker-mae-consumer.yml @@ -45,7 +45,6 @@ jobs: push_to_registries: name: Build and Push Docker Image to Docker Hub runs-on: ubuntu-latest - if: ${{ needs.setup.outputs.publish == 'true' }} needs: setup steps: - name: Check out the repo @@ -61,6 +60,7 @@ jobs: tag-custom: ${{ needs.setup.outputs.tag }} tag-custom-only: true - name: Login to DockerHub + if: ${{ needs.setup.outputs.publish == 'true' }} uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} diff --git a/.github/workflows/docker-mce-consumer.yml b/.github/workflows/docker-mce-consumer.yml index e1034b1069546e..5c69ce5453a652 100644 --- a/.github/workflows/docker-mce-consumer.yml +++ b/.github/workflows/docker-mce-consumer.yml @@ -45,7 +45,6 @@ jobs: push_to_registries: name: Build and Push Docker Image to Docker Hub runs-on: ubuntu-latest - if: ${{ needs.setup.outputs.publish == 'true' }} needs: setup steps: - name: Check out the repo @@ -61,6 +60,7 @@ jobs: tag-custom: ${{ needs.setup.outputs.tag }} tag-custom-only: true - name: Login to DockerHub + if: ${{ needs.setup.outputs.publish == 'true' }} uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md index 13d48dc2d12c96..29696289f30d37 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md @@ -31,7 +31,7 @@ Current chart version is `0.2.0` | global.hostAliases[0].ip | string | `"192.168.0.104"` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"linkedin/datahub-mae-consumer"` | | -| image.tag | string | `"latest"` | | +| image.tag | string | `"v0.8.1"` | | | imagePullSecrets | list | `[]` | | | ingress.annotations | object | `{}` | | | ingress.enabled | bool | `false` | | diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml index 008f75e32f5519..553397a62affc3 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml @@ -54,17 +54,21 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 9091 + protocol: TCP livenessProbe: httpGet: path: /actuator/health - port: 9091 + port: http initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} failureThreshold: {{ .Values.livenessProbe.failureThreshold }} readinessProbe: httpGet: path: /actuator/health - port: 9091 + port: http initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml new file mode 100644 index 00000000000000..4a2178ff52e86a --- /dev/null +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-%s" .Release.Name "datahub-mae-consumer" }} + labels: + {{- include "datahub-mae-consumer.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.global.datahub.mae_consumer.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "datahub-mae-consumer.selectorLabels" . | nindent 4 }} diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml index 16fbd734708428..3da74b68321730 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: linkedin/datahub-mae-consumer pullPolicy: Always - tag: latest + tag: v0.8.1 imagePullSecrets: [] nameOverride: "" @@ -146,7 +146,6 @@ readinessProbe: global: datahub_analytics_enabled: true - datahub_consumer_jobs_enabled: true elasticsearch: host: "elasticsearch" @@ -166,6 +165,10 @@ global: secretRef: "neo4j-secrets" secretKey: "neo4j-password" + datahub: + mae_consumer: + port: "9091" + hostAliases: - ip: "192.168.0.104" hostnames: diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md index 693e5b640fd669..7cae56b3df6e88 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md @@ -24,7 +24,7 @@ Current chart version is `0.2.0` | global.hostAliases[0].ip | string | `"192.168.0.104"` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"linkedin/datahub-mce-consumer"` | | -| image.tag | string | `"latest"` | | +| image.tag | string | `"v0.8.1"` | | | imagePullSecrets | list | `[]` | | | ingress.annotations | object | `{}` | | | ingress.enabled | bool | `false` | | diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml index b98ecfbc25898e..cdcf61ad52f4cc 100644 --- a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: linkedin/datahub-mce-consumer pullPolicy: Always - tag: latest + tag: v0.8.1 imagePullSecrets: [] nameOverride: "" @@ -145,9 +145,6 @@ readinessProbe: failureThreshold: 4 global: - - datahub_consumer_jobs_enabled: true - kafka: bootstrap: server: "broker:9092" diff --git a/datahub-kubernetes/datahub/quickstart-values.yaml b/datahub-kubernetes/datahub/quickstart-values.yaml index 4eee21e2225be7..719598ffd8b284 100644 --- a/datahub-kubernetes/datahub/quickstart-values.yaml +++ b/datahub-kubernetes/datahub/quickstart-values.yaml @@ -15,18 +15,6 @@ datahub-frontend: ingress: enabled: false -datahub-mae-consumer: - enabled: false - image: - repository: linkedin/datahub-mae-consumer - tag: "v0.8.1" - -datahub-mce-consumer: - enabled: false - image: - repository: linkedin/datahub-mce-consumer - tag: "v0.8.1" - elasticsearchSetupJob: enabled: true image: diff --git a/datahub-kubernetes/datahub/values.yaml b/datahub-kubernetes/datahub/values.yaml index 266429249795e8..e7cad25392f06c 100644 --- a/datahub-kubernetes/datahub/values.yaml +++ b/datahub-kubernetes/datahub/values.yaml @@ -16,13 +16,13 @@ datahub-frontend: enabled: false datahub-mae-consumer: - enabled: false + enabled: true image: repository: linkedin/datahub-mae-consumer tag: "v0.8.1" datahub-mce-consumer: - enabled: false + enabled: true image: repository: linkedin/datahub-mce-consumer tag: "v0.8.1" diff --git a/metadata-jobs/mae-consumer-job/README.md b/metadata-jobs/mae-consumer-job/README.md index 664e288a16d8c2..08bf7a2bde5ced 100644 --- a/metadata-jobs/mae-consumer-job/README.md +++ b/metadata-jobs/mae-consumer-job/README.md @@ -22,7 +22,18 @@ However, if you only want to build `MAE Consumer Job` specifically: Before starting `MAE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup), [Elasticsearch](../../docker/elasticsearch), and [Neo4j](../../docker/neo4j) Docker containers are up and running. -## Endpoints TODO: Reevaluate these +## Start via Docker image +Quickest way to try out `MAE Consumer Job` is running the [Docker image](../../docker/datahub-mae-consumer). + +## Start via command line +If you do modify things and want to try it out quickly without building the Docker image, you can also run +the application directly from command line after a successful [build](#build): +``` +./gradlew :metadata-jobs:mae-consumer-job:bootRun +``` + +## Endpoints +Spring boot actuator has been enabled for MAE Application. `healthcheck`, `metrics` and `info` web endpoints are enabled by default. `healthcheck` - http://localhost:9091/actuator/health diff --git a/metadata-jobs/mce-consumer-job/README.md b/metadata-jobs/mce-consumer-job/README.md index 4ca44f76336098..0aefe70d75bad2 100644 --- a/metadata-jobs/mce-consumer-job/README.md +++ b/metadata-jobs/mce-consumer-job/README.md @@ -22,9 +22,29 @@ However, if you only want to build `MCE Consumer Job` specifically: ## Dependencies Before starting `MCE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup) -Docker containers are up and running. MCE Consumer Job runs as part of the `datahub-gms` container. +and [DataHub GMS](../../docker/datahub-gms) Docker containers are up and running. + +## Start via Docker image +Quickest way to try out `MCE Consumer Job` is running the [Docker image](../../docker/datahub-mce-consumer). + +## Start via command line +If you do modify things and want to try it out quickly without building the Docker image, you can also run +the application directly from command line after a successful [build](#build): +``` +./gradlew :metadata-jobs:mce-consumer-job:bootRun +``` + +## Debugging + +To debug with an IDE (i.e. IntelliJ), run the `bootRun` task with the `--debug-jvm` flag. This will launch the app and +listen on port 5005 for a remote debugger. + +``` +./gradlew :metadata-jobs:mce-consumer-job:bootRun --debug-jvm +``` ## Endpoints +Spring boot actuator has been enabled for MCE Application. `healthcheck`, `metrics` and `info` web endpoints are enabled by default. `healthcheck` - http://localhost:9090/actuator/health From 86a98dfee3a416a3bc1850cd239489681362cb87 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 12:49:00 -0700 Subject: [PATCH 09/14] Revert changes to docker readme --- docker/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/README.md b/docker/README.md index 948bb7d6aa38b1..5bcb5ca4310ab1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -25,6 +25,8 @@ DataHub Docker Images: * [linkedin/datahub-gms](https://cloud.docker.com/repository/docker/linkedin/datahub-gms/) * [linkedin/datahub-frontend-react](https://cloud.docker.com/repository/docker/linkedin/datahub-frontend-react/) +* [linkedin/datahub-mae-consumer](https://cloud.docker.com/repository/docker/linkedin/datahub-mae-consumer/) +* [linkedin/datahub-mce-consumer](https://cloud.docker.com/repository/docker/linkedin/datahub-mce-consumer/) Dependencies: * [Kafka, Zookeeper, and Schema Registry](kafka-setup) From 545439474b7b069682bd552ad5cf26d889e6810b Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 13:54:31 -0700 Subject: [PATCH 10/14] Using if not --- .../datahub/charts/datahub-gms/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml index 69219181e50cfb..e0ef91188c135f 100644 --- a/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml +++ b/datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml @@ -73,7 +73,7 @@ spec: periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} env: - {{- if ne .Values.global.datahub_standalone_consumers_enabled true }} + {{- if not .Values.global.datahub_standalone_consumers_enabled }} - name: MCE_CONSUMER_ENABLED value: "true" - name: MAE_CONSUMER_ENABLED From ff2ab17bb70ab73ae87e7fa835c8c1d009c201ba Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 16:52:31 -0700 Subject: [PATCH 11/14] Removing mce mae containers --- metadata-ingestion/src/datahub/check/docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/check/docker.py b/metadata-ingestion/src/datahub/check/docker.py index e7ff5b66b56d8c..162d03b1c9f476 100644 --- a/metadata-ingestion/src/datahub/check/docker.py +++ b/metadata-ingestion/src/datahub/check/docker.py @@ -12,9 +12,7 @@ "elasticsearch-setup", "elasticsearch", "datahub-gms", - "datahub-mce-consumer", "datahub-frontend-react", - "datahub-mae-consumer", "kafka-setup", "schema-registry", "broker", @@ -26,6 +24,8 @@ # "schema-registry-ui", # "kibana", # "kafka-rest-proxy", + # "datahub-mce-consumer", + # "datahub-mae-consumer" ] # Docker seems to under-report memory allocated, so we also need a bit of buffer to account for it. From 3bb9d8cb687617a01f4eeff529079e84a67589e4 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Fri, 11 Jun 2021 17:20:20 -0700 Subject: [PATCH 12/14] Remove MAE Qualification Step --- .../upgrade/nocode/MAEQualificationStep.java | 87 ------------------- .../datahub/upgrade/nocode/NoCodeUpgrade.java | 1 - 2 files changed, 88 deletions(-) delete mode 100644 datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/MAEQualificationStep.java diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/MAEQualificationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/MAEQualificationStep.java deleted file mode 100644 index c68acf53fd5b9f..00000000000000 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/MAEQualificationStep.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.linkedin.datahub.upgrade.nocode; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.linkedin.datahub.upgrade.UpgradeContext; -import com.linkedin.datahub.upgrade.UpgradeStep; -import com.linkedin.datahub.upgrade.UpgradeStepResult; -import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; -import java.util.function.Function; - - -public class MAEQualificationStep implements UpgradeStep { - - private static String convertStreamToString(InputStream is) { - - BufferedReader reader = new BufferedReader(new InputStreamReader(is)); - StringBuilder sb = new StringBuilder(); - - String line = null; - try { - while ((line = reader.readLine()) != null) { - sb.append(line + "\n"); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - is.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return sb.toString(); - } - - MAEQualificationStep() { } - - @Override - public String id() { - return "MAEQualificationStep"; - } - - @Override - public int retryCount() { - return 2; - } - - @Override - public Function executable() { - return (context) -> { - String maeHost = System.getenv("DATAHUB_MAE_CONSUMER_HOST") == null ? "localhost" : System.getenv("DATAHUB_MAE_CONSUMER_HOST"); - String maePort = System.getenv("DATAHUB_MAE_CONSUMER_PORT") == null ? "9091" : System.getenv("DATAHUB_MAE_CONSUMER_PORT"); - try { - String spec = String.format("http://%s:%s/config", maeHost, maePort); - - URLConnection gmsConnection = new URL(spec).openConnection(); - InputStream response = gmsConnection.getInputStream(); - String responseString = convertStreamToString(response); - - ObjectMapper mapper = new ObjectMapper(); - JsonNode configJson = mapper.readTree(responseString); - if (configJson.get("noCode").asBoolean()) { - context.report().addLine("MAE Consumer is running and up to date. Proceeding with upgrade..."); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); - } else { - context.report().addLine(String.format("Failed to qualify MAE Consumer. It is not running on the latest version." - + "Re-run MAE Consumer on the latest datahub release")); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); - } - } catch (Exception e) { - e.printStackTrace(); - context.report().addLine(String.format( - "ERROR: Cannot connect to MAE Consumer" - + "at host %s port %s. Make sure MAE Consumer is on the latest version " - + "and is running at that host before starting the migration.", - maeHost, maePort)); - return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); - } - }; - } -} diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java index 658d0b9ab0f89f..e719bae8f85eba 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java @@ -58,7 +58,6 @@ private List buildUpgradeSteps( final List steps = new ArrayList<>(); steps.add(new RemoveAspectV2TableStep(server)); steps.add(new GMSQualificationStep()); - steps.add(new MAEQualificationStep()); steps.add(new UpgradeQualificationStep(server)); steps.add(new CreateAspectTableStep(server)); steps.add(new IngestDataPlatformsStep(entityService)); From 3d2675ab81c10548afdd499f2680bb71278869e1 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Sun, 13 Jun 2021 19:21:46 -0700 Subject: [PATCH 13/14] fix checkstyle --- .../kafka/config/DataHubUsageEventsProcessorCondition.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java index e61547edfac0d2..0413cd09c36b72 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/DataHubUsageEventsProcessorCondition.java @@ -13,7 +13,7 @@ public boolean matches( AnnotatedTypeMetadata metadata) { Environment env = context.getEnvironment(); return "true".equals(env.getProperty("MAE_CONSUMER_ENABLED")) && ( - env.getProperty("DATAHUB_ANALYTICS_ENABLED") == null || - "true".equals(env.getProperty("DATAHUB_ANALYTICS_ENABLED"))); + env.getProperty("DATAHUB_ANALYTICS_ENABLED") == null + || "true".equals(env.getProperty("DATAHUB_ANALYTICS_ENABLED"))); } } \ No newline at end of file From 4b0f3e2de3eb0376b0d3c126f03be9d2402b5628 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Mon, 14 Jun 2021 17:24:27 -0700 Subject: [PATCH 14/14] Removing mae, mce consumer from quickstart --- .../quickstart/docker-compose.quickstart.yml | 39 +------------------ 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 6b0886e1bad2e2..5f928233112697 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -55,48 +55,13 @@ services: - NEO4J_URI=bolt://neo4j - NEO4J_USERNAME=neo4j - NEO4J_PASSWORD=datahub + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true hostname: datahub-gms image: linkedin/datahub-gms:${DATAHUB_VERSION:-latest} mem_limit: 850m ports: - 8080:8080 - datahub-mae-consumer: - container_name: datahub-mae-consumer - depends_on: - - kafka-setup - - elasticsearch-setup - - neo4j - environment: - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - NEO4J_HOST=http://neo4j:7474 - - NEO4J_URI=bolt://neo4j - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=datahub - - GMS_HOST=datahub-gms - - GMS_PORT=8080 - hostname: datahub-mae-consumer - image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest} - mem_limit: 256m - ports: - - 9091:9091 - datahub-mce-consumer: - container_name: datahub-mce-consumer - depends_on: - - kafka-setup - - datahub-gms - environment: - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - GMS_HOST=datahub-gms - - GMS_PORT=8080 - hostname: datahub-mce-consumer - image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest} - mem_limit: 384m - ports: - - 9090:9090 elasticsearch: container_name: elasticsearch environment: