feat: Converting MCE to a Spring boot Application (datahub-project#1629)

Author: arunvasudevan

build.gradle

@@ -66,6 +66,10 @@ project.ext.externalDependency = [
     'springCore': 'org.springframework:spring-core:5.1.6.RELEASE',
     'springJdbc': 'org.springframework:spring-jdbc:5.1.6.RELEASE',
     'springWeb': 'org.springframework:spring-web:5.1.6.RELEASE',
+    'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.2.RELEASE',
+    'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.2.RELEASE',
+    'springKafka': 'org.springframework.kafka:spring-kafka:2.2.3.RELEASE',
+    'springActuator': 'org.springframework.boot:spring-boot-starter-actuator:2.1.2.RELEASE',
     'testng': 'org.testng:testng:6.9.9'
 ]
 
@@ -102,4 +106,4 @@ subprojects {
       }
     }
   }
-}
+}

docker/mce-consumer/Dockerfile

@@ -4,9 +4,13 @@ MAINTAINER Kerem Sahin [email protected]
 
 COPY . datahub-src
 RUN cd datahub-src && ./gradlew :metadata-jobs:mce-consumer-job:build \
-    && cp metadata-jobs/mce-consumer-job/build/distributions/mce-consumer-job.zip ../mce-consumer-job.zip \
-    && cd .. && rm -rf datahub-src && unzip mce-consumer-job.zip
+    && cp metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar ../mce-consumer-job.jar \
+    && cd .. && rm -rf datahub-src
 
 FROM openjdk:8-jre-alpine
 
-COPY --from=builder /mce-consumer-job /mce-consumer-job/
+COPY --from=builder /mce-consumer-job.jar /mce-consumer-job.jar
+
+EXPOSE 9090
+
+ENTRYPOINT ["java", "-jar", "mce-consumer-job.jar"]

docker/mce-consumer/docker-compose.yml

@@ -5,13 +5,16 @@ services:
     image: keremsahin/datahub-mce-consumer:latest
     hostname: datahub-mce-consumer
     container_name: datahub-mce-consumer
+    ports:
+        - "9090:9090"
     environment:
       - KAFKA_BOOTSTRAP_SERVER=broker:29092
       - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
       - GMS_HOST=datahub-gms
       - GMS_PORT=8080
-    command: "sh -c './mce-consumer-job/bin/mce-consumer-job'"
+      - KAFKA_MCE_TOPIC_NAME=MetadataChangeEvent
+      - KAFKA_FMCE_TOPIC_NAME=FailedMetadataChangeEvent
 
 networks:
   default:
-    name: datahub_network
+    name: datahub_network

docker/quickstart/docker-compose.yml

@@ -241,6 +241,8 @@ services:
     image: keremsahin/datahub-mce-consumer:latest
     hostname: datahub-mce-consumer
     container_name: datahub-mce-consumer
+    ports:
+        - "9090:9090"
     environment:
       - KAFKA_BOOTSTRAP_SERVER=broker:29092
       - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
@@ -250,7 +252,7 @@ services:
       - kafka-setup
       - datahub-gms
     command: "sh -c 'while ping -c1 kafka-setup &>/dev/null; do echo waiting for kafka-setup... && sleep 1; done; \
-              echo kafka-setup done! && ./mce-consumer-job/bin/mce-consumer-job'"
+                  echo kafka-setup done! && java -jar mce-consumer-job.jar'"
 
 networks:
   default:

metadata-jobs/mce-consumer-job/README.md

@@ -29,5 +29,14 @@ Quickest way to try out `MCE Consumer Job` is running the [Docker image](../../d
 If you do modify things and want to try it out quickly without building the Docker image, you can also run
 the application directly from command line after a successful [build](#build):
 ```
-./gradlew :metadata-jobs:mce-consumer-job:run
-```
+./gradlew :metadata-jobs:mce-consumer-job:bootRun
+```
+## Endpoints
+Spring boot actuator has been enabled for MCE Application. 
+`healthcheck`, `metrics` and `info` web endpoints are enabled by default.
+
+`healthcheck` - http://localhost:9090/actuator/health
+`metrics` - http://localhost:9090/actuator/metrics
+
+To retrieve a specific metric - http://localhost:9090/actuator/metrics/kafka.consumer.records.consumed.total
+

metadata-jobs/mce-consumer-job/build.gradle

@@ -1,5 +1,8 @@
-apply plugin: 'application'
-apply plugin: 'java'
+plugins {
+    id 'org.springframework.boot' version '2.1.2.RELEASE'
+    id 'java'
+}
+
 apply plugin: 'pegasus'
 
 configurations {
@@ -23,6 +26,13 @@ dependencies {
     compile externalDependency.kafkaAvroSerde
     compile externalDependency.kafkaStreams
     compile externalDependency.kafkaSerializers
+    compile (externalDependency.springBootStarterWeb) {
+        exclude module: "spring-boot-starter-tomcat"
+    }
+    compile externalDependency.springBootStarterJetty
+    compile externalDependency.springKafka
+
+    compile externalDependency.springActuator
 
     compileOnly externalDependency.lombok
 
@@ -47,4 +57,6 @@ clean {
     project.delete("src/main/resources/avro")
 }
 
-mainClassName = 'com.linkedin.metadata.kafka.MceStreamTask'
+bootJar {
+    mainClassName = 'com.linkedin.metadata.kafka.MceConsumerApplication'
+}

metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java

@@ -0,0 +1,15 @@
+package com.linkedin.metadata.kafka;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientAutoConfiguration;
+import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;
+
+@SpringBootApplication(exclude = {RestClientAutoConfiguration.class, KafkaAutoConfiguration.class})
+public class MceConsumerApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(MceConsumerApplication.class, args);
+    }
+
+}