Fixing mis-applied changes

datahub-project · shirshanka · Jun 15, 2021 · Jun 10, 2021 · Jun 10, 2021 · Jun 10, 2021
commit fea09ccf3ecff35472ed687e67a33ddb4cacf245
diff --git a/.github/workflows/docker-mae-consumer.yml b/.github/workflows/docker-mae-consumer.yml
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/.github/workflows/docker-mce-consumer.yml b/.github/workflows/docker-mce-consumer.yml
@@ -45,7 +45,6 @@ jobs:
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.outputs.publish == 'true' }}
     needs: setup
     steps:
       - name: Check out the repo
@@ -61,6 +60,7 @@ jobs:
           tag-custom: ${{ needs.setup.outputs.tag }}
           tag-custom-only: true
       - name: Login to DockerHub
+        if: ${{ needs.setup.outputs.publish == 'true' }}
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md
@@ -31,7 +31,7 @@ Current chart version is `0.2.0`
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-mae-consumer"` |  |
-| image.tag | string | `"latest"` |  |
+| image.tag | string | `"v0.8.1"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |

diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml
@@ -54,17 +54,21 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 9091
+              protocol: TCP
           livenessProbe:
             httpGet:
               path: /actuator/health
-              port: 9091
+              port: http
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
               path: /actuator/health
-              port: 9091
+              port: http
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-mae-consumer" }}
+  labels:
+    {{- include "datahub-mae-consumer.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.global.datahub.mae_consumer.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "datahub-mae-consumer.selectorLabels" . | nindent 4 }}
diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: linkedin/datahub-mae-consumer
   pullPolicy: Always
-  tag: latest
+  tag: v0.8.1
 
 imagePullSecrets: []
 nameOverride: ""
@@ -146,7 +146,6 @@ readinessProbe:
 
 global:
   datahub_analytics_enabled: true
-  datahub_consumer_jobs_enabled: true
 
   elasticsearch:
     host: "elasticsearch"
@@ -166,6 +165,10 @@ global:
       secretRef: "neo4j-secrets"
       secretKey: "neo4j-password"
 
+  datahub:
+    mae_consumer:
+      port: "9091"
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md
@@ -24,7 +24,7 @@ Current chart version is `0.2.0`
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-mce-consumer"` |  |
-| image.tag | string | `"latest"` |  |
+| image.tag | string | `"v0.8.1"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |

diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: linkedin/datahub-mce-consumer
   pullPolicy: Always
-  tag: latest
+  tag: v0.8.1
 
 imagePullSecrets: []
 nameOverride: ""
@@ -145,9 +145,6 @@ readinessProbe:
   failureThreshold: 4
 
 global:
-
-  datahub_consumer_jobs_enabled: true
-
   kafka:
     bootstrap:
       server: "broker:9092"

diff --git a/datahub-kubernetes/datahub/quickstart-values.yaml b/datahub-kubernetes/datahub/quickstart-values.yaml
@@ -15,18 +15,6 @@ datahub-frontend:
   ingress:
     enabled: false
 
-datahub-mae-consumer:
-  enabled: false
-  image:
-    repository: linkedin/datahub-mae-consumer
-    tag: "v0.8.1"
-
-datahub-mce-consumer:
-  enabled: false
-  image:
-    repository: linkedin/datahub-mce-consumer
-    tag: "v0.8.1"
-
 elasticsearchSetupJob:
   enabled: true
   image:

diff --git a/datahub-kubernetes/datahub/values.yaml b/datahub-kubernetes/datahub/values.yaml
@@ -16,13 +16,13 @@ datahub-frontend:
     enabled: false
 
 datahub-mae-consumer:
-  enabled: false
+  enabled: true
   image:
     repository: linkedin/datahub-mae-consumer
     tag: "v0.8.1"
 
 datahub-mce-consumer:
-  enabled: false
+  enabled: true
   image:
     repository: linkedin/datahub-mce-consumer
     tag: "v0.8.1"

diff --git a/metadata-jobs/mae-consumer-job/README.md b/metadata-jobs/mae-consumer-job/README.md
@@ -22,7 +22,18 @@ However, if you only want to build `MAE Consumer Job` specifically:
 Before starting `MAE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup),
 [Elasticsearch](../../docker/elasticsearch), and [Neo4j](../../docker/neo4j) Docker containers are up and running.
 
-## Endpoints TODO: Reevaluate these
+## Start via Docker image
+Quickest way to try out `MAE Consumer Job` is running the [Docker image](../../docker/datahub-mae-consumer).
+
+## Start via command line
+If you do modify things and want to try it out quickly without building the Docker image, you can also run
+the application directly from command line after a successful [build](#build):
+```
+./gradlew :metadata-jobs:mae-consumer-job:bootRun
+```
+
+## Endpoints
+Spring boot actuator has been enabled for MAE Application. 
 `healthcheck`, `metrics` and `info` web endpoints are enabled by default.
 
 `healthcheck` - http://localhost:9091/actuator/health

diff --git a/metadata-jobs/mce-consumer-job/README.md b/metadata-jobs/mce-consumer-job/README.md
@@ -22,9 +22,29 @@ However, if you only want to build `MCE Consumer Job` specifically:
 
 ## Dependencies
 Before starting `MCE Consumer Job`, you need to make sure that [Kafka, Schema Registry & Zookeeper](../../docker/kafka-setup)
-Docker containers are up and running. MCE Consumer Job runs as part of the `datahub-gms` container.
+and [DataHub GMS](../../docker/datahub-gms) Docker containers are up and running.
+
+## Start via Docker image
+Quickest way to try out `MCE Consumer Job` is running the [Docker image](../../docker/datahub-mce-consumer).
+
+## Start via command line
+If you do modify things and want to try it out quickly without building the Docker image, you can also run
+the application directly from command line after a successful [build](#build):
+```
+./gradlew :metadata-jobs:mce-consumer-job:bootRun
+```
+
+## Debugging
+
+To debug with an IDE (i.e. IntelliJ), run the `bootRun` task with the `--debug-jvm` flag. This will launch the app and
+listen on port 5005 for a remote debugger.
+
+```
+./gradlew :metadata-jobs:mce-consumer-job:bootRun --debug-jvm
+```
 
 ## Endpoints
+Spring boot actuator has been enabled for MCE Application. 
 `healthcheck`, `metrics` and `info` web endpoints are enabled by default.
 
 `healthcheck` - http://localhost:9090/actuator/health