Merge remote-tracking branch 'origin/master' into update-remote-dao

.github/workflows/build-and-test.yml

@@ -65,14 +65,21 @@ jobs:
       - name: Gradle build
         run: ./gradlew build -x check -x docs-website:build
       - name: Smoke test
-        run: |
-          ./docker/dev.sh -d
-          sleep 30
-          ./smoke-test/smoke.sh
+        run: ./smoke-test/smoke.sh
       - name: Slack failure notification
         if: failure() && github.event_name == 'push'
         uses: kpritam/slack-job-status-action@v1
         with:
           job-status: ${{ job.status }}
           slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel: github-activities
+
+  quickstart-compose-validation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.6"
+      - name: Quickstart Compose Validation
+        run: ./docker/quickstart/generate_and_compare.sh

datahub-kubernetes/datahub/Chart.yaml

@@ -20,11 +20,11 @@ dependencies:
   - name: datahub-mae-consumer
     version: 0.2.1
     repository: file://./charts/datahub-mae-consumer
-    condition: datahub-mae-consumer.enabled
+    condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
     version: 0.2.1
     repository: file://./charts/datahub-mce-consumer
-    condition: datahub-mce-consumer.enabled
+    condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron
     version: 0.2.1
     repository: file://./charts/datahub-ingestion-cron

datahub-kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml

@@ -73,6 +73,14 @@ spec:
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           env:
+            {{- if not .Values.global.datahub_standalone_consumers_enabled }}
+            - name: MCE_CONSUMER_ENABLED
+              value: "true"
+            - name: MAE_CONSUMER_ENABLED
+              value: "true"
+            {{- end }}
+            - name: DATAHUB_ANALYTICS_ENABLED
+              value: "{{ .Values.global.datahub_analytics_enabled }}"
             - name: EBEAN_DATASOURCE_USERNAME
               value: "{{ .Values.global.sql.datasource.username }}"
             - name: EBEAN_DATASOURCE_PASSWORD

datahub-kubernetes/datahub/charts/datahub-gms/values.yaml

@@ -146,6 +146,8 @@ readinessProbe:
 #This section is useful if we are installing this chart separately for testing
 # helm install datahub-gms datahub-gms/
 global:
+  datahub_analytics_enabled: true
+
   elasticsearch:
     host: "elasticsearch"
     port: "9200"

datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml

@@ -73,6 +73,8 @@ spec:
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           env:
+            - name: MAE_CONSUMER_ENABLED
+              value: "true"
             - name: GMS_HOST
               value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
             - name: GMS_PORT

datahub-kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml

@@ -69,6 +69,8 @@ spec:
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           env:
+            - name: MCE_CONSUMER_ENABLED
+              value: "true"
             - name: KAFKA_BOOTSTRAP_SERVER
               value: "{{ .Values.global.kafka.bootstrap.server }}"
             - name: KAFKA_SCHEMAREGISTRY_URL

datahub-kubernetes/datahub/quickstart-values.yaml

@@ -15,18 +15,6 @@ datahub-frontend:
   ingress:
     enabled: false
 
-datahub-mae-consumer:
-  enabled: true
-  image:
-    repository: linkedin/datahub-mae-consumer
-    tag: "v0.8.1"
-
-datahub-mce-consumer:
-  enabled: true
-  image:
-    repository: linkedin/datahub-mce-consumer
-    tag: "v0.8.1"
-
 elasticsearchSetupJob:
   enabled: true
   image:

datahub-kubernetes/datahub/values.yaml

@@ -58,7 +58,9 @@ datahubUpgrade:
     tag: "v0.8.1"
 
 global:
+
   datahub_analytics_enabled: true
+  datahub_standalone_consumers_enabled: false
 
   elasticsearch:
     host: "elasticsearch-master"

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java

@@ -58,7 +58,6 @@ private List<UpgradeStep> buildUpgradeSteps(
     final List<UpgradeStep> steps = new ArrayList<>();
     steps.add(new RemoveAspectV2TableStep(server));
     steps.add(new GMSQualificationStep());
-    steps.add(new MAEQualificationStep());
     steps.add(new UpgradeQualificationStep(server));
     steps.add(new CreateAspectTableStep(server));
     steps.add(new IngestDataPlatformsStep(entityService));

docker/datahub-gms/env/docker.env

@@ -13,6 +13,12 @@ NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
 
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true
+
+# Uncomment to disable persistence of client-side analytics events
+# DATAHUB_ANALYTICS_ENABLED=false
+
 # Uncomment to configure kafka topic names
 # Make sure these names are consistent across the whole deployment
 # METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4

docker/datahub-gms/env/docker.mariadb.env

@@ -11,3 +11,5 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true

docker/datahub-gms/env/docker.postgres.env

@@ -11,3 +11,6 @@ NEO4J_HOST=http://neo4j:7474
 NEO4J_URI=bolt://neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=datahub
+MAE_CONSUMER_ENABLED=true
+MCE_CONSUMER_ENABLED=true
+

docker/datahub-mae-consumer/env/docker.env

@@ -1,3 +1,4 @@
+MAE_CONSUMER_ENABLED=true
 KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch

docker/datahub-mce-consumer/env/docker.env

@@ -1,3 +1,4 @@
+MCE_CONSUMER_ENABLED=true
 KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 GMS_HOST=datahub-gms

docker/docker-compose.consumers.dev.yml

@@ -0,0 +1,23 @@
+version: '3.8'
+services:
+  datahub-mae-consumer:
+    image: linkedin/datahub-mae-consumer:debug
+    build:
+      context: datahub-mae-consumer
+      dockerfile: Dockerfile
+      args:
+        APP_ENV: dev
+    volumes:
+      - ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh
+      - ../metadata-jobs/mae-consumer-job/build/libs/:/datahub/datahub-mae-consumer/bin/
+
+  datahub-mce-consumer:
+    image: linkedin/datahub-mce-consumer:debug
+    build:
+      context: datahub-mce-consumer
+      dockerfile: Dockerfile
+      args:
+        APP_ENV: dev
+    volumes:
+      - ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh
+      - ../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin

docker/docker-compose.consumers.yml

@@ -0,0 +1,31 @@
+# Service definitions for standalone Kafka consumer containers.
+version: '3.8'
+services:
+  datahub-mae-consumer:
+    build:
+      context: ../
+      dockerfile: docker/datahub-mae-consumer/Dockerfile
+    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
+    env_file: datahub-mae-consumer/env/docker.env
+    hostname: datahub-mae-consumer
+    container_name: datahub-mae-consumer
+    ports:
+      - "9091:9091"
+    depends_on:
+      - kafka-setup
+      - elasticsearch-setup
+      - neo4j
+
+  datahub-mce-consumer:
+    build:
+      context: ../
+      dockerfile: docker/datahub-mce-consumer/Dockerfile
+    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
+    env_file: datahub-mce-consumer/env/docker.env
+    hostname: datahub-mce-consumer
+    container_name: datahub-mce-consumer
+    ports:
+      - "9090:9090"
+    depends_on:
+      - kafka-setup
+      - datahub-gms

docker/docker-compose.dev.yml

@@ -50,25 +50,3 @@ services:
         APP_ENV: dev
     volumes:
       - ../datahub-frontend/build/stage/datahub-frontend:/datahub-frontend
-
-  datahub-mae-consumer:
-    image: linkedin/datahub-mae-consumer:debug
-    build:
-      context: datahub-mae-consumer
-      dockerfile: Dockerfile
-      args:
-        APP_ENV: dev
-    volumes:
-      - ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh
-      - ../metadata-jobs/mae-consumer-job/build/libs/:/datahub/datahub-mae-consumer/bin/
-
-  datahub-mce-consumer:
-    image: linkedin/datahub-mce-consumer:debug
-    build:
-      context: datahub-mce-consumer
-      dockerfile: Dockerfile
-      args:
-        APP_ENV: dev
-    volumes:
-      - ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh
-      - ../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin

docker/docker-compose.override.yml

@@ -14,6 +14,17 @@ services:
       - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
       - mysqldata:/var/lib/mysql
 
+  mysql-setup:
+    build:
+      context: ../
+      dockerfile: docker/mysql-setup/Dockerfile
+    image: acryldata/datahub-mysql-setup:head
+    env_file: mysql-setup/env/docker.env
+    hostname: mysql-setup
+    container_name: mysql-setup
+    depends_on:
+      - mysql
+
   datahub-gms:
     env_file: datahub-gms/env/docker.env
     depends_on:

docker/docker-compose.yml

@@ -160,35 +160,6 @@ services:
     depends_on:
       - datahub-gms
 
-  datahub-mae-consumer:
-    build:
-      context: ../
-      dockerfile: docker/datahub-mae-consumer/Dockerfile
-    image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
-    env_file: datahub-mae-consumer/env/docker.env
-    hostname: datahub-mae-consumer
-    container_name: datahub-mae-consumer
-    ports:
-        - "9091:9091"
-    depends_on:
-      - kafka-setup
-      - elasticsearch-setup
-      - neo4j
-
-  datahub-mce-consumer:
-    build:
-      context: ../
-      dockerfile: docker/datahub-mce-consumer/Dockerfile
-    image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
-    env_file: datahub-mce-consumer/env/docker.env
-    hostname: datahub-mce-consumer
-    container_name: datahub-mce-consumer
-    ports:
-      - "9090:9090"
-    depends_on:
-      - kafka-setup
-      - datahub-gms
-
 networks:
   default:
     name: datahub_network

docker/elasticsearch/env/docker.env

@@ -1,3 +1,3 @@
 discovery.type=single-node
 xpack.security.enabled=false
-ES_JAVA_OPTS=-Xms1g -Xmx1g
+ES_JAVA_OPTS=-Xms512m -Xmx512m

docker/mysql-setup/env/docker.env

@@ -0,0 +1,5 @@
+MYSQL_HOST=mysql
+MYSQL_PORT=3306
+MYSQL_USERNAME=datahub
+MYSQL_PASSWORD=datahub
+DATAHUB_DB_NAME=datahub