Skip to content

Commit 3786f1a

Browse files
jjoyce0510jjoyce0510
committed
Merge branch 'master' into MergeMAEMCE
2 parents 3d2675a + 7591c89 commit 3786f1a

39 files changed

+1138
-145
lines changed

‎.github/workflows/build-and-test.yml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,21 @@ jobs:
6565
- name: Gradle build
6666
run: ./gradlew build -x check -x docs-website:build
6767
- name: Smoke test
68-
run: |
69-
./docker/dev.sh -d
70-
sleep 30
71-
./smoke-test/smoke.sh
68+
run: ./smoke-test/smoke.sh
7269
- name: Slack failure notification
7370
if: failure() && github.event_name == 'push'
7471
uses: kpritam/slack-job-status-action@v1
7572
with:
7673
job-status: ${{ job.status }}
7774
slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
7875
channel: github-activities
76+
77+
quickstart-compose-validation:
78+
runs-on: ubuntu-latest
79+
steps:
80+
- uses: actions/checkout@v2
81+
- uses: actions/setup-python@v2
82+
with:
83+
python-version: "3.6"
84+
- name: Quickstart Compose Validation
85+
run: ./docker/quickstart/generate_and_compare.sh

‎docker/docker-compose.override.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,17 @@ services:
1414
- ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
1515
- mysqldata:/var/lib/mysql
1616

17+
mysql-setup:
18+
build:
19+
context: ../
20+
dockerfile: docker/mysql-setup/Dockerfile
21+
image: acryldata/datahub-mysql-setup:head
22+
env_file: mysql-setup/env/docker.env
23+
hostname: mysql-setup
24+
container_name: mysql-setup
25+
depends_on:
26+
- mysql
27+
1728
datahub-gms:
1829
env_file: datahub-gms/env/docker.env
1930
depends_on:

‎docker/elasticsearch/env/docker.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
discovery.type=single-node
22
xpack.security.enabled=false
3-
ES_JAVA_OPTS=-Xms1g -Xmx1g
3+
ES_JAVA_OPTS=-Xms512m -Xmx512m

‎docker/mysql-setup/env/docker.env‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
MYSQL_HOST=mysql
2+
MYSQL_PORT=3306
3+
MYSQL_USERNAME=datahub
4+
MYSQL_PASSWORD=datahub
5+
DATAHUB_DB_NAME=datahub

‎docker/quickstart/docker-compose.quickstart.yml‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
networks:
2+
default:
3+
name: datahub_network
4+
services:
5+
broker:
6+
container_name: broker
7+
depends_on:
8+
- zookeeper
9+
environment:
10+
- KAFKA_BROKER_ID=1
11+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
12+
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
13+
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
14+
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
15+
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
16+
hostname: broker
17+
image: confluentinc/cp-kafka:5.4.0
18+
ports:
19+
- 29092:29092
20+
- 9092:9092
21+
datahub-frontend-react:
22+
container_name: datahub-frontend-react
23+
depends_on:
24+
- datahub-gms
25+
environment:
26+
- DATAHUB_GMS_HOST=datahub-gms
27+
- DATAHUB_GMS_PORT=8080
28+
- DATAHUB_SECRET=YouKnowNothing
29+
- DATAHUB_APP_VERSION=1.0
30+
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
31+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
32+
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
33+
- ELASTIC_CLIENT_HOST=elasticsearch
34+
- ELASTIC_CLIENT_PORT=9200
35+
hostname: datahub-frontend-react
36+
image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-latest}
37+
ports:
38+
- 9002:9002
39+
datahub-gms:
40+
container_name: datahub-gms
41+
depends_on:
42+
- mysql
43+
environment:
44+
- DATASET_ENABLE_SCSI=false
45+
- EBEAN_DATASOURCE_USERNAME=datahub
46+
- EBEAN_DATASOURCE_PASSWORD=datahub
47+
- EBEAN_DATASOURCE_HOST=mysql:3306
48+
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
49+
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
50+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
51+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
52+
- ELASTICSEARCH_HOST=elasticsearch
53+
- ELASTICSEARCH_PORT=9200
54+
- NEO4J_HOST=http://neo4j:7474
55+
- NEO4J_URI=bolt://neo4j
56+
- NEO4J_USERNAME=neo4j
57+
- NEO4J_PASSWORD=datahub
58+
hostname: datahub-gms
59+
image: linkedin/datahub-gms:${DATAHUB_VERSION:-latest}
60+
mem_limit: 850m
61+
ports:
62+
- 8080:8080
63+
datahub-mae-consumer:
64+
container_name: datahub-mae-consumer
65+
depends_on:
66+
- kafka-setup
67+
- elasticsearch-setup
68+
- neo4j
69+
environment:
70+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
71+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
72+
- ELASTICSEARCH_HOST=elasticsearch
73+
- ELASTICSEARCH_PORT=9200
74+
- NEO4J_HOST=http://neo4j:7474
75+
- NEO4J_URI=bolt://neo4j
76+
- NEO4J_USERNAME=neo4j
77+
- NEO4J_PASSWORD=datahub
78+
- GMS_HOST=datahub-gms
79+
- GMS_PORT=8080
80+
hostname: datahub-mae-consumer
81+
image: linkedin/datahub-mae-consumer:${DATAHUB_VERSION:-latest}
82+
mem_limit: 256m
83+
ports:
84+
- 9091:9091
85+
datahub-mce-consumer:
86+
container_name: datahub-mce-consumer
87+
depends_on:
88+
- kafka-setup
89+
- datahub-gms
90+
environment:
91+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
92+
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
93+
- GMS_HOST=datahub-gms
94+
- GMS_PORT=8080
95+
hostname: datahub-mce-consumer
96+
image: linkedin/datahub-mce-consumer:${DATAHUB_VERSION:-latest}
97+
mem_limit: 384m
98+
ports:
99+
- 9090:9090
100+
elasticsearch:
101+
container_name: elasticsearch
102+
environment:
103+
- discovery.type=single-node
104+
- xpack.security.enabled=false
105+
- ES_JAVA_OPTS=-Xms512m -Xmx512m
106+
healthcheck:
107+
retries: 4
108+
start_period: 2m
109+
test:
110+
- CMD-SHELL
111+
- curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
112+
|| exit 1
113+
hostname: elasticsearch
114+
image: elasticsearch:7.9.3
115+
mem_limit: 1g
116+
ports:
117+
- 9200:9200
118+
volumes:
119+
- esdata:/usr/share/elasticsearch/data
120+
elasticsearch-setup:
121+
container_name: elasticsearch-setup
122+
depends_on:
123+
- elasticsearch
124+
environment:
125+
- ELASTICSEARCH_HOST=elasticsearch
126+
- ELASTICSEARCH_PORT=9200
127+
- ELASTICSEARCH_PROTOCOL=http
128+
hostname: elasticsearch-setup
129+
image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-latest}
130+
kafka-setup:
131+
container_name: kafka-setup
132+
depends_on:
133+
- broker
134+
- schema-registry
135+
environment:
136+
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
137+
- KAFKA_BOOTSTRAP_SERVER=broker:29092
138+
hostname: kafka-setup
139+
image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-latest}
140+
mysql:
141+
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
142+
container_name: mysql
143+
environment:
144+
- MYSQL_DATABASE=datahub
145+
- MYSQL_USER=datahub
146+
- MYSQL_PASSWORD=datahub
147+
- MYSQL_ROOT_PASSWORD=datahub
148+
hostname: mysql
149+
image: mysql:5.7
150+
ports:
151+
- 3306:3306
152+
volumes:
153+
- ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
154+
- mysqldata:/var/lib/mysql
155+
mysql-setup:
156+
container_name: mysql-setup
157+
depends_on:
158+
- mysql
159+
environment:
160+
- MYSQL_HOST=mysql
161+
- MYSQL_PORT=3306
162+
- MYSQL_USERNAME=datahub
163+
- MYSQL_PASSWORD=datahub
164+
- DATAHUB_DB_NAME=datahub
165+
hostname: mysql-setup
166+
image: acryldata/datahub-mysql-setup:head
167+
neo4j:
168+
container_name: neo4j
169+
environment:
170+
- NEO4J_AUTH=neo4j/datahub
171+
- NEO4J_dbms_default__database=graph.db
172+
- NEO4J_dbms_allow__upgrade=true
173+
hostname: neo4j
174+
image: neo4j:4.0.6
175+
ports:
176+
- 7474:7474
177+
- 7687:7687
178+
volumes:
179+
- neo4jdata:/data
180+
schema-registry:
181+
container_name: schema-registry
182+
depends_on:
183+
- zookeeper
184+
- broker
185+
environment:
186+
- SCHEMA_REGISTRY_HOST_NAME=schemaregistry
187+
- SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
188+
hostname: schema-registry
189+
image: confluentinc/cp-schema-registry:5.4.0
190+
ports:
191+
- 8081:8081
192+
zookeeper:
193+
container_name: zookeeper
194+
environment:
195+
- ZOOKEEPER_CLIENT_PORT=2181
196+
- ZOOKEEPER_TICK_TIME=2000
197+
hostname: zookeeper
198+
image: confluentinc/cp-zookeeper:5.4.0
199+
ports:
200+
- 2181:2181
201+
volumes:
202+
- zkdata:/var/opt/zookeeper
203+
version: '2'
204+
volumes:
205+
esdata: null
206+
mysqldata: null
207+
neo4jdata: null
208+
zkdata: null

‎docker/quickstart/generate_and_compare.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
cd "$DIR"
5+
6+
set -euxo pipefail
7+
8+
python3 -m venv venv
9+
source venv/bin/activate
10+
11+
pip install -r requirements.txt
12+
python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml
13+
14+
if cmp docker-compose.quickstart.yml temp.quickstart.yml; then
15+
printf 'docker-compose.quickstart.yml is up to date.'
16+
exit 0
17+
else
18+
printf 'docker-compose.quickstart.yml is out of date.'
19+
exit 1
20+
fi

0 commit comments

Comments
 (0)