dbt-spark/.circleci/config.yml at ef99d3e84d1e8d79cbda3302134624db1f33a39e · dbt-labs/dbt-spark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
version: 2.1

jobs:
  unit:
    environment:
      DBT_INVOCATION_ENV: circle
    docker:
      - image: fishtownanalytics/test-container:10
    steps:
      - checkout
      - run: tox -e flake8,unit

# Turning off for now due to flaky runs of tests will turn back on at later date.
  # integration-spark-session:
  #   environment:
  #     DBT_INVOCATION_ENV: circle
  #   docker:
  #     - image: godatadriven/pyspark:3.1
  #   steps:
  #     - checkout
  #     - run: apt-get update
  #     - run: python3 -m pip install --upgrade pip
  #     - run: apt-get install -y git gcc g++ unixodbc-dev libsasl2-dev
  #     - run: python3 -m pip install tox
  #     - run:
  #         name: Run integration tests
  #         command: tox -e integration-spark-session
  #         no_output_timeout: 1h
  #     - store_artifacts:
  #         path: ./logs

  integration-spark-thrift:
    environment:
      DBT_INVOCATION_ENV: circle
    docker:
      - image: fishtownanalytics/test-container:10
      - image: godatadriven/spark:3.1.1
        environment:
          WAIT_FOR: localhost:5432
        command: >
          --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
          --name Thrift JDBC/ODBC Server
      - image: postgres:9.6.17-alpine
        environment:
          POSTGRES_USER: dbt
          POSTGRES_PASSWORD: dbt
          POSTGRES_DB: metastore

    steps:
      - checkout

      - run:
          name: Wait for Spark-Thrift
          command: dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s

      - run:
          name: Run integration tests
          command: tox -e integration-spark-thrift
          no_output_timeout: 1h
      - store_artifacts:
          path: ./logs

  integration-spark-databricks-http:
    environment:
      DBT_INVOCATION_ENV: circle
      DBT_DATABRICKS_RETRY_ALL: True
      DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com"
      DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com"
      DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com"
    docker:
      - image: fishtownanalytics/test-container:10
    steps:
      - checkout
      - run:
          name: Run integration tests
          command: tox -e integration-spark-databricks-http
          no_output_timeout: 1h
      - store_artifacts:
          path: ./logs

  integration-spark-databricks-odbc-cluster: &databricks-odbc
    environment:
      DBT_INVOCATION_ENV: circle
      ODBC_DRIVER: Simba # TODO: move env var to Docker image
      DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com"
      DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com"
      DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com"
    docker:
      # image based on `fishtownanalytics/test-container` w/ Simba ODBC Spark driver installed
      - image: 828731156495.dkr.ecr.us-east-1.amazonaws.com/dbt-spark-odbc-test-container:latest
        aws_auth:
          aws_access_key_id: $AWS_ACCESS_KEY_ID_STAGING
          aws_secret_access_key: $AWS_SECRET_ACCESS_KEY_STAGING
    steps:
      - checkout
      - run:
          name: Run integration tests
          command: tox -e integration-spark-databricks-odbc-cluster
          no_output_timeout: 1h
      - store_artifacts:
          path: ./logs

  integration-spark-databricks-odbc-endpoint:
    <<: *databricks-odbc
    steps:
      - checkout
      - run:
          name: Run integration tests
          command: tox -e integration-spark-databricks-odbc-sql-endpoint
          no_output_timeout: 1h
      - store_artifacts:
          path: ./logs

workflows:
  version: 2
  test-everything:
    jobs:
      - unit
      - integration-spark-session:
          requires:
            - unit
      - integration-spark-thrift:
          requires:
            - unit
      - integration-spark-databricks-http:
          requires:
            - integration-spark-thrift
      - integration-spark-databricks-odbc-cluster:
          context: aws-credentials
          requires:
            - integration-spark-thrift
      - integration-spark-databricks-odbc-endpoint:
          context: aws-credentials
          requires:
            - integration-spark-thrift