Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
334a485
Update README
kevinhu Jun 9, 2021
8580c54
Merge branch 'master' of github.com:kevinhu/datahub into glue-etl
kevinhu Jun 9, 2021
499d308
Merge branch 'linkedin:master' into glue-etl
kevinhu Jun 10, 2021
520b099
Merge branch 'glue-etl' of github.com:kevinhu/datahub into glue-etl
kevinhu Jun 10, 2021
1941e93
Merge branch 'linkedin:master' into glue-etl
kevinhu Jun 10, 2021
5270fac
Read transformation DAGs
kevinhu Jun 11, 2021
67c0807
Extract node sources
kevinhu Jun 11, 2021
3ffdb1a
Init glue MCEs
kevinhu Jun 11, 2021
bdae7c1
Refactor job and flow wus
kevinhu Jun 11, 2021
4acc825
Resolve source and sink datasets
kevinhu Jun 11, 2021
68ed8e5
Merge branch 'linkedin:master' into glue-etl
kevinhu Jun 11, 2021
2fd692b
Set URNs correctly
kevinhu Jun 11, 2021
a0bc357
Isort and update snapshot JSONs
kevinhu Jun 11, 2021
e4b4d64
Successful ingestion
kevinhu Jun 11, 2021
7f0eb42
Refactor job listing
kevinhu Jun 11, 2021
fe63ce6
Glue ETL comments
kevinhu Jun 11, 2021
218338e
Clean up s3 naming
kevinhu Jun 11, 2021
5c1c9f2
Add job properties
kevinhu Jun 12, 2021
5e4873a
Fix lint errors
kevinhu Jun 14, 2021
4739367
Temp disable extract_transform in tests
kevinhu Jun 14, 2021
ce58f7b
Fix S3 URN
kevinhu Jun 14, 2021
a37b1db
Stubs for S3
kevinhu Jun 14, 2021
5050b8f
Fix lint errors
kevinhu Jun 14, 2021
ff682a8
Create Glue golden MCE json
kevinhu Jun 15, 2021
5144ffc
Trim Glue golden MCE
kevinhu Jun 15, 2021
ab419a4
Reapply freeze to Glue files
kevinhu Jun 15, 2021
ecd89a6
Fix golden path
kevinhu Jun 15, 2021
06d6c8a
Merge
kevinhu Jun 15, 2021
51ae22a
Fix duplicate MCEs
kevinhu Jun 15, 2021
108a203
Fix outputDatasets
kevinhu Jun 15, 2021
d9d8bc5
Remove S3 URIs
kevinhu Jun 15, 2021
e4a96b4
Expand job names
kevinhu Jun 15, 2021
db9d8ca
Expand job custom props
kevinhu Jun 15, 2021
7e6333b
Update golden
kevinhu Jun 15, 2021
4026917
Remove ownership classes
kevinhu Jun 15, 2021
c436149
Clean up redundant properties
kevinhu Jun 15, 2021
8619918
Merge branch 'linkedin:master' into glue-etl
kevinhu Jun 15, 2021
33dd0ea
Fix topological sort
kevinhu Jun 15, 2021
e0eaf0c
Fix S3 browse paths
kevinhu Jun 15, 2021
6f7b74a
Restore feast
kevinhu Jun 15, 2021
c738a74
Smaller stubs
kevinhu Jun 15, 2021
1d89927
Update README
kevinhu Jun 16, 2021
fe5b087
Resolve golden script conflict
kevinhu Jun 17, 2021
948dd24
Regenerate snapshot JSONs
kevinhu Jun 17, 2021
8a681cd
Merge
kevinhu Jun 17, 2021
95efa7e
Rebuild
kevinhu Jun 17, 2021
55e6619
Refactor node processing
kevinhu Jun 17, 2021
51bdb88
Add links to boto docs
kevinhu Jun 17, 2021
59bbf15
Fix sequence type error
kevinhu Jun 18, 2021
2a4501b
Fix Id typo
kevinhu Jun 18, 2021
8feb874
Merge branch 'linkedin:master' into glue-etl
kevinhu Jun 18, 2021
d173d6c
Types for process_dataflow_graph
kevinhu Jun 18, 2021
1b44baf
Include extension type in glue imports
kevinhu Jun 18, 2021
d80f529
S3 deduplication logic
kevinhu Jun 19, 2021
db1c638
Fix type annotation
kevinhu Jun 19, 2021
8eb3d50
Add comments for deduplication
kevinhu Jun 19, 2021
5b66074
Fix dataset IDs for Glue
kevinhu Jun 21, 2021
9955e06
Merge branch 'linkedin:master' into glue-etl
kevinhu Jun 21, 2021
6335f95
Update golden files
kevinhu Jun 21, 2021
09b9b60
Merge
kevinhu Jun 22, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Merge
  • Loading branch information
kevinhu committed Jun 22, 2021
commit 09b9b60f6e51d1256d9b46f8d6f1c89f4fb9406f
13 changes: 7 additions & 6 deletions metadata-ingestion/scripts/update_golden_files.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@ pytest --basetemp=tmp || true
# Update the golden files.
cp tmp/test_serde_to_json_tests_unit_0/output.json tests/unit/serde/test_serde_large.json
cp tmp/test_serde_to_json_tests_unit_1/output.json tests/unit/serde/test_serde_chart_snapshot.json
cp tmp/test_ldap_ingest0/ldap_mces.json tests/integration/ldap/ldap_mce_golden.json
cp tmp/test_mysql_ingest0/mysql_mces.json tests/integration/mysql/mysql_mce_golden.json
cp tmp/test_mssql_ingest0/mssql_mces.json tests/integration/sql_server/mssql_mce_golden.json
cp tmp/test_mongodb_ingest0/mongodb_mces.json tests/integration/mongodb/mongodb_mce_golden.json
cp tmp/test_feast_ingest0/feast_mces.json tests/integration/feast/feast_mce_golden.json
cp tmp/test_glue_ingest0/glue_mce.json tests/unit/glue/glue_mce_golden.json
cp tmp/test_ldap_ingest0/ldap_mces.json tests/integration/ldap/ldap_mces_golden.json
cp tmp/test_mysql_ingest0/mysql_mces.json tests/integration/mysql/mysql_mces_golden.json
cp tmp/test_mssql_ingest0/mssql_mces.json tests/integration/sql_server/mssql_mces_golden.json
cp tmp/test_mongodb_ingest0/mongodb_mces.json tests/integration/mongodb/mongodb_mces_golden.json
cp tmp/test_feast_ingest0/feast_mces.json tests/integration/feast/feast_mces_golden.json
cp tmp/test_dbt_ingest0/dbt_mces.json tests/integration/dbt/dbt_mces_golden.json
cp tmp/test_glue_ingest0/glue_mces.json tests/unit/glue/glue_mces_golden.json
cp tmp/test_lookml_ingest0/lookml_mces.json tests/integration/lookml/expected_output.json
cp tmp/test_looker_ingest0/looker_mces.json tests/integration/looker/expected_output.json

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker, my_view, PROD)",
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,my_view,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.common.Status": {
Expand Down
6 changes: 3 additions & 3 deletions metadata-ingestion/tests/unit/test_glue_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,13 @@ def test_glue_ingest(tmp_path, pytestconfig):
wu.mce.to_obj() for wu in glue_source_instance.get_workunits()
]

with open(str(tmp_path / "glue_mce.json"), "w") as f:
with open(str(tmp_path / "glue_mces.json"), "w") as f:
json.dump(mce_objects, f, indent=2)

output = mce_helpers.load_json_file(str(tmp_path / "glue_mce.json"))
output = mce_helpers.load_json_file(str(tmp_path / "glue_mces.json"))

test_resources_dir = pytestconfig.rootpath / "tests/unit/glue"
golden = mce_helpers.load_json_file(
str(test_resources_dir / "glue_mce_golden.json")
str(test_resources_dir / "glue_mces_golden.json")
)
mce_helpers.assert_mces_equal(output, golden)
You are viewing a condensed version of this merge commit. You can view the full changes here.