Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
37b5526
early draft
goldmedal Jun 20, 2024
ad1a854
fmt
goldmedal Jun 20, 2024
97ea11c
add example for dynamic file query
goldmedal Jun 21, 2024
2729c49
add test and refactor
goldmedal Jun 21, 2024
6f86577
clippy and add doc
goldmedal Jun 21, 2024
c91cdc6
cargo fmt
goldmedal Jun 21, 2024
3306df6
extract substitute_tilde function
goldmedal Jun 21, 2024
d82b273
fix the error handling
goldmedal Jun 21, 2024
c0491d5
fmt and clippy
goldmedal Jun 21, 2024
a60eeea
fix test
goldmedal Jun 21, 2024
9fa01aa
fix sqllogictests
goldmedal Jun 22, 2024
2ab3639
ignore dirs for windows test
goldmedal Jun 22, 2024
a8ee733
enhance the test for every file format
goldmedal Jun 22, 2024
7faab9f
disable the test for windows
goldmedal Jun 22, 2024
e1f3908
make dynamic file query configurable
goldmedal Jul 1, 2024
cf73ba2
revert array_query.slt
goldmedal Jul 1, 2024
c641e6b
modified the test and add example
goldmedal Jul 1, 2024
0806263
make dirs be optional
goldmedal Jul 1, 2024
f4d24e6
enable dynamic file query in cli
goldmedal Jul 1, 2024
4b71e59
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Jul 1, 2024
9964150
cargo fmt
goldmedal Jul 1, 2024
da1e5d3
modified example
goldmedal Jul 1, 2024
ed670fe
fix test
goldmedal Jul 1, 2024
ea5816e
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Aug 13, 2024
fb8b9e0
fix merge conflict
goldmedal Aug 13, 2024
fa73ae7
tmp
goldmedal Aug 14, 2024
04cc155
tmp
goldmedal Aug 14, 2024
1ede35e
tmp
goldmedal Aug 14, 2024
51b1d41
fix the catalog and schema
goldmedal Aug 15, 2024
75b0b84
move dynamic file catalog to datafusion-catalog
goldmedal Aug 15, 2024
3e8d094
add copyright
goldmedal Aug 15, 2024
4eb8ca5
fix tests
goldmedal Aug 16, 2024
9913405
rename catalog in cli and update lock
goldmedal Aug 16, 2024
5d861b8
enable home_dir feature
goldmedal Aug 16, 2024
ea1c075
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Aug 16, 2024
16be2e7
update lock
goldmedal Aug 16, 2024
db90c28
fix compile
goldmedal Aug 16, 2024
9353123
fix clippy
goldmedal Aug 16, 2024
daa7ed8
fmt toml
goldmedal Aug 16, 2024
e4a2174
fix doc test and add more doc
goldmedal Aug 16, 2024
506d1d6
fix clippy
goldmedal Aug 16, 2024
72ce464
add home_dir feature doc
goldmedal Aug 16, 2024
fb1b6ce
rollback the unused changed
goldmedal Aug 16, 2024
8f0952d
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Aug 18, 2024
f062fec
update lock
goldmedal Aug 18, 2024
b1baa84
fix sqllogictest
goldmedal Aug 18, 2024
76d7fee
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Aug 20, 2024
f0f070b
separate dynamic file test to another slt
goldmedal Aug 20, 2024
6b77b6b
add test for querying url table but disabled this feature
goldmedal Aug 20, 2024
4e51a77
add dynamic_file.slt
goldmedal Aug 20, 2024
fafc9dc
remove home_dir feature
goldmedal Aug 20, 2024
a3a4f4d
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Aug 21, 2024
7dc238f
Merge branch 'main' into feature/10986-dynamic-table-provider
goldmedal Sep 4, 2024
f7b4b8c
update cli lock
goldmedal Sep 4, 2024
25d0ff6
fix msrv check
goldmedal Sep 4, 2024
a78bd3c
fix msrv check
goldmedal Sep 4, 2024
edeff33
rollback the lock change
goldmedal Sep 4, 2024
b1a922c
address review comment and enhance the doc
goldmedal Sep 7, 2024
e5ab14d
remove the legacy comment
goldmedal Sep 7, 2024
87d7503
add missing doc
goldmedal Sep 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
enhance the test for every file format
  • Loading branch information
goldmedal committed Jun 22, 2024
commit a8ee733cde962ef6a3cb1681dc89f4af25a85048
15 changes: 15 additions & 0 deletions datafusion/sqllogictest/test_files/arrow_files.slt
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ SELECT * FROM arrow_simple
3 baz false
4 NULL true

query ITB
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recommend creating a new .slt file explicitly for dynamic file catalog tests rather than extending the existing tests. This would:

  1. Help show that the behavior isn't changed except when dynamic catalog is enabled
  2. make it easier to evaluate how much dynamic file catalog testing there was

Also, is there sufficient negative testing? Specifically that the dynamic catalog provider isn't enabled by default (which would be a security hole?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I create dynamic_file.slt to collect all the tests for URL table. However, I also keep the URL table for each file-related slt to be a negative test to ensure this feature is disabled by default.

SELECT * FROM '../core/tests/data/example.arrow';
----
1 foo true
2 bar NULL
3 baz false
4 NULL true

# ARROW partitioned table
statement ok
CREATE EXTERNAL TABLE arrow_partitioned (
Expand All @@ -64,6 +72,13 @@ SELECT * FROM arrow_partitioned ORDER BY f0;
3 baz true 456
4 NULL NULL 456

# dynamic select arrow file in the folder
query ITB
SELECT * FROM '../core/tests/data/partitioned_table_arrow/part=123' ORDER BY f0;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dynamic query doesn't support the partitioned folder. It could be an enhancement issue for it.

----
1 foo true
2 bar false

# select all fields
query IITB
SELECT part, f0, f1, f2 FROM arrow_partitioned ORDER BY f0;
Expand Down
25 changes: 25 additions & 0 deletions datafusion/sqllogictest/test_files/avro.slt
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,18 @@ SELECT id, CAST(string_col AS varchar) FROM alltypes_plain
0 0
1 1

query IT
SELECT id, CAST(string_col AS varchar) FROM '../../testing/data/avro/alltypes_plain.avro'
----
4 0
5 1
6 0
7 1
2 0
3 1
0 0
1 1

# test avro query with snappy
query IT
SELECT id, CAST(string_col AS varchar) FROM alltypes_plain_snappy
Expand All @@ -149,6 +161,19 @@ SELECT id, CAST(string_col AS varchar) FROM alltypes_plain_snappy
0 0
1 1

# dynamic query snappy avro file
query IT
SELECT id, CAST(string_col AS varchar) FROM '../../testing/data/avro/alltypes_plain.snappy.avro'
----
4 0
5 1
6 0
7 1
2 0
3 1
0 0
1 1

# test avro query with bzip2
query IT
SELECT id, CAST(string_col AS varchar) FROM alltypes_plain_bzip2
Expand Down
17 changes: 17 additions & 0 deletions datafusion/sqllogictest/test_files/csv_files.slt
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,23 @@ id7 value7
id8 value8
id9 value9

# query the csv file dynamically with the config of current session
query TT
select * from '../core/tests/data/quote.csv';
----
c1 c2
~id0~ ~value0~
~id1~ ~value1~
~id2~ ~value2~
~id3~ ~value3~
~id4~ ~value4~
~id5~ ~value5~
~id6~ ~value6~
~id7~ ~value7~
~id8~ ~value8~
~id9~ ~value9~


query TT
select * from csv_with_escape;
----
Expand Down
16 changes: 16 additions & 0 deletions datafusion/sqllogictest/test_files/json.slt
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,22 @@ SELECT a, b FROM json_test
5 -3.5
7 -3.5

query IR rowsort
SELECT a, b FROM '../core/tests/data/2.json'
----
-10 -3.5
1 -3.5
1 0.6
1 0.6
1 2
1 2
1 2
1 2
100000000000000 0.6
2 0.6
5 -3.5
7 -3.5

query TT
EXPLAIN SELECT count(*) from json_test
----
Expand Down
12 changes: 12 additions & 0 deletions datafusion/sqllogictest/test_files/parquet.slt
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,18 @@ SELECT id, CAST(string_col AS varchar) FROM alltypes_plain
0 0
1 1

query IT
SELECT id, CAST(string_col AS varchar) FROM '../../parquet-testing/data/alltypes_plain.parquet';
----
4 0
5 1
6 0
7 1
2 0
3 1
0 0
1 1

# Clean up
statement ok
DROP TABLE alltypes_plain;
Expand Down