Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
a653b15
ExecutionPlan: add APIs for filter pushdown & optimizer rule to apply…
adriangb Apr 3, 2025
a5f998c
wip
adriangb Apr 3, 2025
7e2db66
fix tests
adriangb Apr 3, 2025
cb1f830
fix
adriangb Apr 3, 2025
e92d8b5
fix
adriangb Apr 3, 2025
ca391c1
fix doc
adriangb Apr 3, 2025
c78a590
fix doc
adriangb Apr 3, 2025
34c8285
Improve doc comments of `filter-pushdown-apis` (#22)
alamb Apr 5, 2025
e15374f
Apply suggestions from code review
adriangb Apr 5, 2025
2ceec35
simplify according to pr feedback
adriangb Apr 5, 2025
3fbf379
Add missing file
adriangb Apr 5, 2025
e6721d1
Add tests
adriangb Apr 5, 2025
b7b588b
pipe config in
adriangb Apr 5, 2025
d1f01dd
docstrings
adriangb Apr 5, 2025
5929d03
Update datafusion/physical-plan/src/filter_pushdown.rs
adriangb Apr 5, 2025
24483bc
fix
adriangb Apr 5, 2025
d0295ed
fix
adriangb Apr 6, 2025
2d46289
fmt
adriangb Apr 6, 2025
4318267
fix doc
adriangb Apr 6, 2025
7d29056
add example usage of config
adriangb Apr 6, 2025
d382bd3
fix test
adriangb Apr 6, 2025
2dfa8b8
convert exec API and optimizer rule
berkaysynnada Apr 14, 2025
cda6e8d
re-add docs
adriangb Apr 14, 2025
e4d8a8c
dbg
berkaysynnada Apr 16, 2025
3ec1b2a
dbg 2
berkaysynnada Apr 16, 2025
a2df5e0
avoid clones
adriangb Apr 16, 2025
6938d52
part 3
berkaysynnada Apr 16, 2025
6836dd4
fix lint
adriangb Apr 16, 2025
28bb8ea
Merge branch 'filter-pushdown-apis' into filter-pushdown-apis
berkaysynnada Apr 16, 2025
7e95283
tests pass
berkaysynnada Apr 16, 2025
e2f8c12
Update filter.rs
berkaysynnada Apr 16, 2025
bff47be
update projection tests
berkaysynnada Apr 16, 2025
ce49ad4
update slt files
adriangb Apr 16, 2025
d5792bc
Merge branch 'main' into filter-pushdown-apis
adriangb Apr 16, 2025
834f33e
fix
adriangb Apr 16, 2025
9e59246
fix references
adriangb Apr 16, 2025
57a1230
improve impls and update tests
berkaysynnada Apr 17, 2025
367377f
apply stop logic
berkaysynnada Apr 17, 2025
616165d
update slt's
berkaysynnada Apr 17, 2025
b30953f
update other tests
berkaysynnada Apr 17, 2025
ec54cca
minor
berkaysynnada Apr 17, 2025
6345315
rename modules to match logical optimizer, tweak docs
adriangb Apr 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix test
  • Loading branch information
adriangb committed Apr 6, 2025
commit d382bd394bfb5bd0796f3ffcc842eb327e44ee2e
21 changes: 11 additions & 10 deletions datafusion/core/tests/physical_optimizer/filter_pushdown.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ struct TestSource {
}

impl TestSource {
fn new(support: FilterPushdownSupport) -> Self {
fn new(support: Option<FilterPushdownSupport>) -> Self {
Self {
support: Some(support),
support,
predicate: None,
statistics: None,
}
Expand Down Expand Up @@ -173,7 +173,7 @@ impl FileSource for TestSource {
}
}

fn test_scan(support: FilterPushdownSupport) -> Arc<dyn ExecutionPlan> {
fn test_scan(support: Option<FilterPushdownSupport>) -> Arc<dyn ExecutionPlan> {
let schema = schema();
let source = Arc::new(TestSource::new(support));
let base_config = FileScanConfigBuilder::new(
Expand All @@ -187,7 +187,7 @@ fn test_scan(support: FilterPushdownSupport) -> Arc<dyn ExecutionPlan> {

#[test]
fn test_pushdown_into_scan() {
let scan = test_scan(FilterPushdownSupport::Exact);
let scan = test_scan(Some(FilterPushdownSupport::Exact));
let predicate = col_lit_predicate("a", "foo", schema());
let plan = Arc::new(FilterExec::try_new(predicate, scan).unwrap());

Expand All @@ -209,7 +209,7 @@ fn test_pushdown_into_scan() {
/// Show that we can use config options to determine how to do pushdown.
#[test]
fn test_pushdown_into_scan_with_config_options() {
let scan = test_scan(FilterPushdownSupport::Exact);
let scan = test_scan(None);
let predicate = col_lit_predicate("a", "foo", schema());
let plan = Arc::new(FilterExec::try_new(predicate, scan).unwrap()) as _;

Expand All @@ -228,7 +228,8 @@ fn test_pushdown_into_scan_with_config_options() {
- DataSourceExec: file_groups={0 groups: []}, projection=[a, b, c], file_type=test
output:
Ok:
- DataSourceExec: file_groups={0 groups: []}, projection=[a, b, c], file_type=test, predicate=a@0 = foo
- FilterExec: a@0 = foo
- DataSourceExec: file_groups={0 groups: []}, projection=[a, b, c], file_type=test, predicate=a@0 = foo
"
);

Expand All @@ -254,7 +255,7 @@ fn test_pushdown_into_scan_with_config_options() {
#[test]
fn test_filter_collapse() {
// filter should be pushed down into the parquet scan with two filters
let scan = test_scan(FilterPushdownSupport::Exact);
let scan = test_scan(Some(FilterPushdownSupport::Exact));
let predicate1 = col_lit_predicate("a", "foo", schema());
let filter1 = Arc::new(FilterExec::try_new(predicate1, scan).unwrap());
let predicate2 = col_lit_predicate("b", "bar", schema());
Expand All @@ -277,7 +278,7 @@ fn test_filter_collapse() {

#[test]
fn test_filter_with_projection() {
let scan = test_scan(FilterPushdownSupport::Exact);
let scan = test_scan(Some(FilterPushdownSupport::Exact));
let projection = vec![1, 0];
let projected_schema = Arc::new(schema().project(&projection).unwrap());
let predicate = col_lit_predicate("a", "foo", &projected_schema);
Expand Down Expand Up @@ -308,7 +309,7 @@ fn test_filter_with_projection() {
#[test]
fn test_push_down_through_transparent_nodes() {
// expect the predicate to be pushed down into the DataSource
let scan = test_scan(FilterPushdownSupport::Exact);
let scan = test_scan(Some(FilterPushdownSupport::Exact));
let coalesce = Arc::new(CoalesceBatchesExec::new(scan, 1));
let predicate = col_lit_predicate("a", "foo", schema());
let filter = Arc::new(FilterExec::try_new(predicate, coalesce).unwrap());
Expand Down Expand Up @@ -344,7 +345,7 @@ fn test_no_pushdown_through_aggregates() {
// 1. The outer filter is not pushed down into the aggregate because we haven't
// implemented that yet.
// 2. The inner filter **is** pushed down into the DataSource.
let scan = test_scan(FilterPushdownSupport::Exact);
let scan = test_scan(Some(FilterPushdownSupport::Exact));
let filter = Arc::new(
FilterExec::try_new(col_lit_predicate("a", "foo", schema()), scan.clone())
.unwrap(),
Expand Down