Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
1f93a93
update
XiangpengHao Jul 1, 2025
2e01e56
update
XiangpengHao Jul 1, 2025
0bd08c3
update
XiangpengHao Jul 1, 2025
d6ecbd4
update
XiangpengHao Jul 1, 2025
7cd5518
cleanup
XiangpengHao Jul 2, 2025
4520048
update
XiangpengHao Jul 2, 2025
e6281bc
update
XiangpengHao Jul 2, 2025
6b6d4fc
update
XiangpengHao Jul 2, 2025
b696b66
update
XiangpengHao Jul 2, 2025
f60581f
update
XiangpengHao Jul 2, 2025
1851f0b
clippy and license
XiangpengHao Jul 2, 2025
5e414a8
Merge remote-tracking branch 'apache/main' into pushdown-v4
alamb Jul 7, 2025
58add51
bug fix
XiangpengHao Jul 8, 2025
470cc01
Merge remote-tracking branch 'refs/remotes/origin/pushdown-v3' into p…
XiangpengHao Jul 8, 2025
2bf3d38
clippy
XiangpengHao Jul 8, 2025
2cf1a8f
bug fix
XiangpengHao Jul 8, 2025
86e149c
switch to boolean array for row selection
XiangpengHao Jul 15, 2025
4d24172
Merge remote-tracking branch 'apache/main' into pushdown-v4
alamb Jul 15, 2025
be134d6
Add comments (OCD) and rename some fields
alamb Jul 15, 2025
eecaf99
Merge pull request #4 from alamb/alamb/pushdown_suggestions
XiangpengHao Jul 15, 2025
5537bcb
fmt
XiangpengHao Jul 15, 2025
b835163
fmt
alamb Jul 16, 2025
5132de8
Simplify projection caching
alamb Jul 16, 2025
253dad3
Move cache options construction to ArrayReaderBuilder, add builders
alamb Jul 16, 2025
5d9781e
update memory accounting
XiangpengHao Jul 17, 2025
2e20902
Merge remote-tracking branch 'refs/remotes/origin/pushdown-v4' into p…
XiangpengHao Jul 17, 2025
721d00c
Merge pull request #5 from alamb/alamb/simplify_cache
XiangpengHao Jul 17, 2025
f8aed80
Merge pull request #6 from alamb/alamb/cleaner_api
XiangpengHao Jul 17, 2025
884b591
update
XiangpengHao Jul 17, 2025
4f6b918
array size
XiangpengHao Jul 17, 2025
6c53bfd
add test case
XiangpengHao Jul 17, 2025
8ebe579
fix bug
XiangpengHao Jul 17, 2025
c240a52
clippy & fmt
XiangpengHao Jul 17, 2025
30a0d1c
Add config option for predicate cache memory limit
alamb Jul 23, 2025
ed3ce13
Add option to control predicate cache, documentation, ArrowReaderMetr…
alamb Jul 23, 2025
42d5520
Update parquet/src/arrow/arrow_reader/mod.rs
alamb Jul 24, 2025
6e618b3
Merge pull request #7 from alamb/alamb/test_memory_limit
XiangpengHao Jul 24, 2025
f70e46a
Clarify in documentation that cache is only for async decoder
alamb Jul 25, 2025
15d6826
add comment
alamb Jul 25, 2025
bec6d9c
Revert backwards incompatible changes to the Parquet reader API
alamb Jul 25, 2025
3e05cb2
Merge pull request #9 from alamb/alamb/revert_api_changes
XiangpengHao Jul 25, 2025
4d64dc0
Merge pull request #8 from alamb/alamb/pushdown-v4-cleanup
XiangpengHao Jul 25, 2025
8da582b
Merge remote-tracking branch 'apache/main' into pushdown-v4
alamb Aug 6, 2025
315e463
exclude nested column from cache
XiangpengHao Aug 7, 2025
1db701a
only use expanded selection when the column is one of cache column
XiangpengHao Aug 7, 2025
bea4433
Merge remote-tracking branch 'upstream/main' into pushdown-v4
XiangpengHao Aug 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Clarify in documentation that cache is only for async decoder
  • Loading branch information
alamb committed Jul 25, 2025
commit f70e46af43d17d0dfa2e3d44a8561352cebe0fff
12 changes: 9 additions & 3 deletions parquet/src/arrow/arrow_reader/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -344,14 +344,20 @@ impl<T> ArrowReaderBuilder<T> {
Self { metrics, ..self }
}

/// Set the maximum size (per row group) of the predicate cache in bytes.
/// Set the maximum size (per row group) of the predicate cache in bytes for
/// the async decoder.
///
/// Defaults to 100MB
/// Defaults to 100MB (across all columns). Set to `usize::MAX` to use
/// unlimited cache size.
///
/// This cache is used to store decoded arrays that are used in
/// predicate evaluation ([`Self::with_row_filter`]).
///
/// Set to `usize::MAX` to use unlimited cache size.
/// This cache is only used for the "async" decoder, [`ParquetRecordBatchStream`]. See
/// [this ticket] for more details and alternatives.
///
/// [`ParquetRecordBatchStream`]: https://docs.rs/parquet/latest/parquet/arrow/async_reader/struct.ParquetRecordBatchStream.html
/// [this ticket]: https://github.com/apache/arrow-rs/issues/8000
pub fn with_max_predicate_cache_size(self, max_predicate_cache_size: usize) -> Self {
Self {
max_predicate_cache_size,
Expand Down
20 changes: 12 additions & 8 deletions parquet/tests/arrow_reader/predicate_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ use std::ops::Range;
use std::sync::Arc;
use std::sync::LazyLock;

// TODO file a ticket about the duplication here

#[tokio::test]
async fn test_default_read() {
// The cache is not used without predicates, so we expect 0 records read from cache
Expand All @@ -51,18 +49,24 @@ async fn test_default_read() {
}

#[tokio::test]
async fn test_cache_with_filters() {
async fn test_async_cache_with_filters() {
let test = ParquetPredicateCacheTest::new().with_expected_records_read_from_cache(49);
// TODO The sync reader does not use the cache yet....
// let sync_builder = test.sync_builder();
// let sync_builder = test.add_project_ab_and_filter_b(sync_builder);
// test.run_sync(sync_builder);

let async_builder = test.async_builder().await;
let async_builder = test.add_project_ab_and_filter_b(async_builder);
test.run_async(async_builder).await;
}

#[tokio::test]
async fn test_sync_cache_with_filters() {
let test = ParquetPredicateCacheTest::new()
// The sync reader does not use the cache. See https://github.com/apache/arrow-rs/issues/8000
.with_expected_records_read_from_cache(0);

let sync_builder = test.sync_builder();
let sync_builder = test.add_project_ab_and_filter_b(sync_builder);
test.run_sync(sync_builder);
}

#[tokio::test]
async fn test_cache_disabled_with_filters() {
// expect no records to be read from cache, because the cache is disabled
Expand Down