Skip to content

feat: support incremental scan between 2 snapshots #1470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xxchan wants to merge 4 commits into
base: main
Choose a base branch
from
Open

feat: support incremental scan between 2 snapshots #1470

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3ac5b70
feat: support incremental scan between 2 snapshots
xxchan Mar 25, 2025
157891a
add test
xxchan Mar 25, 2025
550d990
fix
xxchan Jun 27, 2025
d5fb028
refactor: move Ancestors snapshot utilities to util/snapshot.rs
xxchan Jul 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix 
Signed-off-by: xxchan <xxchan22f@gmail.com>
  • Loading branch information
@xxchan
xxchan committed Jul 22, 2025
commit 550d99061bb9e17c54a2d7e8e33c6db28fdc7465
122 changes: 63 additions & 59 deletions crates/iceberg/src/scan/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1899,9 +1899,13 @@ pub mod tests {

assert_eq!(snapshots.len(), 2, "Test fixture should have two snapshots");

// First snapshot is the parent of the second
let first_snapshot_id = snapshots[0].snapshot_id();
let second_snapshot_id = snapshots[1].snapshot_id();
// Determine the correct order by snapshot IDs (since validation requires to_snapshot_id > from_snapshot_id)
// Sort snapshots by snapshot_id to ensure consistent ordering
let mut snapshot_ids: Vec<i64> = snapshots.iter().map(|s| s.snapshot_id()).collect();
snapshot_ids.sort();

let first_snapshot_id = snapshot_ids[0];
let second_snapshot_id = snapshot_ids[1];

// Create an incremental scan from first to second snapshot
let table_scan = fixture
Expand Down Expand Up @@ -2045,67 +2049,67 @@ pub mod tests {

// We need to set up snapshots for the remaining tests
let snapshots = table.metadata().snapshots().collect::<Vec<_>>();
if snapshots.len() >= 2 {
let first_snapshot_id = snapshots[0].snapshot_id();
let second_snapshot_id = snapshots[1].snapshot_id();

// Test case 3: from_snapshot_id doesn't exist but to_snapshot_id does
let result = table
.scan()
.from_snapshot_id(999998) // Non-existent
.to_snapshot_id(second_snapshot_id) // Existent
.build();
assert_eq!(snapshots.len(), 2, "Test fixture should have two snapshots");

assert!(
result.is_err(),
"Should fail when from_snapshot_id does not exist"
);
assert_eq!(
result.unwrap_err().to_string(),
"DataInvalid => from_snapshot_id 999998 not found",
"Should have correct error message for non-existent from_snapshot_id"
);
// Sort snapshots by snapshot_id to ensure consistent ordering
let mut snapshot_ids: Vec<i64> = snapshots.iter().map(|s| s.snapshot_id()).collect();
snapshot_ids.sort();

// Determine which snapshot is newer based on snapshot IDs
let (older_snapshot_id, newer_snapshot_id) = if first_snapshot_id < second_snapshot_id {
(first_snapshot_id, second_snapshot_id)
} else {
(second_snapshot_id, first_snapshot_id)
};
let first_snapshot_id = snapshot_ids[0];
let second_snapshot_id = snapshot_ids[1];

// Test case 4: Reversed snapshot order (to_snapshot_id <= from_snapshot_id)
let result = table
.scan()
.from_snapshot_id(newer_snapshot_id) // Later snapshot
.to_snapshot_id(older_snapshot_id) // Earlier snapshot
.build();
// Test case 3: from_snapshot_id doesn't exist but to_snapshot_id does
let result = table
.scan()
.from_snapshot_id(999998) // Non-existent
.to_snapshot_id(second_snapshot_id) // Existent
.build();

assert!(
result.is_err(),
"Should fail when to_snapshot_id <= from_snapshot_id"
);
assert_eq!(
result.unwrap_err().to_string(),
"DataInvalid => to_snapshot_id must be greater than from_snapshot_id",
"Should have correct error message for reversed snapshot order"
);
assert!(
result.is_err(),
"Should fail when from_snapshot_id does not exist"
);
assert_eq!(
result.unwrap_err().to_string(),
"DataInvalid => from_snapshot_id 999998 not found",
"Should have correct error message for non-existent from_snapshot_id"
);

// Test case 5: Equal snapshot IDs (empty range)
let result = table
.scan()
.from_snapshot_id(older_snapshot_id)
.to_snapshot_id(older_snapshot_id)
.build();
// Determine which snapshot is newer based on snapshot IDs
let (older_snapshot_id, newer_snapshot_id) = (first_snapshot_id, second_snapshot_id);

assert!(
result.is_err(),
"Should fail when from_snapshot_id == to_snapshot_id"
);
assert_eq!(
result.unwrap_err().to_string(),
"DataInvalid => to_snapshot_id must be greater than from_snapshot_id",
"Should have correct error message for equal snapshot IDs"
);
}
// Test case 4: Reversed snapshot order (to_snapshot_id <= from_snapshot_id)
let result = table
.scan()
.from_snapshot_id(newer_snapshot_id) // Later snapshot
.to_snapshot_id(older_snapshot_id) // Earlier snapshot
.build();

assert!(
result.is_err(),
"Should fail when to_snapshot_id <= from_snapshot_id"
);
assert_eq!(
result.unwrap_err().to_string(),
"DataInvalid => to_snapshot_id must be greater than from_snapshot_id",
"Should have correct error message for reversed snapshot order"
);

// Test case 5: Equal snapshot IDs (empty range)
let result = table
.scan()
.from_snapshot_id(older_snapshot_id)
.to_snapshot_id(older_snapshot_id)
.build();

assert!(
result.is_err(),
"Should fail when from_snapshot_id == to_snapshot_id"
);
assert_eq!(
result.unwrap_err().to_string(),
"DataInvalid => to_snapshot_id must be greater than from_snapshot_id",
"Should have correct error message for equal snapshot IDs"
);
}
}