Skip to content

feat: Support configuring datafusion catalog in sqllogictest framework #1865

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kaori-seasons wants to merge 10 commits into
base: main
Choose a base branch
from
Open

feat: Support configuring datafusion catalog in sqllogictest framework #1865

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
98e46b4
feat: support mutli catalog
Nov 17, 2025
13681de
chore: remove useless code
Nov 17, 2025
816a45a
chore: translate chinese to english
Nov 17, 2025
c05fb66
bugfix: fix checkstyle
Nov 25, 2025
794fe71
bugfix: add default impl
Nov 25, 2025
2afae2f
bugfix: fix checkstyle
Nov 25, 2025
4593bcb
bugfix: fix checkstyle
Nov 25, 2025
d65a822
chore: format file
Nov 25, 2025
b7ce6fb
Merge remote-tracking branch 'upstream/main' into issue-1780
Nov 26, 2025
4ce0b5a
chore: fix format
Nov 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: format file
  • Loading branch information
kaori-seasons committed Nov 25, 2025
commit d65a8225ea2dfd3e8dd9e56ef2d303d097a70bce
8 changes: 4 additions & 4 deletions crates/catalog/hms/tests/hms_catalog_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,10 @@ async fn test_list_tables() -> Result<()> {
catalog.create_table(ns.name(), creation).await?;
let result = catalog.list_tables(ns.name()).await?;

assert_eq!(result, vec![TableIdent::new(
ns.name().clone(),
"my_table".to_string()
)]);
assert_eq!(
result,
Copy link
Member

@Xuanwo Xuanwo Dec 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are many format related changes, it's better to avoid that.

vec![TableIdent::new(ns.name().clone(), "my_table".to_string())]
);

Ok(())
}
Expand Down
4 changes: 3 additions & 1 deletion crates/catalog/s3tables/src/catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,9 @@ impl Catalog for S3TablesCatalog {

/// Format AWS SDK error into iceberg error
pub(crate) fn from_aws_sdk_error<T>(error: aws_sdk_s3tables::error::SdkError<T>) -> Error
where T: std::fmt::Debug {
where
T: std::fmt::Debug,
{
Error::new(
ErrorKind::Unexpected,
format!("Operation failed for hitting aws sdk error: {error:?}"),
Expand Down
83 changes: 46 additions & 37 deletions crates/catalog/sql/src/catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,10 +362,10 @@ impl Catalog for SqlCatalog {
);

let namespace_rows = self
.fetch_rows(&all_namespaces_stmt, vec![
Some(&self.name),
Some(&self.name),
])
.fetch_rows(
&all_namespaces_stmt,
vec![Some(&self.name), Some(&self.name)],
)
.await?;

let mut namespaces = HashSet::<NamespaceIdent>::with_capacity(namespace_rows.len());
Expand Down Expand Up @@ -1299,11 +1299,10 @@ mod tests {
let namespace_ident_1 = NamespaceIdent::new("a".into());
let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
let namespace_ident_3 = NamespaceIdent::new("b".into());
create_namespaces(&catalog, &vec![
&namespace_ident_1,
&namespace_ident_2,
&namespace_ident_3,
])
create_namespaces(
&catalog,
&vec![&namespace_ident_1, &namespace_ident_2, &namespace_ident_3],
)
.await;

assert_eq!(
Expand Down Expand Up @@ -1336,11 +1335,10 @@ mod tests {
let namespace_ident_1 = NamespaceIdent::new("a".into());
let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
let namespace_ident_3 = NamespaceIdent::new("c".into());
create_namespaces(&catalog, &vec![
&namespace_ident_1,
&namespace_ident_2,
&namespace_ident_3,
])
create_namespaces(
&catalog,
&vec![&namespace_ident_1, &namespace_ident_2, &namespace_ident_3],
)
.await;

assert_eq!(
Expand All @@ -1366,13 +1364,16 @@ mod tests {
let namespace_ident_3 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
let namespace_ident_4 = NamespaceIdent::from_strs(vec!["a", "c"]).unwrap();
let namespace_ident_5 = NamespaceIdent::new("b".into());
create_namespaces(&catalog, &vec![
&namespace_ident_1,
&namespace_ident_2,
&namespace_ident_3,
&namespace_ident_4,
&namespace_ident_5,
])
create_namespaces(
&catalog,
&vec![
&namespace_ident_1,
&namespace_ident_2,
&namespace_ident_3,
&namespace_ident_4,
&namespace_ident_5,
],
)
.await;

assert_eq!(
Expand Down Expand Up @@ -1673,11 +1674,14 @@ mod tests {
let namespace_ident_a = NamespaceIdent::new("a".into());
let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
create_namespaces(&catalog, &vec![
&namespace_ident_a,
&namespace_ident_a_b,
&namespace_ident_a_b_c,
])
create_namespaces(
&catalog,
&vec![
&namespace_ident_a,
&namespace_ident_a_b,
&namespace_ident_a_b_c,
],
)
.await;

catalog
Expand Down Expand Up @@ -2039,9 +2043,10 @@ mod tests {
.await
.unwrap();

assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
dst_table_ident
],);
assert_eq!(
catalog.list_tables(&namespace_ident).await.unwrap(),
vec![dst_table_ident],
);
}

#[tokio::test]
Expand Down Expand Up @@ -2085,9 +2090,10 @@ mod tests {
.await
.unwrap();

assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
table_ident
],);
assert_eq!(
catalog.list_tables(&namespace_ident).await.unwrap(),
vec![table_ident],
);
}

#[tokio::test]
Expand All @@ -2097,11 +2103,14 @@ mod tests {
let namespace_ident_a = NamespaceIdent::new("a".into());
let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
create_namespaces(&catalog, &vec![
&namespace_ident_a,
&namespace_ident_a_b,
&namespace_ident_a_b_c,
])
create_namespaces(
&catalog,
&vec![
&namespace_ident_a,
&namespace_ident_a_b,
&namespace_ident_a_b_c,
],
)
.await;

let src_table_ident = TableIdent::new(namespace_ident_a_b_c.clone(), "tbl1".into());
Expand Down
18 changes: 9 additions & 9 deletions crates/iceberg/src/arrow/caching_delete_file_loader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -632,9 +632,10 @@ mod tests {
Arc::new(arrow_schema::Schema::new(fields))
};

let equality_deletes_to_write = RecordBatch::try_new(equality_delete_schema.clone(), vec![
col_y, col_z, col_a, col_s,
])
let equality_deletes_to_write = RecordBatch::try_new(
equality_delete_schema.clone(),
vec![col_y, col_z, col_a, col_s],
)
.unwrap();

let path = format!("{}/equality-deletes-1.parquet", &table_location);
Expand Down Expand Up @@ -831,12 +832,11 @@ mod tests {
let file_path_col = Arc::new(StringArray::from_iter_values(&file_path_values));
let pos_col = Arc::new(Int64Array::from_iter_values(vec![0i64, 1, 2, 3]));

let positional_deletes_to_write =
RecordBatch::try_new(positional_delete_schema.clone(), vec![
file_path_col,
pos_col,
])
.unwrap();
let positional_deletes_to_write = RecordBatch::try_new(
positional_delete_schema.clone(),
vec![file_path_col, pos_col],
)
.unwrap();

let props = WriterProperties::builder()
.set_compression(Compression::SNAPPY)
Expand Down
11 changes: 5 additions & 6 deletions crates/iceberg/src/arrow/delete_filter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,12 +281,11 @@ pub(crate) mod tests {
let pos_vals = pos_values.get(n - 1).unwrap();
let pos_col = Arc::new(Int64Array::from_iter_values(pos_vals.clone()));

let positional_deletes_to_write =
RecordBatch::try_new(positional_delete_schema.clone(), vec![
file_path_col.clone(),
pos_col.clone(),
])
.unwrap();
let positional_deletes_to_write = RecordBatch::try_new(
positional_delete_schema.clone(),
vec![file_path_col.clone(), pos_col.clone()],
)
.unwrap();

let file = File::create(format!(
"{}/pos-del-{}.parquet",
Expand Down
11 changes: 7 additions & 4 deletions crates/iceberg/src/arrow/partition_value_calculator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,13 @@ mod tests {
Field::new("name", DataType::Utf8, false),
]));

let batch = RecordBatch::try_new(arrow_schema, vec![
Arc::new(Int32Array::from(vec![10, 20, 30])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
])
let batch = RecordBatch::try_new(
arrow_schema,
vec![
Arc::new(Int32Array::from(vec![10, 20, 30])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
],
)
.unwrap();

// Calculate partition values
Expand Down
Loading