Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9dd9fbc
schema_v1-dataset_builder-add_dimension
dmitriyrepin Jun 24, 2025
f88531e
Merge remote-tracking branch 'upstream/v1' into v1
dmitriyrepin Jun 26, 2025
1358f95
First take on add_dimension(), add_coordinate(), add_variable()
dmitriyrepin Jun 27, 2025
e5261cb
Finished add_dimension, add_coordinate, add_variable
dmitriyrepin Jun 28, 2025
95c01d8
Work on build
dmitriyrepin Jun 30, 2025
46f82f0
Generalize _to_dictionary()
dmitriyrepin Jul 1, 2025
0dc7cc8
build
dmitriyrepin Jul 1, 2025
79863ac
Dataset Build - pass one
dmitriyrepin Jul 2, 2025
ec480f1
Merge the latest TGSAI/mdio-python:v1 branch
dmitriyrepin Jul 2, 2025
fa81ea2
Merge branch 'v1' into v1
tasansal Jul 7, 2025
4b2b163
Revert .container changes
dmitriyrepin Jul 7, 2025
c532c3b
PR review: remove DEVELOPER_NOTES.md
dmitriyrepin Jul 7, 2025
08798cd
PR Review: add_coordinate() should accept only data_type: ScalarType
dmitriyrepin Jul 7, 2025
e8febe4
PR review: add_variable() data_type remove default
dmitriyrepin Jul 7, 2025
0a4be3f
RE review: do not add dimension variable
dmitriyrepin Jul 8, 2025
7b25d6b
PR Review: get api version from the package version
dmitriyrepin Jul 8, 2025
7ca3ed8
PR Review: remove add_dimension_coordinate
dmitriyrepin Jul 9, 2025
4d1ec9c
PR Review: add_coordinate() remove data_type default value
dmitriyrepin Jul 9, 2025
99fcf43
PR Review: improve unit tests by extracting common functionality in v…
dmitriyrepin Jul 9, 2025
0778fdd
Remove the Dockerfile changes. They are not supposed to be a part of …
dmitriyrepin Jul 9, 2025
7e74567
PR Review: run ruff
dmitriyrepin Jul 9, 2025
0aaa5f6
PR Review: fix pre-commit errors
dmitriyrepin Jul 10, 2025
1904dee
remove some noqa overrides
tasansal Jul 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
PR Review: fix pre-commit errors
  • Loading branch information
dmitriyrepin committed Jul 10, 2025
commit 0aaa5f6110eb7afd59a5cb901a747da5fc60cbde
60 changes: 36 additions & 24 deletions src/mdio/schemas/v1/dataset_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,13 @@
from mdio.schemas.v1.variable import Coordinate
from mdio.schemas.v1.variable import Variable

AnyMetadataList: TypeAlias = list[AllUnits |
UserAttributes |
ChunkGridMetadata |
StatisticsMetadata |
DatasetInfo]
AnyMetadataList: TypeAlias = list[
AllUnits | UserAttributes | ChunkGridMetadata | StatisticsMetadata | DatasetInfo
]
CoordinateMetadataList: TypeAlias = list[AllUnits | UserAttributes]
VariableMetadataList: TypeAlias = list[AllUnits |
UserAttributes |
ChunkGridMetadata |
StatisticsMetadata]
VariableMetadataList: TypeAlias = list[
AllUnits | UserAttributes | ChunkGridMetadata | StatisticsMetadata
]
DatasetMetadataList: TypeAlias = list[DatasetInfo | UserAttributes]


Expand All @@ -52,7 +49,7 @@
) -> NamedDimension | None:
"""Get a dimension by name and optional size from the list[NamedDimension]."""
if dimensions is None:
return False

Check warning on line 52 in src/mdio/schemas/v1/dataset_builder.py

View check run for this annotation

Codecov / codecov/patch

src/mdio/schemas/v1/dataset_builder.py#L52

Added line #L52 was not covered by tests
if not isinstance(name, str):
msg = f"Expected str, got {type(name).__name__}"
raise TypeError(msg)
Expand Down Expand Up @@ -97,17 +94,12 @@
"""

def __init__(self, name: str, attributes: UserAttributes | None = None):

try:
api_version = metadata.version("multidimio")
except metadata.PackageNotFoundError:
api_version = "unknown"

Check warning on line 100 in src/mdio/schemas/v1/dataset_builder.py

View check run for this annotation

Codecov / codecov/patch

src/mdio/schemas/v1/dataset_builder.py#L99-L100

Added lines #L99 - L100 were not covered by tests

self._info = DatasetInfo(
name=name,
api_version=api_version,
created_on=datetime.now(UTC)
)
self._info = DatasetInfo(name=name, api_version=api_version, created_on=datetime.now(UTC))
self._attributes = attributes
self._dimensions: list[NamedDimension] = []
self._coordinates: list[Coordinate] = []
Expand All @@ -116,9 +108,7 @@
self._unnamed_variable_counter = 0

def add_dimension( # noqa: PLR0913
self,
name: str,
size: int
self, name: str, size: int
) -> "MDIODatasetBuilder":
"""Add a dimension.

Expand All @@ -128,6 +118,10 @@
name: Name of the dimension
size: Size of the dimension

Raises:
ValueError: If 'name' is not a non-empty string.
if the dimension is already defined.

Returns:
self: Returns self for method chaining
"""
Expand All @@ -146,7 +140,6 @@
self._state = _BuilderState.HAS_DIMENSIONS
return self


def add_coordinate( # noqa: PLR0913
self,
name: str,
Expand All @@ -170,6 +163,13 @@
compressor: Compressor used for the variable (defaults to None)
metadata_info: Optional metadata information for the coordinate

Raises:
ValueError: If no dimensions have been added yet.
If 'name' is not a non-empty string.
If 'dimensions' is not a non-empty list.
If the coordinate is already defined.
If any referenced dimension is not already defined.

Returns:
self: Returns self for method chaining
"""
Expand All @@ -195,7 +195,7 @@
if nd is None:
msg = f"Pre-existing dimension named {dim_name!r} is not found"
raise ValueError(msg)
named_dimensions.append(nd)
named_dimensions.append(nd)

meta_dict = _to_dictionary(metadata_info)
coord = Coordinate(
Expand All @@ -204,7 +204,7 @@
dimensions=named_dimensions,
compressor=compressor,
dataType=data_type,
metadata=meta_dict
metadata=meta_dict,
)
self._coordinates.append(coord)

Expand All @@ -216,7 +216,7 @@
data_type=coord.data_type,
compressor=compressor,
coordinates=[name], # Use the coordinate name as a reference
metadata_info=coord.metadata
metadata_info=coord.metadata,
)

self._state = _BuilderState.HAS_COORDINATES
Expand Down Expand Up @@ -252,6 +252,14 @@
(defaults to None, meaning no coordinates)
metadata_info: Optional metadata information for the variable

Raises:
ValueError: If no dimensions have been added yet.
If 'name' is not a non-empty string.
If 'dimensions' is not a non-empty list.
If the variable is already defined.
If any referenced dimension is not already defined.
If any referenced coordinate is not already defined.

Returns:
self: Returns self for method chaining.
"""
Expand Down Expand Up @@ -293,7 +301,7 @@

# If this is a dimension coordinate variable, embed the Coordinate into it
if coordinates is not None and len(coordinates) == 1 and coordinates[0] == name:
coordinates = coordinate_objs
coordinates = coordinate_objs

meta_dict = _to_dictionary(metadata_info)
var = Variable(
Expand All @@ -303,7 +311,8 @@
data_type=data_type,
compressor=compressor,
coordinates=coordinates,
metadata=meta_dict)
metadata=meta_dict,
)
self._variables.append(var)

self._state = _BuilderState.HAS_VARIABLES
Expand All @@ -315,12 +324,15 @@
This function must be called after at least one dimension is added via add_dimension().
It will create a Dataset object with all added dimensions, coordinates, and variables.

Raises:
ValueError: If no dimensions have been added yet.

Returns:
Dataset: The built dataset with all added dimensions, coordinates, and variables.
"""
if self._state == _BuilderState.INITIAL:
msg = "Must add at least one dimension before building"
raise ValueError(msg)

Check warning on line 335 in src/mdio/schemas/v1/dataset_builder.py

View check run for this annotation

Codecov / codecov/patch

src/mdio/schemas/v1/dataset_builder.py#L334-L335

Added lines #L334 - L335 were not covered by tests

var_meta_dict = _to_dictionary([self._info, self._attributes])
return Dataset(variables=self._variables, metadata=var_meta_dict)
109 changes: 57 additions & 52 deletions tests/unit/v1/helpers.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Helper methods used in unit tests."""

from mdio.schemas.chunk_grid import RegularChunkGrid
from mdio.schemas.chunk_grid import RegularChunkShape
from mdio.schemas.compressors import Blosc
Expand All @@ -23,22 +24,19 @@
from mdio.schemas.v1.variable import Variable


def validate_builder(builder: MDIODatasetBuilder,
state: _BuilderState,
n_dims: int,
n_coords: int,
n_var: int) -> None:
def validate_builder(
builder: MDIODatasetBuilder, state: _BuilderState, n_dims: int, n_coords: int, n_var: int
) -> None:
"""Validate the state of the builder, the number of dimensions, coordinates, and variables."""
assert builder._state == state
assert len(builder._dimensions) == n_dims
assert len(builder._coordinates) == n_coords
assert len(builder._variables) == n_var


def validate_coordinate(builder: MDIODatasetBuilder,
name: str,
dims: list[tuple[str, int]],
dtype: ScalarType) -> Coordinate:
def validate_coordinate(
builder: MDIODatasetBuilder, name: str, dims: list[tuple[str, int]], dtype: ScalarType
) -> Coordinate:
"""Validate existence and the structure of the created coordinate."""
# Validate that coordinate exists
c = next((c for c in builder._coordinates if c.name == name), None)
Expand All @@ -55,11 +53,13 @@
return c


def validate_variable(container: MDIODatasetBuilder | Dataset,
name: str,
dims: list[tuple[str, int]],
coords: list[str],
dtype: ScalarType) -> Variable:
def validate_variable(
container: MDIODatasetBuilder | Dataset,
name: str,
dims: list[tuple[str, int]],
coords: list[str],
dtype: ScalarType,
) -> Variable:
"""Validate existence and the structure of the created variable."""
if isinstance(container, MDIODatasetBuilder):
var_list = container._variables
Expand All @@ -68,8 +68,8 @@
var_list = container.variables
global_coord_list = _get_all_coordinates(container)
else:
err_msg = f"Expected MDIODatasetBuilder or Dataset, got {type(container)}"
raise TypeError(err_msg)

Check warning on line 72 in tests/unit/v1/helpers.py

View check run for this annotation

Codecov / codecov/patch

tests/unit/v1/helpers.py#L71-L72

Added lines #L71 - L72 were not covered by tests

# Validate that the variable exists
v = next((e for e in var_list if e.name == name), None)
Expand All @@ -90,23 +90,24 @@
assert len(v.coordinates) == len(coords)
for coord_name in coords:
assert _get_coordinate(global_coord_list, v.coordinates, coord_name) is not None

assert v.data_type == dtype
return v


def _get_coordinate(
global_coord_list: list[Coordinate],
coordinates_or_references: list[Coordinate] | list[str],
name: str) -> Coordinate | None:
global_coord_list: list[Coordinate],
coordinates_or_references: list[Coordinate] | list[str],
name: str,
) -> Coordinate | None:
"""Get a coordinate by name from the list[Coordinate] | list[str].

The function validates that the coordinate referenced by the name can be found
in the global coordinate list.
The function validates that the coordinate referenced by the name can be found
in the global coordinate list.
If the coordinate is stored as a Coordinate object, it is returned directly.
"""
if coordinates_or_references is None:
return None

Check warning on line 110 in tests/unit/v1/helpers.py

View check run for this annotation

Codecov / codecov/patch

tests/unit/v1/helpers.py#L110

Added line #L110 was not covered by tests

for c in coordinates_or_references:
if isinstance(c, str) and c == name:
Expand All @@ -114,18 +115,17 @@
cc = None
# Find the Coordinate in the global list and return it.
if global_coord_list is not None:
cc = next(
(cc for cc in global_coord_list if cc.name == name), None)
cc = next((cc for cc in global_coord_list if cc.name == name), None)
if cc is None:
msg = f"Pre-existing coordinate named {name!r} is not found"
raise ValueError(msg)

Check warning on line 121 in tests/unit/v1/helpers.py

View check run for this annotation

Codecov / codecov/patch

tests/unit/v1/helpers.py#L120-L121

Added lines #L120 - L121 were not covered by tests
return cc
if isinstance(c, Coordinate) and c.name == name:
# The coordinate is stored as an embedded Coordinate object.
# Return it.
return c

return None

Check warning on line 128 in tests/unit/v1/helpers.py

View check run for this annotation

Codecov / codecov/patch

tests/unit/v1/helpers.py#L128

Added line #L128 was not covered by tests


def _get_all_coordinates(dataset: Dataset) -> list[Coordinate]:
Expand All @@ -142,39 +142,42 @@
"""Create in-memory campos_3d dataset."""
ds = MDIODatasetBuilder(
"campos_3d",
attributes=UserAttributes(attributes={
"textHeader": [
"C01 .......................... ",
"C02 .......................... ",
"C03 .......................... ",
],
"foo": "bar"
}))
attributes=UserAttributes(
attributes={
"textHeader": [
"C01 .......................... ",
"C02 .......................... ",
"C03 .......................... ",
],
"foo": "bar",
}
),
)

# Add dimensions
ds.add_dimension("inline", 256)
ds.add_dimension("crossline", 512)
ds.add_dimension("depth", 384)
ds.add_coordinate("inline", dimensions=["inline"], data_type=ScalarType.UINT32)
ds.add_coordinate("crossline", dimensions=["crossline"], data_type=ScalarType.UINT32)
ds.add_coordinate("depth", dimensions=["depth"], data_type=ScalarType.FLOAT64,
metadata_info=[
AllUnits(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))
])
ds.add_coordinate("inline", dimensions=["inline"], data_type=ScalarType.UINT32)
ds.add_coordinate("crossline", dimensions=["crossline"], data_type=ScalarType.UINT32)
ds.add_coordinate(
"depth",
dimensions=["depth"],
data_type=ScalarType.FLOAT64,
metadata_info=[AllUnits(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))],
)
# Add coordinates
ds.add_coordinate(
"cdp-x",
dimensions=["inline", "crossline"],
data_type=ScalarType.FLOAT32,
metadata_info=[
AllUnits(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))]
metadata_info=[AllUnits(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))],
)
ds.add_coordinate(
"cdp-y",
dimensions=["inline", "crossline"],
data_type=ScalarType.FLOAT32,
metadata_info=[
AllUnits(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))]
metadata_info=[AllUnits(units_v1=LengthUnitModel(length=LengthUnitEnum.METER))],
)

# Add image variable
Expand All @@ -187,7 +190,8 @@
metadata_info=[
ChunkGridMetadata(
chunk_grid=RegularChunkGrid(
configuration=RegularChunkShape(chunk_shape=[128, 128, 128]))
configuration=RegularChunkShape(chunk_shape=[128, 128, 128])
)
),
StatisticsMetadata(
stats_v1=SummaryStatistics(
Expand All @@ -196,13 +200,12 @@
sumSquares=125.12,
min=5.61,
max=10.84,
histogram=CenteredBinHistogram(
binCenters=[1, 2], counts=[10, 15]),
histogram=CenteredBinHistogram(binCenters=[1, 2], counts=[10, 15]),
)
),
UserAttributes(
attributes={"fizz": "buzz", "UnitSystem": "Canonical"}),
])
UserAttributes(attributes={"fizz": "buzz", "UnitSystem": "Canonical"}),
],
)
# Add velocity variable
ds.add_variable(
name="velocity",
Expand All @@ -212,10 +215,10 @@
metadata_info=[
ChunkGridMetadata(
chunk_grid=RegularChunkGrid(
configuration=RegularChunkShape(chunk_shape=[128, 128, 128]))
configuration=RegularChunkShape(chunk_shape=[128, 128, 128])
)
),
AllUnits(units_v1=SpeedUnitModel(
speed=SpeedUnitEnum.METER_PER_SECOND)),
AllUnits(units_v1=SpeedUnitModel(speed=SpeedUnitEnum.METER_PER_SECOND)),
],
)
# Add inline-optimized image variable
Expand All @@ -229,8 +232,10 @@
metadata_info=[
ChunkGridMetadata(
chunk_grid=RegularChunkGrid(
configuration=RegularChunkShape(chunk_shape=[4, 512, 512]))
)]
configuration=RegularChunkShape(chunk_shape=[4, 512, 512])
)
)
],
)
# Add headers variable with structured dtype
ds.add_variable(
Expand All @@ -240,7 +245,7 @@
fields=[
StructuredField(name="cdp-x", format=ScalarType.FLOAT32),
StructuredField(name="cdp-y", format=ScalarType.FLOAT32),
StructuredField(name="inline", format=ScalarType.UINT32),
StructuredField(name="inline", format=ScalarType.UINT32),
StructuredField(name="crossline", format=ScalarType.UINT32),
]
),
Expand Down
Loading