From b6598fff62ff835ba0a5368823b859020541b64e Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 11:38:38 +0100 Subject: [PATCH 01/17] [ENH] Adding logic to test serialization on compute model --- gempy/API/compute_API.py | 13 ++++++++++++- gempy/modules/serialization/save_load.py | 4 +++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gempy/API/compute_API.py b/gempy/API/compute_API.py index db0acf85e..0e921468b 100644 --- a/gempy/API/compute_API.py +++ b/gempy/API/compute_API.py @@ -1,4 +1,7 @@ -from typing import Optional +import dotenv +import os + +from typing import Optional import numpy as np @@ -14,6 +17,8 @@ from ..modules.data_manipulation.engine_factory import interpolation_input_from_structural_frame from ..optional_dependencies import require_gempy_legacy +dotenv.load_dotenv() + def compute_model(gempy_model: GeoModel, engine_config: Optional[GemPyEngineConfig] = None) -> Solutions: """ @@ -56,6 +61,12 @@ def compute_model(gempy_model: GeoModel, engine_config: Optional[GemPyEngineConf case _: raise ValueError(f'Backend {engine_config} not supported') + if os.getenv("VALIDATE_SERIALIZATION", False): + from ..modules.serialization.save_load import save_model + import tempfile + with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp: + save_model(model=gempy_model, path=tmp.name, validate_serialization=True) + return gempy_model.solutions diff --git a/gempy/modules/serialization/save_load.py b/gempy/modules/serialization/save_load.py index 1008b9fad..d43d6d2b8 100644 --- a/gempy/modules/serialization/save_load.py +++ b/gempy/modules/serialization/save_load.py @@ -5,7 +5,7 @@ import os -def save_model(model: GeoModel, path: str, validate_serialization: bool = True): +def save_model(model: GeoModel, path: str | None = None, validate_serialization: bool = True): """ Save a GeoModel to a file with proper extension validation. @@ -23,6 +23,8 @@ def save_model(model: GeoModel, path: str, validate_serialization: bool = True): """ # Define the valid extension for gempy models VALID_EXTENSION = ".gempy" + if path is None: + path = model.meta.name + VALID_EXTENSION # Check if path has an extension path_obj = pathlib.Path(path) From 69c5b58982d2ce75f28bb27874ed6b47a59d4ba6 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 12:04:18 +0100 Subject: [PATCH 02/17] [ENH] Update data validation in structural elements & tests Refactored `StructuralElement` attributes to use Pydantic's `Field` for improved data validation and added `exclude=True` where necessary. Enhanced serialization test to include model save after computation with validation. --- gempy/core/data/structural_element.py | 7 ++++--- test/test_modules/test_serialize_model.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/gempy/core/data/structural_element.py b/gempy/core/data/structural_element.py index 0fff12282..5a1d3b7e8 100644 --- a/gempy/core/data/structural_element.py +++ b/gempy/core/data/structural_element.py @@ -1,5 +1,6 @@ import re from dataclasses import dataclass, field +from pydantic import Field from typing import Optional import numpy as np @@ -29,9 +30,9 @@ class StructuralElement: # Output # ? Should we extract this to a separate class? - vertices: Optional[np.ndarray] = None #: The vertices of the element in 3D space. - edges: Optional[np.ndarray] = None #: The edges of the element in 3D space. - scalar_field_at_interface: Optional[float] = None #: The scalar field value for the element. + vertices: np.ndarray | None = Field(default=None, exclude=True) #: The vertices of the element in 3D space. + edges: np.ndarray | None = Field(default=None, exclude=True) #: The edges of the element in 3D space. + scalar_field_at_interface: float | None = None #: The scalar field value for the element. _id: int = -1 diff --git a/test/test_modules/test_serialize_model.py b/test/test_modules/test_serialize_model.py index 8c3c281ca..8f022d447 100644 --- a/test/test_modules/test_serialize_model.py +++ b/test/test_modules/test_serialize_model.py @@ -22,7 +22,7 @@ def test_generate_horizontal_stratigraphic_model(): f.write(model_json) with loading_model_from_binary( - binary_body=model.structural_frame.input_tables_binary + binary_body=model.structural_frame.input_tables_binary ): model_deserialized = gp.data.GeoModel.model_validate_json(model_json) @@ -50,17 +50,22 @@ def _validate_serialization(original_model, model_deserialized): def test_save_model_to_disk(): model = gp.generate_example_model(ExampleModel.COMBINATION, compute_model=False) save_model(model, "temp/test_save_model_to_disk.gempy") - + # Load the model from disk loaded_model = load_model("temp/test_save_model_to_disk.gempy") _validate_serialization(model, loaded_model) - + gp.compute_model(loaded_model) if True: import gempy_viewer as gpv gpv.plot_3d(loaded_model, image=True) - + # Test save after compute + save_model( + model=model, + path="temp/test_save_model_to_disk.gempy", + validate_serialization=True + ) def test_interpolation_options(): From d79b957825e9fa40af80279d39051c6087028d5a Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 13:00:27 +0100 Subject: [PATCH 03/17] [ENH/WIP] Add warnings for development status and adjust tests Add warnings to indicate that save/load functions are in development and may not work as expected. Updated test cases by commenting unused computations, modifying object serialization, and adding placeholders for future functionality. --- docs/developers_notes/dev_log/2025_05.md | 3 +++ gempy/modules/serialization/save_load.py | 10 ++++++++++ .../_geophysics_TO_UPDATE/test_gravity.py | 11 ++++++++++- ...tal Stratigraphic Model serialization.approved.txt | 4 ---- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/developers_notes/dev_log/2025_05.md b/docs/developers_notes/dev_log/2025_05.md index da6726b24..6016c935f 100644 --- a/docs/developers_notes/dev_log/2025_05.md +++ b/docs/developers_notes/dev_log/2025_05.md @@ -26,5 +26,8 @@ - Solutions (Not sure if I want to save them) - RawArraysSolutions - Solutions +- Testing + - [ ] 10 Test to go in modules + - [ ] test api ## What do I have in the engine server logic? diff --git a/gempy/modules/serialization/save_load.py b/gempy/modules/serialization/save_load.py index d43d6d2b8..6f7f4b9a4 100644 --- a/gempy/modules/serialization/save_load.py +++ b/gempy/modules/serialization/save_load.py @@ -1,3 +1,5 @@ +import warnings + from ...core.data import GeoModel from ...core.data.encoders.converters import loading_model_from_binary from ...optional_dependencies import require_zlib @@ -21,6 +23,10 @@ def save_model(model: GeoModel, path: str | None = None, validate_serialization: ValueError If the file has an extension other than .gempy """ + + # Warning about preview + warnings.warn("This function is still in development. It may not work as expected.") + # Define the valid extension for gempy models VALID_EXTENSION = ".gempy" if path is None: @@ -80,6 +86,10 @@ def load_model(path: str) -> GeoModel: FileNotFoundError If the file doesn't exist """ + + # Warning about preview + warnings.warn("This function is still in development. It may not work as expected.") + VALID_EXTENSION = ".gempy" # Check if path has the valid extension diff --git a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py index f8f8d70d6..b340efdfe 100644 --- a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py +++ b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py @@ -57,7 +57,7 @@ def test_gravity(): structural_frame=frame, ) - gp.compute_model(geo_model) + # gp.compute_model(geo_model) import gempy_viewer as gpv gpv.plot_2d(geo_model, cell_number=0) @@ -75,6 +75,15 @@ def test_gravity(): densities=np.array([2.6, 2.4, 3.2]), ) + model_json = geo_model.model_dump_json(by_alias=True, indent=4) + + return + from pydantic_core import from_json + + json = from_json(model_json, allow_partial=True) + model_deserialized = gp.data.GeoModel.model_validate(json) + + return gp.compute_model(geo_model) print(geo_model.solutions.gravity) diff --git a/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt b/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt index 6e0892088..0fd4ebe30 100644 --- a/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt +++ b/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt @@ -28,8 +28,6 @@ }, "_model_transform": null }, - "vertices": null, - "edges": null, "scalar_field_at_interface": null, "_id": 117776925 }, @@ -51,8 +49,6 @@ }, "_model_transform": null }, - "vertices": null, - "edges": null, "scalar_field_at_interface": null, "_id": 67239155 } From a4126b2ecda2ea0b15a29a687558076622e8f306 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 14:05:09 +0100 Subject: [PATCH 04/17] [ENH][TEST] Improve serialization validation and update tests Add detailed validation for serialized GeoModel instances, including debugging for string differences and hash checks. Introduce `verify_model_serialization` with pre/post-deserialization checks, and update gravity tests for structured verification and data type consistency. --- gempy/modules/serialization/save_load.py | 70 +++++++- .../_geophysics_TO_UPDATE/test_gravity.py | 21 ++- .../2-layers.approved.txt | 159 ++++++++++++++++++ 3 files changed, 235 insertions(+), 15 deletions(-) create mode 100644 test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt diff --git a/gempy/modules/serialization/save_load.py b/gempy/modules/serialization/save_load.py index 6f7f4b9a4..79b06da4c 100644 --- a/gempy/modules/serialization/save_load.py +++ b/gempy/modules/serialization/save_load.py @@ -1,3 +1,5 @@ +from typing import Literal + import warnings from ...core.data import GeoModel @@ -23,10 +25,10 @@ def save_model(model: GeoModel, path: str | None = None, validate_serialization: ValueError If the file has an extension other than .gempy """ - + # Warning about preview warnings.warn("This function is still in development. It may not work as expected.") - + # Define the valid extension for gempy models VALID_EXTENSION = ".gempy" if path is None: @@ -89,7 +91,7 @@ def load_model(path: str) -> GeoModel: # Warning about preview warnings.warn("This function is still in development. It may not work as expected.") - + VALID_EXTENSION = ".gempy" # Check if path has the valid extension @@ -131,10 +133,70 @@ def _to_binary(header_json, body_) -> bytes: def _validate_serialization(original_model, model_deserialized): + if False: + _verify_models(model_deserialized, original_model) + a = hash(original_model.structural_frame.surface_points_copy.data.tobytes()) b = hash(model_deserialized.structural_frame.surface_points_copy.data.tobytes()) o_a = hash(original_model.structural_frame.orientations_copy.data.tobytes()) o_b = hash(model_deserialized.structural_frame.orientations_copy.data.tobytes()) assert a == b, "Hashes for surface points are not equal" assert o_a == o_b, "Hashes for orientations are not equal" - assert model_deserialized.__str__() == original_model.__str__() + original_model___str__ = original_model.__str__() + deserialized___str__ = model_deserialized.__str__() + if original_model___str__ != deserialized___str__: + # Find first char that is not the same + for i in range(min(len(original_model___str__), len(deserialized___str__))): + if original_model___str__[i] != deserialized___str__[i]: + break + print(f"First difference at index {i}:") + i1 = 50 + print(f"Original: {original_model___str__[i - i1:i + i1]}") + print(f"Deserialized: {deserialized___str__[i - i1:i + i1]}") + + assert deserialized___str__ == original_model___str__ + + +def verify_model_serialization(model: GeoModel, verify_moment: Literal["before", "after"], file_name: str): + """ + Verifies the serialization and deserialization process of a GeoModel instance + by ensuring the serialized JSON and binary data match during either the + initial or post-process phase, based on the specified verification moment. + + Args: + model: The GeoModel instance to be verified. + verify_moment: A literal value specifying whether to verify the model + before or after the deserialization process. Accepts "before" + or "after" as valid inputs. + file_name: The filename to associate with the verification process for + logging or output purposes. + + Raises: + ValueError: If `verify_moment` is not set to "before" or "after". + """ + model_json = model.model_dump_json(by_alias=True, indent=4) + + # Compress the binary data + zlib = require_zlib() + compressed_binary = zlib.compress(model.structural_frame.input_tables_binary) + + binary_file = _to_binary(model_json, compressed_binary) + + model_deserialized = _deserialize_binary_file(binary_file) + + original_model = model + original_model.meta.creation_date = "" + model_deserialized.meta.creation_date = "" + from verify_helper import verify_json + if verify_moment == "before": + verify_json( + item=original_model.model_dump_json(by_alias=True, indent=4), + name=file_name + ) + elif verify_moment == "after": + verify_json( + item=model_deserialized.model_dump_json(by_alias=True, indent=4), + name=file_name + ) + else: + raise ValueError("Invalid model parameter") diff --git a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py index b340efdfe..335ca7d72 100644 --- a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py +++ b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.py @@ -4,6 +4,8 @@ # Importing auxiliary libraries import numpy as np +from gempy.modules.serialization.save_load import verify_model_serialization + def test_gravity(): color_generator = gp.data.ColorsGenerator() @@ -64,9 +66,9 @@ def test_gravity(): gp.set_centered_grid( grid=geo_model.grid, - centers=np.array([[6, 0, 4]]), - resolution=np.array([10, 10, 100]), - radius=np.array([16000, 16000, 16000]) # ? This radius makes 0 sense but it is the original one in gempy v2 + centers=np.array([[6, 0, 4]], dtype="float"), + resolution=np.array([10, 10, 100], dtype="float"), + radius=np.array([16000, 16000, 16000], dtype="float") # ? This radius makes 0 sense but it is the original one in gempy v2 ) gravity_gradient = gp.calculate_gravity_gradient(geo_model.grid.centered_grid) @@ -75,15 +77,12 @@ def test_gravity(): densities=np.array([2.6, 2.4, 3.2]), ) - model_json = geo_model.model_dump_json(by_alias=True, indent=4) - - return - from pydantic_core import from_json - - json = from_json(model_json, allow_partial=True) - model_deserialized = gp.data.GeoModel.model_validate(json) + verify_model_serialization( + model=geo_model, + verify_moment="after", + file_name=f"verify/{geo_model.meta.name}" + ) - return gp.compute_model(geo_model) print(geo_model.solutions.gravity) diff --git a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt new file mode 100644 index 000000000..fc8c8ab11 --- /dev/null +++ b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt @@ -0,0 +1,159 @@ +{ + "meta": { + "name": "2-layers", + "creation_date": "", + "last_modification_date": null, + "owner": null + }, + "structural_frame": { + "structural_groups": [ + { + "name": "default", + "elements": [ + { + "name": "surface1", + "is_active": true, + "_color": "#015482", + "surface_points": { + "name_id_map": { + "surface1": 57292991 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "surface1": 57292991 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": -1 + }, + { + "name": "surface2", + "is_active": true, + "_color": "#9f0052", + "surface_points": { + "name_id_map": { + "surface2": 21816406 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": null, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": -1 + } + ], + "structural_relation": 1, + "fault_relations": null, + "faults_input_data": null, + "solution": null + } + ], + "is_dirty": true, + "basement_color": "#ffbe00", + "binary_meta_data": { + "sp_binary_length": 144 + } + }, + "grid": { + "_octree_grid": null, + "_dense_grid": { + "resolution": [ + 500, + 1, + 500 + ], + "extent": [ + 0.0, + 12.0, + -2.0, + 2.0, + 0.0, + 4.0 + ], + "_transform": null + }, + "_custom_grid": null, + "_topography": null, + "_sections": null, + "_centered_grid": { + "centers": [ + [ + 6.0, + 0.0, + 4.0 + ] + ], + "resolution": [ + 10.0, + 10.0, + 100.0 + ], + "radius": [ + 16000.0, + 16000.0, + 16000.0 + ] + }, + "_transform": null, + "_octree_levels": -1, + "active_grids": 1058 + }, + "input_transform": { + "position": [ + -6.0, + -0.0, + -2.51 + ], + "rotation": [ + 0.0, + 0.0, + 0.0 + ], + "scale": [ + 0.08333333333333333, + 0.16778523489932887, + 0.08333333333333333 + ], + "_is_default_transform": false, + "_cached_pivot": null + }, + "_interpolation_options": { + "kernel_options": { + "range": 1.7, + "c_o": 10.0, + "uni_degree": 1, + "i_res": 4.0, + "gi_res": 2.0, + "number_dimensions": 3, + "kernel_function": "cubic", + "kernel_solver": 1, + "compute_condition_number": false, + "optimizing_condition_number": false, + "condition_number": null + }, + "evaluation_options": { + "_number_octree_levels": 1, + "_number_octree_levels_surface": 4, + "octree_curvature_threshold": -1.0, + "octree_error_threshold": 1.0, + "octree_min_level": 2, + "mesh_extraction": true, + "mesh_extraction_masking_options": 3, + "mesh_extraction_fancy": true, + "evaluation_chunk_size": 500000, + "compute_scalar_gradient": false, + "verbose": false + }, + "debug": true, + "cache_mode": 3, + "cache_model_name": "2-layers", + "block_solutions_type": 2, + "sigmoid_slope": 5000000, + "debug_water_tight": false + } +} From 3a290f18a902f0675faf48a974f621b31b5c6fa2 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 14:39:05 +0100 Subject: [PATCH 05/17] [ENH] Update GeoModel to include and reset geophysics input Expose `geophysics_input` and integrate gravity gradient calculation during deserialization. This ensures proper resetting of geophysics-specific fields when grid data is available. --- gempy/core/data/geo_model.py | 13 ++++++++++--- .../2-layers.approved.txt | 8 ++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/gempy/core/data/geo_model.py b/gempy/core/data/geo_model.py index a62315862..9a911c17d 100644 --- a/gempy/core/data/geo_model.py +++ b/gempy/core/data/geo_model.py @@ -15,6 +15,7 @@ from gempy_engine.core.data.interpolation_input import InterpolationInput from gempy_engine.core.data.raw_arrays_solution import RawArraysSolution from gempy_engine.core.data.transforms import Transform, GlobalAnisotropy +from gempy_engine.modules.geophysics.gravity_gradient import calculate_gravity_gradient from .encoders.converters import instantiate_if_necessary from .encoders.json_geomodel_encoder import encode_numpy_array from .grid import Grid @@ -62,7 +63,7 @@ class GeoModel(BaseModel): # region GemPy engine data types _interpolation_options: InterpolationOptions #: The interpolation options provided by the user. - geophysics_input: GeophysicsInput = Field(default=None, exclude=True) #: The geophysics input of the geological model. + geophysics_input: GeophysicsInput = Field(default=None, exclude=False) #: The geophysics input of the geological model. input_transform: Transform = Field(default=None, exclude=False) #: The transformation used in the geological model for input points. interpolation_grid: EngineGrid = Field(default=None, exclude=True) #: ptional grid used for interpolation. Can be seen as a cache field. @@ -303,9 +304,9 @@ def add_surface_points(self, X: Sequence[float], Y: Sequence[float], Z: Sequence @classmethod def deserialize_properties(cls, data: Union["GeoModel", dict], constructor: ModelWrapValidatorHandler["GeoModel"]) -> "GeoModel": match data: - case GeoModel(): + case GeoModel(): return data - case dict(): + case dict(): # instance: GeoModel = constructor(data) instantiate_if_necessary( data=data, @@ -313,6 +314,12 @@ def deserialize_properties(cls, data: Union["GeoModel", dict], constructor: Mode type=InterpolationOptions ) instance._interpolation_options = data.get("_interpolation_options") + + # * Reset geophysics if necessary + centered_grid = instance.grid.centered_grid + if centered_grid is not None and instance.geophysics_input is not None: + instance.geophysics_input.tz = calculate_gravity_gradient(centered_grid) + return instance case _: raise ValidationError diff --git a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt index fc8c8ab11..5fe87bb49 100644 --- a/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt +++ b/test/test_modules/_geophysics_TO_UPDATE/test_gravity.test_gravity.verify/2-layers.approved.txt @@ -103,6 +103,14 @@ "_octree_levels": -1, "active_grids": 1058 }, + "geophysics_input": { + "tz": [], + "densities": [ + 2.6, + 2.4, + 3.2 + ] + }, "input_transform": { "position": [ -6.0, From 61ae4668d32b6313eacc04836096a8e74c13acc9 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 15:17:16 +0100 Subject: [PATCH 06/17] [ENH] Make geophysics input optional --- gempy/core/data/geo_model.py | 2 +- .../test_faults/test_finite_faults.py | 11 +- .../fault.approved.txt | 216 ++++++++++++++++++ 3 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 test/test_modules/test_faults/test_finite_faults.test_finite_fault_scalar_field_on_fault.verify/fault.approved.txt diff --git a/gempy/core/data/geo_model.py b/gempy/core/data/geo_model.py index 9a911c17d..66d35d698 100644 --- a/gempy/core/data/geo_model.py +++ b/gempy/core/data/geo_model.py @@ -63,7 +63,7 @@ class GeoModel(BaseModel): # region GemPy engine data types _interpolation_options: InterpolationOptions #: The interpolation options provided by the user. - geophysics_input: GeophysicsInput = Field(default=None, exclude=False) #: The geophysics input of the geological model. + geophysics_input: GeophysicsInput | None = Field(default=None, exclude=False) #: The geophysics input of the geological model. input_transform: Transform = Field(default=None, exclude=False) #: The transformation used in the geological model for input points. interpolation_grid: EngineGrid = Field(default=None, exclude=True) #: ptional grid used for interpolation. Can be seen as a cache field. diff --git a/test/test_modules/test_faults/test_finite_faults.py b/test/test_modules/test_faults/test_finite_faults.py index 5cfbd5986..33cee774b 100644 --- a/test/test_modules/test_faults/test_finite_faults.py +++ b/test/test_modules/test_faults/test_finite_faults.py @@ -4,6 +4,7 @@ import gempy as gp import gempy_viewer as gpv from gempy.core.data.enumerators import ExampleModel +from gempy.modules.serialization.save_load import verify_model_serialization from gempy_viewer.optional_dependencies import require_pyvista from test.conftest import TEST_SPEED, TestSpeed @@ -32,13 +33,11 @@ def test_finite_fault_scalar_field_on_fault(): radius=scaled_radius, max_slope=k # * This controls the speed of the transition ) - transform = gp.data.Transform( position=np.array([0, 0, 0]), rotation=np.array([0, 60, 0]), scale=np.ones(3) ) - faults_data = gp.data.FaultsData( fault_values_everywhere=np.zeros(0), fault_values_on_sp=np.zeros(0), @@ -53,10 +52,16 @@ def test_finite_fault_scalar_field_on_fault(): ) geo_model.structural_frame.structural_groups[0].faults_input_data = faults_data + + verify_model_serialization( + model=geo_model, + verify_moment="after", + file_name=f"verify/{geo_model.meta.name}" + ) + gp.compute_model(geo_model) # TODO: Try to do this afterwards - # scalar_fault = scalar_funtion(regular_grid.values) if plot_pyvista := True: plot3d = gpv.plot_3d( diff --git a/test/test_modules/test_faults/test_finite_faults.test_finite_fault_scalar_field_on_fault.verify/fault.approved.txt b/test/test_modules/test_faults/test_finite_faults.test_finite_fault_scalar_field_on_fault.verify/fault.approved.txt new file mode 100644 index 000000000..3c1d06bed --- /dev/null +++ b/test/test_modules/test_faults/test_finite_faults.test_finite_fault_scalar_field_on_fault.verify/fault.approved.txt @@ -0,0 +1,216 @@ +{ + "meta": { + "name": "fault", + "creation_date": "", + "last_modification_date": null, + "owner": null + }, + "structural_frame": { + "structural_groups": [ + { + "name": "Fault_Series", + "elements": [ + { + "name": "fault", + "is_active": true, + "_color": "#527682", + "surface_points": { + "name_id_map": { + "fault": 65970106, + "rock1": 167239155, + "rock2": 217776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "fault": 65970106, + "rock1": 167239155, + "rock2": 217776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 65970106 + } + ], + "structural_relation": 3, + "fault_relations": 1, + "faults_input_data": { + "fault_values_everywhere": [], + "fault_values_on_sp": [], + "fault_values_ref": [], + "fault_values_rest": [], + "thickness": null, + "finite_fault_data": { + "implicit_function_transform": { + "position": [ + 0, + 0, + 0 + ], + "rotation": [ + 0, + 60, + 0 + ], + "scale": [ + 1.0, + 1.0, + 1.0 + ], + "_is_default_transform": false, + "_cached_pivot": null + }, + "pivot": [ + 0.0, + 0.0, + 0.0 + ] + } + }, + "solution": null + }, + { + "name": "Strat_Series", + "elements": [ + { + "name": "rock2", + "is_active": true, + "_color": "#ffbe00", + "surface_points": { + "name_id_map": { + "fault": 65970106, + "rock1": 167239155, + "rock2": 217776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "fault": 65970106, + "rock1": 167239155, + "rock2": 217776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 217776925 + }, + { + "name": "rock1", + "is_active": true, + "_color": "#9f0052", + "surface_points": { + "name_id_map": { + "fault": 65970106, + "rock1": 167239155, + "rock2": 217776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "fault": 65970106, + "rock1": 167239155, + "rock2": 217776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 167239155 + } + ], + "structural_relation": 1, + "fault_relations": 3, + "faults_input_data": null, + "solution": null + } + ], + "is_dirty": true, + "basement_color": "#728f02", + "binary_meta_data": { + "sp_binary_length": 792 + } + }, + "grid": { + "_octree_grid": { + "resolution": [ + 64, + 64, + 64 + ], + "extent": [ + 0.0, + 1000.0, + 0.0, + 1000.0, + 0.0, + 1000.0 + ], + "_transform": null + }, + "_dense_grid": null, + "_custom_grid": null, + "_topography": null, + "_sections": null, + "_centered_grid": null, + "_transform": null, + "_octree_levels": -1, + "active_grids": 1025 + }, + "geophysics_input": null, + "input_transform": { + "position": [ + -500.0, + -500.0, + -500.0 + ], + "rotation": [ + 0.0, + 0.0, + 0.0 + ], + "scale": [ + 0.0005, + 0.0005, + 0.0005 + ], + "_is_default_transform": false, + "_cached_pivot": null + }, + "_interpolation_options": { + "kernel_options": { + "range": 1.7, + "c_o": 10.0, + "uni_degree": 1, + "i_res": 4.0, + "gi_res": 2.0, + "number_dimensions": 3, + "kernel_function": "cubic", + "kernel_solver": 1, + "compute_condition_number": false, + "optimizing_condition_number": false, + "condition_number": null + }, + "evaluation_options": { + "_number_octree_levels": 6, + "_number_octree_levels_surface": 4, + "octree_curvature_threshold": -1.0, + "octree_error_threshold": 1.0, + "octree_min_level": 2, + "mesh_extraction": true, + "mesh_extraction_masking_options": 3, + "mesh_extraction_fancy": true, + "evaluation_chunk_size": 500000, + "compute_scalar_gradient": false, + "verbose": false + }, + "debug": true, + "cache_mode": 3, + "cache_model_name": "fault", + "block_solutions_type": 1, + "sigmoid_slope": 5000000, + "debug_water_tight": false + } +} From 727d40e367bc702c2a26f465ad4ac6da8a61b5ee Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 15:22:17 +0100 Subject: [PATCH 07/17] [TEST] Use `tempfile` for temporary file handling in tests Replaced hardcoded file paths with `tempfile.NamedTemporaryFile` to ensure safe and temporary file handling during serialization tests. This improves test reliability and avoids residual files on disk. --- test/test_modules/test_serialize_model.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/test_modules/test_serialize_model.py b/test/test_modules/test_serialize_model.py index 8f022d447..0ccaaed57 100644 --- a/test/test_modules/test_serialize_model.py +++ b/test/test_modules/test_serialize_model.py @@ -1,3 +1,5 @@ +import tempfile + import json import os import pprint @@ -49,10 +51,12 @@ def _validate_serialization(original_model, model_deserialized): def test_save_model_to_disk(): model = gp.generate_example_model(ExampleModel.COMBINATION, compute_model=False) - save_model(model, "temp/test_save_model_to_disk.gempy") + with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp: + tmp_name = tmp.name + ".gempy" # Store the name to use it later + save_model(model, tmp_name) - # Load the model from disk - loaded_model = load_model("temp/test_save_model_to_disk.gempy") + # Load the model from disk + loaded_model = load_model(tmp_name) _validate_serialization(model, loaded_model) gp.compute_model(loaded_model) @@ -60,12 +64,15 @@ def test_save_model_to_disk(): import gempy_viewer as gpv gpv.plot_3d(loaded_model, image=True) + # Test save after compute - save_model( - model=model, - path="temp/test_save_model_to_disk.gempy", - validate_serialization=True - ) + with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp: + tmp_name = tmp.name + ".gempy" # Store the name to use it later + save_model( + model=model, + path=tmp_name, + validate_serialization=True + ) def test_interpolation_options(): From 39ac3de015286d45327b7c1b75630a2e1016da96 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 15:59:07 +0100 Subject: [PATCH 08/17] [WIP] Dealing with custom grids --- gempy/API/compute_API.py | 7 +- gempy/API/grid_API.py | 2 +- gempy/core/data/grid_modules/grid_types.py | 34 ++-- gempy/modules/serialization/save_load.py | 8 +- .../test_grids/test_custom_grid.py | 9 +- .../fold.approved.txt | 148 ++++++++++++++++++ ...tigraphic Model serialization.approved.txt | 1 + 7 files changed, 182 insertions(+), 27 deletions(-) create mode 100644 test/test_modules/test_grids/test_custom_grid.test_custom_grid.verify/fold.approved.txt diff --git a/gempy/API/compute_API.py b/gempy/API/compute_API.py index 0e921468b..3151b6015 100644 --- a/gempy/API/compute_API.py +++ b/gempy/API/compute_API.py @@ -20,7 +20,8 @@ dotenv.load_dotenv() -def compute_model(gempy_model: GeoModel, engine_config: Optional[GemPyEngineConfig] = None) -> Solutions: +def compute_model(gempy_model: GeoModel, engine_config: Optional[GemPyEngineConfig] = None, + **kwargs) -> Solutions: """ Compute the geological model given the provided GemPy model. @@ -61,7 +62,7 @@ def compute_model(gempy_model: GeoModel, engine_config: Optional[GemPyEngineConf case _: raise ValueError(f'Backend {engine_config} not supported') - if os.getenv("VALIDATE_SERIALIZATION", False): + if os.getenv("VALIDATE_SERIALIZATION", False) and kwargs.get("validate_serialization", True): from ..modules.serialization.save_load import save_model import tempfile with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp: @@ -90,7 +91,7 @@ def compute_model_at(gempy_model: GeoModel, at: np.ndarray, xyz_coord=at ) - sol = compute_model(gempy_model, engine_config) + sol = compute_model(gempy_model, engine_config, validate_serialization=False) return sol.raw_arrays.custom diff --git a/gempy/API/grid_API.py b/gempy/API/grid_API.py index 10caf9c81..fd0104733 100644 --- a/gempy/API/grid_API.py +++ b/gempy/API/grid_API.py @@ -86,7 +86,7 @@ def set_topography_from_file(grid: Grid, filepath: str, crop_to_extent: Union[Se def set_custom_grid(grid: Grid, xyz_coord: np.ndarray): - custom_grid = CustomGrid(xyx_coords=xyz_coord) + custom_grid = CustomGrid(values=xyz_coord) grid.custom_grid = custom_grid set_active_grid(grid, [Grid.GridTypes.CUSTOM]) diff --git a/gempy/core/data/grid_modules/grid_types.py b/gempy/core/data/grid_modules/grid_types.py index ae9e771c5..5fd2a8ad1 100644 --- a/gempy/core/data/grid_modules/grid_types.py +++ b/gempy/core/data/grid_modules/grid_types.py @@ -255,6 +255,7 @@ def plot_rotation(regular_grid, pivot, point_x_axis, point_y_axis): plt.show() +@dataclasses.dataclass class Sections: """ Object that creates a grid of cross sections between two points. @@ -364,6 +365,7 @@ def get_section_grid(self, section_name: str): return self.values[l0:l1] +@dataclasses.dataclass class CustomGrid: """Object that contains arbitrary XYZ coordinates. @@ -374,26 +376,20 @@ class CustomGrid: values (np.ndarray): XYZ coordinates """ - def __init__(self, xyx_coords: np.ndarray): - self.values = np.zeros((0, 3)) - self.set_custom_grid(xyx_coords) - - def set_custom_grid(self, custom_grid: np.ndarray): - """ - Give the coordinates of an external generated grid - - Args: - custom_grid (numpy.ndarray like): XYZ (in columns) of the desired coordinates - - Returns: - numpy.ndarray: Unraveled 3D numpy array where every row correspond to the xyz coordinates of a regular - grid - """ - custom_grid = np.atleast_2d(custom_grid) + values: np.ndarray = Field( + exclude=True, + default_factory=lambda: np.zeros((0, 3)), + repr=False + ) + + + def __post_init__(self): + custom_grid = np.atleast_2d(self.values) assert type(custom_grid) is np.ndarray and custom_grid.shape[1] == 3, \ 'The shape of new grid must be (n,3) where n is the number of' \ ' points of the grid' - self.values = custom_grid - self.length = self.values.shape[0] - return self.values + + @property + def length(self): + return self.values.shape[0] diff --git a/gempy/modules/serialization/save_load.py b/gempy/modules/serialization/save_load.py index 79b06da4c..b27983efa 100644 --- a/gempy/modules/serialization/save_load.py +++ b/gempy/modules/serialization/save_load.py @@ -1,3 +1,5 @@ +import re + from typing import Literal import warnings @@ -142,15 +144,15 @@ def _validate_serialization(original_model, model_deserialized): o_b = hash(model_deserialized.structural_frame.orientations_copy.data.tobytes()) assert a == b, "Hashes for surface points are not equal" assert o_a == o_b, "Hashes for orientations are not equal" - original_model___str__ = original_model.__str__() - deserialized___str__ = model_deserialized.__str__() + original_model___str__ = re.sub(r'\s+', ' ', original_model.__str__()) + deserialized___str__ = re.sub(r'\s+', ' ', model_deserialized.__str__()) if original_model___str__ != deserialized___str__: # Find first char that is not the same for i in range(min(len(original_model___str__), len(deserialized___str__))): if original_model___str__[i] != deserialized___str__[i]: break print(f"First difference at index {i}:") - i1 = 50 + i1 = 10 print(f"Original: {original_model___str__[i - i1:i + i1]}") print(f"Deserialized: {deserialized___str__[i - i1:i + i1]}") diff --git a/test/test_modules/test_grids/test_custom_grid.py b/test/test_modules/test_grids/test_custom_grid.py index c786490c9..ed765c2e6 100644 --- a/test/test_modules/test_grids/test_custom_grid.py +++ b/test/test_modules/test_grids/test_custom_grid.py @@ -1,6 +1,7 @@ import numpy as np import pytest +from gempy.modules.serialization.save_load import verify_model_serialization from test.conftest import TEST_SPEED, TestSpeed import gempy as gp from gempy.core.data.enumerators import ExampleModel @@ -32,7 +33,13 @@ def test_custom_grid(): xyz_coord=xyz_coord ) - sol: gp.data.Solutions = gp.compute_model(geo_model) + verify_model_serialization( + model=geo_model, + verify_moment="after", + file_name=f"verify/{geo_model.meta.name}" + ) + + sol: gp.data.Solutions = gp.compute_model(geo_model, validate_serialization=False) np.testing.assert_array_equal( sol.raw_arrays.custom, np.array([3., 3., 3., 3., 1., 1., 1., 1.]) diff --git a/test/test_modules/test_grids/test_custom_grid.test_custom_grid.verify/fold.approved.txt b/test/test_modules/test_grids/test_custom_grid.test_custom_grid.verify/fold.approved.txt new file mode 100644 index 000000000..2a39d7036 --- /dev/null +++ b/test/test_modules/test_grids/test_custom_grid.test_custom_grid.verify/fold.approved.txt @@ -0,0 +1,148 @@ +{ + "meta": { + "name": "fold", + "creation_date": "", + "last_modification_date": null, + "owner": null + }, + "structural_frame": { + "structural_groups": [ + { + "name": "Strat_Series", + "elements": [ + { + "name": "rock2", + "is_active": true, + "_color": "#9f0052", + "surface_points": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 117776925 + }, + { + "name": "rock1", + "is_active": true, + "_color": "#015482", + "surface_points": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 67239155 + } + ], + "structural_relation": 1, + "fault_relations": null, + "faults_input_data": null, + "solution": null + } + ], + "is_dirty": true, + "basement_color": "#ffbe00", + "binary_meta_data": { + "sp_binary_length": 1296 + } + }, + "grid": { + "_octree_grid": { + "resolution": [ + 32, + 32, + 32 + ], + "extent": [ + 0.0, + 1000.0, + 0.0, + 1000.0, + 0.0, + 1000.0 + ], + "_transform": null + }, + "_dense_grid": null, + "_custom_grid": {}, + "_topography": null, + "_sections": null, + "_centered_grid": null, + "_transform": null, + "_octree_levels": -1, + "active_grids": 1029 + }, + "geophysics_input": null, + "input_transform": { + "position": [ + -500.0, + -500.0, + -510.0 + ], + "rotation": [ + 0.0, + 0.0, + 0.0 + ], + "scale": [ + 0.0005, + 0.0005, + 0.0005 + ], + "_is_default_transform": false, + "_cached_pivot": null + }, + "_interpolation_options": { + "kernel_options": { + "range": 1.7, + "c_o": 10.0, + "uni_degree": 1, + "i_res": 4.0, + "gi_res": 2.0, + "number_dimensions": 3, + "kernel_function": "cubic", + "kernel_solver": 1, + "compute_condition_number": false, + "optimizing_condition_number": false, + "condition_number": null + }, + "evaluation_options": { + "_number_octree_levels": 2, + "_number_octree_levels_surface": 4, + "octree_curvature_threshold": -1.0, + "octree_error_threshold": 1.0, + "octree_min_level": 2, + "mesh_extraction": true, + "mesh_extraction_masking_options": 3, + "mesh_extraction_fancy": true, + "evaluation_chunk_size": 500000, + "compute_scalar_gradient": false, + "verbose": false + }, + "debug": true, + "cache_mode": 3, + "cache_model_name": "fold", + "block_solutions_type": 1, + "sigmoid_slope": 5000000, + "debug_water_tight": false + } +} diff --git a/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt b/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt index 0fd4ebe30..a9021a98e 100644 --- a/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt +++ b/test/test_modules/test_serialize_model.test_generate_horizontal_stratigraphic_model.verify/Horizontal Stratigraphic Model serialization.approved.txt @@ -91,6 +91,7 @@ "_octree_levels": -1, "active_grids": 1026 }, + "geophysics_input": null, "input_transform": { "position": [ -500.0, From 5bbf54bc10efb26bcd87e0159b272e17d42bf626 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 16:11:49 +0100 Subject: [PATCH 09/17] [WIP] Making topography a data class --- gempy/core/data/grid_modules/topography.py | 59 +++++++++---------- .../test_grids/test_grids_sections.py | 4 ++ 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/gempy/core/data/grid_modules/topography.py b/gempy/core/data/grid_modules/topography.py index bd7aa21c3..e241117b6 100644 --- a/gempy/core/data/grid_modules/topography.py +++ b/gempy/core/data/grid_modules/topography.py @@ -1,5 +1,8 @@ +import dataclasses + import warnings -from typing import Optional +from pydantic import Field +from typing import Optional, Tuple import numpy as np @@ -7,43 +10,35 @@ from ....modules.grids.create_topography import _LoadDEMArtificial from ....optional_dependencies import require_skimage +from dataclasses import field, dataclass +@dataclass class Topography: - """ - Object to include topography in the model. - - Notes: - This always assumes that the topography we pass fits perfectly the extent """ + Object to include topography in the model. + Notes: + This always assumes that the topography we pass fits perfectly the extent. + """ + + regular_grid: RegularGrid + values_2d: np.ndarray = Field(exclude=True, default_factory=lambda: np.zeros((0, 0, 3))) + source: Optional[str] = None + + # Fields managed internally + values: np.ndarray = field(init=False, default_factory=lambda: np.zeros((0, 3))) + resolution: Tuple[int, int] = field(init=False, default=(0, 0)) + raster_shape: Tuple[int, ...] = field(init=False, default=()) + _mask_topo: Optional[np.ndarray] = field(init=False, default=None, repr=False) + _x: Optional[np.ndarray] = field(init=False, default=None, repr=False) + _y: Optional[np.ndarray] = field(init=False, default=None, repr=False) + + def __post_init__(self): + # if a non-empty array was provided, initialize the flattened values + if self.values_2d.size: + self.set_values(self.values_2d) - def __init__(self, regular_grid: RegularGrid, values_2d: Optional[np.ndarray] = None): - - self._mask_topo = None - self._regular_grid = regular_grid - - # Values (n, 3) - self.values = np.zeros((0, 3)) - - # Values (n, n, 3) - self.values_2d = np.zeros((0, 0, 3)) - - # Shape original - self.raster_shape = tuple() - - # Topography Resolution - self.resolution = np.zeros((0, 3)) - - # Source for the - self.source = None - - # Coords - self._x = None - self._y = None - - if values_2d is not None: - self.set_values(values_2d) @classmethod def from_subsurface_structured_data(cls, structured_data: 'subsurface.StructuredData', regular_grid: RegularGrid): diff --git a/test/test_modules/test_grids/test_grids_sections.py b/test/test_modules/test_grids/test_grids_sections.py index 25e9bd5c8..e7e4d3e53 100644 --- a/test/test_modules/test_grids/test_grids_sections.py +++ b/test/test_modules/test_grids/test_grids_sections.py @@ -20,6 +20,8 @@ def test_section_grids(): evaluation_options=geo_model.interpolation_options.evaluation_options ) + model = geo_model + model_json = model.model_dump_json(by_alias=True, indent=4) gp.set_section_grid( grid=geo_model.grid, section_dict={ @@ -28,6 +30,7 @@ def test_section_grids(): } ) + model_json = model.model_dump_json(by_alias=True, indent=4) gp.set_topography_from_random( grid=geo_model.grid, fractal_dimension=1.2, @@ -35,6 +38,7 @@ def test_section_grids(): topography_resolution=np.array([60, 60]) ) + model_json = model.model_dump_json(by_alias=True, indent=4) gp.compute_model(geo_model) gpv.plot_2d( model=geo_model, From 57b9984f728c6d0008e698c19945b1e03470b4d5 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 16:23:18 +0100 Subject: [PATCH 10/17] [WIP] More work towards converting topography into a proper dataclass --- gempy/core/data/encoders/converters.py | 5 +++++ gempy/core/data/grid_modules/topography.py | 3 ++- gempy/modules/serialization/save_load.py | 5 +++-- test/test_modules/test_grids/test_grids_sections.py | 11 +++++++---- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/gempy/core/data/encoders/converters.py b/gempy/core/data/encoders/converters.py index 76fd2f3d2..a697f48d6 100644 --- a/gempy/core/data/encoders/converters.py +++ b/gempy/core/data/encoders/converters.py @@ -1,3 +1,5 @@ +from typing import Annotated + from contextlib import contextmanager from contextvars import ContextVar @@ -17,6 +19,9 @@ def validate_numpy_array(v): return np.array(v) if v is not None else None +short_array_type = Annotated[np.ndarray, (BeforeValidator(lambda v: np.array(v) if v is not None else None))] + + def instantiate_if_necessary(data: dict, key: str, type: type) -> None: """ Creates instances of the specified type for a dictionary key if the key exists and its diff --git a/gempy/core/data/grid_modules/topography.py b/gempy/core/data/grid_modules/topography.py index e241117b6..1ec26bbe8 100644 --- a/gempy/core/data/grid_modules/topography.py +++ b/gempy/core/data/grid_modules/topography.py @@ -11,6 +11,7 @@ from ....optional_dependencies import require_skimage from dataclasses import field, dataclass +from ..encoders.converters import short_array_type @dataclass @@ -27,7 +28,7 @@ class Topography: source: Optional[str] = None # Fields managed internally - values: np.ndarray = field(init=False, default_factory=lambda: np.zeros((0, 3))) + values: short_array_type = field(init=False, default=np.zeros((0, 3))) resolution: Tuple[int, int] = field(init=False, default=(0, 0)) raster_shape: Tuple[int, ...] = field(init=False, default=()) _mask_topo: Optional[np.ndarray] = field(init=False, default=None, repr=False) diff --git a/gempy/modules/serialization/save_load.py b/gempy/modules/serialization/save_load.py index b27983efa..c379d3d89 100644 --- a/gempy/modules/serialization/save_load.py +++ b/gempy/modules/serialization/save_load.py @@ -184,11 +184,10 @@ def verify_model_serialization(model: GeoModel, verify_moment: Literal["before", binary_file = _to_binary(model_json, compressed_binary) - model_deserialized = _deserialize_binary_file(binary_file) original_model = model original_model.meta.creation_date = "" - model_deserialized.meta.creation_date = "" + from verify_helper import verify_json if verify_moment == "before": verify_json( @@ -196,6 +195,8 @@ def verify_model_serialization(model: GeoModel, verify_moment: Literal["before", name=file_name ) elif verify_moment == "after": + model_deserialized = _deserialize_binary_file(binary_file) + model_deserialized.meta.creation_date = "" verify_json( item=model_deserialized.model_dump_json(by_alias=True, indent=4), name=file_name diff --git a/test/test_modules/test_grids/test_grids_sections.py b/test/test_modules/test_grids/test_grids_sections.py index e7e4d3e53..bfb957438 100644 --- a/test/test_modules/test_grids/test_grids_sections.py +++ b/test/test_modules/test_grids/test_grids_sections.py @@ -4,6 +4,7 @@ import gempy as gp import gempy_viewer as gpv from gempy.core.data.enumerators import ExampleModel +from gempy.modules.serialization.save_load import verify_model_serialization from test.conftest import TEST_SPEED, TestSpeed @@ -20,8 +21,6 @@ def test_section_grids(): evaluation_options=geo_model.interpolation_options.evaluation_options ) - model = geo_model - model_json = model.model_dump_json(by_alias=True, indent=4) gp.set_section_grid( grid=geo_model.grid, section_dict={ @@ -30,7 +29,6 @@ def test_section_grids(): } ) - model_json = model.model_dump_json(by_alias=True, indent=4) gp.set_topography_from_random( grid=geo_model.grid, fractal_dimension=1.2, @@ -38,7 +36,12 @@ def test_section_grids(): topography_resolution=np.array([60, 60]) ) - model_json = model.model_dump_json(by_alias=True, indent=4) + verify_model_serialization( + model=geo_model, + verify_moment="after", + file_name=f"verify/{geo_model.meta.name}" + ) + return gp.compute_model(geo_model) gpv.plot_2d( model=geo_model, From 9fb075e686b1c8677534e92dbdefffa5762ae7a8 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 16:27:01 +0100 Subject: [PATCH 11/17] [CLN] Refactor grids into its own modules --- gempy/core/data/grid_modules/__init__.py | 4 +- gempy/core/data/grid_modules/custom_grid.py | 33 ++++ .../{grid_types.py => regular_grid.py} | 142 +----------------- gempy/core/data/grid_modules/sections_grid.py | 115 ++++++++++++++ gempy/core/data/grid_modules/topography.py | 2 +- 5 files changed, 153 insertions(+), 143 deletions(-) create mode 100644 gempy/core/data/grid_modules/custom_grid.py rename gempy/core/data/grid_modules/{grid_types.py => regular_grid.py} (68%) create mode 100644 gempy/core/data/grid_modules/sections_grid.py diff --git a/gempy/core/data/grid_modules/__init__.py b/gempy/core/data/grid_modules/__init__.py index 25ad81ca5..6ae265d8a 100644 --- a/gempy/core/data/grid_modules/__init__.py +++ b/gempy/core/data/grid_modules/__init__.py @@ -1,2 +1,4 @@ -from .grid_types import Sections, RegularGrid, CustomGrid +from .regular_grid import RegularGrid +from .custom_grid import CustomGrid +from .sections_grid import Sections from .topography import Topography diff --git a/gempy/core/data/grid_modules/custom_grid.py b/gempy/core/data/grid_modules/custom_grid.py new file mode 100644 index 000000000..94e247868 --- /dev/null +++ b/gempy/core/data/grid_modules/custom_grid.py @@ -0,0 +1,33 @@ +import dataclasses +import numpy as np +from pydantic import Field + + +@dataclasses.dataclass +class CustomGrid: + """Object that contains arbitrary XYZ coordinates. + + Args: + xyx_coords (numpy.ndarray like): XYZ (in columns) of the desired coordinates + + Attributes: + values (np.ndarray): XYZ coordinates + """ + + values: np.ndarray = Field( + exclude=True, + default_factory=lambda: np.zeros((0, 3)), + repr=False + ) + + + def __post_init__(self): + custom_grid = np.atleast_2d(self.values) + assert type(custom_grid) is np.ndarray and custom_grid.shape[1] == 3, \ + 'The shape of new grid must be (n,3) where n is the number of' \ + ' points of the grid' + + + @property + def length(self): + return self.values.shape[0] diff --git a/gempy/core/data/grid_modules/grid_types.py b/gempy/core/data/grid_modules/regular_grid.py similarity index 68% rename from gempy/core/data/grid_modules/grid_types.py rename to gempy/core/data/grid_modules/regular_grid.py index 5fd2a8ad1..07e931018 100644 --- a/gempy/core/data/grid_modules/grid_types.py +++ b/gempy/core/data/grid_modules/regular_grid.py @@ -5,11 +5,9 @@ import numpy as np -from ..core_utils import calculate_line_coordinates_2points from ..encoders.converters import numpy_array_short_validator from .... import optional_dependencies -from ....optional_dependencies import require_pandas -from gempy_engine.core.data.transforms import Transform, TransformOpsOrder +from gempy_engine.core.data.transforms import Transform @dataclasses.dataclass @@ -255,141 +253,3 @@ def plot_rotation(regular_grid, pivot, point_x_axis, point_y_axis): plt.show() -@dataclasses.dataclass -class Sections: - """ - Object that creates a grid of cross sections between two points. - - Args: - regular_grid: Model.grid.regular_grid - section_dict: {'section name': ([p1_x, p1_y], [p2_x, p2_y], [xyres, zres])} - """ - - def __init__(self, regular_grid=None, z_ext=None, section_dict=None): - pd = require_pandas() - if regular_grid is not None: - self.z_ext = regular_grid.extent[4:] - else: - self.z_ext = z_ext - - self.section_dict = section_dict - self.names = [] - self.points = [] - self.resolution = [] - self.length = [0] - self.dist = [] - self.df = pd.DataFrame() - self.df['dist'] = self.dist - self.values = np.empty((0, 3)) - self.extent = None - - if section_dict is not None: - self.set_sections(section_dict) - - def _repr_html_(self): - return self.df.to_html() - - def __repr__(self): - return self.df.to_string() - - def show(self): - pass - - def set_sections(self, section_dict, regular_grid=None, z_ext=None): - pd = require_pandas() - self.section_dict = section_dict - if regular_grid is not None: - self.z_ext = regular_grid.extent[4:] - - self.names = np.array(list(self.section_dict.keys())) - - self.get_section_params() - self.calculate_all_distances() - self.df = pd.DataFrame.from_dict(self.section_dict, orient='index', columns=['start', 'stop', 'resolution']) - self.df['dist'] = self.dist - - self.compute_section_coordinates() - - def get_section_params(self): - self.points = [] - self.resolution = [] - self.length = [0] - - for i, section in enumerate(self.names): - points = [self.section_dict[section][0], self.section_dict[section][1]] - assert points[0] != points[ - 1], 'The start and end points of the section must not be identical.' - - self.points.append(points) - self.resolution.append(self.section_dict[section][2]) - self.length = np.append(self.length, self.section_dict[section][2][0] * - self.section_dict[section][2][1]) - self.length = np.array(self.length).cumsum() - - def calculate_all_distances(self): - self.coordinates = np.array(self.points).ravel().reshape(-1, - 4) # axis are x1,y1,x2,y2 - self.dist = np.sqrt(np.diff(self.coordinates[:, [0, 2]]) ** 2 + np.diff( - self.coordinates[:, [1, 3]]) ** 2) - - def compute_section_coordinates(self): - for i in range(len(self.names)): - xy = calculate_line_coordinates_2points(self.coordinates[i, :2], - self.coordinates[i, 2:], - self.resolution[i][0]) - zaxis = np.linspace(self.z_ext[0], self.z_ext[1], self.resolution[i][1], - dtype="float64") - X, Z = np.meshgrid(xy[:, 0], zaxis, indexing='ij') - Y, _ = np.meshgrid(xy[:, 1], zaxis, indexing='ij') - xyz = np.vstack((X.flatten(), Y.flatten(), Z.flatten())).T - if i == 0: - self.values = xyz - else: - self.values = np.vstack((self.values, xyz)) - - def generate_axis_coord(self): - for i, name in enumerate(self.names): - xy = calculate_line_coordinates_2points( - self.coordinates[i, :2], - self.coordinates[i, 2:], - self.resolution[i][0] - ) - yield name, xy - - def get_section_args(self, section_name: str): - where = np.where(self.names == section_name)[0][0] - return self.length[where], self.length[where + 1] - - def get_section_grid(self, section_name: str): - l0, l1 = self.get_section_args(section_name) - return self.values[l0:l1] - - -@dataclasses.dataclass -class CustomGrid: - """Object that contains arbitrary XYZ coordinates. - - Args: - xyx_coords (numpy.ndarray like): XYZ (in columns) of the desired coordinates - - Attributes: - values (np.ndarray): XYZ coordinates - """ - - values: np.ndarray = Field( - exclude=True, - default_factory=lambda: np.zeros((0, 3)), - repr=False - ) - - - def __post_init__(self): - custom_grid = np.atleast_2d(self.values) - assert type(custom_grid) is np.ndarray and custom_grid.shape[1] == 3, \ - 'The shape of new grid must be (n,3) where n is the number of' \ - ' points of the grid' - - - @property - def length(self): - return self.values.shape[0] diff --git a/gempy/core/data/grid_modules/sections_grid.py b/gempy/core/data/grid_modules/sections_grid.py new file mode 100644 index 000000000..4e838d8cd --- /dev/null +++ b/gempy/core/data/grid_modules/sections_grid.py @@ -0,0 +1,115 @@ +import dataclasses +import numpy as np + +from gempy.core.data.core_utils import calculate_line_coordinates_2points +from gempy.optional_dependencies import require_pandas + + +@dataclasses.dataclass +class Sections: + """ + Object that creates a grid of cross sections between two points. + + Args: + regular_grid: Model.grid.regular_grid + section_dict: {'section name': ([p1_x, p1_y], [p2_x, p2_y], [xyres, zres])} + """ + + def __init__(self, regular_grid=None, z_ext=None, section_dict=None): + pd = require_pandas() + if regular_grid is not None: + self.z_ext = regular_grid.extent[4:] + else: + self.z_ext = z_ext + + self.section_dict = section_dict + self.names = [] + self.points = [] + self.resolution = [] + self.length = [0] + self.dist = [] + self.df = pd.DataFrame() + self.df['dist'] = self.dist + self.values = np.empty((0, 3)) + self.extent = None + + if section_dict is not None: + self.set_sections(section_dict) + + def _repr_html_(self): + return self.df.to_html() + + def __repr__(self): + return self.df.to_string() + + def show(self): + pass + + def set_sections(self, section_dict, regular_grid=None, z_ext=None): + pd = require_pandas() + self.section_dict = section_dict + if regular_grid is not None: + self.z_ext = regular_grid.extent[4:] + + self.names = np.array(list(self.section_dict.keys())) + + self.get_section_params() + self.calculate_all_distances() + self.df = pd.DataFrame.from_dict(self.section_dict, orient='index', columns=['start', 'stop', 'resolution']) + self.df['dist'] = self.dist + + self.compute_section_coordinates() + + def get_section_params(self): + self.points = [] + self.resolution = [] + self.length = [0] + + for i, section in enumerate(self.names): + points = [self.section_dict[section][0], self.section_dict[section][1]] + assert points[0] != points[ + 1], 'The start and end points of the section must not be identical.' + + self.points.append(points) + self.resolution.append(self.section_dict[section][2]) + self.length = np.append(self.length, self.section_dict[section][2][0] * + self.section_dict[section][2][1]) + self.length = np.array(self.length).cumsum() + + def calculate_all_distances(self): + self.coordinates = np.array(self.points).ravel().reshape(-1, + 4) # axis are x1,y1,x2,y2 + self.dist = np.sqrt(np.diff(self.coordinates[:, [0, 2]]) ** 2 + np.diff( + self.coordinates[:, [1, 3]]) ** 2) + + def compute_section_coordinates(self): + for i in range(len(self.names)): + xy = calculate_line_coordinates_2points(self.coordinates[i, :2], + self.coordinates[i, 2:], + self.resolution[i][0]) + zaxis = np.linspace(self.z_ext[0], self.z_ext[1], self.resolution[i][1], + dtype="float64") + X, Z = np.meshgrid(xy[:, 0], zaxis, indexing='ij') + Y, _ = np.meshgrid(xy[:, 1], zaxis, indexing='ij') + xyz = np.vstack((X.flatten(), Y.flatten(), Z.flatten())).T + if i == 0: + self.values = xyz + else: + self.values = np.vstack((self.values, xyz)) + + def generate_axis_coord(self): + for i, name in enumerate(self.names): + xy = calculate_line_coordinates_2points( + self.coordinates[i, :2], + self.coordinates[i, 2:], + self.resolution[i][0] + ) + yield name, xy + + def get_section_args(self, section_name: str): + where = np.where(self.names == section_name)[0][0] + return self.length[where], self.length[where + 1] + + def get_section_grid(self, section_name: str): + l0, l1 = self.get_section_args(section_name) + return self.values[l0:l1] diff --git a/gempy/core/data/grid_modules/topography.py b/gempy/core/data/grid_modules/topography.py index 1ec26bbe8..94428d88b 100644 --- a/gempy/core/data/grid_modules/topography.py +++ b/gempy/core/data/grid_modules/topography.py @@ -6,7 +6,7 @@ import numpy as np -from .grid_types import RegularGrid +from .regular_grid import RegularGrid from ....modules.grids.create_topography import _LoadDEMArtificial from ....optional_dependencies import require_skimage From 790924b06a74dc238633c6a4f3f1571ae7cb8dfb Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 16:51:31 +0100 Subject: [PATCH 12/17] [WIP] Converting sections grid into a proper data class --- gempy/API/grid_API.py | 11 +- gempy/core/data/geo_model.py | 5 +- gempy/core/data/grid_modules/sections_grid.py | 119 +++++++++++++----- 3 files changed, 100 insertions(+), 35 deletions(-) diff --git a/gempy/API/grid_API.py b/gempy/API/grid_API.py index fd0104733..e8bbdbfd3 100644 --- a/gempy/API/grid_API.py +++ b/gempy/API/grid_API.py @@ -11,7 +11,10 @@ def set_section_grid(grid: Grid, section_dict: dict): if grid.sections is None: - grid.sections = Sections(regular_grid=grid.regular_grid, section_dict=section_dict) + grid.sections = Sections( + z_ext=grid.regular_grid.extent[4:], + section_dict=section_dict + ) else: grid.sections.set_sections(section_dict, regular_grid=grid.regular_grid) @@ -54,7 +57,7 @@ def set_topography_from_random(grid: Grid, fractal_dimension: float = 2.0, d_z: dz=d_z, fractal_dimension=fractal_dimension ) - + grid.topography = Topography( regular_grid=grid.regular_grid, values_2d=random_topography @@ -70,7 +73,7 @@ def set_topography_from_subsurface_structured_grid(grid: Grid, struct: "subsurfa return grid.topography -def set_topography_from_arrays(grid: Grid, xyz_vertices: np.ndarray): +def set_topography_from_arrays(grid: Grid, xyz_vertices: np.ndarray): grid.topography = Topography.from_unstructured_mesh(grid.regular_grid, xyz_vertices) set_active_grid(grid, [Grid.GridTypes.TOPOGRAPHY]) return grid.topography @@ -88,7 +91,7 @@ def set_topography_from_file(grid: Grid, filepath: str, crop_to_extent: Union[Se def set_custom_grid(grid: Grid, xyz_coord: np.ndarray): custom_grid = CustomGrid(values=xyz_coord) grid.custom_grid = custom_grid - + set_active_grid(grid, [Grid.GridTypes.CUSTOM]) return grid.custom_grid diff --git a/gempy/core/data/geo_model.py b/gempy/core/data/geo_model.py index 66d35d698..fde11585e 100644 --- a/gempy/core/data/geo_model.py +++ b/gempy/core/data/geo_model.py @@ -24,6 +24,8 @@ from .surface_points import SurfacePointsTable from ...modules.data_manipulation.engine_factory import interpolation_input_from_structural_frame +import pandas as pd + """ TODO: - [ ] StructuralFrame will all input points chunked on Elements. Here I will need a property to put all @@ -296,7 +298,8 @@ def add_surface_points(self, X: Sequence[float], Y: Sequence[float], Z: Sequence arbitrary_types_allowed=True, use_enum_values=False, json_encoders={ - np.ndarray: encode_numpy_array + np.ndarray: encode_numpy_array, + pd.DataFrame: lambda df: df.to_dict(orient="list"), } ) diff --git a/gempy/core/data/grid_modules/sections_grid.py b/gempy/core/data/grid_modules/sections_grid.py index 4e838d8cd..a5bed74af 100644 --- a/gempy/core/data/grid_modules/sections_grid.py +++ b/gempy/core/data/grid_modules/sections_grid.py @@ -1,9 +1,27 @@ +from pydantic import Field, model_validator +from typing import Tuple, Dict, List, Optional + import dataclasses import numpy as np from gempy.core.data.core_utils import calculate_line_coordinates_2points from gempy.optional_dependencies import require_pandas +try: + import pandas as pd +except ImportError: + pandas = None + + +@dataclasses.dataclass +class SectionDefinition: + """ + A single cross‐section’s raw parameters. + """ + start: Tuple[float, float] + stop: Tuple[float, float] + resolution: Tuple[int, int] + @dataclasses.dataclass class Sections: @@ -15,26 +33,74 @@ class Sections: section_dict: {'section name': ([p1_x, p1_y], [p2_x, p2_y], [xyres, zres])} """ - def __init__(self, regular_grid=None, z_ext=None, section_dict=None): + """ + Pydantic v2 model of your original Sections class. + All computed fields are initialized with model_validator. + """ + + # user‐provided inputs + + z_ext: Tuple[float, float] + section_dict: Dict[str, tuple[list[int]]] + + # computed/internal (will be serialized too unless excluded) + names: List[str] = Field(default_factory=list) + points: List[List[Tuple[float, float]]] = Field(default_factory=list) + resolution: List[Tuple[int, int]] = Field(default_factory=list) + length: np.ndarray = Field(default_factory=lambda: np.array([0]), exclude=False) + dist: np.ndarray = Field(default_factory=lambda: np.array([]), exclude=False) + df: Optional[pd.DataFrame] = Field(default_factory=None, exclude=False) + values: np.ndarray = Field(default_factory=lambda: np.empty((0, 3)), exclude=False) + extent: Optional[np.ndarray] = None + + # def __init__(self, regular_grid=None, z_ext=None, section_dict=None): + # pd = require_pandas() + # if regular_grid is not None: + # self.z_ext = regular_grid.extent[4:] + # else: + # self.z_ext = z_ext + # + # self.section_dict = section_dict + # self.names = [] + # self.points = [] + # self.resolution = [] + # self.length = [0] + # self.dist = [] + # self.df = pd.DataFrame() + # self.df['dist'] = self.dist + # self.values = np.empty((0, 3)) + # self.extent = None + # + # if section_dict is not None: + # self.set_sections(section_dict) + def __post_init__(self): + self.initialize_computations() + + # @model_validator(mode="after") + # def init_class(self): + # self.initialize_computations() + # return self + + def initialize_computations(self): + # copy names + self.names = list(self.section_dict.keys()) + + # build points/resolution/length + self._get_section_params() + # compute distances + self._calculate_all_distances() + # re-build DataFrame pd = require_pandas() - if regular_grid is not None: - self.z_ext = regular_grid.extent[4:] - else: - self.z_ext = z_ext + df = pd.DataFrame.from_dict( + data=self.section_dict, + orient="index", + columns=["start", "stop", "resolution"], + ) + df["dist"] = self.dist + self.df = df - self.section_dict = section_dict - self.names = [] - self.points = [] - self.resolution = [] - self.length = [0] - self.dist = [] - self.df = pd.DataFrame() - self.df['dist'] = self.dist - self.values = np.empty((0, 3)) - self.extent = None - - if section_dict is not None: - self.set_sections(section_dict) + # compute the XYZ grid + self._compute_section_coordinates() def _repr_html_(self): return self.df.to_html() @@ -50,17 +116,10 @@ def set_sections(self, section_dict, regular_grid=None, z_ext=None): self.section_dict = section_dict if regular_grid is not None: self.z_ext = regular_grid.extent[4:] + + self.initialize_computations() - self.names = np.array(list(self.section_dict.keys())) - - self.get_section_params() - self.calculate_all_distances() - self.df = pd.DataFrame.from_dict(self.section_dict, orient='index', columns=['start', 'stop', 'resolution']) - self.df['dist'] = self.dist - - self.compute_section_coordinates() - - def get_section_params(self): + def _get_section_params(self): self.points = [] self.resolution = [] self.length = [0] @@ -76,13 +135,13 @@ def get_section_params(self): self.section_dict[section][2][1]) self.length = np.array(self.length).cumsum() - def calculate_all_distances(self): + def _calculate_all_distances(self): self.coordinates = np.array(self.points).ravel().reshape(-1, 4) # axis are x1,y1,x2,y2 self.dist = np.sqrt(np.diff(self.coordinates[:, [0, 2]]) ** 2 + np.diff( self.coordinates[:, [1, 3]]) ** 2) - def compute_section_coordinates(self): + def _compute_section_coordinates(self): for i in range(len(self.names)): xy = calculate_line_coordinates_2points(self.coordinates[i, :2], self.coordinates[i, 2:], From cfb0461c1f6a9513d0e8e7c3742f288bd4b1b527 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 17:06:01 +0100 Subject: [PATCH 13/17] [CLN] Refactor `Sections` class and imports for clarity Simplified imports and removed unused `SectionDefinition` class. Adjusted and clarified type annotations while ensuring internal fields are properly excluded from serialization. --- gempy/core/data/grid_modules/sections_grid.py | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/gempy/core/data/grid_modules/sections_grid.py b/gempy/core/data/grid_modules/sections_grid.py index a5bed74af..053ec3610 100644 --- a/gempy/core/data/grid_modules/sections_grid.py +++ b/gempy/core/data/grid_modules/sections_grid.py @@ -4,8 +4,9 @@ import dataclasses import numpy as np -from gempy.core.data.core_utils import calculate_line_coordinates_2points -from gempy.optional_dependencies import require_pandas +from ..core_utils import calculate_line_coordinates_2points +from ..encoders.converters import short_array_type +from ....optional_dependencies import require_pandas try: import pandas as pd @@ -13,16 +14,6 @@ pandas = None -@dataclasses.dataclass -class SectionDefinition: - """ - A single cross‐section’s raw parameters. - """ - start: Tuple[float, float] - stop: Tuple[float, float] - resolution: Tuple[int, int] - - @dataclasses.dataclass class Sections: """ @@ -40,18 +31,25 @@ class Sections: # user‐provided inputs - z_ext: Tuple[float, float] - section_dict: Dict[str, tuple[list[int]]] + z_ext: Tuple[float, float] | short_array_type + section_dict: Dict[ + str, + Tuple[ + Tuple[float, float], # start + Tuple[float, float], # stop + Tuple[int, int] # resolution + ] + ] # computed/internal (will be serialized too unless excluded) - names: List[str] = Field(default_factory=list) - points: List[List[Tuple[float, float]]] = Field(default_factory=list) - resolution: List[Tuple[int, int]] = Field(default_factory=list) - length: np.ndarray = Field(default_factory=lambda: np.array([0]), exclude=False) - dist: np.ndarray = Field(default_factory=lambda: np.array([]), exclude=False) - df: Optional[pd.DataFrame] = Field(default_factory=None, exclude=False) - values: np.ndarray = Field(default_factory=lambda: np.empty((0, 3)), exclude=False) - extent: Optional[np.ndarray] = None + names: List[str] = Field(default_factory=list, exclude=True) + points: List[List[Tuple[float, float]]] = Field(default_factory=list, exclude=True) + resolution: List[Tuple[int, int]] = Field(default_factory=list, exclude=True) + length: np.ndarray = Field(default_factory=lambda: np.array([0]), exclude=True) + dist: np.ndarray = Field(default_factory=lambda: np.array([]), exclude=True) + df: Optional[pd.DataFrame] = Field(default=None, exclude=True) + values: np.ndarray = Field(default_factory=lambda: np.empty((0, 3)), exclude=True) + extent: Optional[np.ndarray] = Field(default=None, exclude=True) # def __init__(self, regular_grid=None, z_ext=None, section_dict=None): # pd = require_pandas() @@ -75,12 +73,13 @@ class Sections: # self.set_sections(section_dict) def __post_init__(self): self.initialize_computations() + pass # @model_validator(mode="after") # def init_class(self): # self.initialize_computations() # return self - + # def initialize_computations(self): # copy names self.names = list(self.section_dict.keys()) From 0d10881c7eb53c85437c8ef98493aec83560c0e6 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 17:08:58 +0100 Subject: [PATCH 14/17] [CLN] Rename regular_grid to _regular_grid in Topography Refactored `regular_grid` to `_regular_grid` to indicate it as a private attribute. This change ensures better encapsulation and aligns with naming conventions. --- gempy/API/grid_API.py | 2 +- gempy/core/data/grid_modules/topography.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gempy/API/grid_API.py b/gempy/API/grid_API.py index e8bbdbfd3..53ec5680a 100644 --- a/gempy/API/grid_API.py +++ b/gempy/API/grid_API.py @@ -59,7 +59,7 @@ def set_topography_from_random(grid: Grid, fractal_dimension: float = 2.0, d_z: ) grid.topography = Topography( - regular_grid=grid.regular_grid, + _regular_grid=grid.regular_grid, values_2d=random_topography ) diff --git a/gempy/core/data/grid_modules/topography.py b/gempy/core/data/grid_modules/topography.py index 94428d88b..4c62e04e5 100644 --- a/gempy/core/data/grid_modules/topography.py +++ b/gempy/core/data/grid_modules/topography.py @@ -23,7 +23,7 @@ class Topography: This always assumes that the topography we pass fits perfectly the extent. """ - regular_grid: RegularGrid + _regular_grid: RegularGrid values_2d: np.ndarray = Field(exclude=True, default_factory=lambda: np.zeros((0, 0, 3))) source: Optional[str] = None From 44d3f1bb34fa7c1be36213648e78809303743a7b Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 17:15:36 +0100 Subject: [PATCH 15/17] [WIP/ENH] Getting the section running --- gempy/core/data/grid_modules/sections_grid.py | 34 +++---------------- .../test_grids/test_grids_sections.py | 3 +- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/gempy/core/data/grid_modules/sections_grid.py b/gempy/core/data/grid_modules/sections_grid.py index 053ec3610..012e25864 100644 --- a/gempy/core/data/grid_modules/sections_grid.py +++ b/gempy/core/data/grid_modules/sections_grid.py @@ -1,8 +1,7 @@ -from pydantic import Field, model_validator -from typing import Tuple, Dict, List, Optional - import dataclasses import numpy as np +from pydantic import Field +from typing import Tuple, Dict, List, Optional from ..core_utils import calculate_line_coordinates_2points from ..encoders.converters import short_array_type @@ -42,7 +41,7 @@ class Sections: ] # computed/internal (will be serialized too unless excluded) - names: List[str] = Field(default_factory=list, exclude=True) + names: short_array_type = Field(default=np.array([]), exclude=True) points: List[List[Tuple[float, float]]] = Field(default_factory=list, exclude=True) resolution: List[Tuple[int, int]] = Field(default_factory=list, exclude=True) length: np.ndarray = Field(default_factory=lambda: np.array([0]), exclude=True) @@ -51,38 +50,13 @@ class Sections: values: np.ndarray = Field(default_factory=lambda: np.empty((0, 3)), exclude=True) extent: Optional[np.ndarray] = Field(default=None, exclude=True) - # def __init__(self, regular_grid=None, z_ext=None, section_dict=None): - # pd = require_pandas() - # if regular_grid is not None: - # self.z_ext = regular_grid.extent[4:] - # else: - # self.z_ext = z_ext - # - # self.section_dict = section_dict - # self.names = [] - # self.points = [] - # self.resolution = [] - # self.length = [0] - # self.dist = [] - # self.df = pd.DataFrame() - # self.df['dist'] = self.dist - # self.values = np.empty((0, 3)) - # self.extent = None - # - # if section_dict is not None: - # self.set_sections(section_dict) def __post_init__(self): self.initialize_computations() pass - # @model_validator(mode="after") - # def init_class(self): - # self.initialize_computations() - # return self - # def initialize_computations(self): # copy names - self.names = list(self.section_dict.keys()) + self.names = np.array(list(self.section_dict.keys())) # build points/resolution/length self._get_section_params() diff --git a/test/test_modules/test_grids/test_grids_sections.py b/test/test_modules/test_grids/test_grids_sections.py index bfb957438..e04899842 100644 --- a/test/test_modules/test_grids/test_grids_sections.py +++ b/test/test_modules/test_grids/test_grids_sections.py @@ -41,8 +41,7 @@ def test_section_grids(): verify_moment="after", file_name=f"verify/{geo_model.meta.name}" ) - return - gp.compute_model(geo_model) + gp.compute_model(geo_model, validate_serialization=False) gpv.plot_2d( model=geo_model, section_names=['section_SW-NE', 'section_NW-SE', 'topography'], From a808559111c851324871ef32fe686d415edace76 Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 17:22:48 +0100 Subject: [PATCH 16/17] [ENH] Removed the pandas hard dependency for sections grid --- gempy/core/data/geo_model.py | 2 -- gempy/core/data/grid_modules/sections_grid.py | 9 +++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/gempy/core/data/geo_model.py b/gempy/core/data/geo_model.py index fde11585e..f481356a9 100644 --- a/gempy/core/data/geo_model.py +++ b/gempy/core/data/geo_model.py @@ -24,7 +24,6 @@ from .surface_points import SurfacePointsTable from ...modules.data_manipulation.engine_factory import interpolation_input_from_structural_frame -import pandas as pd """ TODO: @@ -299,7 +298,6 @@ def add_surface_points(self, X: Sequence[float], Y: Sequence[float], Z: Sequence use_enum_values=False, json_encoders={ np.ndarray: encode_numpy_array, - pd.DataFrame: lambda df: df.to_dict(orient="list"), } ) diff --git a/gempy/core/data/grid_modules/sections_grid.py b/gempy/core/data/grid_modules/sections_grid.py index 012e25864..1509e4afa 100644 --- a/gempy/core/data/grid_modules/sections_grid.py +++ b/gempy/core/data/grid_modules/sections_grid.py @@ -1,16 +1,13 @@ +from __future__ import annotations # Python 3.7+ only import dataclasses import numpy as np from pydantic import Field -from typing import Tuple, Dict, List, Optional +from typing import Tuple, Dict, List, Optional, Any from ..core_utils import calculate_line_coordinates_2points from ..encoders.converters import short_array_type from ....optional_dependencies import require_pandas -try: - import pandas as pd -except ImportError: - pandas = None @dataclasses.dataclass @@ -46,7 +43,7 @@ class Sections: resolution: List[Tuple[int, int]] = Field(default_factory=list, exclude=True) length: np.ndarray = Field(default_factory=lambda: np.array([0]), exclude=True) dist: np.ndarray = Field(default_factory=lambda: np.array([]), exclude=True) - df: Optional[pd.DataFrame] = Field(default=None, exclude=True) + df: Optional[Any] = Field(default=None, exclude=True) values: np.ndarray = Field(default_factory=lambda: np.empty((0, 3)), exclude=True) extent: Optional[np.ndarray] = Field(default=None, exclude=True) From c5041f4d2072d185025275b8545e11b3365546eb Mon Sep 17 00:00:00 2001 From: MigueldelaVarga Date: Wed, 21 May 2025 17:22:57 +0100 Subject: [PATCH 17/17] [TEST] Add approved file for grid sections verification Introduces a new approved test file to validate grid sections in `test_grids_sections`. This ensures consistency and accuracy in grid-related functionality during testing. --- .../fold.approved.txt | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 test/test_modules/test_grids/test_grids_sections.test_section_grids.verify/fold.approved.txt diff --git a/test/test_modules/test_grids/test_grids_sections.test_section_grids.verify/fold.approved.txt b/test/test_modules/test_grids/test_grids_sections.test_section_grids.verify/fold.approved.txt new file mode 100644 index 000000000..496ac8992 --- /dev/null +++ b/test/test_modules/test_grids/test_grids_sections.test_section_grids.verify/fold.approved.txt @@ -0,0 +1,210 @@ +{ + "meta": { + "name": "fold", + "creation_date": "", + "last_modification_date": null, + "owner": null + }, + "structural_frame": { + "structural_groups": [ + { + "name": "Strat_Series", + "elements": [ + { + "name": "rock2", + "is_active": true, + "_color": "#9f0052", + "surface_points": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 117776925 + }, + { + "name": "rock1", + "is_active": true, + "_color": "#015482", + "surface_points": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "orientations": { + "name_id_map": { + "rock1": 67239155, + "rock2": 117776925 + }, + "_model_transform": null + }, + "scalar_field_at_interface": null, + "_id": 67239155 + } + ], + "structural_relation": 1, + "fault_relations": null, + "faults_input_data": null, + "solution": null + } + ], + "is_dirty": true, + "basement_color": "#ffbe00", + "binary_meta_data": { + "sp_binary_length": 1296 + } + }, + "grid": { + "_octree_grid": { + "resolution": [ + 4, + 4, + 4 + ], + "extent": [ + 0.0, + 1000.0, + 0.0, + 1000.0, + 0.0, + 1000.0 + ], + "_transform": null + }, + "_dense_grid": null, + "_custom_grid": null, + "_topography": { + "_regular_grid": { + "resolution": [ + 4, + 4, + 4 + ], + "extent": [ + 0.0, + 1000.0, + 0.0, + 1000.0, + 0.0, + 1000.0 + ], + "_transform": null + }, + "source": null, + "values": [], + "resolution": [ + 0, + 0 + ], + "raster_shape": [], + "_mask_topo": null, + "_x": null, + "_y": null + }, + "_sections": { + "z_ext": [ + 0.0, + 1000.0 + ], + "section_dict": { + "section_SW-NE": [ + [ + 250.0, + 250.0 + ], + [ + 1750.0, + 1750.0 + ], + [ + 100, + 100 + ] + ], + "section_NW-SE": [ + [ + 250.0, + 1750.0 + ], + [ + 1750.0, + 250.0 + ], + [ + 100, + 100 + ] + ] + } + }, + "_centered_grid": null, + "_transform": null, + "_octree_levels": -1, + "active_grids": 1049 + }, + "geophysics_input": null, + "input_transform": { + "position": [ + -500.0, + -500.0, + -510.0 + ], + "rotation": [ + 0.0, + 0.0, + 0.0 + ], + "scale": [ + 0.0005, + 0.0005, + 0.0005 + ], + "_is_default_transform": false, + "_cached_pivot": null + }, + "_interpolation_options": { + "kernel_options": { + "range": 1.7, + "c_o": 10.0, + "uni_degree": 1, + "i_res": 4.0, + "gi_res": 2.0, + "number_dimensions": 3, + "kernel_function": "cubic", + "kernel_solver": 1, + "compute_condition_number": false, + "optimizing_condition_number": false, + "condition_number": null + }, + "evaluation_options": { + "_number_octree_levels": 2, + "_number_octree_levels_surface": 4, + "octree_curvature_threshold": -1.0, + "octree_error_threshold": 1.0, + "octree_min_level": 2, + "mesh_extraction": true, + "mesh_extraction_masking_options": 3, + "mesh_extraction_fancy": true, + "evaluation_chunk_size": 500000, + "compute_scalar_gradient": false, + "verbose": false + }, + "debug": true, + "cache_mode": 3, + "cache_model_name": "fold", + "block_solutions_type": 1, + "sigmoid_slope": 5000000, + "debug_water_tight": false + } +}