Pypsa e2e tests (#114)

* starting work for e2e tests with Antares-Modeler

* e2e tests

* remove series

* refacto

* refacto

* refacto

* format

* format

Author: aoustry

src/andromede/libs/pypsa_models/pypsa_models.yml

@@ -74,10 +74,10 @@ library:
         #- id : type # This parameter is not used in the Gems model
         - id: p_set
           time-dependent: true
-          scenario-dependent: true
+          scenario-dependent: false # Since PyPSA v0.x does not manage simultaneous scenarios at this point
         - id: q_set
           time-dependent: true
-          scenario-dependent: true
+          scenario-dependent: false  # Since PyPSA v0.x does not manage simultaneous scenarios at this point
         - id: sign #default value = -1
           time-dependent: false
           scenario-dependent: false
@@ -338,7 +338,7 @@ library:
           scenario-dependent: false
         - id: inflow
           time-dependent: true
-          scenario-dependent: true
+          scenario-dependent: false
       ##Additional parameters that are not stored in the PyPSA object, but in a Carrier object
         - id: emission_factor
           time_dependent: false

src/andromede/pypsa_converter/pypsa_converter.py

@@ -14,6 +14,7 @@
 from math import inf
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
 from pandas import DataFrame
 from pypsa import Network
@@ -68,6 +69,7 @@ def __init__(
         logger: logging.Logger,
         system_dir: Path,
         series_dir: Path,
+        series_file_format: str,
     ):
         """
         Initialize processor
@@ -79,13 +81,25 @@ def __init__(
         self.pypsalib_id = "pypsa_models"
         self.null_carrier_id = "null"
         self.system_name = pypsa_network.name
+        self.series_file_format = series_file_format
 
+        self.pypsa_components = [
+            "buses",
+            "loads",
+            "generators",
+            "stores",
+            "storage_units",
+            "links",
+            "lines",
+            "transformers",
+        ]
         self._pypsa_network_assertion()
         self._pypsa_network_preprocessing()
-        self._preprocess_pypsa_components("generators", "p_nom")
-        self._preprocess_pypsa_components("stores", "e_nom")
-        self._preprocess_pypsa_components("storage_units", "p_nom")
-        self._preprocess_pypsa_components("links", "p_nom")
+        self._preprocess_pypsa_components("loads", False, "/")
+        self._preprocess_pypsa_components("generators", True, "p_nom")
+        self._preprocess_pypsa_components("stores", True, "e_nom")
+        self._preprocess_pypsa_components("storage_units", True, "p_nom")
+        self._preprocess_pypsa_components("links", True, "p_nom")
 
         self.pypsa_components_data: dict[str, PyPSAComponentData] = {}
         self._register_pypsa_components()
@@ -111,6 +125,9 @@ def _pypsa_network_assertion(self) -> None:
         ### PyPSA components : Links
         if not (all((self.pypsa_network.links["active"] == 1))):
             raise ValueError(f"Converter supports only Links with active = 1")
+        ### PyPSA components : Lines
+        if not len(self.pypsa_network.lines) == 0:
+            raise ValueError(f"Converter does not support Lines yet")
         ### PyPSA components : Storage Units
         if not (all((self.pypsa_network.links["active"] == 1))):
             raise ValueError(f"Converter supports only Storage Units with active = 1")
@@ -146,6 +163,22 @@ def _pypsa_network_assertion(self) -> None:
                 == "co2_emissions"
             )
 
+    def _rename_buses(self) -> None:
+        ### Rename PyPSA buses, to delete spaces
+        if len(self.pypsa_network.buses) > 0:
+            self.pypsa_network.buses.index = self.pypsa_network.buses.index.str.replace(
+                " ", "_"
+            )
+            for _, val in self.pypsa_network.buses_t.items():
+                val.columns = val.columns.str.replace(" ", "_")
+        ### Update the 'bus' columns for the different types of PyPSA components
+        for component_type in self.pypsa_components:
+            df = getattr(self.pypsa_network, component_type)
+            if len(df) > 0:
+                for col in ["bus", "bus0", "bus1"]:
+                    if col in df.columns:
+                        df[col] = df[col].str.replace(" ", "_")
+
     def _pypsa_network_preprocessing(self) -> None:
         ###Add fictitious carrier
         self.pypsa_network.add(
@@ -157,32 +190,31 @@ def _pypsa_network_preprocessing(self) -> None:
         self.pypsa_network.carriers[
             "carrier"
         ] = self.pypsa_network.carriers.index.values
-        ### Rename PyPSA components, to make sure that the names are uniques (used as id in the Gems model)
-        self.pypsa_network.loads.index = (
-            self.pypsa_network.loads.index.astype(str) + "_load"
-        )
-        for key, val in self.pypsa_network.loads_t.items():
-            val.columns = val.columns + "_load"
-
-        self.pypsa_network.generators.index = (
-            self.pypsa_network.generators.index.astype(str) + "_generator"
-        )
-        for key, val in self.pypsa_network.generators_t.items():
-            val.columns = val.columns + "_generator"
+        self._rename_buses()
 
-        self.pypsa_network.storage_units.index = (
-            self.pypsa_network.storage_units.index.astype(str) + "_storage"
-        )
-        for key, val in self.pypsa_network.storage_units_t.items():
-            val.columns = val.columns + "_storage"
+    def _rename_pypsa_components(self, component_type: str) -> None:
+        df = getattr(self.pypsa_network, component_type)
+        if len(df) == 0:
+            return
+        ### Rename PyPSA components, to make sure that the names are uniques (used as id in the Gems model)
+        prefix = component_type[:-1]
+        df.index = prefix + "_" + df.index.str.replace(" ", "_")
+        dictionnary = getattr(self.pypsa_network, component_type + "_t")
+        for _, val in dictionnary.items():
+            val.columns = prefix + "_" + val.columns.str.replace(" ", "_")
 
-        self.pypsa_network.stores.index = (
-            self.pypsa_network.stores.index.astype(str) + "_store"
-        )
-        for key, val in self.pypsa_network.stores_t.items():
-            val.columns = val.columns + "_store"
+    def _fix_capacities(self, component_type: str, capa_str: str) -> None:
+        df = getattr(self.pypsa_network, component_type)
+        if len(df) == 0:
+            return
+        ### Adding min and max capacities to non-extendable objects
+        for field in [capa_str + "_min", capa_str + "_max"]:
+            df.loc[df[capa_str + "_extendable"] == False, field] = df[capa_str]
+            df.loc[df[capa_str + "_extendable"] == False, "capital_cost"] = 0.0
 
-    def _preprocess_pypsa_components(self, component_type: str, capa_str: str) -> None:
+    def _preprocess_pypsa_components(
+        self, component_type: str, extendable: bool, capa_str: str
+    ) -> None:
         ### Handling PyPSA objects without carriers
         df = getattr(self.pypsa_network, component_type)
         for comp in df.index:
@@ -198,11 +230,9 @@ def _preprocess_pypsa_components(self, component_type: str, capa_str: str) -> No
                 rsuffix="_carrier",
             ),
         )
-        df = getattr(self.pypsa_network, component_type)
-        ### Adding min and max capacities to non-extendable objects
-        for field in [capa_str + "_min", capa_str + "_max"]:
-            df.loc[df[capa_str + "_extendable"] == False, field] = df[capa_str]
-        df.loc[df[capa_str + "_extendable"] == False, "capital_cost"] = 0.0
+        self._rename_pypsa_components(component_type)
+        if extendable:
+            self._fix_capacities(component_type, capa_str)
 
     def _register_pypsa_components(self) -> None:
         ### PyPSA components : Generators
@@ -507,7 +537,7 @@ def _write_and_register_timeseries(
                 timeseries_name = self.system_name + "_" + component + "_" + param
                 comp_param_to_timeseries_name[(component, param)] = timeseries_name
                 param_df[[component]].to_csv(
-                    self.series_dir / Path(timeseries_name + ".txt"),
+                    self.series_dir / Path(timeseries_name + self.series_file_format),
                     index=False,
                     header=False,
                 )

src/andromede/pypsa_converter/utils.py

@@ -16,7 +16,7 @@
 import yaml
 from pydantic import BaseModel
 
-PYPSA_CONVERTER_MAX_FLOAT = 1e20
+PYPSA_CONVERTER_MAX_FLOAT = 100000000000
 
 
 def any_to_float(el: Any) -> float: