Merge pull request #84 from MET-OM/add_norkyst_v3

Add norkyst v3

Author: KonstantinChri

.github/workflows/tests.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu", "windows"]
-        python-version: [ "3.11", "3.12" ]
+        python-version: [ "3.12" ]
     steps:
       - uses: actions/checkout@v4
 

metocean_api/ts/internal/metno/met_products.py

@@ -42,6 +42,8 @@ def find_product(name: str) -> Product:
             return NORA3Atm3hrSub(name)
         case "NORKYST800":
             return Norkyst800(name)
+        case "NORKYST800_V3":
+            return Norkyst800_v3(name)
         case "NorkystDA_surface":
             return NorkystDASurface(name)
         case "NorkystDA_zdepth":
@@ -509,6 +511,93 @@ def _flatten_data_structure(self, ds: xr.Dataset, **flatten_dims):
         return super()._flatten_data_structure(ds, **flatten_dims)
 
 
+
+class Norkyst800_v3(MetProduct):
+
+    @property
+    def convention(self) -> Convention:
+        return Convention.OCEANIC
+
+    
+    def get_default_variables(self):
+        return ["salinity", "temperature", "u_eastward", "v_northward", "zeta"]
+
+    def get_dates(self, start_date, end_date):
+        return pd.date_range(start=start_date, end=end_date, freq="D")
+
+    def _get_url_info(self, date: str):
+        if ((date >= pd.Timestamp("2012-01-05 00:00:00")) & (date < pd.Timestamp("2026-01-01 00:00:00"))):
+            print("https://thredds.met.no/thredds/dodsC/romshindcast/norkyst_v3/zdepth/"+date.strftime("%Y")+"/"+date.strftime("%m")+"/norkyst800-"+date.strftime("%Y%m%d")+".nc")
+            return "https://thredds.met.no/thredds/dodsC/romshindcast/norkyst_v3/zdepth/"+date.strftime("%Y")+"/"+date.strftime("%m")+"/norkyst800-"+date.strftime("%Y%m%d")+".nc"
+        else:
+            raise ValueError(f"Unhandled date {str(date)} for product {self.name}. Data only valid from 2012-01-05 onwards.")
+
+    def _get_near_coord(self, url: str, lon: float, lat: float):
+        with xr.open_dataset(url) as ds:
+            x, y = aux_funcs.find_nearest_cart_coord(ds.lon, ds.lat, lon, lat)
+            lon_near = ds.lon.sel(Y=y, X=x).values[0][0]
+            lat_near = ds.lat.sel(Y=y, X=x).values[0][0]
+            return {"X": x, "Y": y}, lon_near, lat_near
+
+    def import_data(self, ts: TimeSeries, save_csv=True, save_nc=False, use_cache=False):
+        """
+        Extract times series of  the nearest gird point (lon,lat) from
+        Norkyst800_v3.
+        """
+        if ts.variable == [] or ts.variable is None:
+            ts.variable = self.get_default_variables()
+        ts.variable.append("lon")  # keep info of regular lon
+        ts.variable.append("lat")  # keep info of regular lat
+        dates = self.get_dates(start_date=ts.start_time, end_date=ts.end_time)
+        print('dates done')
+        tempfiles = aux_funcs.get_tempfiles(ts.product, ts.lon, ts.lat, dates)
+        print('tempfiles done')
+        selection = None
+        lon_near = None
+        lat_near = None
+
+        # extract point and create temp files
+        any_sucessful_download = False
+        failures = []
+        for i in range(len(dates)):
+            url = self._get_url_info(dates[i])
+            print(url)
+            print(f"Downloading {url}")
+
+            selection, lon_near, lat_near = self._get_near_coord(url, ts.lon, ts.lat)
+            
+            if use_cache and os.path.exists(tempfiles[i]):
+                print(f"Found cached file {tempfiles[i]}. Using this instead")
+            else:
+                try:
+                    dataset = xr.open_dataset(url)
+                except Exception as e:
+                    print(f"Could not open {url} due to {e}")
+                    failures.append(i)
+                    if (i>10) and not any_sucessful_download:
+                        raise ValueError("Unable to access any files. This typically means the server is down.")
+                    continue
+                # Reduce to the wanted variables and coordinates
+                dataset = dataset[ts.variable].sel(selection).squeeze(drop=True)
+                dataset.to_netcdf(tempfiles[i])
+                any_sucessful_download = True
+                    
+        ts.lat_data = lat_near
+        ts.lon_data = lon_near
+
+        tempfiles = [tempfiles[i] for i in range(len(dates)) if i not in failures]
+        return self._combine_temporary_files(ts, save_csv, save_nc, use_cache, tempfiles, lon_near, lat_near, height=ts.height)
+
+    def _flatten_data_structure(self, ds: xr.Dataset, **flatten_dims):
+        if "zeta" in ds.variables:
+            # Just use the surface value
+            if "depth" in ds["zeta"].dims:
+                ds["zeta"] = ds.zeta.sel(depth=0)
+
+        return super()._flatten_data_structure(ds, **flatten_dims)
+
+
+
 class NorkystDASurface(MetProduct):
 
     @property

tests/test_extract_data.py

@@ -165,6 +165,15 @@ def test_norkyst_800():
     assert df_ts.data.shape == (48, 65)
     __compare_loaded_data(df_ts)
 
+def test_norkyst_800_v3():
+    df_ts = ts.TimeSeries(lon=3.73, lat=64.60,start_time='2020-09-14', end_time='2020-09-14', product='NORKYST800_V3')
+    # Import data from thredds.met.no
+    df_ts.import_data(save_csv=SAVE_CSV,save_nc=SAVE_NC, use_cache=USE_CACHE)
+    print(df_ts.lat_data, df_ts.lon_data,df_ts.data.shape)
+    assert (df_ts.lat_data, df_ts.lon_data) == (64.5983217588851, 3.7289053730237156)
+    assert df_ts.data.shape == (24, 101)
+    __compare_loaded_data(df_ts)
+
 def test_norkyst_da_zdepth():
     # We want to collect a subset
     depth = [0.0, 500.0, 2500.00]