geocode SLC integration test

.circleci/config.yml

@@ -10,7 +10,14 @@ jobs:
       - run:
           name: "Build COMPASS docker image"
           command: |
-            docker build . -f docker/Dockerfile
+            docker build . -f docker/Dockerfile -t test_image
+      - run:
+          name: "Run integration tests"
+          command: |
+            docker run test_image bash -ci "cd ~/OPERA/COMPASS &&\
+                                            conda init bash &&\
+                                            conda activate COMPASS &&\
+                                            python -m pytest"
 
 workflows:
   build-workflow:

docker/Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get -y update &&\
     apt-get -y install curl git &&\
     adduser --disabled-password compass_user
 
-USER compass_user 
+USER compass_user
 
 COPY docker/specifile.txt /home/compass_user/
 
@@ -52,8 +52,7 @@ RUN echo "Installing OPERA COMPASS" &&\
     chmod -R o+rwx /home/compass_user/scratch &&\
     echo "conda activate COMPASS" >> /home/compass_user/.bashrc
 
-
 WORKDIR /home/compass_user/scratch
 
 #for COMPASS 0.1.1 and future releases
-ENTRYPOINT ["conda", "run", "-n", "COMPASS","s1_cslc.py"]
+#ENTRYPOINT ["python"]

docker/specifile.txt

@@ -91,6 +91,8 @@ https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h238a007_1014.tar.b
 https://conda.anaconda.org/conda-forge/linux-64/libpq-14.2-hd57d9b9_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/nss-3.78-h2350873_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.4.0-hb52868f_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/pytest-7.2.0-py39hf3d152e_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/python-3.9.13-h9a8a25e_0_cpython.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2
@@ -144,4 +146,4 @@ https://conda.anaconda.org/conda-forge/linux-64/gdal-3.4.3-py39hc691d54_0.tar.bz
 https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.9-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/isce3-0.6.0-py39h73072f7_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/requests-2.27.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/conda-4.13.0-py39hf3d152e_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/conda-4.13.0-py39hf3d152e_1.tar.bz2

requirements.txt

@@ -3,5 +3,6 @@ numpy           # sentinel1-reader requirement
 gdal>=3
 #isce3        # since the conda-installed isce3 is not the most updated version, installing isce3 from stratch is recommended, to stay in sync with isce3 development.
 #journal      # as of Mar 2022, journal from conda does not support python3.9; since it is included during isce3 installation above, comment this out temporarily.
+pytest
 ruamel.yaml
 yamale

tests/conftest.py

@@ -0,0 +1,64 @@
+import os
+import pathlib
+import pytest
+import requests
+import types
+
+from s1reader.s1_orbit import check_internet_connection
+
+def download_if_needed(local_path):
+    # check if test inputs and reference files exists; download if not found.
+    if os.path.isfile(local_path):
+        return
+
+    check_internet_connection()
+
+    dataset_url = 'https://zenodo.org/record/6954753/files/'
+    dst_dir, file_name = os.path.split(local_path)
+    print(dst_dir, file_name)
+    os.makedirs(dst_dir, exist_ok=True)
+    target_url = f'{dataset_url}/{file_name}'
+    with open(local_path, 'wb') as f:
+        f.write(requests.get(target_url).content)
+
+@pytest.fixture(scope="session")
+def test_paths():
+    test_paths = types.SimpleNamespace()
+
+    burst_id = 't71_151200_iw1'
+    b_date = '20200511'
+
+    # get test working directory
+    test_path = pathlib.Path(__file__).parent.resolve()
+
+    # set other paths relative to working directory
+    test_data_path = f'{test_path}/data'
+    out_path = f'{test_path}/gburst'
+
+    # paths for template and actual runconfig
+    gslc_template_path = f'{test_data_path}/geo_cslc_s1_template.yaml'
+    test_paths.gslc_cfg_path = f'{test_data_path}/geo_cslc_s1.yaml'
+
+    # read runconfig template, replace pieces, write to runconfig
+    with open(gslc_template_path, 'r') as f_template, \
+            open(test_paths.gslc_cfg_path, 'w') as f_cfg:
+        cfg = f_template.read().replace('@TEST_PATH@', str(test_path)).\
+            replace('@DATA_PATH@', test_data_path).\
+            replace('@BURST_ID@', burst_id)
+        f_cfg.write(cfg)
+
+    # output geocoded SLC paths
+    test_paths.test_gslc = f'{out_path}/{burst_id}/{b_date}/{burst_id}_{b_date}_VV.slc'
+
+    # reference geocoded SLC paths
+    test_paths.ref_gslc = f'{test_data_path}/reference/ref_compass_gslc.slc'
+
+    # check for files and download as needed
+    test_files = ['S1A_IW_SLC__1SSV_20200511T135117_20200511T135144_032518_03C421_7768.zip',
+                  'orbits/S1A_OPER_AUX_POEORB_OPOD_20210318T120818_V20200510T225942_20200512T005942.EOF',
+                  'test_dem.tiff', 'reference/ref_compass_gslc.hdr',
+                  'reference/ref_compass_gslc.slc']
+    for test_file in test_files:
+        download_if_needed(f'{test_data_path}/{test_file}')
+
+    return test_paths

tests/test_s1_geocode_slc.py

@@ -0,0 +1,50 @@
+import numpy as np
+import numpy.testing as npt
+from osgeo import gdal
+
+from compass import s1_geocode_slc
+from compass.utils.geo_runconfig import GeoRunConfig
+
+
+def gdal_get_arr(f):
+    '''
+    extract numpy array from GDAL supported raster
+
+    Parameters
+    ----------
+    f: str
+        path the GDAL supported raster
+
+    Returns
+    -------
+    _: np.ndarray
+        array extracted from GDAL supported raster
+    '''
+    ds = gdal.Open(f, gdal.GA_ReadOnly)
+    arr = np.array([])
+    if ds is not None:
+        arr = ds.GetRasterBand(1).ReadAsArray()
+    return arr
+
+def test_geocode_slc_run(test_paths):
+    '''
+    run s1_geocode_slc to ensure it does not crash
+    '''
+    # load yaml to cfg
+    cfg = GeoRunConfig.load_from_yaml(test_paths.gslc_cfg_path,
+                                      workflow_name='s1_cslc_geo')
+
+    # pass cfg to s1_geocode_slc
+    s1_geocode_slc.run(cfg)
+
+def test_geocode_slc_validate(test_paths):
+    '''
+    check if computed results match golden dataset
+    '''
+    # load test output
+    test_arr = gdal_get_arr(test_paths.test_gslc)
+
+    # load reference output
+    ref_arr = gdal_get_arr(test_paths.ref_gslc)
+
+    npt.assert_array_equal(test_arr, ref_arr)