From 6b02edd7ab2cc425767ad0a88aab69c3ffe8db89 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 8 Aug 2022 10:55:17 -0700 Subject: [PATCH 01/34] initial commit for geocode SLC unit test --- tests/conftest.py | 35 +++++++++++++++++++++++++++++++++ tests/test_s1_geocode_slc.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/test_s1_geocode_slc.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..4c4a01bd --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,35 @@ +import pathlib +import pytest +import types + +@pytest.fixture(scope="session") +def test_paths(): + test_paths = types.SimpleNamespace() + + b_id = 't71_151200_iw1' + b_date = '20200511' + + # get test working directory + test_path = pathlib.Path(__file__).parent.resolve() + + # set other paths relative to working directory + test_data_path = f'{test_path}/data' + out_path = f'{test_path}/gburst' + + # paths for template and actual runconfig + gslc_template_path = f'{test_data_path}/geo_cslc_s1_template.yaml' + test_paths.gslc_cfg_path = f'{test_data_path}/geo_cslc_s1.yaml' + + # read runconfig template, replace pieces, write to runconfig + with open(gslc_template_path, 'r') as f_template, \ + open(test_paths.gslc_cfg_path, 'w') as f_cfg: + cfg = f_template.read().replace('@TEST_PATH@', str(test_path)).\ + replace('@DATA_PATH@', test_data_path).\ + replace('@BURST_ID@', b_id) + f_cfg.write(cfg) + + # output and reference geocoded SLC paths + test_paths.test_gslc = f'{out_path}/{b_id}/{b_date}/{b_id}_{b_date}_VV.slc' + test_paths.ref_gslc = f'{test_path}/data/reference/ref_compass_gslc.slc' + + return test_paths diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py new file mode 100644 index 00000000..f571e3ad --- /dev/null +++ b/tests/test_s1_geocode_slc.py @@ -0,0 +1,38 @@ +import numpy as np +import numpy.testing as npt +from osgeo import gdal + +from compass import s1_geocode_slc +from compass.utils.geo_runconfig import GeoRunConfig + + +def gdal_get_arr(f): + ds = gdal.Open(f, gdal.GA_ReadOnly) + arr = np.array([]) + if ds is not None: + arr = ds.GetRasterBand(1).ReadAsArray() + return arr + +def test_geocode_slc_run(test_paths): + + # load yaml to cfg + cfg = GeoRunConfig.load_from_yaml(test_paths.gslc_cfg_path, + workflow_name='s1_cslc_geo') + + # pass cfg to s1_geocode_slc + s1_geocode_slc.run(cfg) + +def test_geocode_slc_validate(test_paths): + # load test output + test_arr = gdal_get_arr(test_paths.test_gslc) + + # load reference output + ref_arr = gdal_get_arr(test_paths.ref_gslc) + + npt.assert_array_equal(test_arr, ref_arr) + +''' +if __name__ == "__main__": + #test_geocode_slc_run() + #test_geocode_slc_validate() +''' From fd880edce0a46e72a1033fadc6b27d6f413ebfbc Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 15 Aug 2022 14:52:54 -0700 Subject: [PATCH 02/34] download test data from zenodo --- tests/conftest.py | 29 +++++++++++++++++++++++++++-- tests/test_s1_geocode_slc.py | 6 ------ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4c4a01bd..c124875c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,22 @@ +import os import pathlib import pytest +import requests import types +def download_if_needed(local_path): + # check if test inputs and reference files exists; download if not found. + if os.path.isfile(local_path): + return + + dataset_url = 'https://zenodo.org/record/6954753/files/' + dst_dir, file_name = os.path.split(local_path) + print(dst_dir, file_name) + os.makedirs(dst_dir, exist_ok=True) + target_url = f'{dataset_url}/{file_name}' + with open(local_path, 'wb') as f: + f.write(requests.get(target_url).content) + @pytest.fixture(scope="session") def test_paths(): test_paths = types.SimpleNamespace() @@ -28,8 +43,18 @@ def test_paths(): replace('@BURST_ID@', b_id) f_cfg.write(cfg) - # output and reference geocoded SLC paths + # output geocoded SLC paths test_paths.test_gslc = f'{out_path}/{b_id}/{b_date}/{b_id}_{b_date}_VV.slc' - test_paths.ref_gslc = f'{test_path}/data/reference/ref_compass_gslc.slc' + + # reference geocoded SLC paths + test_paths.ref_gslc = f'{test_data_path}/reference/ref_compass_gslc.slc' + + # check for files and download as needed + test_files = ['S1A_IW_SLC__1SSV_20200511T135117_20200511T135144_032518_03C421_7768.zip', + 'orbits/S1A_OPER_AUX_POEORB_OPOD_20210318T120818_V20200510T225942_20200512T005942.EOF', + 'test_dem.tiff', 'reference/ref_compass_gslc.hdr', + 'reference/ref_compass_gslc.slc'] + for test_file in test_files: + download_if_needed(f'{test_data_path}/{test_file}') return test_paths diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index f571e3ad..d6b5d059 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -30,9 +30,3 @@ def test_geocode_slc_validate(test_paths): ref_arr = gdal_get_arr(test_paths.ref_gslc) npt.assert_array_equal(test_arr, ref_arr) - -''' -if __name__ == "__main__": - #test_geocode_slc_run() - #test_geocode_slc_validate() -''' From 9f0b607ce58c20e2d8d2f69636720db99b897079 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 12 Sep 2022 16:44:34 -0700 Subject: [PATCH 03/34] added docstrings, more descriptive variable names, internet check --- tests/conftest.py | 10 +++++++--- tests/test_s1_geocode_slc.py | 20 +++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c124875c..fa06cca3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,11 +4,15 @@ import requests import types +from s1reader.s1_orbit import check_internet_connection + def download_if_needed(local_path): # check if test inputs and reference files exists; download if not found. if os.path.isfile(local_path): return + check_internet_connection() + dataset_url = 'https://zenodo.org/record/6954753/files/' dst_dir, file_name = os.path.split(local_path) print(dst_dir, file_name) @@ -21,7 +25,7 @@ def download_if_needed(local_path): def test_paths(): test_paths = types.SimpleNamespace() - b_id = 't71_151200_iw1' + burst_id = 't71_151200_iw1' b_date = '20200511' # get test working directory @@ -40,11 +44,11 @@ def test_paths(): open(test_paths.gslc_cfg_path, 'w') as f_cfg: cfg = f_template.read().replace('@TEST_PATH@', str(test_path)).\ replace('@DATA_PATH@', test_data_path).\ - replace('@BURST_ID@', b_id) + replace('@BURST_ID@', burst_id) f_cfg.write(cfg) # output geocoded SLC paths - test_paths.test_gslc = f'{out_path}/{b_id}/{b_date}/{b_id}_{b_date}_VV.slc' + test_paths.test_gslc = f'{out_path}/{burst_id}/{b_date}/{burst_id}_{b_date}_VV.slc' # reference geocoded SLC paths test_paths.ref_gslc = f'{test_data_path}/reference/ref_compass_gslc.slc' diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index d6b5d059..c5ee40c0 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -7,6 +7,19 @@ def gdal_get_arr(f): + ''' + extract numpy array from GDAL supported raster + + Parameters + ---------- + f: str + path the GDAL supported raster + + Returns + ------- + _: np.ndarray + array extracted from GDAL supported raster + ''' ds = gdal.Open(f, gdal.GA_ReadOnly) arr = np.array([]) if ds is not None: @@ -14,7 +27,9 @@ def gdal_get_arr(f): return arr def test_geocode_slc_run(test_paths): - + ''' + run s1_geocode_slc to ensure it does not crash + ''' # load yaml to cfg cfg = GeoRunConfig.load_from_yaml(test_paths.gslc_cfg_path, workflow_name='s1_cslc_geo') @@ -23,6 +38,9 @@ def test_geocode_slc_run(test_paths): s1_geocode_slc.run(cfg) def test_geocode_slc_validate(test_paths): + ''' + check if computed results match golden dataset + ''' # load test output test_arr = gdal_get_arr(test_paths.test_gslc) From 2984d4947c9b339a97fa1a17f5f88ccdda71d432 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Thu, 20 Oct 2022 16:14:15 -0700 Subject: [PATCH 04/34] run integration test in circleci --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca5552e6..18ba0a7a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,10 @@ jobs: name: "Build COMPASS docker image" command: | docker build . -f docker/Dockerfile + - run: + name: "Run unit tests" + command: | + docker run pytest workflows: build-workflow: From 1abbfe4d4cfa7b7272ae0d6b70731a0bc6fc51ca Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 24 Oct 2022 07:25:03 -0700 Subject: [PATCH 05/34] adding pytest as requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 074dbb4e..54f1e449 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ numpy # sentinel1-reader requirement gdal>=3 #isce3 # since the conda-installed isce3 is not the most updated version, installing isce3 from stratch is recommended, to stay in sync with isce3 development. #journal # as of Mar 2022, journal from conda does not support python3.9; since it is included during isce3 installation above, comment this out temporarily. +pytest ruamel.yaml yamale From 92daa0fc07665e8db9c7541915247cc3aff6324b Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 28 Oct 2022 15:10:24 -0700 Subject: [PATCH 06/34] adding pytest to docker specfile --- docker/specifile.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/specifile.txt b/docker/specifile.txt index 7da3a2c1..5b38d0e8 100644 --- a/docker/specifile.txt +++ b/docker/specifile.txt @@ -91,6 +91,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h238a007_1014.tar.b https://conda.anaconda.org/conda-forge/linux-64/libpq-14.2-hd57d9b9_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/nss-3.78-h2350873_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.4.0-hb52868f_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pytest-7.2.0-py39hf3d152e_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/python-3.9.13-h9a8a25e_0_cpython.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2 @@ -144,4 +145,4 @@ https://conda.anaconda.org/conda-forge/linux-64/gdal-3.4.3-py39hc691d54_0.tar.bz https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.9-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/isce3-0.6.0-py39h73072f7_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/requests-2.27.1-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/conda-4.13.0-py39hf3d152e_1.tar.bz2 \ No newline at end of file +https://conda.anaconda.org/conda-forge/linux-64/conda-4.13.0-py39hf3d152e_1.tar.bz2 From 01646524303e87232e691344abca312de0140f69 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 28 Oct 2022 15:31:30 -0700 Subject: [PATCH 07/34] another attempt to run pytest in ci --- .circleci/config.yml | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 18ba0a7a..51cf93c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: "Run unit tests" command: | - docker run pytest + docker run python -m pytest workflows: build-workflow: diff --git a/docker/Dockerfile b/docker/Dockerfile index dd3537b9..bfe14bf4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get -y update &&\ apt-get -y install curl git &&\ adduser --disabled-password compass_user -USER compass_user +USER compass_user COPY docker/specifile.txt /home/compass_user/ From ebfea347884153be74fc5815675ed7e30a824ef7 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 28 Oct 2022 15:38:31 -0700 Subject: [PATCH 08/34] yet another attempt to run pytest in ci --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 51cf93c8..293c2af7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,7 @@ jobs: - run: name: "Run unit tests" command: | + docker run conda activate COMPASS docker run python -m pytest workflows: From 13cb5b29b5e83c58d55375f8d827317d6fb9ca4c Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 2 Nov 2022 16:19:08 -0700 Subject: [PATCH 09/34] applying suggestion from rtburns --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 293c2af7..99087e3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,12 +10,11 @@ jobs: - run: name: "Build COMPASS docker image" command: | - docker build . -f docker/Dockerfile + docker build . -f docker/Dockerfile -t test_image - run: - name: "Run unit tests" + name: "Run integration tests" command: | - docker run conda activate COMPASS - docker run python -m pytest + docker run test_image bash -c "python -m pytest" workflows: build-workflow: From 7a5655cb51a47b3738fc05732eabc28cf5e9f992 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 2 Nov 2022 16:57:34 -0700 Subject: [PATCH 10/34] docker command tweak --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 99087e3d..1ef79ad8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash -c "python -m pytest" + docker run test_image bash "python -m pytest" workflows: build-workflow: From 64424020833f58fb91475a3be81948a2c7b6ccc8 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 2 Nov 2022 17:03:16 -0700 Subject: [PATCH 11/34] fix entry point --- .circleci/config.yml | 2 +- docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ef79ad8..480b404b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash "python -m pytest" + docker run test_image bash "-m pytest" workflows: build-workflow: diff --git a/docker/Dockerfile b/docker/Dockerfile index bfe14bf4..46790509 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,4 +56,4 @@ RUN echo "Installing OPERA COMPASS" &&\ WORKDIR /home/compass_user/scratch #for COMPASS 0.1.1 and future releases -ENTRYPOINT ["conda", "run", "-n", "COMPASS","s1_cslc.py"] +ENTRYPOINT ["python"] From cd70f7cfbed75f5d14713f17a03f82f341e3a657 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 4 Nov 2022 14:44:39 -0700 Subject: [PATCH 12/34] remove entry point --- .circleci/config.yml | 2 +- docker/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 480b404b..028a45ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash "-m pytest" + docker run test_image bash -c "cd $HOME/OPERA/COMPASS && python -m pytest" workflows: build-workflow: diff --git a/docker/Dockerfile b/docker/Dockerfile index 46790509..b0c3b818 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -52,8 +52,7 @@ RUN echo "Installing OPERA COMPASS" &&\ chmod -R o+rwx /home/compass_user/scratch &&\ echo "conda activate COMPASS" >> /home/compass_user/.bashrc - WORKDIR /home/compass_user/scratch #for COMPASS 0.1.1 and future releases -ENTRYPOINT ["python"] +#ENTRYPOINT ["python"] From 6e3069c8786955a8515513ffb7561f98c2801e23 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Tue, 8 Nov 2022 12:25:07 -0800 Subject: [PATCH 13/34] fix path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 028a45ae..a1b713bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash -c "cd $HOME/OPERA/COMPASS && python -m pytest" + docker run test_image bash -c "cd ~/OPERA/COMPASS && python -m pytest" workflows: build-workflow: From 8eff671a5eba09190b7258882d0c8a124af348a7 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Tue, 8 Nov 2022 13:48:30 -0800 Subject: [PATCH 14/34] activate environment --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1b713bb..013cf73a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash -c "cd ~/OPERA/COMPASS && python -m pytest" + docker run test_image bash -c "cd ~/OPERA/COMPASS && conda activate COMPASS && python -m pytest" workflows: build-workflow: From 175ab31c9553c29acb123b68d7f53ba2fabe8391 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Tue, 8 Nov 2022 14:20:12 -0800 Subject: [PATCH 15/34] init bash --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 013cf73a..04b48414 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,10 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash -c "cd ~/OPERA/COMPASS && conda activate COMPASS && python -m pytest" + docker run test_image bash -c "cd ~/OPERA/COMPASS &&\ + conda init /bin/bash &&\ + conda activate COMPASS &&\ + python -m pytest" workflows: build-workflow: From c65ace48018fb213faeef1a179497e347cbe1d51 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Tue, 8 Nov 2022 17:17:14 -0800 Subject: [PATCH 16/34] bash path not needed --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04b48414..61564ced 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: name: "Run integration tests" command: | docker run test_image bash -c "cd ~/OPERA/COMPASS &&\ - conda init /bin/bash &&\ + conda init bash &&\ conda activate COMPASS &&\ python -m pytest" From 17a70ffe1592e9195457cf906ca2535743dacc0c Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 9 Nov 2022 14:06:34 -0800 Subject: [PATCH 17/34] docker command fix --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 61564ced..d93e1318 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,10 +14,10 @@ jobs: - run: name: "Run integration tests" command: | - docker run test_image bash -c "cd ~/OPERA/COMPASS &&\ - conda init bash &&\ - conda activate COMPASS &&\ - python -m pytest" + docker run test_image bash -ci "cd ~/OPERA/COMPASS &&\ + conda init bash &&\ + conda activate COMPASS &&\ + python -m pytest" workflows: build-workflow: From dbf457babfb47878c1167559aefce833aa7b7093 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 16 Nov 2022 13:49:33 -0800 Subject: [PATCH 18/34] adding attr to specifile --- docker/specifile.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/specifile.txt b/docker/specifile.txt index 5b38d0e8..b8021959 100644 --- a/docker/specifile.txt +++ b/docker/specifile.txt @@ -92,6 +92,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libpq-14.2-hd57d9b9_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/nss-3.78-h2350873_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.4.0-hb52868f_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/pytest-7.2.0-py39hf3d152e_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/_attr-2.5.1-h166bdaf_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/python-3.9.13-h9a8a25e_0_cpython.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2 From 7e34b3f354d3db89b51849d337f9730c9f73c4bf Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 16 Nov 2022 14:09:32 -0800 Subject: [PATCH 19/34] fix typo in url --- docker/specifile.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/specifile.txt b/docker/specifile.txt index b8021959..8826fbc8 100644 --- a/docker/specifile.txt +++ b/docker/specifile.txt @@ -92,7 +92,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libpq-14.2-hd57d9b9_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/nss-3.78-h2350873_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.4.0-hb52868f_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/pytest-7.2.0-py39hf3d152e_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/_attr-2.5.1-h166bdaf_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/python-3.9.13-h9a8a25e_0_cpython.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2 From 5b4dc9118ed02280f6ae893868805a9307465c1d Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 5 Dec 2022 08:02:15 -0800 Subject: [PATCH 20/34] updated specfile --- docker/specifile.txt | 209 ++++++++++++++++++++++--------------------- 1 file changed, 107 insertions(+), 102 deletions(-) diff --git a/docker/specifile.txt b/docker/specifile.txt index 8826fbc8..3d9492b8 100644 --- a/docker/specifile.txt +++ b/docker/specifile.txt @@ -3,52 +3,54 @@ # platform: linux-64 @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.5.18.1-ha878542_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.9.24-ha878542_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.36.1-hea4e1c9_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-12.1.0-hdcd56e2_16.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-12.1.0-ha89aaad_16.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.39-hcc3a1bd_1.conda +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-12.2.0-h337968e_19.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-12.2.0-h46fd767_19.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.11-hd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tzdata-2022a-h191b570_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-3_cp311.conda +https://conda.anaconda.org/conda-forge/noarch/tzdata-2022f-h191b570_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-12.1.0-h69a702a_16.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgomp-12.1.0-h8d9b700_16.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-12.2.0-h69a702a_19.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-12.2.0-h65d4601_19.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-12.1.0-h8d9b700_16.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-12.2.0-h65d4601_19.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.18.1-h7f98852_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/expat-2.4.8-h27087fc_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-nompi_h77c792f_102.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/freexl-1.0.6-h7f98852_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/geos-3.10.2-h9c3ff4c_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-h27087fc_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/fftw-3.3.10-nompi_hf0379b8_105.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/freexl-1.0.6-h166bdaf_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/geos-3.11.1-h27087fc_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h36c2ea0_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/icu-69.1-h9c3ff4c_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/icu-70.1-h27087fc_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_2.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/lerc-3.0-h9c3ff4c_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.10-h7f98852_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.16-h516909a_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.20-pthreads_h78a6416_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.21-pthreads_h78a6416_3.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.32.1-h7f98852_1000.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.2.2-h7f98852_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.12-h166bdaf_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.2.4-h166bdaf_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-h166bdaf_4.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.3-h9c3ff4c_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h27087fc_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.32-h9c3ff4c_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/openssl-1.1.1o-h166bdaf_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.7-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/pcre-8.45-h9c3ff4c_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.9-hbd366e4_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/tzcode-2022a-h166bdaf_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.9-hbd366e4_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/tzcode-2022f-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.0.10-h7f98852_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.9-h7f98852_0.tar.bz2 @@ -56,94 +58,97 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h7f98852_0.t https://conda.anaconda.org/conda-forge/linux-64/xorg-renderproto-0.11.1-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h7f98852_1002.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.5-h516909a_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/gettext-0.19.8.1-h73d1719_1008.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-15_linux64_openblas.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_openblas.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hf69c175_9.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.47.0-hff17c54_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda +https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-ha49c73b_12.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.40.0-h753d276_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.10.0-hf14f497_3.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.13-h7f98852_1004.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/readline-8.1-h46c0cb4_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.10.3-h7463322_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libzip-1.9.2-hc929e4a_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/readline-8.1.2-h0f457ee_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.3-h8ce2273_4.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.12-h166bdaf_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h8a70e8d_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h6239696_4.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.1-h83bc5f7_3.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.74.0-h6cacc03_7.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h10796ff_3.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/krb5-1.19.3-h3790be6_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-15_linux64_openblas.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libglib-2.70.2-h174f98d_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-15_linux64_openblas.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.47.0-h727a467_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.37-h21135ba_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.10.0-ha56f1ee_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h0fcbabc_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.9.12-h885dcf4_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libzip-1.8.0-h4de3113_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.38.5-h4ff8645_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h9772cbc_5.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.19.3-h08a2579_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h55922b4_4.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0-ha86cf86_0_cpython.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.40.0-h4ff8645_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.7.2-h7f98852_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/freetype-2.10.4-h0708190_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.12-hddcbb42_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.83.1-h7bff187_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h238a007_1014.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libpq-14.2-hd57d9b9_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/attrs-22.1.0-pyh71513ae_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/certifi-2022.9.24-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.0.4-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/iniconfig-1.1.1-pyh9f0ad1d_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-h2283fc2_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libpq-14.5-he2d8382_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.1-py311hc4dbab1_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/nss-3.78-h2350873_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.4.0-hb52868f_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pytest-7.2.0-py39hf3d152e_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/python-3.9.13-h9a8a25e_0_cpython.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py311h7d28db0_0.conda +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.9-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py311hd4cff14_5.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.7-py311hd4cff14_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/backoff-2.1.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.1.0-hd9d235c_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.0.12-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.4-pyh9f0ad1d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/curl-7.83.1-h7bff187_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.0-h8e229c2_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.1-nompi_h2386368_104.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/idna-3.3-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-14.2-h2510834_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/proj-9.0.0-h93bde94_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.9-2_cp39.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/wheel-0.37.1-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha12eb4b_1010.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/certifi-2022.5.18.1-py39hf3d152e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.0-py39h4bc2ebd_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h509b78c_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/kealib-1.4.14-hfe1a663_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libdap4-3.20.6-hd7c4107_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h329d8a1_102.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-ha867d66_15.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.22.4-py39hc58783e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pycosat-0.6.3-py39hb9d737c_1010.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py39h8d5ab89_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.7.1-py39hf3d152e_5.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py39hb9d737c_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.6-py39hb9d737c_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ruamel_yaml-0.15.80-py39hb9d737c_1007.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/setuptools-62.3.2-py39hf3d152e_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.8.3-h1e4a385_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/tqdm-4.64.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py39hb9d737c_1004.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/conda-package-handling-1.8.1-py39hb9d737c_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-37.0.2-py39hd97740a_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/h5py-3.6.0-nompi_py39h7e08c79_100.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pip-22.1.2-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/poppler-22.04.0-h1434ded_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.21-py39hb9d737c_1.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.8.1-py39he49c0e8_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/shapely-1.8.2-py39h73b9895_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda +https://conda.anaconda.org/conda-forge/linux-64/curl-7.86.0-h2283fc2_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_100.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/packaging-21.3-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-14.5-ha105346_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h93bde94_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.21-py311hd4cff14_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.9.3-py311h69910c8_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/shapely-1.8.5-py311h2b60523_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.4.3-h56144a5_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.0.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.4.3-py39hc691d54_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.9-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/isce3-0.6.0-py39h73072f7_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/requests-2.27.1-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/conda-4.13.0-py39hf3d152e_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py311hd4cff14_1005.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/cryptography-38.0.3-py311h42a1071_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-ha76d385_4.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/h5py-3.7.0-nompi_py311hbe7f6d8_102.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/kealib-1.4.15-ha7026e8_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libdap4-3.20.6-hd7c4107_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h261ec11_106.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h7c8129e_22.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/poppler-22.11.0-h92391eb_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.0-py311hb0e1098_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py311hbbb8f27_3.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.0-pyhd8ed1ab_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.11.3-h3f4058f_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.5.3-hb03553a_4.conda +https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.1.0-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.5.3-py311hd39052d_4.conda +https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.11-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/isce3-0.8.0-py311hdc7d59a_2.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 From 07c83e4a987052be1f20bacc46c9a53d7d497e2f Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 5 Dec 2022 08:04:07 -0800 Subject: [PATCH 21/34] account for multiburst, new burst ID, geocode entire burst --- src/compass/utils/runconfig.py | 7 +++++-- tests/conftest.py | 6 +++--- tests/test_s1_geocode_slc.py | 8 ++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/compass/utils/runconfig.py b/src/compass/utils/runconfig.py index ed1c7e5f..4f2beffb 100644 --- a/src/compass/utils/runconfig.py +++ b/src/compass/utils/runconfig.py @@ -199,14 +199,16 @@ def runconfig_to_bursts(cfg: SimpleNamespace, auto_download: bool = False) -> li for pol, i_subswath in pol_subswath_index_pairs: # loop over burst objs extracted from SAFE zip - for burst in load_bursts(safe_file, orbit_path, i_subswath, pol): + loaded_bursts = load_bursts(safe_file, orbit_path, i_subswath, pol) + for burst in loaded_bursts: # get burst ID - burst_id = burst.burst_id + burst_id = str(burst.burst_id) # include ALL bursts if no burst IDs given # is burst_id wanted? skip if not given in config if (cfg.input_file_group.burst_id is not None and burst_id not in cfg.input_file_group.burst_id): + print(burst_id, cfg.input_file_group.burst_id, burst_id in cfg.input_file_group.burst_id) continue # get polarization and save as tuple with burst ID @@ -230,6 +232,7 @@ def runconfig_to_bursts(cfg: SimpleNamespace, auto_download: bool = False) -> li # if reference burst, ok to add if id+pol combo does not exist # no duplicate id+pol combos for reference bursts if not_ref or not burst_id_pol_exist: + print(f'{burst_id} found') burst_ids_found.append(burst_id) bursts.append(burst) diff --git a/tests/conftest.py b/tests/conftest.py index fa06cca3..23863e33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ def download_if_needed(local_path): def test_paths(): test_paths = types.SimpleNamespace() - burst_id = 't71_151200_iw1' + burst_id = 't071_151200_iw1' b_date = '20200511' # get test working directory @@ -33,7 +33,7 @@ def test_paths(): # set other paths relative to working directory test_data_path = f'{test_path}/data' - out_path = f'{test_path}/gburst' + out_path = f'{test_path}/product' # paths for template and actual runconfig gslc_template_path = f'{test_data_path}/geo_cslc_s1_template.yaml' @@ -48,7 +48,7 @@ def test_paths(): f_cfg.write(cfg) # output geocoded SLC paths - test_paths.test_gslc = f'{out_path}/{burst_id}/{b_date}/{burst_id}_{b_date}_VV.slc' + test_paths.test_gslc = f'{out_path}/{burst_id}/{b_date}/{burst_id}_VV.slc' # reference geocoded SLC paths test_paths.ref_gslc = f'{test_data_path}/reference/ref_compass_gslc.slc' diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index c5ee40c0..958afdff 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -37,6 +37,7 @@ def test_geocode_slc_run(test_paths): # pass cfg to s1_geocode_slc s1_geocode_slc.run(cfg) + def test_geocode_slc_validate(test_paths): ''' check if computed results match golden dataset @@ -44,7 +45,14 @@ def test_geocode_slc_validate(test_paths): # load test output test_arr = gdal_get_arr(test_paths.test_gslc) + # place holder test since reference dataset no longer valid + # because COMPASS now only geocodes entire bursts + ref_shape = (19470, 4495) + np.assert_array_equal(test_array.shape, ref_shape) + + ''' # load reference output ref_arr = gdal_get_arr(test_paths.ref_gslc) npt.assert_array_equal(test_arr, ref_arr) + ''' From b30f823c3591617fb014101096884b30d5b09180 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 5 Dec 2022 16:34:53 -0800 Subject: [PATCH 22/34] use main instead of tag because of missing features --- docker/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b0c3b818..53d09734 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,10 +33,7 @@ SHELL ["conda", "run", "-n", "COMPASS", "/bin/bash", "-c"] RUN echo "Installing OPERA s1-reader" &&\ mkdir -p $HOME/OPERA &&\ cd $HOME/OPERA &&\ - curl -sSL https://github.com/opera-adt/s1-reader/archive/refs/tags/v0.1.2.tar.gz -o s1_reader_src.tar.gz &&\ - tar -xvf s1_reader_src.tar.gz &&\ - ln -s s1-reader-0.1.2 s1-reader &&\ - rm s1_reader_src.tar.gz &&\ + git clone git@github.com:opera-adt/s1-reader.git &&\ python -m pip install ./s1-reader COPY . /home/compass_user/OPERA/COMPASS From 18c9cb4a73b8167d681a0980e1319b911ab19e3f Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 5 Dec 2022 21:05:41 -0800 Subject: [PATCH 23/34] clone hhtps not ssh --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 53d09734..58666c2b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ SHELL ["conda", "run", "-n", "COMPASS", "/bin/bash", "-c"] RUN echo "Installing OPERA s1-reader" &&\ mkdir -p $HOME/OPERA &&\ cd $HOME/OPERA &&\ - git clone git@github.com:opera-adt/s1-reader.git &&\ + git clone https://github.com/opera-adt/s1-reader.git &&\ python -m pip install ./s1-reader COPY . /home/compass_user/OPERA/COMPASS From 4ddf5f83a3fff03846ae84d01160d6b83c61790a Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 5 Dec 2022 21:17:21 -0800 Subject: [PATCH 24/34] template yaml --- tests/data/geo_cslc_s1_template.yaml | 43 ++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/data/geo_cslc_s1_template.yaml diff --git a/tests/data/geo_cslc_s1_template.yaml b/tests/data/geo_cslc_s1_template.yaml new file mode 100644 index 00000000..bd518e5c --- /dev/null +++ b/tests/data/geo_cslc_s1_template.yaml @@ -0,0 +1,43 @@ +runconfig: + groups: + dynamic_ancillary_file_group: + dem_file: @DATA_PATH@/test_dem.tiff + burst_database_file: @DATA_PATH@/burst_map_bbox_only.sqlite3 + input_file_group: + burst_id: + - @BURST_ID@ + orbit_file_path: + - @DATA_PATH@/orbits/S1A_OPER_AUX_POEORB_OPOD_20210318T120818_V20200510T225942_20200512T005942.EOF + safe_file_path: + - @DATA_PATH@/S1A_IW_SLC__1SSV_20200511T135117_20200511T135144_032518_03C421_7768.zip + pge_name_group: + pge_name: CSLC_S1_PGE + primary_executable: + product_type: CSLC_S1 + processing: + rdr2geo: + enabled: false + geo2rdr: + lines_per_block: 1000 + numiter: 25 + threshold: 1.0e-08 + geocoding: + flatten: true + lines_per_block: 1000 + output_format: GTiff + x_posting: 5 + x_snap: null + y_posting: 10 + y_snap: null + polarization: co-pol # dual-pol + range_split_spectrum: + enabled: False + product_path_group: + product_path: @TEST_PATH@/product + sas_output_file: @TEST_PATH@/product + scratch_path: @TEST_PATH@/scratch + worker: + gpu_enabled: false + gpu_id: 0 + internet_access: false + name: cslc_s1_workflow_default From 8e101006f6d8a5c97ed54cff10e863ac7c7e16dc Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Tue, 21 Feb 2023 08:37:44 -0800 Subject: [PATCH 25/34] update unit tests for smaller dataset --- tests/conftest.py | 11 ++++++----- tests/data/geo_cslc_s1_template.yaml | 10 +++++----- tests/test_s1_geocode_slc.py | 15 +-------------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b0ff9dbc..2424d034 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ def download_if_needed(local_path): def test_paths(): test_paths = types.SimpleNamespace() - burst_id = 't071_151200_iw1' + burst_id = 't064_135523_iw2' b_date = '20200511' # get test working directory @@ -55,12 +55,13 @@ def test_paths(): test_paths.ref_gslc = f'{test_data_path}/reference/ref_compass_gslc.slc' # check for files and download as needed - test_files = ['S1A_IW_SLC__1SSV_20200511T135117_20200511T135144_032518_03C421_7768.zip', - 'orbits/S1A_OPER_AUX_POEORB_OPOD_20210318T120818_V20200510T225942_20200512T005942.EOF', - 'test_dem.tiff', 'reference/ref_compass_gslc.hdr', - 'reference/ref_compass_gslc.slc'] + test_files = ['S1A_IW_SLC__1SDV_20221016T015043_20221016T015111_045461_056FC0_6681.zip', + 'orbits/S1A_OPER_AUX_POEORB_OPOD_20221105T083813_V20221015T225942_20221017T005942.EOF', + 'test_dem.tiff'] + ''' for test_file in test_files: download_if_needed(f'{test_data_path}/{test_file}') + ''' return test_paths diff --git a/tests/data/geo_cslc_s1_template.yaml b/tests/data/geo_cslc_s1_template.yaml index bd518e5c..c07024e5 100644 --- a/tests/data/geo_cslc_s1_template.yaml +++ b/tests/data/geo_cslc_s1_template.yaml @@ -2,14 +2,15 @@ runconfig: groups: dynamic_ancillary_file_group: dem_file: @DATA_PATH@/test_dem.tiff - burst_database_file: @DATA_PATH@/burst_map_bbox_only.sqlite3 + static_ancillary_file_group: + burst_database_file: @DATA_PATH@/test_burst_map.sqlite3 input_file_group: burst_id: - @BURST_ID@ orbit_file_path: - - @DATA_PATH@/orbits/S1A_OPER_AUX_POEORB_OPOD_20210318T120818_V20200510T225942_20200512T005942.EOF + - @DATA_PATH@/orbits/S1A_OPER_AUX_POEORB_OPOD_20221105T083813_V20221015T225942_20221017T005942.EOF safe_file_path: - - @DATA_PATH@/S1A_IW_SLC__1SSV_20200511T135117_20200511T135144_032518_03C421_7768.zip + - @DATA_PATH@/S1A_IW_SLC__1SDV_20221016T015043_20221016T015111_045461_056FC0_6681.zip pge_name_group: pge_name: CSLC_S1_PGE primary_executable: @@ -24,12 +25,11 @@ runconfig: geocoding: flatten: true lines_per_block: 1000 - output_format: GTiff x_posting: 5 x_snap: null y_posting: 10 y_snap: null - polarization: co-pol # dual-pol + polarization: co-pol range_split_spectrum: enabled: False product_path_group: diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index 958afdff..0c6aae80 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -42,17 +42,4 @@ def test_geocode_slc_validate(test_paths): ''' check if computed results match golden dataset ''' - # load test output - test_arr = gdal_get_arr(test_paths.test_gslc) - - # place holder test since reference dataset no longer valid - # because COMPASS now only geocodes entire bursts - ref_shape = (19470, 4495) - np.assert_array_equal(test_array.shape, ref_shape) - - ''' - # load reference output - ref_arr = gdal_get_arr(test_paths.ref_gslc) - - npt.assert_array_equal(test_arr, ref_arr) - ''' + pass From 88f0487c1b109f2c28df77c7123ad0f6fb5075f8 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 22 Feb 2023 19:19:41 -0800 Subject: [PATCH 26/34] parallel test data download update test data URL update validation to check for presense of reflectors --- tests/conftest.py | 41 +++++++++--------- tests/test_s1_geocode_slc.py | 83 +++++++++++++++++++++++++----------- 2 files changed, 79 insertions(+), 45 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2424d034..a51b657b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,15 @@ +from functools import partial +import multiprocessing as mp import os import pathlib -import requests import types -from compass.utils import iono import pytest +import requests from s1reader.s1_orbit import check_internet_connection +from compass.utils import iono + def download_if_needed(local_path): # check if test inputs and reference files exists; download if not found. if os.path.isfile(local_path): @@ -14,27 +17,26 @@ def download_if_needed(local_path): check_internet_connection() - dataset_url = 'https://zenodo.org/record/6954753/files/' + dataset_url = 'https://zenodo.org/record/7668410/files/' dst_dir, file_name = os.path.split(local_path) - print(dst_dir, file_name) - os.makedirs(dst_dir, exist_ok=True) + if dst_dir: + os.makedirs(dst_dir, exist_ok=True) target_url = f'{dataset_url}/{file_name}' with open(local_path, 'wb') as f: f.write(requests.get(target_url).content) @pytest.fixture(scope="session") -def test_paths(): +def unit_test_paths(): test_paths = types.SimpleNamespace() burst_id = 't064_135523_iw2' - b_date = '20200511' + b_date = '20221016' # get test working directory test_path = pathlib.Path(__file__).parent.resolve() # set other paths relative to working directory test_data_path = f'{test_path}/data' - out_path = f'{test_path}/product' # paths for template and actual runconfig gslc_template_path = f'{test_data_path}/geo_cslc_s1_template.yaml' @@ -48,20 +50,21 @@ def test_paths(): replace('@BURST_ID@', burst_id) f_cfg.write(cfg) - # output geocoded SLC paths - test_paths.test_gslc = f'{out_path}/{burst_id}/{b_date}/{burst_id}_VV.slc' - - # reference geocoded SLC paths - test_paths.ref_gslc = f'{test_data_path}/reference/ref_compass_gslc.slc' - # check for files and download as needed test_files = ['S1A_IW_SLC__1SDV_20221016T015043_20221016T015111_045461_056FC0_6681.zip', 'orbits/S1A_OPER_AUX_POEORB_OPOD_20221105T083813_V20221015T225942_20221017T005942.EOF', - 'test_dem.tiff'] - ''' - for test_file in test_files: - download_if_needed(f'{test_data_path}/{test_file}') - ''' + 'test_dem.tiff', 'test_burst_map.sqlite3', + '2022-10-16_0000_Rosamond-corner-reflectors.csv'] + #for test_file in test_files: + # download_if_needed(f'{test_data_path}/{test_file}') + pool = mp.Pool(len(test_files)) + _ = pool.map(download_if_needed, test_files) + pool.close() + pool.join() + + test_paths.corner_coord_csv_path = f'{test_data_path}/{test_files[-1]}' + test_paths.output_hdf5 = f'{test_path}/product/{burst_id}/{b_date}/{burst_id}_{b_date}.h5' + test_paths.grid_group_path = '/science/SENTINEL1/CSLC/grids' return test_paths diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index 0c6aae80..5cdbeb68 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -1,45 +1,76 @@ +import csv + +import h5py +import isce3 import numpy as np -import numpy.testing as npt from osgeo import gdal from compass import s1_geocode_slc from compass.utils.geo_runconfig import GeoRunConfig -def gdal_get_arr(f): - ''' - extract numpy array from GDAL supported raster - - Parameters - ---------- - f: str - path the GDAL supported raster - - Returns - ------- - _: np.ndarray - array extracted from GDAL supported raster - ''' - ds = gdal.Open(f, gdal.GA_ReadOnly) - arr = np.array([]) - if ds is not None: - arr = ds.GetRasterBand(1).ReadAsArray() - return arr - -def test_geocode_slc_run(test_paths): +def test_geocode_slc_run(unit_test_paths): ''' run s1_geocode_slc to ensure it does not crash ''' # load yaml to cfg - cfg = GeoRunConfig.load_from_yaml(test_paths.gslc_cfg_path, + cfg = GeoRunConfig.load_from_yaml(unit_test_paths.gslc_cfg_path, workflow_name='s1_cslc_geo') # pass cfg to s1_geocode_slc s1_geocode_slc.run(cfg) +def get_nearest_index(arr, val): + return np.abs(arr - val).argmin() + +def get_reflectors_extents_slice(unit_test_paths, margin=50): + ''' + get max and min lat, lon + ''' + # extract from HDF5 + with h5py.File(unit_test_paths.output_hdf5, 'r') as h5_obj: + grid_group = h5_obj[unit_test_paths.grid_group_path] + + # create projection to covert from UTM to LLH + epsg = int(grid_group['projection'][()]) + proj = isce3.core.UTM(epsg) + + x_coords_utm = grid_group['x_coordinates'][()] + y_coords_utm = grid_group['y_coordinates'][()] + + lons = np.array([np.degrees(proj.inverse([x, y_coords_utm[0], 0])[0]) + for x in x_coords_utm]) + + lats = np.array([np.degrees(proj.inverse([x_coords_utm[0], y, 0])[1]) + for y in y_coords_utm]) + + # extract all lat/lon corner reflector coordinates + corner_lats = [] + corner_lons = [] + with open(unit_test_paths.corner_coord_csv_path, 'r') as csvfile: + corner_reader = csv.DictReader(csvfile) + for row in corner_reader: + corner_lats.append(float(row['Latitude (deg)'])) + corner_lons.append(float(row['Longitude (deg)'])) -def test_geocode_slc_validate(test_paths): + i_max_lat = get_nearest_index(lats, np.max(corner_lats)) + i_min_lat = get_nearest_index(lats, np.min(corner_lats)) + i_max_lon = get_nearest_index(lons, np.max(corner_lons)) + i_min_lon = get_nearest_index(lons, np.min(corner_lons)) + + return np.s_[i_max_lat - margin:i_min_lat + margin, + i_min_lon - margin:i_max_lon + margin] + +def test_geocode_slc_validate(unit_test_paths): ''' - check if computed results match golden dataset + check for reflectors in geocoded output ''' - pass + s_ = get_reflectors_extents_slice(unit_test_paths) + + with h5py.File(unit_test_paths.output_hdf5, 'r') as h5_obj: + src_path = f'{unit_test_paths.grid_group_path}/VV' + arr = h5_obj[src_path][()][s_] + print(arr.shape, s_) + + corner_reflector_threshold = 3e3 + assert np.any(arr > corner_reflector_threshold) From d438b583fb30b11be7624063273f1598b0cae752 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 22 Feb 2023 23:40:26 -0800 Subject: [PATCH 27/34] update specifile fix test file paths update test data URL --- docker/specifile.txt | 56 ++++++++++++++++++++++++-------------------- requirements.txt | 5 +++- tests/conftest.py | 9 +++---- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/docker/specifile.txt b/docker/specifile.txt index 3e96bb93..ab76390d 100644 --- a/docker/specifile.txt +++ b/docker/specifile.txt @@ -8,10 +8,10 @@ https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.39-hcc3a1bd_1.conda +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-12.2.0-h337968e_19.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-12.2.0-h46fd767_19.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.11-hd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-3_cp311.conda https://conda.anaconda.org/conda-forge/noarch/tzdata-2022g-h191b570_0.conda https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2 @@ -29,7 +29,7 @@ https://conda.anaconda.org/conda-forge/linux-64/geos-3.11.1-h27087fc_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h36c2ea0_2.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/icu-70.1-h27087fc_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h0b41bf4_3.conda https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2 @@ -43,10 +43,10 @@ https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.21-pthreads_h78a https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.32.1-h7f98852_1000.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.2.4-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-h166bdaf_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.3-h9c3ff4c_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h27087fc_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.7-h0b41bf4_1.conda +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.8-h0b41bf4_0.conda https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.9-hbd366e4_2.tar.bz2 @@ -56,7 +56,7 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.0.10-h7f98852_0.ta https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.9-h7f98852_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h7f98852_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-renderproto-0.11.1-h7f98852_1002.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h7f98852_1002.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h0b41bf4_1003.conda https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2 @@ -75,7 +75,7 @@ https://conda.anaconda.org/conda-forge/linux-64/readline-8.1.2-h0f457ee_0.tar.bz https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h3eb15da_5.conda +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h3eb15da_6.conda https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.3-hafa529b_0.conda https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda @@ -85,7 +85,8 @@ https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openbl https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h6adf6a1_2.conda -https://conda.anaconda.org/conda-forge/linux-64/nss-3.82-he02c5a1_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/nss-3.88-he45b914_0.conda https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0-he550d4f_1_cpython.conda https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.40.0-h4ff8645_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.7.2-h7f98852_0.tar.bz2 @@ -97,56 +98,61 @@ https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.con https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.0-pyhd8ed1ab_0.conda -https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda -https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.87.0-hdc1c0ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.1-hdc1c0ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_3.conda -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.1-py311hbde0eaa_0.conda +https://conda.anaconda.org/conda-forge/linux-64/libpq-15.2-hb675445_0.conda +https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.2-py311h14a6109_0.conda +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.2-py311h8e6699e_0.conda https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda https://conda.anaconda.org/conda-forge/noarch/packaging-23.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py311hd4cff14_5.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.7-py311h2582759_1.conda -https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/noarch/setuptools-67.4.0-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_3.conda https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda -https://conda.anaconda.org/conda-forge/linux-64/curl-7.87.0-hdc1c0ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.1-hdc1c0ab_0.conda https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_101.conda -https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_3.conda +https://conda.anaconda.org/conda-forge/noarch/pip-23.0.1-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.2-h3248436_0.conda https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.1-h8ffa02c_2.conda https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.1-pyhd8ed1ab_0.conda +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.21-py311hd4cff14_2.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.0-py311h0f577a2_0.conda +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311h0f577a2_0.conda https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2 https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py311hd4cff14_1005.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-39.0.0-py311h9b4c7bb_0.conda +https://conda.anaconda.org/conda-forge/linux-64/cryptography-39.0.1-py311h9b4c7bb_0.conda https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h7a142b4_6.conda -https://conda.anaconda.org/conda-forge/linux-64/h5py-3.7.0-nompi_py311hbe7f6d8_102.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/h5py-3.8.0-nompi_py311h1db17ec_100.conda https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.0-ha7026e8_0.conda -https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h261ec11_106.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.1-nompi_h34a3ff0_100.conda https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h221c8f1_23.conda +https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.3-py311h2872171_0.conda https://conda.anaconda.org/conda-forge/linux-64/poppler-23.01.0-h091648b_0.conda https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py311h945b3ca_1.conda https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py311hbbb8f27_3.tar.bz2 https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.2-hd532e3d_0.conda -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-h8c90c07_5.conda +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-hea5766e_7.conda https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py311hadb6153_5.conda +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py311hadb6153_7.conda https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.10.0-py311h8e6699e_0.conda -https://conda.anaconda.org/conda-forge/linux-64/isce3-0.9.0-py311h0802494_0.conda +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.10.0-py311h8e6699e_2.conda +https://conda.anaconda.org/conda-forge/linux-64/isce3-0.10.0-py311h0802494_0.conda diff --git a/requirements.txt b/requirements.txt index 4eaf116f..2558154f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,15 @@ python>=3.9 # sentinel1-reader requirement numpy # sentinel1-reader requirement +lxml # sentinel1-reader requirement gdal>=3 #isce3 # since the conda-installed isce3 is not the most updated version, installing isce3 from stratch is recommended, to stay in sync with isce3 development. #journal # as of Mar 2022, journal from conda does not support python3.9; since it is included during isce3 installation above, comment this out temporarily. -lxml pandas pyproj pytest ruamel.yaml scipy yamale +h5py +shapely +requests diff --git a/tests/conftest.py b/tests/conftest.py index a51b657b..838ecaea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ def download_if_needed(local_path): check_internet_connection() - dataset_url = 'https://zenodo.org/record/7668410/files/' + dataset_url = 'https://zenodo.org/record/7668411/files/' dst_dir, file_name = os.path.split(local_path) if dst_dir: os.makedirs(dst_dir, exist_ok=True) @@ -55,14 +55,15 @@ def unit_test_paths(): 'orbits/S1A_OPER_AUX_POEORB_OPOD_20221105T083813_V20221015T225942_20221017T005942.EOF', 'test_dem.tiff', 'test_burst_map.sqlite3', '2022-10-16_0000_Rosamond-corner-reflectors.csv'] - #for test_file in test_files: - # download_if_needed(f'{test_data_path}/{test_file}') + test_files = [f'{test_data_path}/{test_file}' for test_file in test_files] + + # parallel download of test files pool = mp.Pool(len(test_files)) _ = pool.map(download_if_needed, test_files) pool.close() pool.join() - test_paths.corner_coord_csv_path = f'{test_data_path}/{test_files[-1]}' + test_paths.corner_coord_csv_path = test_files[-1] test_paths.output_hdf5 = f'{test_path}/product/{burst_id}/{b_date}/{burst_id}_{b_date}.h5' test_paths.grid_group_path = '/science/SENTINEL1/CSLC/grids' From dd2e841033f6c06969d0e7491c57e043482177e3 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Thu, 23 Feb 2023 00:08:55 -0800 Subject: [PATCH 28/34] remove unused imports --- tests/conftest.py | 1 - tests/test_s1_geocode_slc.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 838ecaea..0c9d3787 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -from functools import partial import multiprocessing as mp import os import pathlib diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index 5cdbeb68..90e97e4a 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -3,7 +3,6 @@ import h5py import isce3 import numpy as np -from osgeo import gdal from compass import s1_geocode_slc from compass.utils.geo_runconfig import GeoRunConfig From e693f5129acf3c675a968b817aa422ff03f8ed9c Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Thu, 23 Feb 2023 08:34:21 -0800 Subject: [PATCH 29/34] more comments/docstrings add environment yaml to help create clean/minimal environment add bounds check to margin application --- .gitignore | 10 +++- compass_env.yml | 19 ++++++++ src/compass/utils/runconfig.py | 5 +- tests/conftest.py | 58 ++++++++++++++++------ tests/test_s1_geocode_slc.py | 89 +++++++++++++++++++++++++--------- 5 files changed, 139 insertions(+), 42 deletions(-) create mode 100755 compass_env.yml diff --git a/.gitignore b/.gitignore index 92663c8a..bca12c9e 100644 --- a/.gitignore +++ b/.gitignore @@ -151,4 +151,12 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ - +# unit test files/directories +tests/data/geo_cslc_s1.yaml +tests/data/2022-10-16_0000_Rosamond-corner-reflectors.csv +tests/data/S1A_IW_SLC__1SDV_20221016T015043_20221016T015111_045461_056FC0_6681.zip +tests/data/orbits +tests/data/test_dem.tiff +tests/data/test_burst_map.sqlite3 +tests/product +tests/scratch diff --git a/compass_env.yml b/compass_env.yml new file mode 100755 index 00000000..2ab520b9 --- /dev/null +++ b/compass_env.yml @@ -0,0 +1,19 @@ +name: compass +channels: + - conda-forge + - nodefaults +dependencies: + # isce3 can be added if forge version is sufficient + - gdal>=3.0 + - h5py + - lxml + - numpy + - pandas + - pyproj + - pytest + - python>=3.9 + - requests + - ruamel.yaml + - scipy + - shapely + - yamale diff --git a/src/compass/utils/runconfig.py b/src/compass/utils/runconfig.py index ff375645..540cacb1 100644 --- a/src/compass/utils/runconfig.py +++ b/src/compass/utils/runconfig.py @@ -206,8 +206,7 @@ def runconfig_to_bursts(cfg: SimpleNamespace) -> list[Sentinel1BurstSlc]: for pol, i_subswath in pol_subswath_index_pairs: # loop over burst objs extracted from SAFE zip - loaded_bursts = load_bursts(safe_file, orbit_path, i_subswath, pol) - for burst in loaded_bursts: + for burst in load_bursts(safe_file, orbit_path, i_subswath, pol): # get burst ID burst_id = str(burst.burst_id) @@ -215,7 +214,6 @@ def runconfig_to_bursts(cfg: SimpleNamespace) -> list[Sentinel1BurstSlc]: # is burst_id wanted? skip if not given in config if (cfg.input_file_group.burst_id is not None and burst_id not in cfg.input_file_group.burst_id): - print(burst_id, cfg.input_file_group.burst_id, burst_id in cfg.input_file_group.burst_id) continue # get polarization and save as tuple with burst ID @@ -239,7 +237,6 @@ def runconfig_to_bursts(cfg: SimpleNamespace) -> list[Sentinel1BurstSlc]: # if reference burst, ok to add if id+pol combo does not exist # no duplicate id+pol combos for reference bursts if not_ref or not burst_id_pol_exist: - print(f'{burst_id} found') burst_ids_found.append(burst_id) bursts.append(burst) diff --git a/tests/conftest.py b/tests/conftest.py index 0c9d3787..f49fd04c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,25 +9,48 @@ from compass.utils import iono + def download_if_needed(local_path): - # check if test inputs and reference files exists; download if not found. + ''' + Check if given path to file exists. Download if it from zenodo does not. + + Parameters + ---------- + local_path: str + Path to file + ''' + # return if file is found if os.path.isfile(local_path): return check_internet_connection() - dataset_url = 'https://zenodo.org/record/7668411/files/' dst_dir, file_name = os.path.split(local_path) + + # create destination directory if it does not exist if dst_dir: os.makedirs(dst_dir, exist_ok=True) + + # download data + dataset_url = 'https://zenodo.org/record/7668411/files/' target_url = f'{dataset_url}/{file_name}' with open(local_path, 'wb') as f: f.write(requests.get(target_url).content) + @pytest.fixture(scope="session") -def unit_test_paths(): - test_paths = types.SimpleNamespace() +def geocode_slc_params(): + ''' + Parameters to be used by geocode SLC unit test + + Returns + ------- + test_params: SimpleNamespace + SimpleNamespace containing geocode SLC unit test parameters + ''' + test_params = types.SimpleNamespace() + # burst ID and date of burst burst_id = 't064_135523_iw2' b_date = '20221016' @@ -39,34 +62,40 @@ def unit_test_paths(): # paths for template and actual runconfig gslc_template_path = f'{test_data_path}/geo_cslc_s1_template.yaml' - test_paths.gslc_cfg_path = f'{test_data_path}/geo_cslc_s1.yaml' + test_params.gslc_cfg_path = f'{test_data_path}/geo_cslc_s1.yaml' # read runconfig template, replace pieces, write to runconfig with open(gslc_template_path, 'r') as f_template, \ - open(test_paths.gslc_cfg_path, 'w') as f_cfg: + open(test_params.gslc_cfg_path, 'w') as f_cfg: cfg = f_template.read().replace('@TEST_PATH@', str(test_path)).\ replace('@DATA_PATH@', test_data_path).\ replace('@BURST_ID@', burst_id) f_cfg.write(cfg) - # check for files and download as needed + # files needed for geocode SLC unit test test_files = ['S1A_IW_SLC__1SDV_20221016T015043_20221016T015111_045461_056FC0_6681.zip', 'orbits/S1A_OPER_AUX_POEORB_OPOD_20221105T083813_V20221015T225942_20221017T005942.EOF', 'test_dem.tiff', 'test_burst_map.sqlite3', '2022-10-16_0000_Rosamond-corner-reflectors.csv'] test_files = [f'{test_data_path}/{test_file}' for test_file in test_files] - # parallel download of test files + # parallel download of test files (if necessary) pool = mp.Pool(len(test_files)) _ = pool.map(download_if_needed, test_files) pool.close() pool.join() - test_paths.corner_coord_csv_path = test_files[-1] - test_paths.output_hdf5 = f'{test_path}/product/{burst_id}/{b_date}/{burst_id}_{b_date}.h5' - test_paths.grid_group_path = '/science/SENTINEL1/CSLC/grids' + # path to file containing corner reflectors + test_params.corner_coord_csv_path = test_files[-1] + + # path the output HDF5 + test_params.output_hdf5 = f'{test_path}/product/{burst_id}/{b_date}/{burst_id}_{b_date}.h5' - return test_paths + # path to groups and datasets in output HDF5 + test_params.grid_group_path = '/science/SENTINEL1/CSLC/grids' + test_params.raster_path = f'{test_params.grid_group_path}/VV' + + return test_params @pytest.fixture(scope='session') def ionex_params(download_data=True): @@ -81,9 +110,8 @@ def ionex_params(download_data=True): Returns ------- - tec_file: str - Path to local or downloaded TEC file to - use in the unit test + tec_file: SimpleNamespace + SimpleNamespace containing parameters needed for ionex unit test ''' test_params = types.SimpleNamespace() diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index 90e97e4a..60aed39c 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -8,27 +8,55 @@ from compass.utils.geo_runconfig import GeoRunConfig -def test_geocode_slc_run(unit_test_paths): +def test_geocode_slc_run(geocode_slc_params): ''' - run s1_geocode_slc to ensure it does not crash + Run s1_geocode_slc to ensure it does not crash + + Parameters + ---------- + geocode_slc_params: SimpleNamespace + SimpleNamespace containing geocode SLC unit test parameters ''' # load yaml to cfg - cfg = GeoRunConfig.load_from_yaml(unit_test_paths.gslc_cfg_path, + cfg = GeoRunConfig.load_from_yaml(geocode_slc_params.gslc_cfg_path, workflow_name='s1_cslc_geo') # pass cfg to s1_geocode_slc s1_geocode_slc.run(cfg) -def get_nearest_index(arr, val): + +def _get_nearest_index(arr, val): + ''' + Find index of element in given array closest to given value + + Parameters + ---------- + arr: np.ndarray + 1D array to be searched + val: float + Number to be searched for + + Returns + ------- + _: int + Index of element in arr where val is closest + ''' return np.abs(arr - val).argmin() -def get_reflectors_extents_slice(unit_test_paths, margin=50): + +def _get_reflectors_bounding_slice(geocode_slc_params): ''' - get max and min lat, lon + Get latitude, longitude slice that contains all the corner reflectors in + CSV list of corner reflectors + + Parameters + ---------- + geocode_slc_params: SimpleNamespace + SimpleNamespace containing geocode SLC unit test parameters ''' # extract from HDF5 - with h5py.File(unit_test_paths.output_hdf5, 'r') as h5_obj: - grid_group = h5_obj[unit_test_paths.grid_group_path] + with h5py.File(geocode_slc_params.output_hdf5, 'r') as h5_obj: + grid_group = h5_obj[geocode_slc_params.grid_group_path] # create projection to covert from UTM to LLH epsg = int(grid_group['projection'][()]) @@ -43,33 +71,50 @@ def get_reflectors_extents_slice(unit_test_paths, margin=50): lats = np.array([np.degrees(proj.inverse([x_coords_utm[0], y, 0])[1]) for y in y_coords_utm]) + # get array shape for later check of slice with margins applied + height, width = h5_obj[geocode_slc_params.raster_path].shape + # extract all lat/lon corner reflector coordinates corner_lats = [] corner_lons = [] - with open(unit_test_paths.corner_coord_csv_path, 'r') as csvfile: + with open(geocode_slc_params.corner_coord_csv_path, 'r') as csvfile: corner_reader = csv.DictReader(csvfile) for row in corner_reader: corner_lats.append(float(row['Latitude (deg)'])) corner_lons.append(float(row['Longitude (deg)'])) - i_max_lat = get_nearest_index(lats, np.max(corner_lats)) - i_min_lat = get_nearest_index(lats, np.min(corner_lats)) - i_max_lon = get_nearest_index(lons, np.max(corner_lons)) - i_min_lon = get_nearest_index(lons, np.min(corner_lons)) + # find nearest index for min/max of lats/lons and apply margin + # apply margin to bounding box and ensure raster bounds are not exceeded + # application of margin y indices reversed due descending order lats vector + margin = 50 + i_max_y = max(_get_nearest_index(lats, np.max(corner_lats)) - margin, 0) + i_min_y = min(_get_nearest_index(lats, np.min(corner_lats)) + margin, + height - 1) + i_max_x = min(_get_nearest_index(lons, np.max(corner_lons)) + margin, + width - 1) + i_min_x = max(_get_nearest_index(lons, np.min(corner_lons)) - margin, 0) - return np.s_[i_max_lat - margin:i_min_lat + margin, - i_min_lon - margin:i_max_lon + margin] + # return as slice + # y indices reversed to account for descending order lats vector + return np.s_[i_max_y:i_min_y, i_min_x:i_max_x] -def test_geocode_slc_validate(unit_test_paths): + +def test_geocode_slc_validate(geocode_slc_params): ''' - check for reflectors in geocoded output + Check for reflectors in geocoded output + + Parameters + ---------- + geocode_slc_params: SimpleNamespace + SimpleNamespace containing geocode SLC unit test parameters ''' - s_ = get_reflectors_extents_slice(unit_test_paths) + # get slice where corner reflectors should be + s_ = _get_reflectors_bounding_slice(geocode_slc_params) - with h5py.File(unit_test_paths.output_hdf5, 'r') as h5_obj: - src_path = f'{unit_test_paths.grid_group_path}/VV' - arr = h5_obj[src_path][()][s_] - print(arr.shape, s_) + # slice raster array + with h5py.File(geocode_slc_params.output_hdf5, 'r') as h5_obj: + arr = h5_obj[geocode_slc_params.raster_path][()][s_] + # check for bright spots in sliced array corner_reflector_threshold = 3e3 assert np.any(arr > corner_reflector_threshold) From aeb64ac5259a1d2ebd69a547a7512855787b375c Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Sat, 25 Feb 2023 14:31:46 -0800 Subject: [PATCH 30/34] cleaner mp pool Co-authored-by: Scott Staniewicz --- tests/conftest.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f49fd04c..b895baa0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,10 +80,8 @@ def geocode_slc_params(): test_files = [f'{test_data_path}/{test_file}' for test_file in test_files] # parallel download of test files (if necessary) - pool = mp.Pool(len(test_files)) - _ = pool.map(download_if_needed, test_files) - pool.close() - pool.join() + with mp.Pool(len(test_files)) as pool: + pool.map(download_if_needed, test_files) # path to file containing corner reflectors test_params.corner_coord_csv_path = test_files[-1] From 3f72ad374278adebb9c93498551542bcce2b72d4 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Sat, 25 Feb 2023 14:32:19 -0800 Subject: [PATCH 31/34] fix docstring Co-authored-by: Scott Staniewicz --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index b895baa0..04ab575c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,7 +108,7 @@ def ionex_params(download_data=True): Returns ------- - tec_file: SimpleNamespace + test_params : SimpleNamespace SimpleNamespace containing parameters needed for ionex unit test ''' test_params = types.SimpleNamespace() From b95f45cf125d7e3a9a0755930ca92fcba48a337b Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Sat, 25 Feb 2023 15:20:28 -0800 Subject: [PATCH 32/34] load just the slice not EVERYTHING --- tests/test_s1_geocode_slc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index 60aed39c..06fb3f34 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -113,7 +113,7 @@ def test_geocode_slc_validate(geocode_slc_params): # slice raster array with h5py.File(geocode_slc_params.output_hdf5, 'r') as h5_obj: - arr = h5_obj[geocode_slc_params.raster_path][()][s_] + arr = h5_obj[geocode_slc_params.raster_path][s_] # check for bright spots in sliced array corner_reflector_threshold = 3e3 From dd7b5e1c3480a1a16930678616b2ffcc0f8d2c9d Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 1 Mar 2023 17:36:17 -0800 Subject: [PATCH 33/34] clarity in file related variable --- compass_env.yml | 19 ------------------- tests/conftest.py | 6 ++++-- tests/test_s1_geocode_slc.py | 6 +++--- 3 files changed, 7 insertions(+), 24 deletions(-) delete mode 100755 compass_env.yml diff --git a/compass_env.yml b/compass_env.yml deleted file mode 100755 index 2ab520b9..00000000 --- a/compass_env.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: compass -channels: - - conda-forge - - nodefaults -dependencies: - # isce3 can be added if forge version is sufficient - - gdal>=3.0 - - h5py - - lxml - - numpy - - pandas - - pyproj - - pytest - - python>=3.9 - - requests - - ruamel.yaml - - scipy - - shapely - - yamale diff --git a/tests/conftest.py b/tests/conftest.py index 04ab575c..e37dc6db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,7 +52,7 @@ def geocode_slc_params(): # burst ID and date of burst burst_id = 't064_135523_iw2' - b_date = '20221016' + burst_date = '20221016' # get test working directory test_path = pathlib.Path(__file__).parent.resolve() @@ -87,7 +87,9 @@ def geocode_slc_params(): test_params.corner_coord_csv_path = test_files[-1] # path the output HDF5 - test_params.output_hdf5 = f'{test_path}/product/{burst_id}/{b_date}/{burst_id}_{b_date}.h5' + output_path = f'{test_path}/product/{burst_id}/{burst_date}' + output_file_name = f'{burst_id}_{burst_date}.h5' + test_params.output_hdf5_path = f'{output_path}/{output_file_name}.h5' # path to groups and datasets in output HDF5 test_params.grid_group_path = '/science/SENTINEL1/CSLC/grids' diff --git a/tests/test_s1_geocode_slc.py b/tests/test_s1_geocode_slc.py index 06fb3f34..1fbaa494 100644 --- a/tests/test_s1_geocode_slc.py +++ b/tests/test_s1_geocode_slc.py @@ -55,7 +55,7 @@ def _get_reflectors_bounding_slice(geocode_slc_params): SimpleNamespace containing geocode SLC unit test parameters ''' # extract from HDF5 - with h5py.File(geocode_slc_params.output_hdf5, 'r') as h5_obj: + with h5py.File(geocode_slc_params.output_hdf5_path, 'r') as h5_obj: grid_group = h5_obj[geocode_slc_params.grid_group_path] # create projection to covert from UTM to LLH @@ -101,7 +101,7 @@ def _get_reflectors_bounding_slice(geocode_slc_params): def test_geocode_slc_validate(geocode_slc_params): ''' - Check for reflectors in geocoded output + Check for presence of any reflectors in geocoded output Parameters ---------- @@ -112,7 +112,7 @@ def test_geocode_slc_validate(geocode_slc_params): s_ = _get_reflectors_bounding_slice(geocode_slc_params) # slice raster array - with h5py.File(geocode_slc_params.output_hdf5, 'r') as h5_obj: + with h5py.File(geocode_slc_params.output_hdf5_path, 'r') as h5_obj: arr = h5_obj[geocode_slc_params.raster_path][s_] # check for bright spots in sliced array From fa33f64642550f3228e88e6da2c2b9343a4284cb Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 1 Mar 2023 17:39:22 -0800 Subject: [PATCH 34/34] fix typo in path --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index e37dc6db..e416f240 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,7 +89,7 @@ def geocode_slc_params(): # path the output HDF5 output_path = f'{test_path}/product/{burst_id}/{burst_date}' output_file_name = f'{burst_id}_{burst_date}.h5' - test_params.output_hdf5_path = f'{output_path}/{output_file_name}.h5' + test_params.output_hdf5_path = f'{output_path}/{output_file_name}' # path to groups and datasets in output HDF5 test_params.grid_group_path = '/science/SENTINEL1/CSLC/grids'