Skip to content

Commit d96ddd5

Browse files
alejandrobodasalejandrobodas
authored
Merge pull request #123 from CFMIP/fix_download_kgos
2 parents 9afb7a4 + 9af7e8c commit d96ddd5

File tree

3 files changed

+50
-91
lines changed

3 files changed

+50
-91
lines changed

‎.github/workflows/containerized-ci.yml‎

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,9 @@ jobs:
1212
# Flags and KGOs for Intel Fortran Compiler Classic
1313
- compiler: ifort
1414
fcflags: -m64 -g -traceback -heap-arrays -assume realloc_lhs -extend-source 132 -check bounds,uninit,pointers,stack -stand f08
15-
gdkgo1: https://docs.google.com/uc?export=download&id=1R6uvtWno-bhdgxACCwdcaRNXs9l5_G4w
16-
gdkgo2: https://docs.google.com/uc?export=download&id=1bVU1_dx6hXcV_fHX2cLUn-D03T-Hij8z
1715
# Flags and KGOs for Intel Fortran Compiler
1816
- compiler: ifx
1917
fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08
20-
gdkgo1: https://docs.google.com/uc?export=download&id=1CLPJ1ikAcGBEiZ3WAGqL-zpaPIgC__po
21-
gdkgo2: https://docs.google.com/uc?export=download&id=1JOrLFbt0BYgpD0h6A3tsoRD4iTJ4jDvZ
2218
# Set container images
2319
- compiler: ifort
2420
image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:oneapi
@@ -41,8 +37,6 @@ jobs:
4137
ATOL: 0.0
4238
RTOL: 0.0
4339
KGO_VERSION: ${{ matrix.kgo_version }}
44-
GDKGO1: ${{ matrix.gdkgo1 }}
45-
GDKGO2: ${{ matrix.gdkgo2 }}
4640
steps:
4741
#
4842
# Checks-out repository under $GITHUB_WORKSPACE
@@ -71,30 +65,11 @@ jobs:
7165
cd build
7266
make -j driver
7367
# Retrieve and expand large data files
74-
- name: Retrieve input files
75-
run: |
76-
GDFILE='https://docs.google.com/uc?export=download&id=17eK4_DVEvFOE9Uf6siXJDpWZJKT1aqkU'
77-
OUTPATH=driver/data/inputs/UKMO/cosp_input.um_global.nc.gz
78-
curl -sSfL -o $OUTPATH $GDFILE
79-
gunzip ${OUTPATH}
80-
cd driver/data/inputs/UKMO
81-
md5sum -c cosp_input.um_global.nc.md5
82-
- name: Retrieve KGOs for basic test
83-
run: |
84-
cd ${GITHUB_WORKSPACE}
85-
OUTPATH=driver/data/outputs/UKMO/cosp2_output_um.$F90.kgo.$KGO_VERSION.nc.gz
86-
curl -sSfL -o $OUTPATH $GDKGO1
87-
gunzip ${OUTPATH}
88-
cd driver/data/outputs/UKMO
89-
# md5sum -c cosp2_output_um.${F90}.kgo.$KGO_VERSION.nc.md5
90-
- name: Retrieve KGOs for global test
68+
- name: Retrieve input files and KGOs
9169
run: |
70+
cd driver
71+
. download_test_data.sh ${F90}
9272
cd ${GITHUB_WORKSPACE}
93-
OUTPATH=driver/data/outputs/UKMO/cosp2_output.um_global.${F90}.kgo.$KGO_VERSION.nc.gz
94-
curl -sSfL -o $OUTPATH $GDKGO2
95-
gunzip ${OUTPATH}
96-
cd driver/data/outputs/UKMO
97-
# md5sum -c cosp2_output.um_global.${F90}.kgo.$KGO_VERSION.nc.md5
9873
###############################################################################
9974
# Run COSP2 tests. Basic test and UM global snapshot
10075
###############################################################################

‎.github/workflows/continuous_integration.yml‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,6 @@ jobs:
5353
ATOL: 0.0
5454
RTOL: 0.0
5555
KGO_VERSION: ${{ matrix.kgo_version }}
56-
# KGO1 is cosp2_output_um.gfortran.kgo.vNNN.nc.gz
57-
GDKGO1: https://docs.google.com/uc?export=download&id=1kC9RViPBdAsGcOivpYXxs3hHJ11Kv8IN
58-
# KGO2 is cosp2_output.um_global.gfortran.kgo.vNNN.nc.gz
59-
GDKGO2: https://docs.google.com/uc?export=download&id=1X_oOzvY2lf-kyAR-D1E6JfuefkGg1idn
60-
# KGO3 is cosp2_output.um_global_model_levels.gfortran.kgo.vNNN.nc.gz
61-
GDKGO3: https://docs.google.com/uc?export=download&id=1c14qBf9VwYJWYVGCu-Cw35F-qqHx0mSg
6256
F90_SHORT_NAME: ${{ matrix.compiler_short_name }}
6357
# Sequence of tasks that will be executed as part of the job
6458
steps:
@@ -92,39 +86,11 @@ jobs:
9286
cd build
9387
make -j driver
9488
# Retrieve and expand large data files
95-
- name: Retrieve input files
96-
run: |
97-
GDFILE='https://docs.google.com/uc?export=download&id=17eK4_DVEvFOE9Uf6siXJDpWZJKT1aqkU'
98-
OUTPATH=driver/data/inputs/UKMO/cosp_input.um_global.nc.gz
99-
curl -sSfL -o $OUTPATH $GDFILE
100-
gunzip ${OUTPATH}
101-
cd driver/data/inputs/UKMO
102-
md5sum -c cosp_input.um_global.nc.md5
103-
- name: Retrieve KGOs for basic test
104-
run: |
105-
cd ${GITHUB_WORKSPACE}
106-
OUTPATH=driver/data/outputs/UKMO/cosp2_output_um.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc.gz
107-
curl -sSfL -o $OUTPATH $GDKGO1
108-
gunzip ${OUTPATH}
109-
cd driver/data/outputs/UKMO
110-
md5sum -c cosp2_output_um.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc.md5
111-
- name: Retrieve KGOs for global test
112-
run: |
113-
cd ${GITHUB_WORKSPACE}
114-
OUTPATH=driver/data/outputs/UKMO/cosp2_output.um_global.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc.gz
115-
curl -sSfL -o $OUTPATH $GDKGO2
116-
gunzip ${OUTPATH}
117-
cd driver/data/outputs/UKMO
118-
md5sum -c cosp2_output.um_global.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc.md5
119-
cd ${GITHUB_WORKSPACE}
120-
- name: Retrieve KGOs for global test with outputs on model levels
89+
- name: Retrieve input files and KGOs
12190
run: |
91+
cd driver
92+
. download_test_data.sh ${F90_SHORT_NAME}
12293
cd ${GITHUB_WORKSPACE}
123-
OUTPATH=driver/data/outputs/UKMO/cosp2_output.um_global_model_levels.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc.gz
124-
curl -sSfL -o $OUTPATH $GDKGO3
125-
gunzip ${OUTPATH}
126-
cd driver/data/outputs/UKMO
127-
md5sum -c cosp2_output.um_global_model_levels.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc.md5
12894
###############################################################################
12995
# Run COSP2 tests. We could run both tests in one step, but
13096
# doing it this way the output is easier to interpret.

‎driver/download_test_data.sh‎

Lines changed: 44 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,58 @@
11
#/bin/bash
22

3-
if [[ -n ${KGO_VERSION} ]] ; then
4-
echo "KGO_VERSION=$KGO_VERSION"
5-
else
6-
export KGO_VERSION=v003
7-
echo "set KGO_VERSION=$KGO_VERSION"
8-
fi
3+
COMPILER=$1
94

5+
case $COMPILER in
106

11-
kgo_links=( "https://docs.google.com/uc?export=download&id=1oQBJGFg0F8k-LhRGsCYn-qWzmMMVfQ6K" \
12-
"https://docs.google.com/uc?export=download&id=1b7qwJWqDzoZGcIP0qyUprTV_LErCGpT6" \
13-
"https://docs.google.com/uc?export=download&id=1NvTo3bYaGpz-FUpZ4jta_kRzA04xTCsn" )
7+
"gfortran")
8+
KGO_VERSION=v004
9+
kgo_links=( "https://docs.google.com/uc?export=download&id=1kC9RViPBdAsGcOivpYXxs3hHJ11Kv8IN" \
10+
"https://docs.google.com/uc?export=download&id=1X_oOzvY2lf-kyAR-D1E6JfuefkGg1idn" \
11+
"https://docs.google.com/uc?export=download&id=1c14qBf9VwYJWYVGCu-Cw35F-qqHx0mSg" )
12+
out_type=( "cosp2_output_um.${COMPILER}.kgo" \
13+
"cosp2_output.um_global.${COMPILER}.kgo" \
14+
"cosp2_output.um_global_model_levels.${COMPILER}.kgo" )
15+
;;
1416

15-
out_type=( "cosp2_output_um.gfortran.kgo" \
16-
"cosp2_output.um_global.gfortran.kgo" \
17-
"cosp2_output.um_global_model_levels.gfortran.kgo" )
17+
"ifort")
18+
KGO_VERSION=v005
19+
kgo_links=( "https://docs.google.com/uc?export=download&id=1R6uvtWno-bhdgxACCwdcaRNXs9l5_G4w" \
20+
"https://docs.google.com/uc?export=download&id=1bVU1_dx6hXcV_fHX2cLUn-D03T-Hij8z" )
21+
out_type=( "cosp2_output_um.${COMPILER}.kgo" \
22+
"cosp2_output.um_global.${COMPILER}.kgo" )
23+
;;
24+
25+
"ifx")
26+
KGO_VERSION=v005
27+
kgo_links=( "https://docs.google.com/uc?export=download&id=1CLPJ1ikAcGBEiZ3WAGqL-zpaPIgC__po" \
28+
"https://docs.google.com/uc?export=download&id=1JOrLFbt0BYgpD0h6A3tsoRD4iTJ4jDvZ" )
29+
out_type=( "cosp2_output_um.${COMPILER}.kgo" \
30+
"cosp2_output.um_global.${COMPILER}.kgo" )
31+
;;
32+
33+
*)
34+
echo "ERROR: Invalid compiler"
35+
exit 1
36+
;;
37+
esac
38+
39+
echo "KGO_VERSION=$KGO_VERSION"
1840

1941
# Input: global UM
2042
DRIVER_DIR=${PWD}
2143
GDFILE='https://docs.google.com/uc?export=download&id=17eK4_DVEvFOE9Uf6siXJDpWZJKT1aqkU'
2244
OUTPATH=data/inputs/UKMO/cosp_input.um_global.nc.gz
23-
wget --no-check-certificate $GDFILE -O $OUTPATH
45+
curl -sSfL -o $OUTPATH $GDFILE
2446
gunzip -f ${OUTPATH}
2547
cd data/inputs/UKMO
2648
md5sum -c cosp_input.um_global.nc.md5
2749

28-
if [[ ${KGO_VERSION} == "v002" ]] ; then
29-
for i in ${!kgo_links[@]}; do
30-
cd ${DRIVER_DIR}
31-
GDFILE=${kgo_links[$i]}
32-
OUTPATH=data/outputs/UKMO/${out_type[$i]}.$KGO_VERSION.nc.gz
33-
wget --no-check-certificate $GDFILE -O $OUTPATH
34-
gunzip -f ${OUTPATH}
35-
cd data/outputs/UKMO
36-
md5sum -c ${out_type[$i]}.$KGO_VERSION.nc.md5
37-
done
38-
else
39-
echo "wrong KGO_VERSION supplied, try 'export KGO_VERSION=v002'"
40-
fi
50+
for i in ${!kgo_links[@]}; do
51+
cd ${DRIVER_DIR}
52+
GDFILE=${kgo_links[$i]}
53+
OUTPATH=data/outputs/UKMO/${out_type[$i]}.$KGO_VERSION.nc.gz
54+
curl -sSfL -o $OUTPATH $GDFILE
55+
gunzip -f ${OUTPATH}
56+
cd data/outputs/UKMO
57+
md5sum -c ${out_type[$i]}.$KGO_VERSION.nc.md5
58+
done

0 commit comments

Comments
 (0)