diff --git a/.circleci/config.yml b/.circleci/config.yml index 00a0f11..a3012a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: cd .. # deploy public Lambda layers pip install awscli - for region in us-east-1 us-west-2 eu-central-1 + for region in us-east-1 us-west-2 eu-central-1 eu-west-2 eu-north-1 do LVERSION="$(aws lambda publish-layer-version --region ${region} \ --layer-name geolambda --license-info 'MIT' \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 484033b..2f437fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,39 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added +- Added deployments to `eu-west-2` and `eu-north-1` + +### Fixed +- README updates regarding versions + +### Changed +- Keep symlinks when zipping, resulting in smaller deploy package +- Python Lambda Layers no longer need the base GeoLambda layer in addition to the python layer +- Manually compile sqlite3 used in Proj, GDAL, nghttp2 +- GDAL 3.2.1 +- Proj 7.2.1 +- GEOS 3.8.1 +- GeoTIFF 1.5.1 +- HDF4 4.2.15 +- HDF5 1.10.7 +- NetCDF 4.7.4 +- Nghttp2 1.41.0 +- OpenJPEG 2.4.0 +- libJPEG Turbo 2.4.0 +- Curl 7.73.0 +- Webp 1.1.0 +- Zstd 1.4.5 +- OpenSSL 1.1.1 +- In python layer: + - Python 3.7.4 -> 3.7.9 + - rasterio 1.1.0 -> 1.2.0 + - shapely 1.6.4.post2 -> 1.7.1 + - pyproj 2.4.0 -> 3.0.0.post1 + +### Removed +- GDAL Elasticsearch driver + ## [v2.0.0] - 2019-10-25 @@ -85,6 +118,7 @@ Package Versions - GDAL_VERSION=2.3.1 [Unreleased]: https://github.com/sat-utils/sat-stac/compare/master...develop +[v2.1.0]: https://github.com/developmentseed/geolambda/compare/2.0.0...2.1.0 [v2.0.0]: https://github.com/developmentseed/geolambda/compare/1.2.0...2.0.0 [v1.2.0]: https://github.com/developmentseed/geolambda/compare/1.1.0...1.2.0 [v1.1.0]: https://github.com/developmentseed/geolambda/compare/1.0.0...1.1.0 diff --git a/Dockerfile b/Dockerfile index 61497da..17508f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,29 +6,29 @@ LABEL authors="Matthew Hanson " # install system libraries RUN \ yum makecache fast; \ - yum install -y wget libpng-devel nasm; \ + yum install -y wget libpng-devel nasm rsync; \ yum install -y bash-completion --enablerepo=epel; \ yum clean all; \ yum autoremove # versions of packages ENV \ - GDAL_VERSION=3.0.1 \ - PROJ_VERSION=6.2.0 \ - GEOS_VERSION=3.8.0 \ - GEOTIFF_VERSION=1.5.1 \ - HDF4_VERSION=4.2.14 \ - HDF5_VERSION=1.10.5 \ - NETCDF_VERSION=4.7.1 \ - NGHTTP2_VERSION=1.39.2 \ - OPENJPEG_VERSION=2.3.1 \ - CURL_VERSION=7.66.0 \ - LIBJPEG_TURBO_VERSION=2.0.3 \ + GDAL_VERSION=3.2.1 \ + PROJ_VERSION=7.2.1 \ + GEOS_VERSION=3.8.1 \ + GEOTIFF_VERSION=1.6.0 \ + HDF4_VERSION=4.2.15 \ + HDF5_VERSION=1.10.7 \ + NETCDF_VERSION=4.7.4 \ + NGHTTP2_VERSION=1.41.0 \ + OPENJPEG_VERSION=2.4.0 \ + LIBJPEG_TURBO_VERSION=2.0.6 \ + CURL_VERSION=7.73.0 \ PKGCONFIG_VERSION=0.29.2 \ SZIP_VERSION=2.1.1 \ - WEBP_VERSION=1.0.3 \ - ZSTD_VERSION=1.4.3 \ - OPENSSL_VERSION=1.0.2 + WEBP_VERSION=1.1.0 \ + ZSTD_VERSION=1.4.5 \ + OPENSSL_VERSION=1.1.1 # Paths to things ENV \ @@ -37,9 +37,9 @@ ENV \ PREFIX=/usr/local \ GDAL_CONFIG=/usr/local/bin/gdal-config \ LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \ - PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig:/usr/lib64/pkgconfig \ - GDAL_DATA=${PREFIX}/share/gdal \ - PROJ_LIB=${PREFIX}/share/proj + PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_LIB=/usr/local/share/proj # switch to a build directory WORKDIR /build @@ -53,14 +53,25 @@ RUN \ make -j ${NPROC} install; \ cd ../; rm -rf pkg-config +# sqlite3 (required by proj) +RUN \ + mkdir sqlite3; \ + wget -qO- https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz \ + | tar xvz -C sqlite3 --strip-components=1; cd sqlite3; \ + ./configure --prefix=$PREFIX; \ + make; make install; \ + cd ../; rm -rf sqlite3; + # proj RUN \ mkdir proj; \ - wget -qO- http://download.osgeo.org/proj/proj-$PROJ_VERSION.tar.gz | tar xvz -C proj --strip-components=1; cd proj; \ - ./configure --prefix=$PREFIX; \ + wget -qO- http://download.osgeo.org/proj/proj-$PROJ_VERSION.tar.gz \ + | tar xvz -C proj --strip-components=1; cd proj; \ + SQLITE3_LIBS="=L$PREFIX/lib -lsqlite3" SQLITE3_INCLUDE_DIRS=$PREFIX/include/proj ./configure --prefix=$PREFIX; \ make -j ${NPROC} install; \ cd ..; rm -rf proj + # nghttp2 RUN \ mkdir nghttp2; \ @@ -194,9 +205,11 @@ RUN \ --with-zstd=${PREFIX} \ --with-jpeg=${PREFIX} \ --with-threads=yes \ + --with-sqlite3=$PREFIX \ --with-curl=${PREFIX}/bin/curl-config \ --without-python \ --without-libtool \ + --disable-driver-elastic \ --with-geos=$PREFIX/bin/geos-config \ --with-hide-internal-symbols=yes \ CFLAGS="-O2 -Os" CXXFLAGS="-O2 -Os" \ diff --git a/README.md b/README.md index 70a18fe..965f0fb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # GeoLambda: geospatial AWS Lambda Layer -The GeoLambda project provides public Docker images and AWS Lambda Layers containing common geospatial native libraries. GeoLambda contains the libraries PROJ.5, GEOS, GeoTIFF, HDF4/5, SZIP, NetCDF, OpenJPEG, WEBP, ZSTD, and GDAL. For some applications you may wish to minimize the size of the libraries by excluding unused libraries, or you may wish to add other libraries. In this case this repository can be used as a template to create your own Docker image or Lambda Layer following the instructions in this README. +The GeoLambda project provides public Docker images and AWS Lambda Layers containing common geospatial native libraries. GeoLambda contains the libraries for GDAL, Proj, GEOS, GeoTIFF, HDF4/5, SZIP, NetCDF, OpenJPEG, WEBP, ZSTD, and others. For some applications you may wish to minimize the size of the libraries by excluding unused libraries, or you may wish to add other libraries. In this case this repository can be used as a template to create your own Docker image or Lambda Layer following the instructions in this README. -This repository also contains additional images and layers for specific runtimes. Using them as a Layer assumes the use of the basee GeoLambda layer. +This repository also contains additional images and layers for specific runtimes. Using them as a Layer assumes the use of the base GeoLambda layer. - [Python](python/README.md) @@ -16,6 +16,7 @@ While GeoLambda was initially intended for AWS Lambda it is also useful as a bas | 1.1.0 | 2.4.1 | | | 1.2.0 | 2.4.2 | Separate Python (3.7.4) image and Lambda Layer added | | 2.0.0 | 3.0.1 | libgeotiff 1.5.1, proj 6.2.0 | +| 2.1.0 | 3.2.1 | libgeotiff 1.6.0, proj 7.2.1, openjpeg 2.4.0, Python layer 3.7.9 | #### Environment variables @@ -26,7 +27,30 @@ When using GeoLambda some environment variables need to be set. These are set in ### Lambda Layers -If you just wish to use the publicly available Lambda layers you will need the ARN for the layer in the same region as your Lambda function. Currently, GeoLambda layers are available in `us-east-1`, `us-west-2`, and `eu-central-1`. If you want to use it in another region please file an issue or you can also create your own layer using this repository (see instructions below on 'Create a new version'). +If you just wish to use the publicly available Lambda layers you will need the ARN for the layer in the same region as your Lambda function. Currently, the latest GeoLambda layers are deployed in `us-east-1`, `us-west-2`, `eu-central-1`, `eu-west-2`, and `eu-north-1`. If you want to use it in another region please file an issue or you can also create your own layer using this repository (see instructions below on 'Create a new version'). + +#### v2.1.0 + +| Region | ARN | +| ------ | --- | +| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:4 | +| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:4 | +| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:4 | +| eu-west-2 | | +| eu-north-1 | | + +#### v2.1.0-python + +See the [GeoLambda Python README](python/README.md). The Python Lambda Layer includes the libraries `numpy`, `rasterio`, `GDAL`, `pyproj`, and `shapely`, plus everything in the standard GeoLambda layer. Note this is a change from v2.0.0 where both Layers needed to be included in a Lambda. + +| Region | ARN | +| ------ | --- | +| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-python:3 | +| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-python:3 | +| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-python:3 | +| eu-west-2 | | +| eu-north-1 | | + #### v2.0.0 diff --git a/VERSION b/VERSION index 227cea2..7ec1d6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 +2.1.0 diff --git a/bin/package.sh b/bin/package.sh index ba2dc0e..19640ed 100755 --- a/bin/package.sh +++ b/bin/package.sh @@ -3,8 +3,6 @@ # directory used for deployment export DEPLOY_DIR=lambda -echo Creating deploy package - # make deployment directory and add lambda handler mkdir -p $DEPLOY_DIR/lib @@ -21,4 +19,4 @@ rsync -ax $PREFIX/share/proj $DEPLOY_DIR/share/ # zip up deploy package cd $DEPLOY_DIR -zip -ruq ../lambda-deploy.zip ./ +zip --symlinks -ruq ../lambda-deploy.zip ./ diff --git a/build-and-test.sh b/build-and-test.sh index bc7309a..26ac040 100755 --- a/build-and-test.sh +++ b/build-and-test.sh @@ -1,6 +1,7 @@ #!/bin/bash VERSION=$(cat VERSION) +PYVERSION=$(cat python/PYVERSION) docker build . -t developmentseed/geolambda:${VERSION} docker run --rm -v $PWD:/home/geolambda -it developmentseed/geolambda:${VERSION} package.sh @@ -9,4 +10,5 @@ cd python docker build . --build-arg VERSION=${VERSION} -t developmentseed/geolambda:${VERSION}-python docker run -v ${PWD}:/home/geolambda -t developmentseed/geolambda:${VERSION}-python package-python.sh -docker run --rm -v ${PWD}/lambda:/var/task -v ${PWD}/../lambda:/opt lambci/lambda:python3.7 lambda_function.lambda_handler '{}' +docker run -e GDAL_DATA=/opt/share/gdal -e PROJ_LIB=/opt/share/proj \ + --rm -v ${PWD}/lambda:/var/task lambci/lambda:python3.7 lambda_function.lambda_handler '{}' diff --git a/python/Dockerfile b/python/Dockerfile index 6d947b7..1e63278 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -1,10 +1,10 @@ -ARG VERSION=1.2.0 +ARG VERSION=2.1.0 FROM developmentseed/geolambda:${VERSION} LABEL maintainer="Development Seed " LABEL authors="Matthew Hanson " -ARG PYVERSION=3.7.4 +ARG PYVERSION=3.7.9 # install Python ENV \ diff --git a/python/PYVERSION b/python/PYVERSION new file mode 100644 index 0000000..c77a7de --- /dev/null +++ b/python/PYVERSION @@ -0,0 +1 @@ +3.7.9 diff --git a/python/README.md b/python/README.md index b199342..f7028b6 100644 --- a/python/README.md +++ b/python/README.md @@ -19,11 +19,11 @@ An example Lambda handler is located at [lambda/lambda_function.py](lambda/lambd Now, use the [Dockerfile](Dockerfile) can be used to create a new Docker image based on any version of GeoLambda with any version of Python by providing the versions as build arguments to `docker run`. This will install the specified version of Python along with any Python packages provided in [requirements.txt](requirements.txt). ``` -$ VERSION=1.2.0 -$ docker build . --build-arg VERSION=${VERSION} --build-arg PYVERSION=3.7.4 -t :latest +$ VERSION=2.1.0 +$ docker build . --build-arg VERSION=${VERSION} --build-arg PYVERSION=3.7.9 -t :latest ``` -If not provided, `VERSION` (the version of GeoLambda to use) will default to `latest` and `PYVERSION` (Python version) will default to `3.7.4`. +If not provided, `VERSION` (the version of GeoLambda to use) will default to `latest` and `PYVERSION` (Python version) will default to `3.7.9`. **4. Set up development environment and lambda layer zip file** @@ -39,7 +39,7 @@ This will create geolambda base layer (`lambda-deploy.zip`) file and `/geolambda $ docker run --rm -v $PWD:/home/geolambda -it developmentseed/geolambda:${VERSION} package.sh ``` -This will copy site-packages in /geolambda/python/lambda directory needed for development. Also, it will create a `lambda-layer-deploy.zip` file in the current directory. +This will copy site-packages in /geolambda/python/lambda directory needed for development. Also, it will create a `lambda-deploy.zip` file in the current directory. Contents of the zip file are following AWS conventions: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path ``` @@ -55,7 +55,8 @@ You can use the [LambCI Docker images](https://github.com/lambci/docker-lambda) ``` # current dir: geolambda/python -$ docker run --rm -v ${PWD}/lambda:/var/task -v ${PWD}/../lambda:/opt lambci/lambda:python3.7 lambda_function.lambda_handler '{}' +$ docker run -e GDAL_DATA=/opt/share/gdal -e PROJ_LIB=/opt/share/proj \ + --rm -v ${PWD}/lambda:/var/task lambci/lambda:python3.7 lambda_function.lambda_handler '{}' ``` The last argument is a JSON string that will be passed as the event payload to the handler function. @@ -68,7 +69,7 @@ Deploy the Lambda layer by using AWS CLI. $ aws lambda publish-layer-version \ --layer-name geolambda-python \ --description "Python bindings for GDAL" \ - --zip-file fileb://lambda-layer-deploy.zip + --zip-file fileb://lambda-deploy.zip ``` ### Pre-built images diff --git a/python/bin/package-python.sh b/python/bin/package-python.sh index fc6f7ce..6dd86fc 100755 --- a/python/bin/package-python.sh +++ b/python/bin/package-python.sh @@ -3,6 +3,20 @@ # directory used for development export DEPLOY_DIR=lambda +# make deployment directory and add lambda handler +mkdir -p $DEPLOY_DIR/lib + +# copy libs +cp -P ${PREFIX}/lib/*.so* $DEPLOY_DIR/lib/ +cp -P ${PREFIX}/lib64/libjpeg*.so* $DEPLOY_DIR/lib/ + +strip $DEPLOY_DIR/lib/* || true + +# copy GDAL_DATA files over +mkdir -p $DEPLOY_DIR/share +rsync -ax $PREFIX/share/gdal $DEPLOY_DIR/share/ +rsync -ax $PREFIX/share/proj $DEPLOY_DIR/share/ + # Get Python version PYVERSION=$(cat /root/.pyenv/version) MAJOR=${PYVERSION%%.*} @@ -27,7 +41,7 @@ cp -r $DEPLOY_DIR ./python rm ./python/lambda_function.py # zip up deploy package -zip -ruq lambda-deploy.zip ./python +zip --symlinks -ruq lambda-deploy.zip ./python # cleanup rm -rf ./python \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt index 4f9177b..67ea0fc 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,4 +1,4 @@ -GDAL==3.0.1 -rasterio==1.1.0 --no-binary rasterio -shapely==1.6.4.post2 -pyproj==2.4.0 \ No newline at end of file +GDAL==3.2.1 +rasterio==1.2.0 --no-binary rasterio +shapely==1.7.1 +pyproj==3.0.0.post1