diff --git a/images/full-history/Dockerfile b/images/full-history/Dockerfile index 7f2875c6..3c608c1e 100644 --- a/images/full-history/Dockerfile +++ b/images/full-history/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.964.h5e9b11b +FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99 VOLUME /mnt/data COPY ./start.sh / diff --git a/images/full-history/start.sh b/images/full-history/start.sh index acd82174..b4cc0dbb 100755 --- a/images/full-history/start.sh +++ b/images/full-history/start.sh @@ -101,13 +101,22 @@ upload_planet_file() { # =============================== if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then - download_dump_file - echo "Generating history planet file with planet-dump-ng..." - export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE" - planet-dump-ng \ - --dump-file "$dumpFile" \ - --history-pbf "$local_planetHistoryPBFFile" - + download_dump_file + echo "Generating history planet file with planet-dump-ng..." + export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE" + + if [ -n "$PLANET_DUMP_NG_METADATA_URL" ]; then + echo "Downloading metadata file..." + curl "$PLANET_DUMP_NG_METADATA_URL" -o metadata.yml + planet-dump-ng \ + --dump-file "$dumpFile" \ + --history-pbf "$local_planetHistoryPBFFile" \ + -M metadata.yml + else + planet-dump-ng \ + --dump-file "$dumpFile" \ + --history-pbf "$local_planetHistoryPBFFile" + fi elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then echo "Generating history planet file with osmosis..." # Creating full history diff --git a/images/osm-processor/Dockerfile b/images/osm-processor/Dockerfile index 041b01be..70754d0c 100644 --- a/images/osm-processor/Dockerfile +++ b/images/osm-processor/Dockerfile @@ -45,6 +45,7 @@ RUN set -ex \ tmux \ zsh \ git \ + curl \ libxml2 \ libboost-filesystem1.74.0 \ libboost-program-options1.74.0 \ @@ -56,4 +57,4 @@ RUN set -ex \ libosmpbf1 \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng \ No newline at end of file +COPY --from=builder /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng diff --git a/images/planet-dump/Dockerfile b/images/planet-dump/Dockerfile index 7b6c91aa..a7352b8d 100644 --- a/images/planet-dump/Dockerfile +++ b/images/planet-dump/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.964.h5e9b11b +FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99 VOLUME /mnt/data COPY ./start.sh / diff --git a/images/planet-dump/start.sh b/images/planet-dump/start.sh index f48085d8..7b0902e9 100755 --- a/images/planet-dump/start.sh +++ b/images/planet-dump/start.sh @@ -100,11 +100,20 @@ upload_planet_file() { # =============================== if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then - download_dump_file - echo "Generating planet file with planet-dump-ng..." - planet-dump-ng \ - --dump-file "$dumpFile" \ - --pbf "$local_planetPBFFile" + download_dump_file + echo "Generating planet file with planet-dump-ng..." + + if [ -n "$PLANET_DUMP_NG_METADATA_URL" ]; then + curl "$PLANET_DUMP_NG_METADATA_URL" -o metadata.yml + planet-dump-ng \ + --dump-file "$dumpFile" \ + --pbf "$local_planetPBFFile" \ + -M metadata.yml + else + planet-dump-ng \ + --dump-file "$dumpFile" \ + --pbf "$local_planetPBFFile" + fi elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then echo "Generating planet file with osmosis..." if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then diff --git a/osm-seed/templates/jobs/full-history-job.yaml b/osm-seed/templates/jobs/full-history-job.yaml index 87c7353a..a0812929 100644 --- a/osm-seed/templates/jobs/full-history-job.yaml +++ b/osm-seed/templates/jobs/full-history-job.yaml @@ -51,6 +51,8 @@ spec: value: {{ .Values.fullHistory.env.DUMP_CLOUD_URL | quote}} - name: PLANET_EPOCH_DATE value: {{ .Values.fullHistory.env.PLANET_EPOCH_DATE | quote}} + - name: PLANET_DUMP_NG_METADATA_URL + value: {{ .Values.fullHistory.env.PLANET_DUMP_NG_METADATA_URL | quote}} # In case cloudProvider=aws {{- if eq .Values.cloudProvider "aws" }} - name: AWS_S3_BUCKET diff --git a/osm-seed/templates/jobs/planet-dump-job.yaml b/osm-seed/templates/jobs/planet-dump-job.yaml index e0bef29a..21a75be9 100644 --- a/osm-seed/templates/jobs/planet-dump-job.yaml +++ b/osm-seed/templates/jobs/planet-dump-job.yaml @@ -55,6 +55,8 @@ spec: value: {{ .Values.planetDump.env.DUMP_CLOUD_URL | quote}} - name: PLANET_EPOCH_DATE value: {{ .Values.planetDump.env.PLANET_EPOCH_DATE | quote}} + - name: PLANET_DUMP_NG_METADATA_URL + value: {{ .Values.planetDump.env.PLANET_DUMP_NG_METADATA_URL | quote}} # In case cloudProvider=aws {{- if eq .Values.cloudProvider "aws" }} - name: AWS_S3_BUCKET diff --git a/osm-seed/templates/tasking-manager-api/tm-configMap.yaml b/osm-seed/templates/tasking-manager-api/tm-configMap.yaml index a98b55a4..204729af 100644 --- a/osm-seed/templates/tasking-manager-api/tm-configMap.yaml +++ b/osm-seed/templates/tasking-manager-api/tm-configMap.yaml @@ -9,7 +9,7 @@ data: POSTGRES_PASSWORD: {{ quote .Values.tmDb.env.POSTGRES_PASSWORD | default "" }} POSTGRES_USER: {{ quote .Values.tmDb.env.POSTGRES_USER | default "" }} POSTGRES_PORT: "5432" - TM_DB: {{ quote (printf "postgresql://%s:%s@%s-tasking-manager-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB) }} + TM_DB: {{ quote (printf "postgresql+asyncpg://%s:%s@%s-tasking-manager-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB) }} TM_ORG_NAME: {{ quote .Values.tmApi.env.TM_ORG_NAME | default "" }} TM_ORG_CODE: {{ quote .Values.tmApi.env.TM_ORG_CODE | default "" }} TM_ORG_URL: {{ quote .Values.tmApi.env.TM_ORG_URL | default "" }} @@ -44,4 +44,6 @@ data: TM_REDIRECT_URI: {{ quote .Values.tmApi.env.TM_REDIRECT_URI | default "" }} TM_SCOPE: {{ quote .Values.tmApi.env.TM_SCOPE | default "" }} TM_APP_API_VERSION: {{ quote .Values.tmApi.env.TM_APP_API_VERSION | default "v2" }} -{{- end }} \ No newline at end of file + OHSOME_STATS_TOKEN: {{ quote .Values.tmApi.env.OHSOME_STATS_TOKEN }} + POSTGRES_TEST_DB: {{ quote .Values.tmApi.env.POSTGRES_TEST_DB }} +{{- end }} diff --git a/osm-seed/templates/tasking-manager-api/tm-deployment.yaml b/osm-seed/templates/tasking-manager-api/tm-deployment.yaml index 88e98f06..8ba044da 100644 --- a/osm-seed/templates/tasking-manager-api/tm-deployment.yaml +++ b/osm-seed/templates/tasking-manager-api/tm-deployment.yaml @@ -26,7 +26,12 @@ spec: initContainers: - name: db-upgrade image: "{{ .Values.tmApi.image.name }}:{{ .Values.tmApi.image.tag }}" - command: ["flask", "db", "upgrade"] + command: + - alembic + - -c + - /usr/src/app/migrations/alembic.ini + - upgrade + - head envFrom: - configMapRef: name: {{ .Release.Name }}-tm-common-env diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 05ebbe3c..6b31fab9 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -262,9 +262,10 @@ fullHistory: schedule: "* * * * *" env: OVERWRITE_FHISTORY_FILE: false - PLANET_EXPORT_METHOD: osmosis + PLANET_EXPORT_METHOD: planet-dump-ng DUMP_CLOUD_URL: s3://osm-seed/db.dump PLANET_EPOCH_DATE: '1970-01-01' + PLANET_DUMP_NG_METADATA_URL: https://osmseed.org/metadata/planet-meta.yaml resources: enabled: false requests: @@ -348,7 +349,9 @@ planetDump: env: OVERWRITE_PLANET_FILE: false DUMP_CLOUD_URL : s3://osm-seed/db.dump - PLANET_EXPORT_METHOD: osmosis + PLANET_EXPORT_METHOD: planet-dump-ng + PLANET_EPOCH_DATE: '1970-01-01' + PLANET_DUMP_NG_METADATA_URL: https://osmseed.org/metadata/planet-meta.yaml resources: enabled: false requests: @@ -726,6 +729,8 @@ tmApi: TM_DEFAULT_LOCALE: "en" TM_APP_API_URL: "" TM_APP_BASE_URL: "" + OHSOME_STATS_TOKEN: "dummy-token-for-migrations" + POSTGRES_TEST_DB: "postgresql://user:pass@db:5432/test_db" resources: enabled: false requests: