Update TM-api and planet dump configurations

images/full-history/Dockerfile

@@ -1,4 +1,4 @@
-FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.964.h5e9b11b
+FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99
 
 VOLUME /mnt/data
 COPY ./start.sh /

images/full-history/start.sh

@@ -101,13 +101,22 @@ upload_planet_file() {
 # ===============================
 
 if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
-	download_dump_file
-	echo "Generating history planet file with planet-dump-ng..."
-	export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE"
-	planet-dump-ng \
-		--dump-file "$dumpFile" \
-		--history-pbf "$local_planetHistoryPBFFile"
-
+    download_dump_file
+    echo "Generating history planet file with planet-dump-ng..."
+    export PLANET_EPOCH_DATE="$PLANET_EPOCH_DATE"
+
+    if [ -n "$PLANET_DUMP_NG_METADATA_URL" ]; then
+        echo "Downloading metadata file..."
+        curl "$PLANET_DUMP_NG_METADATA_URL" -o metadata.yml
+        planet-dump-ng \
+            --dump-file "$dumpFile" \
+            --history-pbf "$local_planetHistoryPBFFile" \
+            -M metadata.yml
+    else
+        planet-dump-ng \
+            --dump-file "$dumpFile" \
+            --history-pbf "$local_planetHistoryPBFFile"
+    fi
 elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
 	echo "Generating history planet file with osmosis..."
 	# Creating full history

images/osm-processor/Dockerfile

@@ -45,6 +45,7 @@ RUN set -ex \
         tmux \
         zsh \
         git \
+        curl \
         libxml2 \
         libboost-filesystem1.74.0 \
         libboost-program-options1.74.0 \
@@ -56,4 +57,4 @@ RUN set -ex \
         libosmpbf1 \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=builder /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng
+COPY --from=builder /opt/planet-dump-ng/planet-dump-ng /usr/local/bin/planet-dump-ng

images/planet-dump/Dockerfile

@@ -1,4 +1,4 @@
-FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.964.h5e9b11b
+FROM developmentseed/osmseed-osm-processor:0.1.0-0.dev.git.968.hb111c99
 
 VOLUME /mnt/data
 COPY ./start.sh /

images/planet-dump/start.sh

@@ -100,11 +100,20 @@ upload_planet_file() {
 # ===============================
 
 if [ "$PLANET_EXPORT_METHOD" == "planet-dump-ng" ]; then
-	download_dump_file
-	echo "Generating planet file with planet-dump-ng..."
-	planet-dump-ng \
-		--dump-file "$dumpFile" \
-		--pbf "$local_planetPBFFile"
+    download_dump_file
+    echo "Generating planet file with planet-dump-ng..."
+
+    if [ -n "$PLANET_DUMP_NG_METADATA_URL" ]; then
+        curl "$PLANET_DUMP_NG_METADATA_URL" -o metadata.yml
+        planet-dump-ng \
+            --dump-file "$dumpFile" \
+            --pbf "$local_planetPBFFile" \
+            -M metadata.yml
+    else
+        planet-dump-ng \
+            --dump-file "$dumpFile" \
+            --pbf "$local_planetPBFFile"
+    fi
 elif [ "$PLANET_EXPORT_METHOD" == "osmosis" ]; then
 	echo "Generating planet file with osmosis..."
 	if [ -z "$MEMORY_JAVACMD_OPTIONS" ]; then

osm-seed/templates/jobs/full-history-job.yaml

@@ -51,6 +51,8 @@ spec:
                 value: {{ .Values.fullHistory.env.DUMP_CLOUD_URL | quote}}
               - name: PLANET_EPOCH_DATE
                 value: {{ .Values.fullHistory.env.PLANET_EPOCH_DATE | quote}}
+              - name: PLANET_DUMP_NG_METADATA_URL
+                value: {{ .Values.fullHistory.env.PLANET_DUMP_NG_METADATA_URL | quote}}
               # In case cloudProvider=aws
               {{- if eq .Values.cloudProvider "aws" }}
               - name: AWS_S3_BUCKET

osm-seed/templates/jobs/planet-dump-job.yaml

@@ -55,6 +55,8 @@ spec:
                 value: {{ .Values.planetDump.env.DUMP_CLOUD_URL | quote}}
               - name: PLANET_EPOCH_DATE
                 value: {{ .Values.planetDump.env.PLANET_EPOCH_DATE | quote}}
+              - name: PLANET_DUMP_NG_METADATA_URL
+                value: {{ .Values.planetDump.env.PLANET_DUMP_NG_METADATA_URL | quote}}
               # In case cloudProvider=aws
               {{- if eq .Values.cloudProvider "aws" }}
               - name: AWS_S3_BUCKET

osm-seed/templates/tasking-manager-api/tm-configMap.yaml

@@ -9,7 +9,7 @@ data:
   POSTGRES_PASSWORD: {{ quote .Values.tmDb.env.POSTGRES_PASSWORD | default "" }}
   POSTGRES_USER: {{ quote .Values.tmDb.env.POSTGRES_USER | default "" }}
   POSTGRES_PORT: "5432"
-  TM_DB: {{ quote (printf "postgresql://%s:%s@%s-tasking-manager-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB) }}
+  TM_DB: {{ quote (printf "postgresql+asyncpg://%s:%s@%s-tasking-manager-db/%s" .Values.tmDb.env.POSTGRES_USER .Values.tmDb.env.POSTGRES_PASSWORD .Release.Name .Values.tmDb.env.POSTGRES_DB) }}
   TM_ORG_NAME: {{ quote .Values.tmApi.env.TM_ORG_NAME | default "" }}
   TM_ORG_CODE: {{ quote .Values.tmApi.env.TM_ORG_CODE | default "" }}
   TM_ORG_URL: {{ quote .Values.tmApi.env.TM_ORG_URL | default "" }}
@@ -44,4 +44,6 @@ data:
   TM_REDIRECT_URI: {{ quote .Values.tmApi.env.TM_REDIRECT_URI | default "" }}
   TM_SCOPE: {{ quote .Values.tmApi.env.TM_SCOPE | default "" }}
   TM_APP_API_VERSION: {{ quote .Values.tmApi.env.TM_APP_API_VERSION | default "v2" }}
-{{- end }}
+  OHSOME_STATS_TOKEN: {{ quote .Values.tmApi.env.OHSOME_STATS_TOKEN }}
+  POSTGRES_TEST_DB: {{ quote .Values.tmApi.env.POSTGRES_TEST_DB }}
+{{- end }}

osm-seed/templates/tasking-manager-api/tm-deployment.yaml

@@ -26,7 +26,12 @@ spec:
       initContainers:
         - name: db-upgrade
           image: "{{ .Values.tmApi.image.name }}:{{ .Values.tmApi.image.tag }}"
-          command: ["flask", "db", "upgrade"]
+          command:
+            - alembic
+            - -c
+            - /usr/src/app/migrations/alembic.ini
+            - upgrade
+            - head
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-tm-common-env

osm-seed/values.yaml

@@ -262,9 +262,10 @@ fullHistory:
   schedule: "* * * * *"
   env:
     OVERWRITE_FHISTORY_FILE: false
-    PLANET_EXPORT_METHOD: osmosis
+    PLANET_EXPORT_METHOD: planet-dump-ng
     DUMP_CLOUD_URL: s3://osm-seed/db.dump
     PLANET_EPOCH_DATE: '1970-01-01'
+    PLANET_DUMP_NG_METADATA_URL: https://osmseed.org/metadata/planet-meta.yaml 
   resources:
     enabled: false
     requests:
@@ -348,7 +349,9 @@ planetDump:
   env:
     OVERWRITE_PLANET_FILE: false
     DUMP_CLOUD_URL : s3://osm-seed/db.dump
-    PLANET_EXPORT_METHOD: osmosis
+    PLANET_EXPORT_METHOD: planet-dump-ng
+    PLANET_EPOCH_DATE: '1970-01-01'
+    PLANET_DUMP_NG_METADATA_URL: https://osmseed.org/metadata/planet-meta.yaml 
   resources:
     enabled: false
     requests:
@@ -726,6 +729,8 @@ tmApi:
     TM_DEFAULT_LOCALE: "en"
     TM_APP_API_URL: ""
     TM_APP_BASE_URL: ""
+    OHSOME_STATS_TOKEN: "dummy-token-for-migrations"
+    POSTGRES_TEST_DB: "postgresql://user:pass@db:5432/test_db"
   resources:
     enabled: false
     requests: