feat: Feature Store

ors-api/src/main/java/org/heigit/ors/api/services/DynamicDataService.java

@@ -14,10 +14,9 @@
 
 import java.sql.*;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.List;
-import java.util.TimeZone;
+import java.util.*;
+
+import static org.heigit.ors.api.util.Utils.isJUnitTest;
 
 @Service
 public class DynamicDataService {
@@ -28,6 +27,7 @@ public class DynamicDataService {
     private final String storePassword;
     private final Boolean enabled;
     private final List<RoutingProfile> enabledProfiles = new ArrayList<>();
+    private final Map<String, Instant> lastUpdateTimestamps = new HashMap<>();
 
     @Autowired
     public DynamicDataService(EngineProperties engineProperties, @Value("${ors.engine.dynamic_data.enabled:false}") Boolean enabled, @Value("${ors.engine.dynamic_data.store_url:}") String storeURL, @Value("${ors.engine.dynamic_data.store_user:}") String storeUser, @Value("${ors.engine.dynamic_data.store_pass:}") String storePassword) {
@@ -66,20 +66,20 @@ private void initialize() {
         });
         if (enabledProfiles.isEmpty()) {
             LOGGER.warn("Dynamic data module activated but no profile has custom models enabled.");
-        } else {
-            for (RoutingProfile profile : enabledProfiles) {
-                for (String datasetName : profile.getProfileConfiguration().getService().getDynamicData().getEnabledDynamicDatasets()) {
-                    LOGGER.info("Adding dynamic data support for dataset '" + datasetName + "' to profile '" + profile.name() + "'.");
-                    profile.addDynamicData(datasetName);
-                }
-                fetchDynamicData(profile);
+        }
+        for (RoutingProfile profile : enabledProfiles) {
+            for (String datasetName : profile.getProfileConfiguration().getService().getDynamicData().getEnabledDynamicDatasets()) {
+                LOGGER.info("Adding dynamic data support for dataset '" + datasetName + "' to profile '" + profile.name() + "'.");
+                profile.addDynamicData(datasetName);
             }
-            LOGGER.info("Dynamic data service initialized for profiles: " + enabledProfiles.stream().map(RoutingProfile::name).toList());
+            fetchDynamicData(profile);
         }
+        LOGGER.info("Dynamic data service initialized for profiles: " + enabledProfiles.stream().map(RoutingProfile::name).toList());
     }
 
     public void reinitialize() {
         enabledProfiles.clear();
+        lastUpdateTimestamps.clear();
         this.initialize();
     }
 
@@ -97,7 +97,7 @@ public void update() {
     private void fetchDynamicData(RoutingProfile profile) {
         if (StringUtility.isNullOrEmpty(storeURL))
             return;
-        String graphDate = profile.getGraphhopper().getGraphHopperStorage().getProperties().get("datareader.data.date");
+        String graphDate = getGraphDate(profile);
         try (Connection con = DriverManager.getConnection(storeURL, storeUser, storePassword)) {
             if (con == null) {
                 LOGGER.error("Database connection is null, cannot fetch dynamic data.");
@@ -109,24 +109,44 @@ private void fetchDynamicData(RoutingProfile profile) {
                         WHERE dataset_key = ?
                         AND profile = ?
                         AND graph_timestamp = ?
+                        AND timestamp > ?
                     """)) {
                 stmt.setTimestamp(3, Timestamp.from(Instant.parse(graphDate)), Calendar.getInstance(TimeZone.getTimeZone("UTC")));
                 for (String key : profile.getDynamicDatasets()) {
-                    LOGGER.debug("Fetching dynamic data for profile '" + profile.name() + "', dataset '" + key + "', graph date '" + graphDate + "'.");
+                    String lastUpdateKey = profile.name() + "." + key;
+                    Instant lastUpdateTimestamp = lastUpdateTimestamps.getOrDefault(lastUpdateKey, Instant.EPOCH);
+                    int fetchedCount = 0;
                     stmt.setString(1, key);
                     stmt.setString(2, profile.name());
+                    stmt.setTimestamp(4, Timestamp.from(lastUpdateTimestamp), Calendar.getInstance(TimeZone.getTimeZone("UTC")));
                     ResultSet result = stmt.executeQuery();
                     while (result.next()) {
                         int edgeID = result.getInt("edge_id");
                         String value = result.getString("value");
+                        Instant ts = result.getTimestamp("timestamp", Calendar.getInstance(TimeZone.getTimeZone("UTC"))).toInstant();
                         LOGGER.trace("Update dynamic data in dataset '" + key + "' for profile '" + profile.name() + "': edge ID " + edgeID + " -> value '" + value + "'");
-                        profile.updateDynamicData(key, edgeID, value);
+                        if (result.getBoolean("deleted")) {
+                            profile.unsetDynamicData(key, edgeID);
+                        } else {
+                            profile.updateDynamicData(key, edgeID, value);
+                        }
+                        fetchedCount++;
+                        if (lastUpdateTimestamp.isBefore(ts)) {
+                            lastUpdateTimestamp = ts;
+                            lastUpdateTimestamps.put(lastUpdateKey, ts);
+                        }
                     }
+                    LOGGER.debug("Fetched " + fetchedCount + " rows for profile '" + profile.name() + "', dataset '" + key + "', graph date '" + graphDate + "', lastUpdateTimestamp '" + lastUpdateTimestamp + "'.");
                 }
             }
-            // TODO implement more sophisticated update strategy, e.g. only update changed values, inform feature store about new graph date, etc.
         } catch (SQLException e) {
             LOGGER.error("Error during dynamic data update: " + e.getMessage(), e);
         }
     }
+
+    private static String getGraphDate(RoutingProfile profile) {
+        return isJUnitTest() ?
+                "2024-09-08T20:21:00Z" :
+                profile.getGraphhopper().getGraphHopperStorage().getProperties().get("datareader.import.date");
+    }
 }

ors-api/src/main/java/org/heigit/ors/api/services/MatchingService.java

@@ -13,11 +13,17 @@
 import org.heigit.ors.routing.RoutingProfile;
 import org.heigit.ors.routing.RoutingProfileManager;
 import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
 import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryFactory;
 import org.locationtech.jts.io.geojson.GeoJsonReader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
 @Service
 public class MatchingService extends ApiService {
 
@@ -32,7 +38,7 @@ public MatchingInfo generateMatchingInformation(String profileName) throws Statu
         if (rp == null) {
             throw new InternalServerException(MatchingErrorCodes.UNKNOWN, "Unable to find an appropriate routing profile.");
         }
-        String graphTimestamp = rp.getGraphhopper().getGraphHopperStorage().getProperties().get("datareader.data.date");
+        String graphTimestamp = rp.getGraphhopper().getGraphHopperStorage().getProperties().get("datareader.import.date");
         return new MatchingInfo(graphTimestamp);
     }
 
@@ -79,16 +85,18 @@ private MatchingRequest convertMatchingRequest(MatchingApiRequest matchingApiReq
 
         GeoJsonReader reader = new GeoJsonReader();
         try {
-            Geometry geometry = reader.read(features.toJSONString());
-            if (geometry == null) {
-                throw new StatusCodeException(StatusCode.BAD_REQUEST, MatchingErrorCodes.INVALID_PARAMETER_VALUE, "geometry is null");
-            } else if (geometry.isEmpty()) {
-                throw new StatusCodeException(StatusCode.BAD_REQUEST, MatchingErrorCodes.INVALID_PARAMETER_VALUE, "geometry is empty");
-            }
-            if (geometry.getGeometryType().equals("GeometryCollection") && geometry.getNumGeometries() == 1) {
-                geometry = geometry.getGeometryN(0);
+            List<Geometry> geometries = new ArrayList<>();
+            for (Map<String, Object> feature : (List<Map<String, Object>>) features.get("features")) {
+                Geometry geometry = reader.read(JSONValue.toJSONString(feature));
+                if (geometry == null) {
+                    throw new StatusCodeException(StatusCode.BAD_REQUEST, MatchingErrorCodes.INVALID_PARAMETER_VALUE, "geometry is null");
+                } else if (geometry.isEmpty()) {
+                    throw new StatusCodeException(StatusCode.BAD_REQUEST, MatchingErrorCodes.INVALID_PARAMETER_VALUE, "geometry is empty");
+                }
+                geometry.setUserData(feature.get("properties"));
+                geometries.add(geometry);
             }
-            matchingRequest.setGeometry(geometry);
+            matchingRequest.setGeometry(new GeometryFactory().createGeometryCollection(geometries.toArray(new Geometry[0])));
         } catch (Exception e) {
             throw new StatusCodeException(StatusCode.BAD_REQUEST, MatchingErrorCodes.INVALID_PARAMETER_VALUE, "invalid GeoJSON format: %s".formatted(e.getMessage()));
         }

ors-api/src/main/java/org/heigit/ors/api/util/Utils.java

@@ -0,0 +1,32 @@
+/*
+ *  Licensed to GIScience Research Group, Heidelberg University (GIScience)
+ *
+ *   	 http://www.giscience.uni-hd.de
+ *   	 http://www.heigit.org
+ *
+ *  under one or more contributor license agreements. See the NOTICE file
+ *  distributed with this work for additional information regarding copyright
+ *  ownership. The GIScience licenses this file to you under the Apache License,
+ *  Version 2.0 (the "License"); you may not use this file except in compliance
+ *  with the License. You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.heigit.ors.api.util;
+
+public class Utils {
+    public static boolean isJUnitTest() {
+        for (StackTraceElement element : Thread.currentThread().getStackTrace()) {
+            if (element.getClassName().startsWith("org.junit.")) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

ors-api/src/main/resources/application.yml

@@ -206,6 +206,8 @@ ors:
             WaySurfaceType:
             RoadAccessRestrictions:
               use_for_warnings: true
+          encoded_values:
+            road_environment: true
         service:
           execution:
             methods:

ors-api/src/test/java/org/heigit/ors/apitests/dynamicrouting/ResultTest.java

@@ -72,29 +72,35 @@ CREATE TABLE features (
                         feature_id INTEGER NOT NULL,
                         dataset_key VARCHAR(255) NOT NULL,
                         value VARCHAR(20) NOT NULL,
+                        geom GEOMETRY(Geometry, 4326) NOT NULL,
+                        geojson JSONB,
+                        timestamp TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+                        deleted BOOLEAN DEFAULT FALSE,
                         PRIMARY KEY (feature_id, dataset_key)
                     );
                     """);
             connection.createStatement().execute("""
                     CREATE TABLE mappings (
                         feature_id INTEGER NOT NULL,
-                        graph_timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+                        graph_timestamp TIMESTAMP NOT NULL,
                         profile VARCHAR(20) NOT NULL,
                         edge_id INTEGER NOT NULL,
+                        timestamp TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+                        deleted BOOLEAN DEFAULT FALSE,
                         PRIMARY KEY (feature_id, graph_timestamp, profile, edge_id)
                     );
                     """);
             connection.createStatement().execute("""
-                    CREATE VIEW feature_map AS
-                    SELECT f.feature_id, f.dataset_key, m.graph_timestamp, m.profile, m.edge_id, f.value
-                    FROM features f
-                    JOIN mappings m ON f.feature_id = m.feature_id
+                        CREATE VIEW feature_map AS
+                        SELECT f.feature_id, f.dataset_key, m.graph_timestamp, m.profile, m.edge_id, f.value, m.deleted, m.timestamp
+                        FROM features f
+                        JOIN mappings m ON f.feature_id = m.feature_id
                     """);
             connection.createStatement().execute("""
                     INSERT INTO features VALUES
-                    (1, 'logie_borders','CLOSED'),
-                    (2, 'logie_bridges','RESTRICTED'),
-                    (3, 'logie_roads','RESTRICTED');
+                    (1, 'logie_borders','CLOSED', 'POINT(0 0)'),
+                    (2, 'logie_bridges','RESTRICTED', 'POINT(0 0)'),
+                    (3, 'logie_roads','RESTRICTED', 'POINT(0 0)');
                     """);
             connection.createStatement().execute("""
                     INSERT INTO mappings VALUES