From 5e22af5a6bcbee0f81fd884ca9fd5521d61fe962 Mon Sep 17 00:00:00 2001 From: takb Date: Thu, 27 Nov 2025 17:37:59 +0100 Subject: [PATCH 01/11] feat: when running in preparation_mode, pack graph --- .../java/org/heigit/ors/api/Application.java | 5 +- .../listeners/ORSInitContextListener.java | 20 ++++-- .../listeners/ORSInitContextListenerTest.java | 6 +- .../ors/apitests/PreparationModeTest.java | 61 ++++++++++++++++++ .../application-preparation-mode-test.yml | 33 ++++++++++ .../ors/config/profile/BuildProperties.java | 7 +++ .../heigit/ors/routing/RoutingProfile.java | 62 ++++++++++++++++++- .../ors/routing/RoutingProfileManager.java | 9 +-- .../manage/PersistedGraphBuildInfo.java | 3 + .../manage/local/ORSGraphFileManager.java | 13 ++++ 10 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java create mode 100644 ors-api/src/test/resources/application-preparation-mode-test.yml diff --git a/ors-api/src/main/java/org/heigit/ors/api/Application.java b/ors-api/src/main/java/org/heigit/ors/api/Application.java index 89b99b5ceb..b5bea6c902 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/Application.java +++ b/ors-api/src/main/java/org/heigit/ors/api/Application.java @@ -14,6 +14,7 @@ import org.springframework.boot.web.servlet.ServletListenerRegistrationBean; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.context.annotation.Bean; +import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @@ -40,9 +41,9 @@ public static void main(String[] args) { } @Bean("orsInitContextListenerBean") - public ServletListenerRegistrationBean createORSInitContextListenerBean(ApiEngineProperties apiEngineProperties, GraphService graphService) { + public ServletListenerRegistrationBean createORSInitContextListenerBean(ApiEngineProperties apiEngineProperties, GraphService graphService, Environment environment) { ServletListenerRegistrationBean bean = new ServletListenerRegistrationBean<>(); - bean.setListener(new ORSInitContextListener(apiEngineProperties, graphService)); + bean.setListener(new ORSInitContextListener(apiEngineProperties, graphService, environment)); return bean; } } diff --git a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java index 5822c20dcc..1208534cd4 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java +++ b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java @@ -35,6 +35,7 @@ import org.heigit.ors.routing.graphhopper.extensions.manage.ORSGraphManager; import org.heigit.ors.util.FormatUtility; import org.heigit.ors.util.StringUtility; +import org.springframework.core.env.Environment; import org.springframework.core.io.ClassPathResource; import java.io.FileOutputStream; @@ -46,6 +47,7 @@ public class ORSInitContextListener implements ServletContextListener { private static final Logger LOGGER = Logger.getLogger(ORSInitContextListener.class); private final EngineProperties engineProperties; private final GraphService graphService; + private Environment environment; @Override public void contextInitialized(ServletContextEvent contextEvent) { @@ -59,6 +61,15 @@ public void contextInitialized(ServletContextEvent contextEvent) { LOGGER.info("Initializing ORS..."); graphService.setIsActivatingGraphs(true); RoutingProfileManager routingProfileManager = new RoutingProfileManager(engineProperties, AppInfo.GRAPH_VERSION); + if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { + LOGGER.info("Running in preparation mode, all enabled graphs are built, job is done."); + if (environment.getActiveProfiles().length == 0) { // only exit if no active profile is set (i.e., not in test mode) + RoutingProfileManagerStatus.setShutdown(true); + } + } + if (RoutingProfileManagerStatus.isShutdown()) { + System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); + } for (RoutingProfile profile : routingProfileManager.getUniqueProfiles()) { ORSGraphManager orsGraphManager = profile.getGraphhopper().getOrsGraphManager(); if (orsGraphManager != null && orsGraphManager.useGraphRepository()) { @@ -66,10 +77,6 @@ public void contextInitialized(ServletContextEvent contextEvent) { graphService.addGraphManagerInstance(orsGraphManager); } } - if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { - LOGGER.info("Running in preparation mode, all enabled graphs are built, job is done."); - System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); - } } catch (Exception e) { LOGGER.warn("Unable to initialize ORS due to an unexpected exception: " + e); } finally { @@ -103,10 +110,11 @@ public void contextDestroyed(ServletContextEvent contextEvent) { try { LOGGER.info("Shutting down openrouteservice %s and releasing resources.".formatted(AppInfo.getEngineInfo())); FormatUtility.unload(); - if (RoutingProfileManagerStatus.isReady()) - RoutingProfileManager.getInstance().destroy(); StatisticsProviderFactory.releaseProviders(); LogFactory.release(Thread.currentThread().getContextClassLoader()); + RoutingProfileManager.getInstance().destroy(); + } catch (UnsupportedOperationException e) { + LOGGER.warn("RoutingProfileManager was not initialized, nothing to destroy."); } catch (Exception e) { LOGGER.error(e.getMessage()); } diff --git a/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java b/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java index ab6fd38daa..4a11bb2104 100644 --- a/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java +++ b/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -14,9 +15,12 @@ class ORSInitContextListenerTest { @Autowired private GraphService graphService; + @Autowired + private Environment environment; + @Test void testConfigurationOutputTarget() { - ORSInitContextListener orsInitContextListener = new ORSInitContextListener(new EngineProperties(), graphService); + ORSInitContextListener orsInitContextListener = new ORSInitContextListener(new EngineProperties(), graphService, environment); EngineProperties engineProperties = new EngineProperties(); assertNull(orsInitContextListener.configurationOutputTarget(engineProperties), "default should return null"); diff --git a/ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java b/ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java new file mode 100644 index 0000000000..cb43fd0a23 --- /dev/null +++ b/ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java @@ -0,0 +1,61 @@ +package org.heigit.ors.apitests; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.heigit.ors.api.Application; +import org.heigit.ors.config.EngineProperties; +import org.heigit.ors.routing.RoutingProfileManager; +import org.heigit.ors.routing.RoutingProfileManagerStatus; +import org.heigit.ors.routing.graphhopper.extensions.manage.PersistedGraphBuildInfo; +import org.heigit.ors.util.AppInfo; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.*; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = Application.class) +@ActiveProfiles("preparation-mode-test") +@DirtiesContext +@Order(value = Integer.MAX_VALUE) +class PreparationModeTest { + + @Autowired + private EngineProperties engineProperties; + + @Test + void testPrepMode() throws IOException { + assertTrue(engineProperties.getPreparationMode()); + await().atMost(Duration.ofSeconds(60)).until(RoutingProfileManagerStatus::isReady); + assertTrue(RoutingProfileManagerStatus.isReady()); + + Path graphPath = RoutingProfileManager.getInstance().getRoutingProfile("driving-car-preparation").getProfileProperties().getGraphPath(); + String fileName = "test_heidelberg_%s_driving-car".formatted(AppInfo.GRAPH_VERSION); + File yamlFile = Paths.get(graphPath.toString(), fileName + ".yml").toFile(); + assertTrue(yamlFile.exists(), "Graph info YAML should exist"); + assertTrue(Paths.get(graphPath.toString(), fileName + ".ghz").toFile().exists(), "Packed graph should exist"); + assertFalse(Paths.get(graphPath.toString(), "driving-car-preparation").toFile().exists(), "Build graph dir should not exist"); + + // parse yaml file and check for expected values + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + PersistedGraphBuildInfo graphBuildInfo = mapper.readValue(yamlFile, PersistedGraphBuildInfo.class); + assertNotNull(graphBuildInfo.getGraphBuildDate(), "Graph build date should be set"); + assertNotNull(graphBuildInfo.getOsmDate(), "OSM date should be set"); + assertNotNull(graphBuildInfo.getGraphVersion(), "Graph version should be set"); + assertNotNull(graphBuildInfo.getGraphSizeBytes(), "Graph size bytes should be set"); + assertNotNull(graphBuildInfo.getProfileProperties(), "Profile properties should be set"); + } +} + + diff --git a/ors-api/src/test/resources/application-preparation-mode-test.yml b/ors-api/src/test/resources/application-preparation-mode-test.yml new file mode 100644 index 0000000000..98dace0b3a --- /dev/null +++ b/ors-api/src/test/resources/application-preparation-mode-test.yml @@ -0,0 +1,33 @@ +ors: + engine: + preparation_mode: true + profile_default: + graph_path: graphs-apitests + build: + source_file: ./src/test/files/heidelberg.test.pbf + coverage: heidelberg + profile_group: test + elevation: false + preparation: + methods: + fastisochrones: + enabled: false + ch: + enabled: false + lm: + enabled: false + core: + enabled: false + profiles: + driving-car-preparation: + enabled: true + encoder_name: driving-car + build: + encoder_options: + turn_costs: false + block_fords: false + use_acceleration: false + maximum_grade_level: 1 + conditional_access: false + conditional_speed: false + enable_custom_models: false diff --git a/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java b/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java index 1da286ba64..0c845717e7 100644 --- a/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java +++ b/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java @@ -17,6 +17,11 @@ public class BuildProperties { @JsonIgnore private Path sourceFile; + @JsonIgnore + private String profileGroup; + @JsonIgnore + private String coverage; + @JsonProperty("elevation") private Boolean elevation; @JsonProperty("elevation_smoothing") @@ -56,6 +61,8 @@ public BuildProperties(String ignored) { } public void merge(BuildProperties other) { + profileGroup = ofNullable(profileGroup).orElse(other.profileGroup); + coverage = ofNullable(coverage).orElse(other.coverage); elevation = ofNullable(elevation).orElse(other.elevation); elevationSmoothing = ofNullable(elevationSmoothing).orElse(other.elevationSmoothing); encoderFlagsSize = ofNullable(encoderFlagsSize).orElse(other.encoderFlagsSize); diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index 27894a37aa..b0f0957679 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -26,16 +26,28 @@ import org.heigit.ors.routing.graphhopper.extensions.storages.builders.BordersGraphStorageBuilder; import org.heigit.ors.routing.graphhopper.extensions.storages.builders.GraphStorageBuilder; import org.heigit.ors.routing.pathprocessors.ORSPathProcessorFactory; +import org.heigit.ors.util.AppInfo; import org.heigit.ors.util.TimeUtility; import org.json.simple.JSONObject; +import org.springframework.util.FileSystemUtils; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; import java.util.function.Function; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; /** * This class generates {@link RoutingProfile} classes and is used by mostly all service classes e.g. @@ -141,9 +153,55 @@ public ORSGraphHopper initGraphHopper(RoutingProfileLoadContext loadCntx) throws if (!file2.exists()) Files.write(pathTimestamp, Long.toString(file.length()).getBytes()); } + + if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { + prepareGeneratedGraphForUpload(); + } return gh; } + private void prepareGeneratedGraphForUpload() { + LOGGER.info("Running in preparation_mode, preparing graph for upload"); + String graphName = String.join("_", + profileProperties.getBuild().getProfileGroup(), + profileProperties.getBuild().getCoverage(), + AppInfo.GRAPH_VERSION, + profileProperties.getEncoderName().toString() + ); + Path graphFilesPath = profileProperties.getGraphPath().resolve(profileProperties.getProfileName()); + Path graphInfoSrc = graphFilesPath.resolve("graph_build_info.yml"); + Path graphInfoDst = profileProperties.getGraphPath().resolve(graphName + ".yml"); + Path graphArchiveDst = profileProperties.getGraphPath().resolve(graphName + ".ghz"); + try { + Files.copy(graphInfoSrc, graphInfoDst, REPLACE_EXISTING); + LOGGER.info("Copied graph info from %s to %s".formatted(graphInfoSrc.toString(), graphInfoDst.toString())); + // create a zip archive of all files in graphFilesPath with .ghz extension + try (FileOutputStream fos = new FileOutputStream(graphArchiveDst.toFile()); ZipOutputStream zos = new ZipOutputStream(fos)) { + try (Stream src = Files.walk(graphFilesPath)) { + src.filter(path -> !Files.isDirectory(path)) + .forEach((path -> { + try (FileInputStream fis = new FileInputStream((path).toFile())) { + ZipEntry zipEntry = new ZipEntry(graphFilesPath.relativize(path).toString()); + zos.putNextEntry(zipEntry); + byte[] buffer = new byte[1024]; + int len; + while ((len = fis.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } catch (IOException e) { + LOGGER.error("Failed to add file %s to archive: %s".formatted(path.toString(), e.getMessage())); + } + })); + } + } + LOGGER.info("Created archive %s".formatted(graphArchiveDst.toString())); + FileSystemUtils.deleteRecursively(graphFilesPath); + LOGGER.info("Deleted graph files in %s".formatted(graphFilesPath.toString())); + } catch (IOException e) { + LOGGER.error("Failed preparing files: %s".formatted(e.getMessage())); + } + } + public boolean hasCHProfile(String profileName) { boolean hasCHProfile = false; for (CHProfile chProfile : getGraphhopper().getCHPreparationHandler().getCHProfiles()) { @@ -211,7 +269,7 @@ public List getDynamicDatasets() { } public void updateDynamicData(String key, int edgeID, String value) { - Function stateFromString = null; + Function stateFromString = null; switch (key) { case LogieBorders.KEY: stateFromString = s -> LogieBorders.valueOf(s.replace(" ", "_").toUpperCase()); diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index 02a99391a6..15fb2c93f0 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -34,13 +34,8 @@ public class RoutingProfileManager { private static RoutingProfileManager instance; public RoutingProfileManager(EngineProperties config, String graphVersion) { - if (instance == null) { - instance = this; - initialize(config, graphVersion); - if (RoutingProfileManagerStatus.isShutdown()) { - System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); - } - } + instance = this; + initialize(config, graphVersion); } public static synchronized RoutingProfileManager getInstance() { diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java index 0e014dd39f..31d04a4c49 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java @@ -25,6 +25,9 @@ public class PersistedGraphBuildInfo { @JsonProperty("graph_version") private String graphVersion; + @JsonProperty("graph_size_bytes") + private Long graphSizeBytes; + @JsonProperty("profile_properties") private ProfileProperties profileProperties; diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java index 3054920a15..c546bc0d9f 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java @@ -357,9 +357,22 @@ public void writeOrsGraphBuildInfoFileIfNotExists(ORSGraphHopper gh) { persistedGraphBuildInfo.setGraphVersion(AppInfo.GRAPH_VERSION); persistedGraphBuildInfo.setProfileProperties(profileProperties); + // Add the total size of the graph folder + File graphFolder = getActiveGraphDirectory(); + persistedGraphBuildInfo.setGraphSizeBytes(calculateFolderSizeInBytes(graphFolder)); + ORSGraphFileManager.writeOrsGraphBuildInfo(persistedGraphBuildInfo, activeGraphBuildInfoFile); } + private long calculateFolderSizeInBytes(File graphFolder) { + try { + return FileUtils.sizeOfDirectory(graphFolder); + } catch (Exception e) { + LOGGER.error("Could not calculate size of graph folder %s: %s".formatted(graphFolder.getAbsolutePath(), e.getMessage())); + return 0; + } + } + Date getDateFromGhProperty(GraphHopper gh, String ghProperty) { try { String importDateString = gh.getGraphHopperStorage().getProperties().get(ghProperty); From 684e66fb29dd0db31a41e2b6f2823e71e60a965d Mon Sep 17 00:00:00 2001 From: takb Date: Thu, 27 Nov 2025 18:09:40 +0100 Subject: [PATCH 02/11] fix: call it graph_extent instead of coverage, corresponding to how it's called in the repo client settings --- .../src/test/resources/application-preparation-mode-test.yml | 2 +- .../java/org/heigit/ors/config/profile/BuildProperties.java | 4 ++-- .../src/main/java/org/heigit/ors/routing/RoutingProfile.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ors-api/src/test/resources/application-preparation-mode-test.yml b/ors-api/src/test/resources/application-preparation-mode-test.yml index 98dace0b3a..dbceb40bda 100644 --- a/ors-api/src/test/resources/application-preparation-mode-test.yml +++ b/ors-api/src/test/resources/application-preparation-mode-test.yml @@ -5,7 +5,7 @@ ors: graph_path: graphs-apitests build: source_file: ./src/test/files/heidelberg.test.pbf - coverage: heidelberg + graph_extent: heidelberg profile_group: test elevation: false preparation: diff --git a/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java b/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java index 0c845717e7..eb7a473f6e 100644 --- a/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java +++ b/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java @@ -20,7 +20,7 @@ public class BuildProperties { @JsonIgnore private String profileGroup; @JsonIgnore - private String coverage; + private String graphExtent; @JsonProperty("elevation") private Boolean elevation; @@ -62,7 +62,7 @@ public BuildProperties(String ignored) { public void merge(BuildProperties other) { profileGroup = ofNullable(profileGroup).orElse(other.profileGroup); - coverage = ofNullable(coverage).orElse(other.coverage); + graphExtent = ofNullable(graphExtent).orElse(other.graphExtent); elevation = ofNullable(elevation).orElse(other.elevation); elevationSmoothing = ofNullable(elevationSmoothing).orElse(other.elevationSmoothing); encoderFlagsSize = ofNullable(encoderFlagsSize).orElse(other.encoderFlagsSize); diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index b0f0957679..ca45ad6054 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -164,7 +164,7 @@ private void prepareGeneratedGraphForUpload() { LOGGER.info("Running in preparation_mode, preparing graph for upload"); String graphName = String.join("_", profileProperties.getBuild().getProfileGroup(), - profileProperties.getBuild().getCoverage(), + profileProperties.getBuild().getGraphExtent(), AppInfo.GRAPH_VERSION, profileProperties.getEncoderName().toString() ); From e52c3e47e22190fc97f64fa34325aaf61ccf2f66 Mon Sep 17 00:00:00 2001 From: takb Date: Thu, 27 Nov 2025 18:10:21 +0100 Subject: [PATCH 03/11] docs: configuration YAML changes, CHANGEDOC --- CHANGELOG.md | 1 + .../configuration/engine/profiles/build.md | 6 ++++-- .../configuration/engine/profiles/repo.md | 16 ++++++++-------- .../technical-details/graph-repo-client/index.md | 4 ++-- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8da3559434..4805d0cd37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Releasing is documented in RELEASE.md ### Changed - enhanced status endpoint to expose dynamic data statistics and feature store metrics ([#2156](https://github.com/GIScience/openrouteservice/pull/2156)) +- when running in `preparation_mode`, pack the graph for upload ([#2185](https://github.com/GIScience/openrouteservice/pull/2185)) ### Deprecated diff --git a/docs/run-instance/configuration/engine/profiles/build.md b/docs/run-instance/configuration/engine/profiles/build.md index 611496fdc3..5074a70b2d 100644 --- a/docs/run-instance/configuration/engine/profiles/build.md +++ b/docs/run-instance/configuration/engine/profiles/build.md @@ -6,6 +6,8 @@ graphs for the specified profile. | key | type | description | default value | |----------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------| | source_file | string | The OSM file to be used, supported formats are `.osm`, `.osm.gz`, `.osm.zip` and `.pbf` | `ors-api/src/test/files/heidelberg.test.pbf` | +| graph_extent | string | Used only in `preparation_mode` to determine the `graph_extent` part of the filename of the packed archive. See [graph repo client](/technical-details/graph-repo-client/) for more information. | _NA_ | +| profile_group | string | Used only in `preparation_mode` to determine the `profile_group` part of the filename of the packed archive. See [graph repo client](/technical-details/graph-repo-client/) for more information. | _NA_ | | elevation | boolean | Specifies whether to download and use elevation data. If true, `cache_path` and `provider` must be set in ors.engine.elevation as well. | `false` | | elevation_smoothing | boolean | Smooth out elevation data | `false` | | traffic | boolean | Use traffic data if available | `false` | @@ -34,7 +36,7 @@ Properties beneath `ors.engine.profiles..build.encoder_options`: | problematic_speed_factor | number | wheelchair | Travel speeds on edges classified as problematic for wheelchair users are multiplied by this factor, use to set slow traveling speeds on such ways | `0.7` | | turn_costs | boolean | car, hgv, bike-* | Should turn restrictions be respected | `true` | | use_acceleration | boolean | car, hgv | Models how a vehicle would accelerate on the road segment to the maximum allowed speed. In practice it reduces speed on shorter road segments such as ones between nearby intersections in a city | `true` | -| enable_custom_models | boolean | * | Enables whether the profile is prepared to support custom models. Also see the corresponding parameter `allow_custom_models` in the [service properties](service.md). | `false` | +| enable_custom_models | boolean | * | Enables whether the profile is prepared to support custom models. Also see the corresponding parameter `allow_custom_models` in the [service properties](service.md). | `false` | ## `preparation` @@ -160,7 +162,7 @@ country borders, compatible with any profile type. | key | type | description | example value | |--------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------| | boundaries | string | The path to a GeoJSON file containing polygons representing country borders. Ignored if `preprocessed = true` is set. | `borders.geojson.tar.gz` | -| preprocessed | boolean | Indicates whether the source OSM file has been enriched with country data. If set to `true` then country codes are read from `country` node tags rather than being resolved at build time based on geometries provided in `boundaries`. | `true` | +| preprocessed | boolean | Indicates whether the source OSM file has been enriched with country data. If set to `true` then country codes are read from `country` node tags rather than being resolved at build time based on geometries provided in `boundaries`. | `true` | | ids | string | Path to a csv file containing a unique id for each country, its local name and its English name | `ids.csv` | | openborders | string | Path to a csv file containing pairs of countries where the borders are open (i.e. Schengen borders) | `openborders.csv` | diff --git a/docs/run-instance/configuration/engine/profiles/repo.md b/docs/run-instance/configuration/engine/profiles/repo.md index 4c7e43e47c..77a940b798 100644 --- a/docs/run-instance/configuration/engine/profiles/repo.md +++ b/docs/run-instance/configuration/engine/profiles/repo.md @@ -6,14 +6,14 @@ and if graph management is enabled [ors.engine.graph_management.enabled](/run-instance/configuration/engine/graph-management.md), openrouteservice will use the specified repository to load the graph data. -| key | type | description | example values | -|--------------------------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| repository_uri | string | The base URL (for a http or minio repository) or path (for a file system repository) | * `http://some.domain.ors`
* `file:///absolute/path`
* `/absolute/path`
* `relative/path` | -| repository_name | string | An editorial name representing the target group or vendor for the graphs. | `public`, `my-organization` | -| repository_profile_group | string | A group of profiles with specific characteristics lke the usage of traffic data or the pre-calculation of advanced data structures e.g. for fastisochrones | `traffic`, `fastiso` | -| graph_extent | string | The geographic region covered by the graph. This corresponds to the OSM PBF file used for graph building. | `planet` | -| repository_user | string | The user name (key_id) for authentication at the repository. Currently only used by the minio client. | `username` | -| repository_password | string | The password (access_key) for authentication at the repository. Currently only used by the minio client. | `password` | +| key | type | description | example values | +|--------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| repository_uri | string | The base URL (for a http or minio repository) or path (for a file system repository) | * `http://some.domain.ors`
* `file:///absolute/path`
* `/absolute/path`
* `relative/path` | +| repository_name | string | An editorial name representing the target group or vendor for the graphs. | `public`, `my-organization` | +| repository_profile_group | string | A group of profiles with specific characteristics like the usage of traffic data or the pre-calculation of advanced data structures e.g. for fastisochrones | `traffic`, `fastiso` | +| graph_extent | string | The geographic region covered by the graph. This corresponds to the OSM PBF file used for graph building. | `planet` | +| repository_user | string | The user name (key_id) for authentication at the repository. Currently only used by the minio client. | `username` | +| repository_password | string | The password (access_key) for authentication at the repository. Currently only used by the minio client. | `password` | Each routing profile can have its individual repository parameters, which makes it possible to have routing profiles from different repositories diff --git a/docs/technical-details/graph-repo-client/index.md b/docs/technical-details/graph-repo-client/index.md index e60f6efe53..3e977c6a23 100644 --- a/docs/technical-details/graph-repo-client/index.md +++ b/docs/technical-details/graph-repo-client/index.md @@ -103,7 +103,7 @@ In the following graph repository example there are graphs for three target grou http://repos.provider.com/graphs/ # repo base URL ├── project-b # repo name │ └── basic # profile group -│ └── switzerland # coverage +│ └── switzerland # graph extent │ └── 2 # graph version │ ├── switzerland_2_cycling-mountain.ghz # compacted graph │ ├── switzerland_2_cycling-mountain.yml # metadata @@ -113,7 +113,7 @@ http://repos.provider.com/graphs/ # repo base URL │ └── switzerland_2_cycling-street.yml # metadata ├── public # repo name │ └── basic # profile group -│ └── europe # coverage +│ └── europe # graph extent │ ├── 1 # graph version │ │ ├── europe_1_driving-car.ghz # compacted graph │ │ ├── europe_1_driving-car.yml # metadata From def1687eb56c89d3622d0bb9c4a4c97f8b047498 Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 00:45:02 +0100 Subject: [PATCH 04/11] fix: apitests spanning multiple contexts --- .../src/main/java/org/heigit/ors/routing/RoutingProfile.java | 3 +++ .../java/org/heigit/ors/routing/RoutingProfileManager.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index ca45ad6054..b95fb0f930 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -228,6 +228,9 @@ public ProfileProperties getProfileConfiguration() { } public void close() { + synchronized (lockObj) { + profileIdentifier = 0; + } mGraphHopper.close(); } diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index 15fb2c93f0..31d57de70c 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -34,6 +34,11 @@ public class RoutingProfileManager { private static RoutingProfileManager instance; public RoutingProfileManager(EngineProperties config, String graphVersion) { + if (instance != null) { + // TODO: We should really refactor and avoid singleton pattern here + LOGGER.warn("Re-initializing RoutingProfileManager, this should only happen during tests!"); + instance.destroy(); + } instance = this; initialize(config, graphVersion); } From ecda2e92cec57171cd782ced2adf1ad52b0b77e3 Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 10:15:26 +0100 Subject: [PATCH 05/11] fix: make RoutingProfileManager singleton pattern a bit safer --- .../heigit/ors/routing/RoutingProfileManager.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index 31d57de70c..8bef39c252 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -34,13 +34,15 @@ public class RoutingProfileManager { private static RoutingProfileManager instance; public RoutingProfileManager(EngineProperties config, String graphVersion) { - if (instance != null) { - // TODO: We should really refactor and avoid singleton pattern here - LOGGER.warn("Re-initializing RoutingProfileManager, this should only happen during tests!"); - instance.destroy(); + synchronized (RoutingProfileManager.class) { + if (instance != null) { + // TODO: We should really refactor and avoid singleton pattern here + LOGGER.warn("Re-initializing RoutingProfileManager, this should only happen during tests!"); + instance.destroy(); + } + instance = this; + initialize(config, graphVersion); } - instance = this; - initialize(config, graphVersion); } public static synchronized RoutingProfileManager getInstance() { From 16c0e010f92d93b9a5840e99cbf385de09df7c5c Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 12:50:06 +0100 Subject: [PATCH 06/11] fix: make prepareGeneratedGraphForUpload more safe and readable, add test --- .../heigit/ors/routing/RoutingProfile.java | 93 +++++++---- .../PrepareGeneratedGraphForUploadTest.java | 151 ++++++++++++++++++ pom.xml | 26 ++- 3 files changed, 234 insertions(+), 36 deletions(-) create mode 100644 ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index b95fb0f930..164c637696 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -13,6 +13,7 @@ */ package org.heigit.ors.routing; +import com.google.common.base.Strings; import com.graphhopper.config.CHProfile; import com.graphhopper.routing.ev.*; import com.graphhopper.storage.GraphHopperStorage; @@ -42,8 +43,8 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.function.Function; -import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -155,50 +156,80 @@ public ORSGraphHopper initGraphHopper(RoutingProfileLoadContext loadCntx) throws } if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { - prepareGeneratedGraphForUpload(); + prepareGeneratedGraphForUpload(profileProperties, AppInfo.GRAPH_VERSION); } return gh; } - private void prepareGeneratedGraphForUpload() { + /** + * Prepares the generated graph for upload when running in preparation mode. + * + *

This is a static helper method which expects a fully-initialized {@link ProfileProperties} + * instance and prepares the graph files located under {@code profileProperties.getGraphPath()/profileProperties.getProfileName()}. + * The method performs the following steps: + *

    + *
  • Constructs a graph name using the profile group, graph extent, the application graph version and the encoder name, separated by underscores.
    • + *
  • Copies the graph build info file ("graph_build_info.yml") from the profile directory to the graph path, renaming it to "{graphName}.yml".
    • + *
  • Creates a ZIP archive of the graph directory, saving it as "{graphName}.ghz" in the graph path.
    • + *
  • Deletes the original graph directory after archiving.
    • + *
+ * + *

Important behaviour and error handling: + *

    + *
  • The method is defensive: IO errors while copying the graph build info or while creating the archive are logged and cause an early return so no further steps are executed.
    • + *
  • Errors while deleting the original graph directory are caught and logged; the method does not rethrow them.
    • + *
  • This method mutates the filesystem (creates files and archives, and attempts to delete directories). Tests that exercise it should use temporary directories.
    • + *
+ * + * @param profileProperties profile properties holding graph path and profile name; must not be null and must include a non-null graph path and profile name + */ + public static void prepareGeneratedGraphForUpload(ProfileProperties profileProperties, String graphVersion) { LOGGER.info("Running in preparation_mode, preparing graph for upload"); - String graphName = String.join("_", - profileProperties.getBuild().getProfileGroup(), - profileProperties.getBuild().getGraphExtent(), - AppInfo.GRAPH_VERSION, - profileProperties.getEncoderName().toString() - ); + String profileGroup = Strings.isNullOrEmpty(profileProperties.getBuild().getProfileGroup()) ? "unknownGroup" : profileProperties.getBuild().getProfileGroup(); + String graphExtent = Strings.isNullOrEmpty(profileProperties.getBuild().getGraphExtent()) ? "unknownExtent" : profileProperties.getBuild().getGraphExtent(); + String encoderName = Strings.isNullOrEmpty(profileProperties.getEncoderName().toString()) ? "unknownEncoder" : profileProperties.getEncoderName().toString(); + String graphName = String.join("_", profileGroup, graphExtent, graphVersion, encoderName); Path graphFilesPath = profileProperties.getGraphPath().resolve(profileProperties.getProfileName()); - Path graphInfoSrc = graphFilesPath.resolve("graph_build_info.yml"); - Path graphInfoDst = profileProperties.getGraphPath().resolve(graphName + ".yml"); - Path graphArchiveDst = profileProperties.getGraphPath().resolve(graphName + ".ghz"); + + // copy graph_build_info.yml to {graphName}.yml try { + Path graphInfoSrc = graphFilesPath.resolve("graph_build_info.yml"); + Path graphInfoDst = profileProperties.getGraphPath().resolve(graphName + ".yml"); Files.copy(graphInfoSrc, graphInfoDst, REPLACE_EXISTING); LOGGER.info("Copied graph info from %s to %s".formatted(graphInfoSrc.toString(), graphInfoDst.toString())); - // create a zip archive of all files in graphFilesPath with .ghz extension - try (FileOutputStream fos = new FileOutputStream(graphArchiveDst.toFile()); ZipOutputStream zos = new ZipOutputStream(fos)) { - try (Stream src = Files.walk(graphFilesPath)) { - src.filter(path -> !Files.isDirectory(path)) - .forEach((path -> { - try (FileInputStream fis = new FileInputStream((path).toFile())) { - ZipEntry zipEntry = new ZipEntry(graphFilesPath.relativize(path).toString()); - zos.putNextEntry(zipEntry); - byte[] buffer = new byte[1024]; - int len; - while ((len = fis.read(buffer)) > 0) { - zos.write(buffer, 0, len); - } - } catch (IOException e) { - LOGGER.error("Failed to add file %s to archive: %s".formatted(path.toString(), e.getMessage())); - } - })); + } catch (IOException e) { + LOGGER.error("Failed to copy graph build info: %s".formatted(e.getMessage())); + return; + } + + // create a zip archive of all files in graphFilesPath with .ghz extension + Path graphArchiveDst = profileProperties.getGraphPath().resolve(graphName + ".ghz"); + try (FileOutputStream fos = new FileOutputStream(graphArchiveDst.toFile()); ZipOutputStream zos = new ZipOutputStream(fos)) { + for (File file : Objects.requireNonNull(graphFilesPath.toFile().listFiles())) { + if (!Files.isDirectory(file.toPath())) { + try (FileInputStream fis = new FileInputStream(file)) { + ZipEntry zipEntry = new ZipEntry(graphFilesPath.relativize(file.toPath()).toString()); + zos.putNextEntry(zipEntry); + byte[] buffer = new byte[1024]; + int len; + while ((len = fis.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } } } LOGGER.info("Created archive %s".formatted(graphArchiveDst.toString())); + } catch (IOException e) { + LOGGER.error("Failed to create archive: %s".formatted(e.getMessage())); + return; + } + + // delete original graph files + try { FileSystemUtils.deleteRecursively(graphFilesPath); - LOGGER.info("Deleted graph files in %s".formatted(graphFilesPath.toString())); + LOGGER.info("Deleted original graph files at %s after archiving.".formatted(graphFilesPath.toString())); } catch (IOException e) { - LOGGER.error("Failed preparing files: %s".formatted(e.getMessage())); + LOGGER.error("Failed to delete graph files: %s".formatted(e.getMessage())); } } diff --git a/ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java b/ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java new file mode 100644 index 0000000000..bc259e089a --- /dev/null +++ b/ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java @@ -0,0 +1,151 @@ +package org.heigit.ors.routing; + +import org.heigit.ors.common.EncoderNameEnum; +import org.heigit.ors.config.profile.BuildProperties; +import org.heigit.ors.config.profile.ProfileProperties; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.util.FileSystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for {@link RoutingProfile#prepareGeneratedGraphForUpload(ProfileProperties, String)} + * covering success and error branches. + */ +class PrepareGeneratedGraphForUploadTest { + private Path tempRoot; + private static final String GRAPH_VERSION = "x"; + + @AfterEach + void cleanup() { + if (tempRoot != null && Files.exists(tempRoot)) { + try (Stream paths = Files.walk(tempRoot)) { + paths.map(Path::toFile) + .sorted(Comparator.reverseOrder()) + .forEach(File::delete); + } catch (IOException ignored) { + // ignore + } + } + } + + private ProfileProperties makeProfileProps(Path graphsRoot, String profileName, String group, String extent) { + ProfileProperties props = new ProfileProperties(); + BuildProperties build = new BuildProperties(); + build.setProfileGroup(group); + build.setGraphExtent(extent); + props.setBuild(build); + props.setEncoderName(EncoderNameEnum.DEFAULT); + props.setGraphPath(graphsRoot); + props.setProfileName(profileName); + return props; + } + + @Test + void success_createsArchive_and_deletesSource() throws Exception { + tempRoot = Files.createTempDirectory("pgu-success"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileA"); + Files.createDirectories(profileDir); + + // graph_build_info.yml must exist and at least one file to be archived + Files.writeString(profileDir.resolve("graph_build_info.yml"), "info"); + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileA", "group", "extent"); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archive = graphsRoot.resolve(graphName + ".ghz"); + + assertTrue(Files.exists(archive), "Archive file should be created on success"); + assertFalse(Files.exists(profileDir), "Source profile directory should be deleted after successful archive"); + } + + @Test + void missing_graph_build_info_will_not_create_archive_and_will_not_delete_source() throws Exception { + tempRoot = Files.createTempDirectory("pgu-missinginfo"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileB"); + Files.createDirectories(profileDir); + + // note: intentionally NOT creating graph_build_info.yml + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileB", "group", "extent"); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archive = graphsRoot.resolve(graphName + ".ghz"); + + assertFalse(Files.exists(archive), "Archive should NOT be created when graph_build_info.yml is missing"); + assertTrue(Files.exists(profileDir), "Source profile directory should remain when preparation aborted due to missing info file"); + } + + @Test + void archive_creation_failure_is_handled_and_source_is_kept() throws Exception { + tempRoot = Files.createTempDirectory("pgu-archivefail"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileC"); + Files.createDirectories(profileDir); + + Files.writeString(profileDir.resolve("graph_build_info.yml"), "info"); + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileC", "group", "extent"); + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archiveDir = graphsRoot.resolve(graphName + ".ghz"); + // create DIRECTORY where archive file should be to force a failure when opening FileOutputStream + Files.createDirectories(archiveDir); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + + assertTrue(Files.exists(archiveDir) && Files.isDirectory(archiveDir), "Archive destination was a directory and should remain (archive creation failed)"); + assertTrue(Files.exists(profileDir), "Source profile directory should remain when archive creation fails"); + } + + @Test + void delete_failure_is_handled_archive_still_exists_and_source_is_kept() throws Exception { + tempRoot = Files.createTempDirectory("pgu-deletefail"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileD"); + Files.createDirectories(profileDir); + + Files.writeString(profileDir.resolve("graph_build_info.yml"), "info"); + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileD", "group", "extent"); + + // mock static FileSystemUtils.deleteRecursively to throw IOException when called with profileDir + try (MockedStatic mocked = Mockito.mockStatic(FileSystemUtils.class)) { + mocked.when(() -> FileSystemUtils.deleteRecursively(profileDir)).thenThrow(new IOException("simulated delete failure")); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + } + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archive = graphsRoot.resolve(graphName + ".ghz"); + + assertTrue(Files.exists(archive), "Archive should still be created even if delete fails"); + assertTrue(Files.exists(profileDir), "Source directory should still exist when delete fails (method must catch and log the exception)"); + } +} + diff --git a/pom.xml b/pom.xml index 4df460730d..f311274121 100644 --- a/pom.xml +++ b/pom.xml @@ -79,16 +79,16 @@ 1.3.2 1.3.5 - 5.12.2 - 5.12.0 + 3.5.4 + 3.5.4 + 6.0.1 + 5.20.0 5.5.1 1.21.0 - 3.12.4 2.1.1 1.1.1 UTF-8 2.27.2 - 3.5.2 5.4.4 3.13.4 4.13.0 @@ -599,7 +599,19 @@ **/integrationtests/** - -Duser.language=en -Duser.region=US -Dillegal-access=permit ${surefireArgLine} + + -javaagent:${org.mockito:mockito-core:jar} -Duser.language=en -Duser.region=US -Dillegal-access=permit ${surefireArgLine} + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar + @@ -696,6 +708,10 @@ maven-surefire-plugin ${surefire.version} + + maven-failsafe-plugin + ${failsafe.version} + maven-war-plugin 3.3.2 From c7f10734402f28b20fdf7326404fdad9cae71f79 Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 13:52:13 +0100 Subject: [PATCH 07/11] fix: resolve ORSInitContextListener anti-pattern with RoutingProfileManager.isInitialized --- .../ors/api/servlet/listeners/ORSInitContextListener.java | 7 +++---- .../main/java/org/heigit/ors/routing/RoutingProfile.java | 6 +++--- .../java/org/heigit/ors/routing/RoutingProfileManager.java | 7 ++++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java index 1208534cd4..f3f4c7a837 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java +++ b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java @@ -112,11 +112,10 @@ public void contextDestroyed(ServletContextEvent contextEvent) { FormatUtility.unload(); StatisticsProviderFactory.releaseProviders(); LogFactory.release(Thread.currentThread().getContextClassLoader()); - RoutingProfileManager.getInstance().destroy(); - } catch (UnsupportedOperationException e) { - LOGGER.warn("RoutingProfileManager was not initialized, nothing to destroy."); + if (RoutingProfileManager.isInitialized()) + RoutingProfileManager.getInstance().destroy(); } catch (Exception e) { - LOGGER.error(e.getMessage()); + LOGGER.error(e.toString()); } } } diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index 164c637696..f25ac359a4 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -198,7 +198,7 @@ public static void prepareGeneratedGraphForUpload(ProfileProperties profilePrope Files.copy(graphInfoSrc, graphInfoDst, REPLACE_EXISTING); LOGGER.info("Copied graph info from %s to %s".formatted(graphInfoSrc.toString(), graphInfoDst.toString())); } catch (IOException e) { - LOGGER.error("Failed to copy graph build info: %s".formatted(e.getMessage())); + LOGGER.error("Failed to copy graph build info: %s".formatted(e.toString())); return; } @@ -220,7 +220,7 @@ public static void prepareGeneratedGraphForUpload(ProfileProperties profilePrope } LOGGER.info("Created archive %s".formatted(graphArchiveDst.toString())); } catch (IOException e) { - LOGGER.error("Failed to create archive: %s".formatted(e.getMessage())); + LOGGER.error("Failed to create archive: %s".formatted(e.toString())); return; } @@ -229,7 +229,7 @@ public static void prepareGeneratedGraphForUpload(ProfileProperties profilePrope FileSystemUtils.deleteRecursively(graphFilesPath); LOGGER.info("Deleted original graph files at %s after archiving.".formatted(graphFilesPath.toString())); } catch (IOException e) { - LOGGER.error("Failed to delete graph files: %s".formatted(e.getMessage())); + LOGGER.error("Failed to delete graph files: %s".formatted(e.toString())); } } diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index 8bef39c252..664459d71d 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -46,12 +46,13 @@ public RoutingProfileManager(EngineProperties config, String graphVersion) { } public static synchronized RoutingProfileManager getInstance() { - if (instance == null) { - throw new UnsupportedOperationException("RoutingProfileManager has not been initialized!"); - } return instance; } + public static synchronized boolean isInitialized() { + return instance != null; + } + public void initialize(EngineProperties config, String graphVersion) { RuntimeUtility.printRAMInfo("", LOGGER); long startTime = System.currentTimeMillis(); From 2f4cd4551a8de81366963c058accbfff58803413 Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 17:31:19 +0100 Subject: [PATCH 08/11] test: remove apitest for preparation_mode in favor of unit tests --- .../ors/apitests/PreparationModeTest.java | 61 ------------------- .../heigit/ors/routing/RoutingProfile.java | 3 - .../ors/routing/RoutingProfileManager.java | 3 +- pom.xml | 20 +----- 4 files changed, 3 insertions(+), 84 deletions(-) delete mode 100644 ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java diff --git a/ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java b/ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java deleted file mode 100644 index cb43fd0a23..0000000000 --- a/ors-api/src/test/java/org/heigit/ors/apitests/PreparationModeTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.heigit.ors.apitests; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.heigit.ors.api.Application; -import org.heigit.ors.config.EngineProperties; -import org.heigit.ors.routing.RoutingProfileManager; -import org.heigit.ors.routing.RoutingProfileManagerStatus; -import org.heigit.ors.routing.graphhopper.extensions.manage.PersistedGraphBuildInfo; -import org.heigit.ors.util.AppInfo; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; - -import static org.junit.jupiter.api.Assertions.*; -import static org.testcontainers.shaded.org.awaitility.Awaitility.await; - - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = Application.class) -@ActiveProfiles("preparation-mode-test") -@DirtiesContext -@Order(value = Integer.MAX_VALUE) -class PreparationModeTest { - - @Autowired - private EngineProperties engineProperties; - - @Test - void testPrepMode() throws IOException { - assertTrue(engineProperties.getPreparationMode()); - await().atMost(Duration.ofSeconds(60)).until(RoutingProfileManagerStatus::isReady); - assertTrue(RoutingProfileManagerStatus.isReady()); - - Path graphPath = RoutingProfileManager.getInstance().getRoutingProfile("driving-car-preparation").getProfileProperties().getGraphPath(); - String fileName = "test_heidelberg_%s_driving-car".formatted(AppInfo.GRAPH_VERSION); - File yamlFile = Paths.get(graphPath.toString(), fileName + ".yml").toFile(); - assertTrue(yamlFile.exists(), "Graph info YAML should exist"); - assertTrue(Paths.get(graphPath.toString(), fileName + ".ghz").toFile().exists(), "Packed graph should exist"); - assertFalse(Paths.get(graphPath.toString(), "driving-car-preparation").toFile().exists(), "Build graph dir should not exist"); - - // parse yaml file and check for expected values - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - PersistedGraphBuildInfo graphBuildInfo = mapper.readValue(yamlFile, PersistedGraphBuildInfo.class); - assertNotNull(graphBuildInfo.getGraphBuildDate(), "Graph build date should be set"); - assertNotNull(graphBuildInfo.getOsmDate(), "OSM date should be set"); - assertNotNull(graphBuildInfo.getGraphVersion(), "Graph version should be set"); - assertNotNull(graphBuildInfo.getGraphSizeBytes(), "Graph size bytes should be set"); - assertNotNull(graphBuildInfo.getProfileProperties(), "Profile properties should be set"); - } -} - - diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index f25ac359a4..f561ae7161 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -259,9 +259,6 @@ public ProfileProperties getProfileConfiguration() { } public void close() { - synchronized (lockObj) { - profileIdentifier = 0; - } mGraphHopper.close(); } diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index 664459d71d..7d323ad5a6 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -37,8 +37,7 @@ public RoutingProfileManager(EngineProperties config, String graphVersion) { synchronized (RoutingProfileManager.class) { if (instance != null) { // TODO: We should really refactor and avoid singleton pattern here - LOGGER.warn("Re-initializing RoutingProfileManager, this should only happen during tests!"); - instance.destroy(); + throw new UnsupportedOperationException("RoutingProfileManager is already initialized."); } instance = this; initialize(config, graphVersion); diff --git a/pom.xml b/pom.xml index f311274121..bd97b1edb3 100644 --- a/pom.xml +++ b/pom.xml @@ -80,9 +80,9 @@ 1.3.5 3.5.4 - 3.5.4 6.0.1 5.20.0 + 1.17.8 5.5.1 1.21.0 2.1.1 @@ -599,19 +599,7 @@ **/integrationtests/** - - -javaagent:${org.mockito:mockito-core:jar} -Duser.language=en -Duser.region=US -Dillegal-access=permit ${surefireArgLine} - - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - - -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar - + -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar -Duser.language=en -Duser.region=US -Dillegal-access=permit ${surefireArgLine} @@ -708,10 +696,6 @@ maven-surefire-plugin ${surefire.version} - - maven-failsafe-plugin - ${failsafe.version} - maven-war-plugin 3.3.2 From 2372fb659b8917e52626759b2a7885b8f02681b1 Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 17:46:50 +0100 Subject: [PATCH 09/11] fix: mockito javaagent seems to cause trouble in the github runner --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd97b1edb3..a1ef52b860 100644 --- a/pom.xml +++ b/pom.xml @@ -599,7 +599,7 @@ **/integrationtests/** - -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar -Duser.language=en -Duser.region=US -Dillegal-access=permit ${surefireArgLine} + -Duser.language=en -Duser.region=US -Dillegal-access=permit ${surefireArgLine} From 9a8f5e26421a0cb09c4b990496aab4ac694aae8b Mon Sep 17 00:00:00 2001 From: takb Date: Fri, 28 Nov 2025 18:46:24 +0100 Subject: [PATCH 10/11] fix: remove bytebuddy version variable no longer needed --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index a1ef52b860..642700b420 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,6 @@ 3.5.4 6.0.1 5.20.0 - 1.17.8 5.5.1 1.21.0 2.1.1 From 1acc0d708d5c8a154855b2c727c6af69a7c20853 Mon Sep 17 00:00:00 2001 From: takb Date: Sat, 29 Nov 2025 12:06:06 +0100 Subject: [PATCH 11/11] refactor: extract method to reduce complexity --- .../listeners/ORSInitContextListener.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java index f3f4c7a837..30ef54f9fd 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java +++ b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java @@ -56,33 +56,35 @@ public void contextInitialized(ServletContextEvent contextEvent) { copyDefaultConfigurationToFile(outputTarget); return; } - new Thread(() -> { - try { - LOGGER.info("Initializing ORS..."); - graphService.setIsActivatingGraphs(true); - RoutingProfileManager routingProfileManager = new RoutingProfileManager(engineProperties, AppInfo.GRAPH_VERSION); - if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { - LOGGER.info("Running in preparation mode, all enabled graphs are built, job is done."); - if (environment.getActiveProfiles().length == 0) { // only exit if no active profile is set (i.e., not in test mode) - RoutingProfileManagerStatus.setShutdown(true); - } - } - if (RoutingProfileManagerStatus.isShutdown()) { - System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); + new Thread(this::initializeORS, "ORS-Init").start(); + } + + private void initializeORS() { + try { + LOGGER.info("Initializing ORS..."); + graphService.setIsActivatingGraphs(true); + RoutingProfileManager routingProfileManager = new RoutingProfileManager(engineProperties, AppInfo.GRAPH_VERSION); + if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { + LOGGER.info("Running in preparation mode, all enabled graphs are built, job is done."); + if (environment.getActiveProfiles().length == 0) { // only exit if no active profile is set (i.e., not in test mode) + RoutingProfileManagerStatus.setShutdown(true); } - for (RoutingProfile profile : routingProfileManager.getUniqueProfiles()) { - ORSGraphManager orsGraphManager = profile.getGraphhopper().getOrsGraphManager(); - if (orsGraphManager != null && orsGraphManager.useGraphRepository()) { - LOGGER.debug("Adding orsGraphManager for profile %s with encoder %s to GraphService".formatted(profile.getProfileConfiguration().getProfileName(), profile.getProfileConfiguration().getEncoderName())); - graphService.addGraphManagerInstance(orsGraphManager); - } + } + if (RoutingProfileManagerStatus.isShutdown()) { + System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); + } + for (RoutingProfile profile : routingProfileManager.getUniqueProfiles()) { + ORSGraphManager orsGraphManager = profile.getGraphhopper().getOrsGraphManager(); + if (orsGraphManager != null && orsGraphManager.useGraphRepository()) { + LOGGER.debug("Adding orsGraphManager for profile %s with encoder %s to GraphService".formatted(profile.getProfileConfiguration().getProfileName(), profile.getProfileConfiguration().getEncoderName())); + graphService.addGraphManagerInstance(orsGraphManager); } - } catch (Exception e) { - LOGGER.warn("Unable to initialize ORS due to an unexpected exception: " + e); - } finally { - graphService.setIsActivatingGraphs(false); } - }, "ORS-Init").start(); + } catch (Exception e) { + LOGGER.warn("Unable to initialize ORS due to an unexpected exception: " + e); + } finally { + graphService.setIsActivatingGraphs(false); + } } public String configurationOutputTarget(EngineProperties engineProperties) {