diff --git a/CHANGELOG.md b/CHANGELOG.md index 42943076e8..b098caa311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Releasing is documented in RELEASE.md ### Changed - enhanced status endpoint to expose dynamic data statistics and feature store metrics ([#2156](https://github.com/GIScience/openrouteservice/pull/2156)) +- when running in `preparation_mode`, pack the graph for upload ([#2185](https://github.com/GIScience/openrouteservice/pull/2185)) ### Deprecated diff --git a/docs/run-instance/configuration/engine/profiles/build.md b/docs/run-instance/configuration/engine/profiles/build.md index 611496fdc3..5074a70b2d 100644 --- a/docs/run-instance/configuration/engine/profiles/build.md +++ b/docs/run-instance/configuration/engine/profiles/build.md @@ -6,6 +6,8 @@ graphs for the specified profile. | key | type | description | default value | |----------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------| | source_file | string | The OSM file to be used, supported formats are `.osm`, `.osm.gz`, `.osm.zip` and `.pbf` | `ors-api/src/test/files/heidelberg.test.pbf` | +| graph_extent | string | Used only in `preparation_mode` to determine the `graph_extent` part of the filename of the packed archive. See [graph repo client](/technical-details/graph-repo-client/) for more information. | _NA_ | +| profile_group | string | Used only in `preparation_mode` to determine the `profile_group` part of the filename of the packed archive. See [graph repo client](/technical-details/graph-repo-client/) for more information. | _NA_ | | elevation | boolean | Specifies whether to download and use elevation data. If true, `cache_path` and `provider` must be set in ors.engine.elevation as well. | `false` | | elevation_smoothing | boolean | Smooth out elevation data | `false` | | traffic | boolean | Use traffic data if available | `false` | @@ -34,7 +36,7 @@ Properties beneath `ors.engine.profiles..build.encoder_options`: | problematic_speed_factor | number | wheelchair | Travel speeds on edges classified as problematic for wheelchair users are multiplied by this factor, use to set slow traveling speeds on such ways | `0.7` | | turn_costs | boolean | car, hgv, bike-* | Should turn restrictions be respected | `true` | | use_acceleration | boolean | car, hgv | Models how a vehicle would accelerate on the road segment to the maximum allowed speed. In practice it reduces speed on shorter road segments such as ones between nearby intersections in a city | `true` | -| enable_custom_models | boolean | * | Enables whether the profile is prepared to support custom models. Also see the corresponding parameter `allow_custom_models` in the [service properties](service.md). | `false` | +| enable_custom_models | boolean | * | Enables whether the profile is prepared to support custom models. Also see the corresponding parameter `allow_custom_models` in the [service properties](service.md). | `false` | ## `preparation` @@ -160,7 +162,7 @@ country borders, compatible with any profile type. | key | type | description | example value | |--------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------| | boundaries | string | The path to a GeoJSON file containing polygons representing country borders. Ignored if `preprocessed = true` is set. | `borders.geojson.tar.gz` | -| preprocessed | boolean | Indicates whether the source OSM file has been enriched with country data. If set to `true` then country codes are read from `country` node tags rather than being resolved at build time based on geometries provided in `boundaries`. | `true` | +| preprocessed | boolean | Indicates whether the source OSM file has been enriched with country data. If set to `true` then country codes are read from `country` node tags rather than being resolved at build time based on geometries provided in `boundaries`. | `true` | | ids | string | Path to a csv file containing a unique id for each country, its local name and its English name | `ids.csv` | | openborders | string | Path to a csv file containing pairs of countries where the borders are open (i.e. Schengen borders) | `openborders.csv` | diff --git a/docs/run-instance/configuration/engine/profiles/repo.md b/docs/run-instance/configuration/engine/profiles/repo.md index 4c7e43e47c..77a940b798 100644 --- a/docs/run-instance/configuration/engine/profiles/repo.md +++ b/docs/run-instance/configuration/engine/profiles/repo.md @@ -6,14 +6,14 @@ and if graph management is enabled [ors.engine.graph_management.enabled](/run-instance/configuration/engine/graph-management.md), openrouteservice will use the specified repository to load the graph data. -| key | type | description | example values | -|--------------------------|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| repository_uri | string | The base URL (for a http or minio repository) or path (for a file system repository) | * `http://some.domain.ors`
* `file:///absolute/path`
* `/absolute/path`
* `relative/path` | -| repository_name | string | An editorial name representing the target group or vendor for the graphs. | `public`, `my-organization` | -| repository_profile_group | string | A group of profiles with specific characteristics lke the usage of traffic data or the pre-calculation of advanced data structures e.g. for fastisochrones | `traffic`, `fastiso` | -| graph_extent | string | The geographic region covered by the graph. This corresponds to the OSM PBF file used for graph building. | `planet` | -| repository_user | string | The user name (key_id) for authentication at the repository. Currently only used by the minio client. | `username` | -| repository_password | string | The password (access_key) for authentication at the repository. Currently only used by the minio client. | `password` | +| key | type | description | example values | +|--------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| repository_uri | string | The base URL (for a http or minio repository) or path (for a file system repository) | * `http://some.domain.ors`
* `file:///absolute/path`
* `/absolute/path`
* `relative/path` | +| repository_name | string | An editorial name representing the target group or vendor for the graphs. | `public`, `my-organization` | +| repository_profile_group | string | A group of profiles with specific characteristics like the usage of traffic data or the pre-calculation of advanced data structures e.g. for fastisochrones | `traffic`, `fastiso` | +| graph_extent | string | The geographic region covered by the graph. This corresponds to the OSM PBF file used for graph building. | `planet` | +| repository_user | string | The user name (key_id) for authentication at the repository. Currently only used by the minio client. | `username` | +| repository_password | string | The password (access_key) for authentication at the repository. Currently only used by the minio client. | `password` | Each routing profile can have its individual repository parameters, which makes it possible to have routing profiles from different repositories diff --git a/docs/technical-details/graph-repo-client/index.md b/docs/technical-details/graph-repo-client/index.md index e60f6efe53..3e977c6a23 100644 --- a/docs/technical-details/graph-repo-client/index.md +++ b/docs/technical-details/graph-repo-client/index.md @@ -103,7 +103,7 @@ In the following graph repository example there are graphs for three target grou http://repos.provider.com/graphs/ # repo base URL ├── project-b # repo name │ └── basic # profile group -│ └── switzerland # coverage +│ └── switzerland # graph extent │ └── 2 # graph version │ ├── switzerland_2_cycling-mountain.ghz # compacted graph │ ├── switzerland_2_cycling-mountain.yml # metadata @@ -113,7 +113,7 @@ http://repos.provider.com/graphs/ # repo base URL │ └── switzerland_2_cycling-street.yml # metadata ├── public # repo name │ └── basic # profile group -│ └── europe # coverage +│ └── europe # graph extent │ ├── 1 # graph version │ │ ├── europe_1_driving-car.ghz # compacted graph │ │ ├── europe_1_driving-car.yml # metadata diff --git a/ors-api/src/main/java/org/heigit/ors/api/Application.java b/ors-api/src/main/java/org/heigit/ors/api/Application.java index 89b99b5ceb..b5bea6c902 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/Application.java +++ b/ors-api/src/main/java/org/heigit/ors/api/Application.java @@ -14,6 +14,7 @@ import org.springframework.boot.web.servlet.ServletListenerRegistrationBean; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.context.annotation.Bean; +import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @@ -40,9 +41,9 @@ public static void main(String[] args) { } @Bean("orsInitContextListenerBean") - public ServletListenerRegistrationBean createORSInitContextListenerBean(ApiEngineProperties apiEngineProperties, GraphService graphService) { + public ServletListenerRegistrationBean createORSInitContextListenerBean(ApiEngineProperties apiEngineProperties, GraphService graphService, Environment environment) { ServletListenerRegistrationBean bean = new ServletListenerRegistrationBean<>(); - bean.setListener(new ORSInitContextListener(apiEngineProperties, graphService)); + bean.setListener(new ORSInitContextListener(apiEngineProperties, graphService, environment)); return bean; } } diff --git a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java index 5822c20dcc..30ef54f9fd 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java +++ b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java @@ -35,6 +35,7 @@ import org.heigit.ors.routing.graphhopper.extensions.manage.ORSGraphManager; import org.heigit.ors.util.FormatUtility; import org.heigit.ors.util.StringUtility; +import org.springframework.core.env.Environment; import org.springframework.core.io.ClassPathResource; import java.io.FileOutputStream; @@ -46,6 +47,7 @@ public class ORSInitContextListener implements ServletContextListener { private static final Logger LOGGER = Logger.getLogger(ORSInitContextListener.class); private final EngineProperties engineProperties; private final GraphService graphService; + private Environment environment; @Override public void contextInitialized(ServletContextEvent contextEvent) { @@ -54,28 +56,35 @@ public void contextInitialized(ServletContextEvent contextEvent) { copyDefaultConfigurationToFile(outputTarget); return; } - new Thread(() -> { - try { - LOGGER.info("Initializing ORS..."); - graphService.setIsActivatingGraphs(true); - RoutingProfileManager routingProfileManager = new RoutingProfileManager(engineProperties, AppInfo.GRAPH_VERSION); - for (RoutingProfile profile : routingProfileManager.getUniqueProfiles()) { - ORSGraphManager orsGraphManager = profile.getGraphhopper().getOrsGraphManager(); - if (orsGraphManager != null && orsGraphManager.useGraphRepository()) { - LOGGER.debug("Adding orsGraphManager for profile %s with encoder %s to GraphService".formatted(profile.getProfileConfiguration().getProfileName(), profile.getProfileConfiguration().getEncoderName())); - graphService.addGraphManagerInstance(orsGraphManager); - } + new Thread(this::initializeORS, "ORS-Init").start(); + } + + private void initializeORS() { + try { + LOGGER.info("Initializing ORS..."); + graphService.setIsActivatingGraphs(true); + RoutingProfileManager routingProfileManager = new RoutingProfileManager(engineProperties, AppInfo.GRAPH_VERSION); + if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { + LOGGER.info("Running in preparation mode, all enabled graphs are built, job is done."); + if (environment.getActiveProfiles().length == 0) { // only exit if no active profile is set (i.e., not in test mode) + RoutingProfileManagerStatus.setShutdown(true); } - if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { - LOGGER.info("Running in preparation mode, all enabled graphs are built, job is done."); - System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); + } + if (RoutingProfileManagerStatus.isShutdown()) { + System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); + } + for (RoutingProfile profile : routingProfileManager.getUniqueProfiles()) { + ORSGraphManager orsGraphManager = profile.getGraphhopper().getOrsGraphManager(); + if (orsGraphManager != null && orsGraphManager.useGraphRepository()) { + LOGGER.debug("Adding orsGraphManager for profile %s with encoder %s to GraphService".formatted(profile.getProfileConfiguration().getProfileName(), profile.getProfileConfiguration().getEncoderName())); + graphService.addGraphManagerInstance(orsGraphManager); } - } catch (Exception e) { - LOGGER.warn("Unable to initialize ORS due to an unexpected exception: " + e); - } finally { - graphService.setIsActivatingGraphs(false); } - }, "ORS-Init").start(); + } catch (Exception e) { + LOGGER.warn("Unable to initialize ORS due to an unexpected exception: " + e); + } finally { + graphService.setIsActivatingGraphs(false); + } } public String configurationOutputTarget(EngineProperties engineProperties) { @@ -103,12 +112,12 @@ public void contextDestroyed(ServletContextEvent contextEvent) { try { LOGGER.info("Shutting down openrouteservice %s and releasing resources.".formatted(AppInfo.getEngineInfo())); FormatUtility.unload(); - if (RoutingProfileManagerStatus.isReady()) - RoutingProfileManager.getInstance().destroy(); StatisticsProviderFactory.releaseProviders(); LogFactory.release(Thread.currentThread().getContextClassLoader()); + if (RoutingProfileManager.isInitialized()) + RoutingProfileManager.getInstance().destroy(); } catch (Exception e) { - LOGGER.error(e.getMessage()); + LOGGER.error(e.toString()); } } } diff --git a/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java b/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java index ab6fd38daa..4a11bb2104 100644 --- a/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java +++ b/ors-api/src/test/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListenerTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -14,9 +15,12 @@ class ORSInitContextListenerTest { @Autowired private GraphService graphService; + @Autowired + private Environment environment; + @Test void testConfigurationOutputTarget() { - ORSInitContextListener orsInitContextListener = new ORSInitContextListener(new EngineProperties(), graphService); + ORSInitContextListener orsInitContextListener = new ORSInitContextListener(new EngineProperties(), graphService, environment); EngineProperties engineProperties = new EngineProperties(); assertNull(orsInitContextListener.configurationOutputTarget(engineProperties), "default should return null"); diff --git a/ors-api/src/test/resources/application-preparation-mode-test.yml b/ors-api/src/test/resources/application-preparation-mode-test.yml new file mode 100644 index 0000000000..dbceb40bda --- /dev/null +++ b/ors-api/src/test/resources/application-preparation-mode-test.yml @@ -0,0 +1,33 @@ +ors: + engine: + preparation_mode: true + profile_default: + graph_path: graphs-apitests + build: + source_file: ./src/test/files/heidelberg.test.pbf + graph_extent: heidelberg + profile_group: test + elevation: false + preparation: + methods: + fastisochrones: + enabled: false + ch: + enabled: false + lm: + enabled: false + core: + enabled: false + profiles: + driving-car-preparation: + enabled: true + encoder_name: driving-car + build: + encoder_options: + turn_costs: false + block_fords: false + use_acceleration: false + maximum_grade_level: 1 + conditional_access: false + conditional_speed: false + enable_custom_models: false diff --git a/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java b/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java index 1da286ba64..eb7a473f6e 100644 --- a/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java +++ b/ors-engine/src/main/java/org/heigit/ors/config/profile/BuildProperties.java @@ -17,6 +17,11 @@ public class BuildProperties { @JsonIgnore private Path sourceFile; + @JsonIgnore + private String profileGroup; + @JsonIgnore + private String graphExtent; + @JsonProperty("elevation") private Boolean elevation; @JsonProperty("elevation_smoothing") @@ -56,6 +61,8 @@ public BuildProperties(String ignored) { } public void merge(BuildProperties other) { + profileGroup = ofNullable(profileGroup).orElse(other.profileGroup); + graphExtent = ofNullable(graphExtent).orElse(other.graphExtent); elevation = ofNullable(elevation).orElse(other.elevation); elevationSmoothing = ofNullable(elevationSmoothing).orElse(other.elevationSmoothing); encoderFlagsSize = ofNullable(encoderFlagsSize).orElse(other.encoderFlagsSize); diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java index 27894a37aa..f561ae7161 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfile.java @@ -13,6 +13,7 @@ */ package org.heigit.ors.routing; +import com.google.common.base.Strings; import com.graphhopper.config.CHProfile; import com.graphhopper.routing.ev.*; import com.graphhopper.storage.GraphHopperStorage; @@ -26,16 +27,28 @@ import org.heigit.ors.routing.graphhopper.extensions.storages.builders.BordersGraphStorageBuilder; import org.heigit.ors.routing.graphhopper.extensions.storages.builders.GraphStorageBuilder; import org.heigit.ors.routing.pathprocessors.ORSPathProcessorFactory; +import org.heigit.ors.util.AppInfo; import org.heigit.ors.util.TimeUtility; import org.json.simple.JSONObject; +import org.springframework.util.FileSystemUtils; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Objects; import java.util.function.Function; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; /** * This class generates {@link RoutingProfile} classes and is used by mostly all service classes e.g. @@ -141,9 +154,85 @@ public ORSGraphHopper initGraphHopper(RoutingProfileLoadContext loadCntx) throws if (!file2.exists()) Files.write(pathTimestamp, Long.toString(file.length()).getBytes()); } + + if (Boolean.TRUE.equals(engineProperties.getPreparationMode())) { + prepareGeneratedGraphForUpload(profileProperties, AppInfo.GRAPH_VERSION); + } return gh; } + /** + * Prepares the generated graph for upload when running in preparation mode. + * + *

This is a static helper method which expects a fully-initialized {@link ProfileProperties} + * instance and prepares the graph files located under {@code profileProperties.getGraphPath()/profileProperties.getProfileName()}. + * The method performs the following steps: + *

    + *
  • Constructs a graph name using the profile group, graph extent, the application graph version and the encoder name, separated by underscores.
    • + *
  • Copies the graph build info file ("graph_build_info.yml") from the profile directory to the graph path, renaming it to "{graphName}.yml".
    • + *
  • Creates a ZIP archive of the graph directory, saving it as "{graphName}.ghz" in the graph path.
    • + *
  • Deletes the original graph directory after archiving.
    • + *
+ * + *

Important behaviour and error handling: + *

    + *
  • The method is defensive: IO errors while copying the graph build info or while creating the archive are logged and cause an early return so no further steps are executed.
    • + *
  • Errors while deleting the original graph directory are caught and logged; the method does not rethrow them.
    • + *
  • This method mutates the filesystem (creates files and archives, and attempts to delete directories). Tests that exercise it should use temporary directories.
    • + *
+ * + * @param profileProperties profile properties holding graph path and profile name; must not be null and must include a non-null graph path and profile name + */ + public static void prepareGeneratedGraphForUpload(ProfileProperties profileProperties, String graphVersion) { + LOGGER.info("Running in preparation_mode, preparing graph for upload"); + String profileGroup = Strings.isNullOrEmpty(profileProperties.getBuild().getProfileGroup()) ? "unknownGroup" : profileProperties.getBuild().getProfileGroup(); + String graphExtent = Strings.isNullOrEmpty(profileProperties.getBuild().getGraphExtent()) ? "unknownExtent" : profileProperties.getBuild().getGraphExtent(); + String encoderName = Strings.isNullOrEmpty(profileProperties.getEncoderName().toString()) ? "unknownEncoder" : profileProperties.getEncoderName().toString(); + String graphName = String.join("_", profileGroup, graphExtent, graphVersion, encoderName); + Path graphFilesPath = profileProperties.getGraphPath().resolve(profileProperties.getProfileName()); + + // copy graph_build_info.yml to {graphName}.yml + try { + Path graphInfoSrc = graphFilesPath.resolve("graph_build_info.yml"); + Path graphInfoDst = profileProperties.getGraphPath().resolve(graphName + ".yml"); + Files.copy(graphInfoSrc, graphInfoDst, REPLACE_EXISTING); + LOGGER.info("Copied graph info from %s to %s".formatted(graphInfoSrc.toString(), graphInfoDst.toString())); + } catch (IOException e) { + LOGGER.error("Failed to copy graph build info: %s".formatted(e.toString())); + return; + } + + // create a zip archive of all files in graphFilesPath with .ghz extension + Path graphArchiveDst = profileProperties.getGraphPath().resolve(graphName + ".ghz"); + try (FileOutputStream fos = new FileOutputStream(graphArchiveDst.toFile()); ZipOutputStream zos = new ZipOutputStream(fos)) { + for (File file : Objects.requireNonNull(graphFilesPath.toFile().listFiles())) { + if (!Files.isDirectory(file.toPath())) { + try (FileInputStream fis = new FileInputStream(file)) { + ZipEntry zipEntry = new ZipEntry(graphFilesPath.relativize(file.toPath()).toString()); + zos.putNextEntry(zipEntry); + byte[] buffer = new byte[1024]; + int len; + while ((len = fis.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } + } + } + LOGGER.info("Created archive %s".formatted(graphArchiveDst.toString())); + } catch (IOException e) { + LOGGER.error("Failed to create archive: %s".formatted(e.toString())); + return; + } + + // delete original graph files + try { + FileSystemUtils.deleteRecursively(graphFilesPath); + LOGGER.info("Deleted original graph files at %s after archiving.".formatted(graphFilesPath.toString())); + } catch (IOException e) { + LOGGER.error("Failed to delete graph files: %s".formatted(e.toString())); + } + } + public boolean hasCHProfile(String profileName) { boolean hasCHProfile = false; for (CHProfile chProfile : getGraphhopper().getCHPreparationHandler().getCHProfiles()) { @@ -211,7 +300,7 @@ public List getDynamicDatasets() { } public void updateDynamicData(String key, int edgeID, String value) { - Function stateFromString = null; + Function stateFromString = null; switch (key) { case LogieBorders.KEY: stateFromString = s -> LogieBorders.valueOf(s.replace(" ", "_").toUpperCase()); diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index 02a99391a6..7d323ad5a6 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -34,22 +34,24 @@ public class RoutingProfileManager { private static RoutingProfileManager instance; public RoutingProfileManager(EngineProperties config, String graphVersion) { - if (instance == null) { + synchronized (RoutingProfileManager.class) { + if (instance != null) { + // TODO: We should really refactor and avoid singleton pattern here + throw new UnsupportedOperationException("RoutingProfileManager is already initialized."); + } instance = this; initialize(config, graphVersion); - if (RoutingProfileManagerStatus.isShutdown()) { - System.exit(RoutingProfileManagerStatus.hasFailed() ? 1 : 0); - } } } public static synchronized RoutingProfileManager getInstance() { - if (instance == null) { - throw new UnsupportedOperationException("RoutingProfileManager has not been initialized!"); - } return instance; } + public static synchronized boolean isInitialized() { + return instance != null; + } + public void initialize(EngineProperties config, String graphVersion) { RuntimeUtility.printRAMInfo("", LOGGER); long startTime = System.currentTimeMillis(); diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java index 0e014dd39f..31d04a4c49 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/PersistedGraphBuildInfo.java @@ -25,6 +25,9 @@ public class PersistedGraphBuildInfo { @JsonProperty("graph_version") private String graphVersion; + @JsonProperty("graph_size_bytes") + private Long graphSizeBytes; + @JsonProperty("profile_properties") private ProfileProperties profileProperties; diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java index 3054920a15..c546bc0d9f 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/graphhopper/extensions/manage/local/ORSGraphFileManager.java @@ -357,9 +357,22 @@ public void writeOrsGraphBuildInfoFileIfNotExists(ORSGraphHopper gh) { persistedGraphBuildInfo.setGraphVersion(AppInfo.GRAPH_VERSION); persistedGraphBuildInfo.setProfileProperties(profileProperties); + // Add the total size of the graph folder + File graphFolder = getActiveGraphDirectory(); + persistedGraphBuildInfo.setGraphSizeBytes(calculateFolderSizeInBytes(graphFolder)); + ORSGraphFileManager.writeOrsGraphBuildInfo(persistedGraphBuildInfo, activeGraphBuildInfoFile); } + private long calculateFolderSizeInBytes(File graphFolder) { + try { + return FileUtils.sizeOfDirectory(graphFolder); + } catch (Exception e) { + LOGGER.error("Could not calculate size of graph folder %s: %s".formatted(graphFolder.getAbsolutePath(), e.getMessage())); + return 0; + } + } + Date getDateFromGhProperty(GraphHopper gh, String ghProperty) { try { String importDateString = gh.getGraphHopperStorage().getProperties().get(ghProperty); diff --git a/ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java b/ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java new file mode 100644 index 0000000000..bc259e089a --- /dev/null +++ b/ors-engine/src/test/java/org/heigit/ors/routing/PrepareGeneratedGraphForUploadTest.java @@ -0,0 +1,151 @@ +package org.heigit.ors.routing; + +import org.heigit.ors.common.EncoderNameEnum; +import org.heigit.ors.config.profile.BuildProperties; +import org.heigit.ors.config.profile.ProfileProperties; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.util.FileSystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for {@link RoutingProfile#prepareGeneratedGraphForUpload(ProfileProperties, String)} + * covering success and error branches. + */ +class PrepareGeneratedGraphForUploadTest { + private Path tempRoot; + private static final String GRAPH_VERSION = "x"; + + @AfterEach + void cleanup() { + if (tempRoot != null && Files.exists(tempRoot)) { + try (Stream paths = Files.walk(tempRoot)) { + paths.map(Path::toFile) + .sorted(Comparator.reverseOrder()) + .forEach(File::delete); + } catch (IOException ignored) { + // ignore + } + } + } + + private ProfileProperties makeProfileProps(Path graphsRoot, String profileName, String group, String extent) { + ProfileProperties props = new ProfileProperties(); + BuildProperties build = new BuildProperties(); + build.setProfileGroup(group); + build.setGraphExtent(extent); + props.setBuild(build); + props.setEncoderName(EncoderNameEnum.DEFAULT); + props.setGraphPath(graphsRoot); + props.setProfileName(profileName); + return props; + } + + @Test + void success_createsArchive_and_deletesSource() throws Exception { + tempRoot = Files.createTempDirectory("pgu-success"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileA"); + Files.createDirectories(profileDir); + + // graph_build_info.yml must exist and at least one file to be archived + Files.writeString(profileDir.resolve("graph_build_info.yml"), "info"); + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileA", "group", "extent"); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archive = graphsRoot.resolve(graphName + ".ghz"); + + assertTrue(Files.exists(archive), "Archive file should be created on success"); + assertFalse(Files.exists(profileDir), "Source profile directory should be deleted after successful archive"); + } + + @Test + void missing_graph_build_info_will_not_create_archive_and_will_not_delete_source() throws Exception { + tempRoot = Files.createTempDirectory("pgu-missinginfo"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileB"); + Files.createDirectories(profileDir); + + // note: intentionally NOT creating graph_build_info.yml + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileB", "group", "extent"); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archive = graphsRoot.resolve(graphName + ".ghz"); + + assertFalse(Files.exists(archive), "Archive should NOT be created when graph_build_info.yml is missing"); + assertTrue(Files.exists(profileDir), "Source profile directory should remain when preparation aborted due to missing info file"); + } + + @Test + void archive_creation_failure_is_handled_and_source_is_kept() throws Exception { + tempRoot = Files.createTempDirectory("pgu-archivefail"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileC"); + Files.createDirectories(profileDir); + + Files.writeString(profileDir.resolve("graph_build_info.yml"), "info"); + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileC", "group", "extent"); + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archiveDir = graphsRoot.resolve(graphName + ".ghz"); + // create DIRECTORY where archive file should be to force a failure when opening FileOutputStream + Files.createDirectories(archiveDir); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + + assertTrue(Files.exists(archiveDir) && Files.isDirectory(archiveDir), "Archive destination was a directory and should remain (archive creation failed)"); + assertTrue(Files.exists(profileDir), "Source profile directory should remain when archive creation fails"); + } + + @Test + void delete_failure_is_handled_archive_still_exists_and_source_is_kept() throws Exception { + tempRoot = Files.createTempDirectory("pgu-deletefail"); + Path graphsRoot = tempRoot.resolve("graphs"); + Files.createDirectories(graphsRoot); + Path profileDir = graphsRoot.resolve("profileD"); + Files.createDirectories(profileDir); + + Files.writeString(profileDir.resolve("graph_build_info.yml"), "info"); + Files.writeString(profileDir.resolve("data.bin"), "payload"); + + ProfileProperties props = makeProfileProps(graphsRoot, "profileD", "group", "extent"); + + // mock static FileSystemUtils.deleteRecursively to throw IOException when called with profileDir + try (MockedStatic mocked = Mockito.mockStatic(FileSystemUtils.class)) { + mocked.when(() -> FileSystemUtils.deleteRecursively(profileDir)).thenThrow(new IOException("simulated delete failure")); + + RoutingProfile.prepareGeneratedGraphForUpload(props, GRAPH_VERSION); + } + + String graphName = String.join("_", "group", "extent", GRAPH_VERSION, props.getEncoderName().toString()); + Path archive = graphsRoot.resolve(graphName + ".ghz"); + + assertTrue(Files.exists(archive), "Archive should still be created even if delete fails"); + assertTrue(Files.exists(profileDir), "Source directory should still exist when delete fails (method must catch and log the exception)"); + } +} + diff --git a/pom.xml b/pom.xml index 4df460730d..642700b420 100644 --- a/pom.xml +++ b/pom.xml @@ -79,16 +79,15 @@ 1.3.2 1.3.5 - 5.12.2 - 5.12.0 + 3.5.4 + 6.0.1 + 5.20.0 5.5.1 1.21.0 - 3.12.4 2.1.1 1.1.1 UTF-8 2.27.2 - 3.5.2 5.4.4 3.13.4 4.13.0