diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage new file mode 100644 index 0000000..ec59c15 --- /dev/null +++ b/Dockerfile.build_sysimage @@ -0,0 +1,84 @@ +#============================================================================== +# ReefGuide Worker: Sysimage Builder +#============================================================================== +# Builds a Julia system image for faster ReefGuide Worker startup times. +# The system image is compiled using PackageCompiler.jl and can be extracted +# from the resulting container. +# +# Following advice found in this Discourse thread: +# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 +# +# Also found this relevant issue: +# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 +# +# Build: +# docker build --target export-sysimage -f Dockerfile.build_sysimage -t reefguide-sysimage . +# +# Extract sysimage: +# docker create --name temp-sysimage reefguide-sysimage +# docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +# docker rm temp-sysimage +#============================================================================== + +ARG JULIA_VERSION="1.11.5" +FROM julia:${JULIA_VERSION}-bookworm AS internal-base + +# Since 1.9.0 Julia, the CPU target is set to "native" by default. This settings +# avoids the need to compile the Julia packages for the specific CPU +# architecture of the host machine Make sure the image can be used on any x86_64 +# machine by setting JULIA_CPU_TARGET to the same value used by the generic +# julia binaries, see +# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" + +ENV JULIA_VERSION=1.11.5 +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=/usr/local/share/julia +ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app +ENV APP_SRC_DIR=/usr/local/src/app +ENV JULIA_PKG_USE_CLI_GIT=true + +# Update all pre-installed OS packages (to get security updates) +# and add a few extra utilities +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get install --no-install-recommends -y \ + git \ + openssl \ + libssl-dev \ + g++ \ + curl \ + ca-certificates \ + gdal-bin \ + libgdal-dev \ + libfftw3-dev \ + && apt-get clean \ + && apt-get autoremove --purge \ + && rm -rf /var/lib/apt/lists/* + +# Setup shared environment and add packages +RUN mkdir -p "${JULIA_DEPOT_PATH}" \ + && chmod 0755 "${JULIA_DEPOT_PATH}" + +WORKDIR "${APP_SRC_DIR}" + +COPY Project.toml Manifest*.toml ./ +COPY src/ src/ + +# Build sysimage +RUN julia -t auto --project=@app -e \ + 'using Pkg; \ + Pkg.add(["PackageCompiler", "Infiltrator", "Revise"]); \ + Pkg.develop(PackageSpec(path=pwd())); \ + Pkg.instantiate(); ' + +# Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation +# https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 +RUN julia --project=@app -t auto -e 'include("src/sysimage.jl")' + +# Export Julia sysimage to host filesystem +# From project root +# docker build --target export-sysimage -t reefguide-sysimage -f sandbox/smaller_sysimage/Dockerfile . +FROM scratch AS export-sysimage +COPY --from=internal-base /usr/local/src/app/reefguide_img.so ./reefguide_img.so diff --git a/Manifest.toml b/Manifest.toml index ed303ec..f71790f 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -6,9 +6,9 @@ project_hash = "1838a064a4b97911b698a470075b190fe5ff193c" [[deps.AWS]] deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Random", "SHA", "Sockets", "URIs", "UUIDs", "XMLDict"] -git-tree-sha1 = "1da539ec68c473173595ca545e28a4cc1e15d896" +git-tree-sha1 = "7f2817f567eb0c348b86b6329f9b7c00105c82d9" uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" -version = "1.94.0" +version = "1.94.1" [[deps.AWSS3]] deps = ["AWS", "ArrowTypes", "Base64", "Compat", "Dates", "EzXML", "FilePathsBase", "HTTP", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "SymDict", "URIs", "UUIDs", "XMLDict"] @@ -16,17 +16,6 @@ git-tree-sha1 = "e70068e13a472090b66750a9a6c5147c1c2dfc51" uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" version = "0.11.4" -[[deps.AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.5.0" -weakdeps = ["ChainRulesCore", "Test"] - - [deps.AbstractFFTs.extensions] - AbstractFFTsChainRulesCoreExt = "ChainRulesCore" - AbstractFFTsTestExt = "Test" - [[deps.AbstractTrees]] git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" @@ -73,18 +62,20 @@ uuid = "35492f91-a3bd-45ad-95db-fcad7dcfedb7" version = "1.2.0" [[deps.ArchGDAL]] -deps = ["CEnum", "ColorTypes", "Dates", "DiskArrays", "Extents", "GDAL", "GeoFormatTypes", "GeoInterface", "GeoInterfaceMakie", "GeoInterfaceRecipes", "ImageCore", "Tables"] -git-tree-sha1 = "706bc126751ce5468cb2b31b5620eb765a3275cf" +deps = ["CEnum", "ColorTypes", "Dates", "DiskArrays", "Extents", "GDAL", "GeoFormatTypes", "GeoInterface", "ImageCore", "Tables"] +git-tree-sha1 = "7048c7ffbdac14fe76f50ad3cce335cbff8b7816" uuid = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" -version = "0.10.8" +version = "0.10.10" [deps.ArchGDAL.extensions] ArchGDALJLD2Ext = "JLD2" ArchGDALMakieExt = "Makie" + ArchGDALRecipesBaseExt = "RecipesBase" [deps.ArchGDAL.weakdeps] JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [[deps.ArgCheck]] git-tree-sha1 = "f9e9a66c9b7be1ad7372bbd9b062d9230c30c5ce" @@ -128,10 +119,10 @@ version = "7.19.0" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" [[deps.ArrayLayouts]] -deps = ["FillArrays", "LinearAlgebra"] -git-tree-sha1 = "4e25216b8fea1908a0ce0f5d87368587899f75be" +deps = ["FillArrays", "LinearAlgebra", "StaticArrays"] +git-tree-sha1 = "120e392af69350960b1d3b89d41dcc1d66543858" uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" -version = "1.11.1" +version = "1.11.2" weakdeps = ["SparseArrays"] [deps.ArrayLayouts.extensions] @@ -226,9 +217,9 @@ version = "0.5.0" [[deps.CFTime]] deps = ["Dates", "Printf"] -git-tree-sha1 = "937628bf8b377208ac359f57314fd85d3e0165d9" +git-tree-sha1 = "9b37e9e51aeea9763eea65b9b3aa1728fca94ffc" uuid = "179af706-886a-5703-950a-314cd64e0468" -version = "0.1.4" +version = "0.2.1" [[deps.CRlibm]] deps = ["CRlibm_jll"] @@ -244,48 +235,48 @@ version = "1.0.1+0" [[deps.ChainRulesCore]] deps = ["Compat", "LinearAlgebra"] -git-tree-sha1 = "06ee8d1aa558d2833aa799f6f0b31b30cada405f" +git-tree-sha1 = "e4c6a16e77171a5f5e25e9646617ab1c276c5607" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.25.2" +version = "1.26.0" weakdeps = ["SparseArrays"] [deps.ChainRulesCore.extensions] ChainRulesCoreSparseArraysExt = "SparseArrays" [[deps.ChunkCodecCore]] -git-tree-sha1 = "1667d15ab0285b4f8f4cfb8ea10ba2e48c6e32b4" +git-tree-sha1 = "e4a8d39a846ef288256a2cb94a60eb95c78e300a" uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" -version = "0.5.0" +version = "0.5.3" [[deps.ChunkCodecLibBrotli]] deps = ["ChunkCodecCore", "brotli_jll"] -git-tree-sha1 = "af76f3a42ef46fcb2942cd3c91c9c26f0ae3c01c" +git-tree-sha1 = "042987bfdeb5e814f0fe85ad54f1da3425d3275e" uuid = "653b0ff7-85b5-4442-93c1-dcc330d3ec7d" -version = "0.2.0" +version = "0.2.1" [[deps.ChunkCodecLibLz4]] deps = ["ChunkCodecCore", "Lz4_jll"] -git-tree-sha1 = "ad4336c7daaea2e589ee2f03a7f4d45702516267" +git-tree-sha1 = "36e0b65111bf84e3fcfbdbbeb1e1c23204e6404c" uuid = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" -version = "0.2.1" +version = "0.2.2" [[deps.ChunkCodecLibSnappy]] deps = ["ChunkCodecCore", "snappy_jll"] -git-tree-sha1 = "3014a20590582608bf4d4839cbbae5d0f2f600a5" +git-tree-sha1 = "7a5939207a97438b7f1c3939c9f7fc634b9e684f" uuid = "eac87354-86d5-4a5b-ab5f-a6ee56b239b3" -version = "0.2.0" +version = "0.2.1" [[deps.ChunkCodecLibZlib]] deps = ["ChunkCodecCore", "Zlib_jll"] -git-tree-sha1 = "5ac2692623c9beb04e35292e2daf678d980c128d" +git-tree-sha1 = "5866bf08bebfb3743e40c17ce805fbf03f85dbf4" uuid = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" -version = "0.2.0" +version = "0.2.1" [[deps.ChunkCodecLibZstd]] deps = ["ChunkCodecCore", "Zstd_jll"] -git-tree-sha1 = "e851a6d1a53b6246820c3ee49a703bd54e4a8ad5" +git-tree-sha1 = "6225e84baab33a74d6b16186c4465c46cb6b035a" uuid = "55437552-ac27-4d47-9aa3-63184e8fd398" -version = "0.2.0" +version = "0.2.1" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] @@ -295,33 +286,41 @@ version = "0.7.8" [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" +git-tree-sha1 = "67e11ee83a43eb71ddc950302c53bf33f0690dfe" uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.11.5" +version = "0.12.1" +weakdeps = ["StyledStrings"] + + [deps.ColorTypes.extensions] + StyledStringsExt = "StyledStrings" [[deps.ColorVectorSpace]] -deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "SpecialFunctions", "Statistics", "TensorCore"] -git-tree-sha1 = "600cc5508d66b78aae350f7accdb58763ac18589" +deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "Requires", "Statistics", "TensorCore"] +git-tree-sha1 = "8b3b6f87ce8f65a2b4f857528fd8d70086cd72b1" uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" -version = "0.9.10" +version = "0.11.0" +weakdeps = ["SpecialFunctions"] + + [deps.ColorVectorSpace.extensions] + SpecialFunctionsExt = "SpecialFunctions" [[deps.Colors]] deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] -git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" +git-tree-sha1 = "37ea44092930b1811e666c3bc38065d7d87fcc74" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.11" +version = "0.13.1" [[deps.CommonDataModel]] deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] -git-tree-sha1 = "358bf5a7d5c1387b995a43577673290c5d344758" +git-tree-sha1 = "a4f9a314202585fcdce4f1a3c4b86ce988ce76b1" uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" -version = "0.3.8" +version = "0.3.9" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +git-tree-sha1 = "0037835448781bb46feb39866934e243886d756a" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.16.0" +version = "4.18.0" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] @@ -426,34 +425,38 @@ uuid = "927a84f5-c5f4-47a5-9785-b46e178433df" version = "1.6.4" [[deps.DimensionalData]] -deps = ["Adapt", "ArrayInterface", "ConstructionBase", "DataAPI", "Dates", "Extents", "Interfaces", "IntervalSets", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "PrecompileTools", "Random", "RecipesBase", "SparseArrays", "Statistics", "TableTraits", "Tables"] -git-tree-sha1 = "b628bd06173897d44ab5cb5122e4a31509997c5a" +deps = ["Adapt", "ArrayInterface", "ConstructionBase", "DataAPI", "Dates", "Extents", "Interfaces", "IntervalSets", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "PrecompileTools", "Random", "RecipesBase", "Statistics", "TableTraits", "Tables"] +git-tree-sha1 = "5f488556227868b98a976f7bc2ad8dc47e82e6c5" uuid = "0703355e-b756-11e9-17c0-8b28908087d0" -version = "0.29.17" +version = "0.29.21" [deps.DimensionalData.extensions] + DimensionalDataAbstractFFTsExt = "AbstractFFTs" DimensionalDataAlgebraOfGraphicsExt = "AlgebraOfGraphics" DimensionalDataCategoricalArraysExt = "CategoricalArrays" DimensionalDataDiskArraysExt = "DiskArrays" DimensionalDataMakie = "Makie" DimensionalDataNearestNeighborsExt = "NearestNeighbors" DimensionalDataPythonCall = "PythonCall" + DimensionalDataSparseArraysExt = "SparseArrays" DimensionalDataStatsBase = "StatsBase" [deps.DimensionalData.weakdeps] + AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c" AlgebraOfGraphics = "cbdf2221-f076-402e-a563-3d30da359d67" CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" NearestNeighbors = "b8a86587-4115-5ab1-83bc-aa920d37bbce" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" [[deps.DiskArrays]] deps = ["ConstructionBase", "LRUCache", "Mmap", "OffsetArrays"] -git-tree-sha1 = "9ec4e6e7d6ef5599340f48bfbfeca4e730162b31" +git-tree-sha1 = "bfde0790720fcac006a3d62149309a685fc3aa13" uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" -version = "0.4.13" +version = "0.4.15" [[deps.Distances]] deps = ["LinearAlgebra", "Statistics", "StatsAPI"] @@ -548,9 +551,9 @@ version = "0.1.6" [[deps.EzXML]] deps = ["Printf", "XML2_jll"] -git-tree-sha1 = "f6f44ab51d253f851d2084c1ac761bb679798408" +git-tree-sha1 = "680a39c9aadce7c721b68d979e66dc65d2021aa6" uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" -version = "1.2.1" +version = "1.2.2" [[deps.FLoops]] deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] @@ -663,33 +666,32 @@ version = "0.4.4" [[deps.GeoInterface]] deps = ["DataAPI", "Extents", "GeoFormatTypes"] -git-tree-sha1 = "294e99f19869d0b0cb71aef92f19d03649d028d5" +git-tree-sha1 = "0f265264b9287a19715dc5d491dbe3aff00c1e71" uuid = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" -version = "1.4.1" +version = "1.5.0" -[[deps.GeoInterfaceMakie]] -deps = ["GeoInterface", "GeometryBasics", "MakieCore"] -git-tree-sha1 = "378afe561ba990392146e1a7abd472c7db7f1479" -uuid = "0edc0954-3250-4c18-859d-ec71c1660c08" -version = "0.1.9" + [deps.GeoInterface.extensions] + GeoInterfaceMakieExt = ["Makie", "GeometryBasics"] + GeoInterfaceRecipesBaseExt = "RecipesBase" -[[deps.GeoInterfaceRecipes]] -deps = ["GeoInterface", "RecipesBase"] -git-tree-sha1 = "fb1156076f24f1dfee45b3feadb31d05730a49ac" -uuid = "0329782f-3d07-4b52-b9f6-d3137cf03c7a" -version = "1.0.2" + [deps.GeoInterface.weakdeps] + GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326" + Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [[deps.GeoJSON]] -deps = ["Extents", "GeoFormatTypes", "GeoInterface", "GeoInterfaceMakie", "GeoInterfaceRecipes", "JSON3", "StructTypes", "Tables"] -git-tree-sha1 = "c803640368acf4ae02bb5a69a986cecc4a5cb984" +deps = ["Extents", "GeoFormatTypes", "GeoInterface", "JSON3", "StructTypes", "Tables"] +git-tree-sha1 = "ce64817b826c36b30493b31be2ce53c55a277835" uuid = "61d90e0f-e114-555e-ac52-39dfb47a3ef9" -version = "0.8.3" +version = "0.8.4" [deps.GeoJSON.extensions] GeoJSONMakieExt = "Makie" + GeoJSONRecipesBaseExt = "RecipesBase" [deps.GeoJSON.weakdeps] Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [[deps.GeoParquet]] deps = ["DataFrames", "Extents", "GeoFormatTypes", "GeoInterface", "JSON3", "Parquet2", "StructTypes", "Tables", "WellKnownGeometry"] @@ -705,10 +707,14 @@ version = "0.3.0" QuackIO = "7b8617ff-c43a-4c37-a28b-ad5a1791f8ae" [[deps.GeometryBasics]] -deps = ["EarCut_jll", "Extents", "GeoInterface", "IterTools", "LinearAlgebra", "PrecompileTools", "Random", "StaticArrays"] -git-tree-sha1 = "2670cf32dcf0229c9893b895a9afe725edb23545" +deps = ["EarCut_jll", "Extents", "IterTools", "LinearAlgebra", "PrecompileTools", "Random", "StaticArrays"] +git-tree-sha1 = "1f5a80f4ed9f5a4aada88fc2db456e637676414b" uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" -version = "0.5.9" +version = "0.5.10" +weakdeps = ["GeoInterface"] + + [deps.GeometryBasics.extensions] + GeometryBasicsGeoInterfaceExt = "GeoInterface" [[deps.GeometryOps]] deps = ["AbstractTrees", "AdaptivePredicates", "CoordinateTransformations", "DataAPI", "DelaunayTriangulation", "ExactPredicates", "Extents", "GeoFormatTypes", "GeoInterface", "GeometryOpsCore", "LinearAlgebra", "SortTileRecursiveTree", "StaticArrays", "Statistics", "Tables"] @@ -744,26 +750,20 @@ version = "5.2.3+0" [[deps.GitHub]] deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal", "URIs"] -git-tree-sha1 = "7ee730a8484d673a8ce21d8536acfe6494475994" +git-tree-sha1 = "2501485472c50b24874cc949f0fce9bd3c4a1738" uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" -version = "5.9.0" +version = "5.10.0" [[deps.Glob]] git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" version = "1.3.1" -[[deps.Graphics]] -deps = ["Colors", "LinearAlgebra", "NaNMath"] -git-tree-sha1 = "a641238db938fff9b2f60d08ed9030387daf428c" -uuid = "a2bd30eb-e257-5431-a919-1863eab51364" -version = "1.1.3" - [[deps.HDF4_jll]] deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Zlib_jll", "libaec_jll"] -git-tree-sha1 = "b4936dcc929748932ae40ebb3762255654602bc6" +git-tree-sha1 = "ea9eff9cfef5f45b771096e5c2de3de0eab937c3" uuid = "818ab7a1-5177-5f44-ba99-6e845030c6cb" -version = "4.3.1+0" +version = "4.3.2+0" [[deps.HDF5_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] @@ -773,9 +773,9 @@ version = "1.14.6+0" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "f93655dc73d7a0b4a368e3c0bce296ae035ad76e" +git-tree-sha1 = "ed5e9c58612c4e081aecdb6e1a479e18462e041e" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.10.16" +version = "1.10.17" [[deps.Hwloc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -797,33 +797,33 @@ version = "0.2.0" [[deps.ImageAxes]] deps = ["AxisArrays", "ImageBase", "ImageCore", "Reexport", "SimpleTraits"] -git-tree-sha1 = "2e4520d67b0cef90865b3ef727594d2a58e0e1f8" +git-tree-sha1 = "e12629406c6c4442539436581041d372d69c55ba" uuid = "2803e5a7-5153-5ecf-9a86-9b4c37f5f5ac" -version = "0.6.11" +version = "0.6.12" [[deps.ImageBase]] deps = ["ImageCore", "Reexport"] -git-tree-sha1 = "b51bb8cae22c66d0f6357e3bcb6363145ef20835" +git-tree-sha1 = "eb49b82c172811fd2c86759fa0553a2221feb909" uuid = "c817782e-172a-44cc-b673-b171935fbb9e" -version = "0.1.5" +version = "0.1.7" [[deps.ImageCore]] -deps = ["AbstractFFTs", "ColorVectorSpace", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] -git-tree-sha1 = "acf614720ef026d38400b3817614c45882d75500" +deps = ["ColorVectorSpace", "Colors", "FixedPointNumbers", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "PrecompileTools", "Reexport"] +git-tree-sha1 = "8c193230235bbcee22c8066b0374f63b5683c2d3" uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" -version = "0.9.4" +version = "0.10.5" [[deps.ImageIO]] -deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs"] -git-tree-sha1 = "437abb322a41d527c197fa800455f79d414f0a3c" +deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs", "WebP"] +git-tree-sha1 = "696144904b76e1ca433b886b4e7edd067d76cbf7" uuid = "82e4d734-157c-48bb-816b-45c225c6df19" -version = "0.6.8" +version = "0.6.9" [[deps.ImageMetadata]] deps = ["AxisArrays", "ImageAxes", "ImageBase", "ImageCore"] -git-tree-sha1 = "355e2b974f2e3212a75dfb60519de21361ad3cb7" +git-tree-sha1 = "2a81c3897be6fbcde0802a0ebe6796d0562f63ec" uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" -version = "0.9.9" +version = "0.9.10" [[deps.Imath_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -950,9 +950,9 @@ version = "1.0.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.7.1" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] @@ -1022,9 +1022,9 @@ version = "1.4.0" [[deps.LazyArrays]] deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "SparseArrays"] -git-tree-sha1 = "866ce84b15e54d758c11946aacd4e5df0e60b7a3" +git-tree-sha1 = "76627adb8c542c6b73f68d4bfd0aa71c9893a079" uuid = "5078a376-72f3-5289-bfd5-ec5146d43c02" -version = "2.6.1" +version = "2.6.2" [deps.LazyArrays.extensions] LazyArraysBandedMatricesExt = "BandedMatrices" @@ -1059,16 +1059,18 @@ uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" version = "8.6.0+0" [[deps.LibGEOS]] -deps = ["CEnum", "Extents", "GEOS_jll", "GeoInterface", "GeoInterfaceMakie", "GeoInterfaceRecipes"] -git-tree-sha1 = "a38d03335c2628dcc6a287affd1083f8764adc30" +deps = ["CEnum", "Extents", "GEOS_jll", "GeoInterface"] +git-tree-sha1 = "943871d7f1486a6d108f9be884b0b6c2fc63e8f9" uuid = "a90b1aa1-3769-5649-ba7e-abc5a9d163eb" -version = "0.9.4" +version = "0.9.5" [deps.LibGEOS.extensions] LibGEOSMakieExt = "Makie" + LibGEOSRecipesBaseExt = "RecipesBase" [deps.LibGEOS.weakdeps] Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [[deps.LibGit2]] deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] @@ -1115,9 +1117,9 @@ version = "4.7.1+0" [[deps.LightBSON]] deps = ["DataStructures", "Dates", "DecFP", "FNVHash", "JSON3", "Sockets", "StructTypes", "Transducers", "UUIDs", "UnsafeArrays", "WeakRefStrings"] -git-tree-sha1 = "3146f61d1f95f38f185c19b77e6e1aba9e4c0ab2" +git-tree-sha1 = "cef85bcff3b022934329dcf2a9c3bcec9bbacb14" uuid = "a4a7f996-b3a6-4de6-b9db-2fa5f350df41" -version = "1.3.0" +version = "1.3.1" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] @@ -1190,12 +1192,6 @@ git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" version = "0.5.16" -[[deps.MakieCore]] -deps = ["ColorTypes", "GeometryBasics", "IntervalSets", "Observables"] -git-tree-sha1 = "c3159eb1e3aa3e409edbb71f4035ed8b1fc16e23" -uuid = "20f20a25-4f0e-4fdf-b5d1-57303727442b" -version = "0.9.5" - [[deps.MappedArrays]] git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" @@ -1261,12 +1257,6 @@ version = "0.3.4" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" version = "2023.12.12" -[[deps.NaNMath]] -deps = ["OpenLibm_jll"] -git-tree-sha1 = "9b8215b1ee9e78a293f99797cd31375471b2bcae" -uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "1.1.3" - [[deps.NameResolution]] deps = ["PrettyPrint"] git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" @@ -1275,9 +1265,9 @@ version = "0.1.5" [[deps.NearestNeighbors]] deps = ["Distances", "StaticArrays"] -git-tree-sha1 = "8a3271d8309285f4db73b4f662b1b290c715e85e" +git-tree-sha1 = "ca7e18198a166a1f3eb92a3650d53d94ed8ca8a1" uuid = "b8a86587-4115-5ab1-83bc-aa920d37bbce" -version = "0.4.21" +version = "0.4.22" [[deps.NetCDF_jll]] deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "TOML", "XML2_jll", "Zlib_jll", "Zstd_jll", "libaec_jll", "libzip_jll"] @@ -1295,11 +1285,6 @@ version = "1.1.1" uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" -[[deps.Observables]] -git-tree-sha1 = "7438a59546cf62428fc9d1bc94729146d37a7225" -uuid = "510215fc-4207-5dde-b226-833fc4488ee2" -version = "0.5.5" - [[deps.OffsetArrays]] git-tree-sha1 = "117432e406b5c023f665fa73dc26e79ec3630151" uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" @@ -1357,9 +1342,9 @@ version = "1.5.0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "9216a80ff3682833ac4b733caa8c00390620ba5d" +git-tree-sha1 = "2ae7d4ddec2e13ad3bddf5c0796f7547cf682391" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.0+0" +version = "3.5.2+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] @@ -1441,9 +1426,9 @@ version = "1.2.1" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.0" [[deps.PrettyPrint]] git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" @@ -1469,9 +1454,9 @@ version = "1.10.4" [[deps.Proj]] deps = ["CEnum", "CoordinateTransformations", "GeoFormatTypes", "GeoInterface", "NetworkOptions", "PROJ_jll"] -git-tree-sha1 = "7fbdd2a507a06cf577ee465c4a6ec8d951b4bcee" +git-tree-sha1 = "61188669db4f5b400173e4ec60da8bcb72d6e749" uuid = "c94c279d-25a6-4763-9509-64d165bea63e" -version = "1.8.1" +version = "1.9.0" [[deps.QOI]] deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] @@ -1502,9 +1487,9 @@ version = "0.3.2" [[deps.Rasters]] deps = ["Adapt", "ColorTypes", "CommonDataModel", "ConstructionBase", "Dates", "DimensionalData", "DiskArrays", "Extents", "FillArrays", "Flatten", "GeoFormatTypes", "GeoInterface", "GeometryOpsCore", "LinearAlgebra", "Missings", "Mmap", "OffsetArrays", "ProgressMeter", "RecipesBase", "Reexport", "Setfield", "Statistics"] -git-tree-sha1 = "8692a80775808eb436b431b1493a91a24a5690c1" +git-tree-sha1 = "e741f9f9e51a7fd4527cf6df07251cc7a339ffc5" uuid = "a3a2b9e3-a471-40c9-b274-f788e487c689" -version = "0.14.5" +version = "0.14.6" [deps.Rasters.extensions] RastersArchGDALExt = "ArchGDAL" @@ -1629,15 +1614,15 @@ version = "1.2.0" [[deps.SimpleTraits]] deps = ["InteractiveUtils", "MacroTools"] -git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +git-tree-sha1 = "be8eeac05ec97d379347584fa9fe2f5f76795bcb" uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" -version = "0.9.4" +version = "0.9.5" [[deps.Sixel]] deps = ["Dates", "FileIO", "ImageCore", "IndirectArrays", "OffsetArrays", "REPL", "libsixel_jll"] -git-tree-sha1 = "2da10356e31327c7096832eb9cd86307a50b1eb6" +git-tree-sha1 = "0494aed9501e7fb65daba895fb7fd57cc38bc743" uuid = "45858cf5-a6b0-47a3-bbea-62219f50df47" -version = "0.1.3" +version = "0.1.5" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" @@ -1657,9 +1642,9 @@ version = "0.1.4" [[deps.SortingAlgorithms]] deps = ["DataStructures"] -git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.1" +version = "1.2.2" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] @@ -1701,9 +1686,9 @@ version = "0.1.2" [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] -git-tree-sha1 = "0feb6b9031bd5c51f9072393eb5ab3efd31bf9e4" +git-tree-sha1 = "cbea8a6bd7bed51b1619658dec70035e07b8502f" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.9.13" +version = "1.9.14" weakdeps = ["ChainRulesCore", "Statistics"] [deps.StaticArrays.extensions] @@ -1820,10 +1805,10 @@ uuid = "e0b8ae26-5307-5830-91fd-398402328850" version = "0.21.0+0" [[deps.TiffImages]] -deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "ProgressMeter", "SIMD", "UUIDs"] -git-tree-sha1 = "38f139cc4abf345dd4f22286ec000728d5e8e097" +deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "PrecompileTools", "ProgressMeter", "SIMD", "UUIDs"] +git-tree-sha1 = "02aca429c9885d1109e58f400c333521c13d48a0" uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" -version = "0.10.2" +version = "0.11.4" [[deps.TranscodingStreams]] git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" @@ -1853,9 +1838,9 @@ version = "0.4.84" Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" [[deps.URIs]] -git-tree-sha1 = "cbbebadbcc76c5ca1cc4b4f3b0614b3e603b5000" +git-tree-sha1 = "bef26fb046d031353ef97a82e3fdb6afe7f21b1a" uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.5.2" +version = "1.6.1" [[deps.UUIDs]] deps = ["Random", "SHA"] @@ -1877,6 +1862,12 @@ git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" version = "1.4.2" +[[deps.WebP]] +deps = ["CEnum", "ColorTypes", "FileIO", "FixedPointNumbers", "ImageCore", "libwebp_jll"] +git-tree-sha1 = "aa1ca3c47f119fbdae8770c29820e5e6119b83f2" +uuid = "e3aaa7dc-3e4b-44e0-be63-ffb868ccd7c1" +version = "0.1.3" + [[deps.WellKnownGeometry]] deps = ["GeoFormatTypes", "GeoInterface"] git-tree-sha1 = "73c83e60acd25f48573deea39cb41b7a5632fd0c" @@ -1985,9 +1976,9 @@ version = "100.702.400+0" [[deps.libpng_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "cd155272a3738da6db765745b89e466fa64d0830" +git-tree-sha1 = "07b6a107d926093898e82b3b1db657ebe33134ec" uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" -version = "1.6.49+0" +version = "1.6.50+0" [[deps.libsixel_jll]] deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "libpng_jll"] @@ -2003,9 +1994,9 @@ version = "1.0.21+0" [[deps.libwebp_jll]] deps = ["Artifacts", "Giflib_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libglvnd_jll", "Libtiff_jll", "libpng_jll"] -git-tree-sha1 = "d2408cac540942921e7bd77272c32e58c33d8a77" +git-tree-sha1 = "4e4282c4d846e11dce56d74fa8040130b7a95cb3" uuid = "c5f90fcd-3b7e-5836-afba-fc50a0988cb2" -version = "1.5.0+0" +version = "1.6.0+0" [[deps.libzip_jll]] deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "OpenSSL_jll", "XZ_jll", "Zlib_jll", "Zstd_jll"] diff --git a/sandbox/start.sh b/sandbox/start.sh index 6025dd2..f787c6d 100755 --- a/sandbox/start.sh +++ b/sandbox/start.sh @@ -1 +1 @@ -julia -t auto --project=. -e "using Revise; using DotEnv; using Pkg; Pkg.develop(path=\"..\"); using ReefGuideWorker; DotEnv.load!();" -i +julia -t auto --project=. -e "using Revise; using DotEnv; using ReefGuideWorker; DotEnv.load!(); ReefGuideWorker.start_worker();" -i diff --git a/src/ReefGuideWorker.jl b/src/ReefGuideWorker.jl index c40acb3..46dc448 100644 --- a/src/ReefGuideWorker.jl +++ b/src/ReefGuideWorker.jl @@ -34,7 +34,7 @@ function start_worker() @info "Initializing worker from environment variables..." worker = create_worker_from_env() - @info "Warming up regional data (expensive op which is cached in memory and disk)" + @info "Warming up regional data (expensive op which is cached in memory)" try get_regional_data(; data_path=worker.config.data_path, diff --git a/src/sysimage.jl b/src/sysimage.jl index 17c047d..2419b34 100644 --- a/src/sysimage.jl +++ b/src/sysimage.jl @@ -12,7 +12,7 @@ end # cpu_targets taken from: https://docs.julialang.org/en/v1/devdocs/sysimg/#Specifying-multiple-system-image-targets create_sysimage( - ["ReefGuideWorker"]; + ["ReefGuideWorker", "Revise", "Infiltrator"]; sysimage_path="reefguide_img.$ext", sysimage_build_args=`--strip-metadata`, # `--strip-ir --strip-metadata --incremental=false` cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)", # ;x86_64; haswell;skylake;skylake-avx512;tigerlake diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index fb92108..f28694b 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -61,6 +61,7 @@ function check_existing_regional_data_from_disk(; else @debug "No disk cache file found" expected_path = reg_cache_filename end + # No cache available or load failed return nothing end @@ -91,13 +92,6 @@ function initialize_data_with_cache(; REGIONAL_DATA = local_data return nothing end - - # Try disk cache second (faster than full reload) - disk_data = check_existing_regional_data_from_disk(; cache_path=cache_path) - if !isnothing(disk_data) - REGIONAL_DATA = disk_data - return nothing - end else @info "Cache invalidation forced - reloading from source files" end @@ -109,21 +103,6 @@ function initialize_data_with_cache(; # Update global cache REGIONAL_DATA = regional_data - # Save to disk for future use - @info "Saving regional data to disk cache" cache_path - try - # Ensure cache directory exists - mkpath(cache_path) - - serialize( - joinpath(cache_path, REGIONAL_DATA_CACHE_FILENAME), - regional_data - ) - @info "Successfully saved regional data cache to disk" - catch e - @warn "Failed to save regional data cache to disk" error = e - end - return nothing end @@ -144,7 +123,8 @@ function get_regional_data(; data_path::String, cache_path::String)::ReefGuide.R @debug "Getting regional data with automatic cache management" data_path cache_path # Ensure data is loaded (with caching) - initialize_data_with_cache(; data_path=data_path, cache_path=cache_path) + ts = @elapsed initialize_data_with_cache(; data_path=data_path, cache_path=cache_path) + @info "Took $(ts) seconds to initialize" # Return cached data return REGIONAL_DATA