Add soft link and external links in JLD2

Author: JonasIsensee

CHANGELOG.md

@@ -20,6 +20,7 @@ makedocs(;
         "Internals & Design" => "internals.md",
         "HDF5 Compatibility" => "hdf5compat.md",
         "Advanced Usage" => "advanced.md",
+        "Dataset Links" => "external_links.md",
         "Legacy" => "legacy.md",
         "Troubleshooting" => "troubleshooting.md"
     ],

docs/make.jl

@@ -0,0 +1,31 @@
+# Links in JLD2
+
+JLD2 supports three types of links compatible with the HDF5 specification:
+
+- **Hard Links** (default): Direct pointers to objects within the same file
+- **Soft Links**: Path-based symbolic links resolved at access time
+- **External Links**: References to objects in different files
+
+## Usage
+
+```@example
+using JLD2
+jldopen("file.jld2", "w") do f
+    f["data"] = [1, 2, 3, 4, 5]
+
+    # Soft link (within same file)
+    f["data_alias"] = JLD2.Link("/data")
+
+    # External link (to different file)
+    f["remote_data"] = JLD2.Link("/dataset"; file="other.jld2")
+end
+
+# Create external file (before or after)
+jldsave("other.jld2"; dataset="external data")
+
+# Access links transparently
+jldopen("file.jld2", "r") do f
+    f["data_alias"]   # Returns [1, 2, 3, 4, 5]
+    f["remote_data"]  # Loads from other.jld2
+end
+```

docs/src/external_links.md

@@ -10,6 +10,7 @@ export jldopen, @load, @save, save_object, load_object, jldsave
 export Shuffle, Deflate, ZstdFilter
 
 include("types.jl")
+include("links.jl")
 include("macros_utils.jl")
 include("io/mmapio.jl")
 include("io/bufferedio.jl")
@@ -59,13 +60,14 @@ mutable struct Group{T}
     next_link_offset::Int64
     est_num_entries::Int
     est_link_name_len::Int
-    unwritten_links::OrderedDict{String,RelOffset}
+    unwritten_links::OrderedDict{String,Link}
     unwritten_child_groups::OrderedDict{String,Group{T}}
-    written_links::OrderedDict{String,RelOffset}
+    written_links::OrderedDict{String,Link}
 
     Group{T}(f; est_num_entries::Int=4, est_link_name_len::Int=8) where T =
         new(f, -1, -1, -1, -1, est_num_entries, est_link_name_len,
-        OrderedDict{String,RelOffset}(), OrderedDict{String,Group{T}}())
+        OrderedDict{String,Link}(), OrderedDict{String,Group{T}}(),
+        OrderedDict{String,Link}())
 
     Group{T}(f, last_chunk_start_offset, continuation_message_goes_here,
              last_chunk_checksum_offset, next_link_offset,
@@ -180,7 +182,7 @@ function jldopen(fname::AbstractString, wr::Bool, create::Bool, truncate::Bool,
     parallel_read::Bool=false,
     plain::Bool=false
 ) where T<:Union{Type{IOStream},Type{MmapIO}}
-  
+
     mmaparrays && @warn "mmaparrays keyword is currently ignored" maxlog = 1
     filters = Filters.normalize_filters(compress)
 
@@ -263,10 +265,12 @@ function initialize_fileobject!(f::JLDFile)
     end
     f.root_group = load_group(f, f.root_group_offset)
 
-    types_offset = get(f.root_group.written_links, "_types", UNDEFINED_ADDRESS)
-    if types_offset != UNDEFINED_ADDRESS
+    # Use lookup_link directly instead of lookup_offset
+    types_offset = getoffset(f.root_group, lookup_link(f.root_group, "_types"))
+    if types_offset !== UNDEFINED_ADDRESS
         f.types_group = f.loaded_groups[types_offset] = load_group(f, types_offset)
-        for (i, offset::RelOffset) in enumerate(values(f.types_group.written_links))
+        for (i, link) in enumerate(values(f.types_group.written_links))
+            offset = getoffset(f.types_group, link)
             f.datatype_locations[offset] = CommittedDatatype(offset, i)
         end
         resize!(f.datatypes, length(f.datatype_locations))
@@ -409,6 +413,8 @@ Base.keys(f::JLDFile) = filter!(x->x != "_types", keys(f.root_group))
 Base.keytype(f::JLDFile) = String
 Base.length(f::Union{JLDFile, Group}) = length(keys(f))
 
+lookup_link(f::JLDFile, name::AbstractString) = lookup_link(f.root_group, name)
+
 Base.get(default::Function, f::Union{JLDFile, Group}, name::AbstractString) =
     haskey(f, name) ? f[name] : default()
 Base.get(f::Union{JLDFile, Group}, name::AbstractString, default) =

src/JLD2.jl

@@ -315,15 +315,15 @@ function get_dataset(g::Group, name::String)
     f.n_times_opened == 0 && throw(ArgumentError("file is closed"))
 
     (g, name) = pathize(g, name, false)
-    roffset = lookup_offset(g, name)
-    if roffset == UNDEFINED_ADDRESS
-        if isempty(name)
-            # this is a group
-            return get_dataset(f, group_offset(g), g, name)
-        end
-        throw(KeyError(name))
+
+    if isempty(name)
+        # this is a group
+        return get_dataset(f, group_offset(g), g, name)
     end
-    get_dataset(f, roffset, g, name)
+
+    link = lookup_link(g, name)
+    offset = getoffset(g, link)
+    return get_dataset(f, offset, g, name)
 end
 
 function get_dataset(f::JLDFile, offset::RelOffset, g=f.root_group, name="")
@@ -362,8 +362,53 @@ end
 
 # Links
 message_size(msg::Pair{String, RelOffset}) = jlsizeof(Val(HmLinkMessage); link_name=msg.first)
+message_size(msg::Pair{String, Link}) = message_size_for_link(msg.first, msg.second)
+
 write_header_message(io, f, msg::Pair{String, RelOffset}, _=nothing) =
     write_header_message(io, Val(HmLinkMessage); link_name=msg.first, target=msg.second)
+write_header_message(io, f, msg::Pair{String, Link}, _=nothing) =
+    write_link_message(io, msg.first, msg.second)
+
+"""
+    message_size_for_link(name::String, link::Link) -> Int
+
+Calculate the size of a link message for the given link type.
+"""
+function message_size_for_link(link_name::String, link::Link)
+    is_hard_link(link) && return jlsizeof(Val(HmLinkMessage); link_name)
+
+    flags = UInt8(0x10 | 0x08 | size_flag(sizeof(link_name)))
+    if is_soft_link(link)
+        jlsizeof(Val(HmLinkMessage); link_name, flags, link_type=UInt8(1),
+                 link_info_size=sizeof(link.path), soft_link=UInt8[])
+    else  # external link
+        jlsizeof(Val(HmLinkMessage); link_name, flags, link_type=UInt8(64),
+                 link_info_size=3+sizeof(link.external_file)+sizeof(link.path),
+                 external_link=UInt8[])
+    end
+end
+
+"""
+    write_link_message(io, name::String, link::Link)
+
+Write a link message for the given link type to the I/O stream.
+"""
+function write_link_message(io, link_name::String, link::Link)
+    if is_hard_link(link)
+        return write_header_message(io, Val(HmLinkMessage); link_name, target=link.offset)
+    end
+    flags = UInt8(0x10 | 0x08 | size_flag(sizeof(link_name)))
+    if is_soft_link(link)
+        soft_link = Vector{UInt8}(link.path)
+        write_header_message(io, Val(HmLinkMessage); link_name, flags, link_type=1, soft_link)
+    elseif is_external_link(link)
+        # External link data: two null-terminated strings
+        external_link = vcat(0x00, Vector{UInt8}(link.external_file), 0x00,
+                            Vector{UInt8}(link.path), 0x00)
+        write_header_message(io, Val(HmLinkMessage); link_name, flags, link_type=64,
+                           external_link)
+    end
+end
 
 function attach_message(f::JLDFile, offset, messages, wsession=JLDWriteSession();
     chunk_start,