feat: add decompression support for CDF files (#8)

Author: Beforerr

.gitignore

@@ -5,4 +5,6 @@
 /docs/Manifest*.toml
 /docs/build/
 ref/
-.claude/
+.claude/
+data/
+!data/a_*.cdf

CLAUDE.md AGENTS.mdCLAUDE.md renamed to AGENTS.md

@@ -1,4 +1,6 @@
 
 There are two main reference implementation in ref/ folder. When having trouble implement the related function, refer to the reference implementation to understand the logic and the details. 
 - CDFpp is a C++ implementation. 
-- cdflib is a python implementation. 
+- cdflib is a python implementation. 
+
+Simple and clear implementation is preferred than verbose and complex implementation. 

Project.toml

@@ -4,15 +4,19 @@ version = "0.1.0"
 authors = ["Beforerr <zzj956959688@gmail.com> and contributors"]
 
 [deps]
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Dictionaries = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4"
+LibDeflate = "9255714d-24a7-4b30-8ea3-d46a97f7e13b"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 StaticStrings = "4db0a0c5-418a-4e1d-8806-cb305fe13294"
 UnixTimes = "ab1a18e7-b408-4913-896c-624bb82ed7f4"
 
 [compat]
+CodecZlib = "0.7"
 Dates = "1"
 Dictionaries = "0.4"
+LibDeflate = "0.4.3"
 Mmap = "1"
 StaticStrings = "0.2.6"
 UnixTimes = "1.7.2"

README.md

@@ -36,9 +36,6 @@ println("Variables: ", keys(cdf))
 
 # Access a variable
 var = cdf["temperature"]
-println("Variable data: ", var.data)
-println("Data type: ", var.data_type)
-println("Dimensions: ", var.dimensions)
 ```
 
 ## Elsewhere

data/a_cdf_with_compressed_vars.cdf

@@ -4,6 +4,10 @@ using Dates, UnixTimes
 using Mmap
 using Dictionaries
 using StaticStrings
+using Base.Threads
+using CodecZlib: GzipDecompressor, transcode
+using LibDeflate
+using LibDeflate: GzipDecompressResult
 
 export CDFDataset, CDFVariable
 export Majority, CompressionType, DataType
@@ -12,9 +16,10 @@ export Epoch, Epoch16, TT2000
 include("epochs.jl")
 include("enums.jl")
 include("parsing.jl")
+include("decompress.jl")
 include("records/records.jl")
-include("dataset.jl")
 include("variable.jl")
+include("dataset.jl")
 include("loading/attribute.jl")
 include("loading/variable.jl")
 

data/a_compressed_cdf.cdf

@@ -26,10 +26,12 @@ function CDFDataset(filename)
         magic_bytes = read_be(buffer, 1, UInt32)
         @assert validate_cdf_magic(magic_bytes)
 
-        # Read compression info
-        compression_bytes = read_be(buffer, 5, UInt32)
-        compression = CompressionType(compression_bytes)
-        RecordSizeType = is_cdf_v3(magic_bytes) ? UInt64 : UInt32
+        RecordSizeType = is_cdf_v3(magic_bytes) ? Int64 : Int32
+        compression_flag = read_be(buffer, 5, UInt32)
+        compression = NoCompression
+        if compression_flag != 0x0000FFFF
+            buffer, compression = decompress_bytes(buffer, RecordSizeType)
+        end
         # Parse CDF header
         cdr = CDR(buffer, 9, RecordSizeType)
         gdr = GDR(buffer, cdr.gdr_offset, RecordSizeType)
@@ -98,3 +100,15 @@ function Base.keys(cdf::CDFDataset)
 end
 
 Base.haskey(cdf::CDFDataset, var_name::String) = var_name in keys(cdf)
+
+Base.iterate(cdf::CDFDataset, state = 1) = state > length(cdf) ? nothing : (cdf[keys(cdf)[state]], state + 1)
+
+function Base.show(io::IO, ::MIME"text/plain", cdf::CDFDataset)
+    println(io, typeof(cdf), ":", cdf.filename)
+    println(io, "variables")
+    for var in keys(cdf)
+        println(io, "  $var")
+    end
+    println(io, cdf.cdr)
+    return
+end

data/a_rle_compressed_cdf.cdf

@@ -0,0 +1,52 @@
+include("decompress/rle.jl")
+include("decompress/gzip.jl")
+
+
+function decompress_bytes(buffer::Vector{UInt8}, RecordSizeType)
+    ccr = CCR(buffer, 8, RecordSizeType)
+    cpr = CPR(buffer, Int(ccr.cpr_offset), RecordSizeType)
+    compression = CompressionType(cpr.compression_type)
+    payload = data_view(ccr, buffer)
+    expected = Int(ccr.uncompressed_size)
+    decompressed = decompress_bytes(payload, compression; expected_bytes = expected)
+    new_size = 8 + length(decompressed)
+    new_buffer = Vector{UInt8}(undef, new_size)
+    copyto!(new_buffer, 1, buffer, 1, 4)
+    new_buffer[5] = 0x00
+    new_buffer[6] = 0x00
+    new_buffer[7] = 0xFF
+    new_buffer[8] = 0xFF
+    copyto!(new_buffer, 9, decompressed, 1, length(decompressed))
+    return new_buffer, compression
+end
+
+function decompress_bytes(data::AbstractVector{UInt8}, compression::CompressionType; expected_bytes::Union{Nothing, Int} = nothing)
+    compression == NoCompression && return data
+    @assert compression in (GzipCompression, RLECompression)
+    if compression == GzipCompression
+        result = transcode(GzipDecompressor, Vector{UInt8}(data))
+    elseif compression == RLECompression
+        isnothing(expected_bytes) && throw(ArgumentError("RLE decompression requires expected size"))
+        result = _rle_decompress(data, expected_bytes)
+    end
+    if !isnothing(expected_bytes) && length(result) != expected_bytes
+        throw(ArgumentError("Decompressed payload size mismatch (expected $(expected_bytes), got $(length(result)))"))
+    end
+    return result
+end
+
+function decompress_bytes!(decompressor, dest, doffs, src::AbstractVector{UInt8}, soffs, N, n_in, compression::CompressionType)
+    if compression == NoCompression
+        _copy_to!(dest, doffs, src, soffs, N)
+        return
+    end
+    @assert compression in (GzipCompression, RLECompression)
+    n_out = N * sizeof(eltype(dest))
+    out_ptr = pointer(dest, doffs)
+    in_ptr = pointer(src, soffs)
+    return if compression == GzipCompression
+        out = _unsafe_gzip_decompress!(decompressor, out_ptr, n_out, in_ptr, n_in)
+        @assert !(out isa LibDeflateError) out
+    elseif compression == RLECompression
+    end
+end