From df8d948095e9ec1479bc0ace895c22ba6d6adf8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 7 Mar 2026 12:56:30 +0100 Subject: [PATCH 1/3] Replace hex_erl_tar fork with OTP master's erl_tar Copy erl_tar.erl and erl_tar.hrl from OTP commit 013041bd68c2 with minimal modifications (module rename, include path, remove -doc/-moduledoc attributes for OTP 24 compatibility). This eliminates divergence from upstream. The fork's add/5 is replaced with OTP's add/4 using the {mode, Mode} option. hex_filename.erl is removed since filelib:safe_relative_path/2 is available since OTP 23. --- src/hex_erl_tar.erl | 346 +++++++++++++++++++------------------ src/hex_erl_tar.hrl | 71 ++++---- src/hex_filename.erl | 58 ------- src/hex_tarball.erl | 2 +- test/hex_tarball_SUITE.erl | 2 +- 5 files changed, 218 insertions(+), 261 deletions(-) delete mode 100644 src/hex_filename.erl diff --git a/src/hex_erl_tar.erl b/src/hex_erl_tar.erl index 28ac0d29..b77634c8 100644 --- a/src/hex_erl_tar.erl +++ b/src/hex_erl_tar.erl @@ -1,20 +1,15 @@ -%% @private -%% Copied from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/erl_tar.erl -%% with modifications: -%% - Change module name to `hex_erl_tar` -%% - Set tar mtimes to 0 and remove dependency on :os.system_time/1 -%% - Preserve modes when building tarball -%% - Do not crash if failing to write tar -%% - Allow setting file_info opts on :hex_erl_tar.add -%% - Add safe_relative_path_links/2 to check directory traversal vulnerability when extracting files, -%% it differs from OTP's current fix (2020-02-04) in that it checks regular files instead of -%% symlink targets. This allows creating symlinks with relative path targets such as `../tmp/log` -%% - Remove ram_file usage (backported from OTP master) - +%% This file is a copy of erl_tar.erl from OTP with the following modifications: +%% 1. Module renamed from erl_tar to hex_erl_tar +%% 2. -include changed from erl_tar.hrl to hex_erl_tar.hrl +%% 3. -doc and -moduledoc attributes removed for OTP 24 compatibility +%% +%% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% SPDX-License-Identifier: Apache-2.0 +%% +%% Copyright Ericsson AB 1997-2025. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,18 +37,20 @@ %% http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html -module(hex_erl_tar). + -export([init/3, create/2, create/3, extract/1, extract/2, table/1, table/2, t/1, tt/1, open/2, close/1, - add/3, add/4, add/5, + add/3, add/4, format_error/1]). -include_lib("kernel/include/file.hrl"). --include_lib("hex_erl_tar.hrl"). +-include("hex_erl_tar.hrl"). %% Converts the short error reason to a descriptive string. + -spec format_error(term()) -> string(). format_error(invalid_tar_checksum) -> "Checksum failed"; @@ -92,17 +89,21 @@ format_error(Term) -> lists:flatten(io_lib:format("~tp", [Term])). %% Initializes a new reader given a custom file handle and I/O wrappers --spec init(handle(), write | read, file_op()) -> {ok, reader()} | {error, badarg}. -init(Handle, AccessMode, Fun) when is_function(Fun, 2) -> - Reader = #reader{handle=Handle,access=AccessMode,func=Fun}, + + +-spec init(UserData :: user_data(), write | read, file_op()) -> + {ok, tar_descriptor()} | {error, badarg}. +init(UserData, AccessMode, Fun) when is_function(Fun, 2) -> + Reader = #reader{handle=UserData,access=AccessMode,func=Fun}, {ok, Pos, Reader2} = do_position(Reader, {cur, 0}), {ok, Reader2#reader{pos=Pos}}; -init(_Handle, _AccessMode, _Fun) -> +init(_UserData, _AccessMode, _Fun) -> {error, badarg}. %%%================================================================ %% Extracts all files from the tar file Name. --spec extract(open_handle()) -> ok | {error, term()}. + +-spec extract(Open :: open_type()) -> ok | {error, term()}. extract(Name) -> extract(Name, []). @@ -115,10 +116,11 @@ extract(Name) -> %% - {files, ListOfFilesToExtract}: Only extract ListOfFilesToExtract %% - verbose: Prints verbose information about the extraction, %% - {cwd, AbsoluteDir}: Sets the current working directory for the extraction --spec extract(open_handle(), [extract_opt()]) -> - ok - | {ok, [{string(), binary()}]} - | {error, term()}. + +-spec extract(Open :: open_type(), []) -> + ok | {error, term()}; + (Open :: open_type(), [extract_opt(), ...]) -> + ok | {ok, [{string(), binary()}]} | {error, term()}. extract({binary, Bin}, Opts) when is_list(Opts) -> do_extract({binary, Bin}, Opts); extract({file, Fd}, Opts) when is_list(Opts) -> @@ -178,7 +180,6 @@ check_extract(Name, #read_opts{files=Files}) -> %%%================================================================ %% The following table functions produce a list of information about %% the files contained in the archive. --type filename() :: string(). -type typeflag() :: regular | link | symlink | char | block | directory | fifo | reserved | unknown. @@ -186,23 +187,25 @@ check_extract(Name, #read_opts{files=Files}) -> -type uid() :: non_neg_integer(). -type gid() :: non_neg_integer(). --type tar_entry() :: {filename(), - typeflag(), - non_neg_integer(), - tar_time(), - mode(), - uid(), - gid()}. +-type tar_entry() :: {Name :: name_in_archive(), + Type :: typeflag(), + Size :: non_neg_integer(), + MTime :: tar_time(), + Mode :: mode(), + Uid :: uid(), + Gid :: gid()}. %% Returns a list of names of the files in the tar file Name. --spec table(open_handle()) -> {ok, [string()]} | {error, term()}. + +-spec table(Open :: open_type()) -> {ok, [name_in_archive()]} | {error, term()}. table(Name) -> table(Name, []). %% Returns a list of names of the files in the tar file Name. %% Options accepted: compressed, verbose, cooked. --spec table(open_handle(), [compressed | verbose | cooked]) -> - {ok, [tar_entry()]} | {error, term()}. + +-spec table(Open :: open_type(), [compressed | verbose | cooked]) -> + {ok, [name_in_archive() | tar_entry()]} | {error, term()}. table(Name, Opts) when is_list(Opts) -> foldl_read(Name, fun table1/4, [], table_opts(Opts)). @@ -242,6 +245,7 @@ typeflag(_) -> unknown. %% meant to be invoked from the shell. %% Prints each filename in the archive + -spec t(file:filename()) -> ok | {error, term()}. t(Name) when is_list(Name); is_binary(Name) -> case table(Name) of @@ -252,7 +256,8 @@ t(Name) when is_list(Name); is_binary(Name) -> end. %% Prints verbose information about each file in the archive --spec tt(open_handle()) -> ok | {error, term()}. + +-spec tt(open_type()) -> ok | {error, term()}. tt(Name) -> case table(Name, [verbose]) of {ok, List} -> @@ -314,11 +319,12 @@ month(12) -> "Dec". %%%================================================================ %% The open function with friends is to keep the file and binary api of this module --type open_handle() :: file:filename() +-type open_type() :: file:filename_all() | {binary, binary()} - | {file, term()}. --spec open(open_handle(), [write | compressed | cooked]) -> - {ok, reader()} | {error, term()}. + | {file, file:io_device()}. + +-spec open(Open :: open_type(), [write | compressed | cooked]) -> + {ok, tar_descriptor()} | {error, term()}. open({binary, Bin}, Mode) when is_binary(Bin) -> do_open({binary, Bin}, Mode); open({file, Fd}, Mode) -> @@ -334,32 +340,53 @@ do_open(Name, Mode) when is_list(Mode) -> {error, {Name, Reason}} end. -open1({binary,Bin0}, read, _Raw, Opts) when is_binary(Bin0) -> +open1({binary,Bin0}=Handle, read, _Raw, Opts) when is_binary(Bin0) -> Bin = case lists:member(compressed, Opts) of true -> + %% emulate file:open with Modes = [compressed_one ...] + Z = zlib:open(), + zlib:inflateInit(Z, 31, cut), try - zlib:gunzip(Bin0) + IoList = zlib:inflate(Z, Bin0), + zlib:inflateEnd(Z), + iolist_to_binary(IoList) catch - _:_ -> Bin0 + _:_ -> Bin0 + after + zlib:close(Z) end; false -> Bin0 end, + case file:open(Bin, [ram,binary,read]) of {ok,File} -> {ok, #reader{handle=File,access=read,func=fun file_op/2}}; - Error -> - Error + {error, Reason} -> + {error, {Handle, Reason}} end; -open1({file, Fd}, read, _Raw, _Opts) -> - Reader = #reader{handle=Fd,access=read,func=fun file_op/2}, - case do_position(Reader, {cur, 0}) of - {ok, Pos, Reader2} -> - {ok, Reader2#reader{pos=Pos}}; - {error, _} = Err -> - Err +open1({file, Fd}=Handle, read, [raw], Opts) -> + case not lists:member(compressed, Opts) of + true -> + Reader = #reader{handle=Fd,access=read,func=fun file_op/2}, + case do_position(Reader, {cur, 0}) of + {ok, Pos, Reader2} -> + {ok, Reader2#reader{pos=Pos}}; + {error, Reason} -> + {error, {Handle, Reason}} + end; + false -> + {error, {Handle, {incompatible_option, compressed}}} end; -open1(Name, Access, Raw, Opts) when is_list(Name) or is_binary(Name) -> +open1({file, _Fd}=Handle, read, [], _Opts) -> + {error, {Handle, {incompatible_option, cooked}}}; +open1(Name, Access, Raw, Opts0) when is_list(Name); is_binary(Name) -> + Opts = case lists:member(compressed, Opts0) andalso Access == read of + true -> + [compressed_one | (Opts0 -- [compressed])]; + false -> + Opts0 + end, case file:open(Name, Raw ++ [binary, Access|Opts]) of {ok, File} -> {ok, #reader{handle=File,access=Access,func=fun file_op/2}}; @@ -379,7 +406,7 @@ open_mode([read|Rest], false, Raw, Opts) -> open_mode([write|Rest], false, Raw, Opts) -> open_mode(Rest, write, Raw, Opts); open_mode([compressed|Rest], Access, Raw, Opts) -> - open_mode(Rest, Access, Raw, [compressed|Opts]); + open_mode(Rest, Access, Raw, [compressed,read_ahead|Opts]); open_mode([cooked|Rest], Access, _Raw, Opts) -> open_mode(Rest, Access, [], Opts); open_mode([], Access, Raw, Opts) -> @@ -397,7 +424,8 @@ file_op(close, Fd) -> file:close(Fd). %% Closes a tar archive. --spec close(reader()) -> ok | {error, term()}. + +-spec close(TarDescriptor :: tar_descriptor()) -> ok | {error, term()}. close(#reader{access=read}=Reader) -> ok = do_close(Reader); close(#reader{access=write}=Reader) -> @@ -418,16 +446,18 @@ pad_file(#reader{pos=Pos}=Reader) -> %% Creation/modification of tar archives %% Creates a tar file Name containing the given files. --spec create(file:filename(), filelist()) -> ok | {error, {string(), term()}}. + +-spec create(file:filename_all(), filelist()) -> ok | {error, {string(), term()}}. create(Name, FileList) when is_list(Name); is_binary(Name) -> create(Name, FileList, []). %% Creates a tar archive Name containing the given files. %% Accepted options: verbose, compressed, cooked --spec create(file:filename(), filelist(), [create_opt()]) -> + +-spec create(file:filename_all(), filelist(), [create_opt()]) -> ok | {error, term()} | {error, {string(), term()}}. create(Name, FileList, Options) when is_list(Name); is_binary(Name) -> - Mode = lists:filter(fun(X) -> (X=:=compressed) or (X=:=cooked) + Mode = lists:filter(fun(X) -> X =:= compressed orelse X =:= cooked end, Options), case open(Name, [write|Mode]) of {ok, TarFile} -> @@ -456,49 +486,44 @@ do_create(TarFile, [Name|Rest], Opts) -> end. %% Adds a file to a tape archive. --type add_type() :: string() - | {string(), string()} - | {string(), binary()}. --spec add(reader(), add_type(), [add_opt()]) -> ok | {error, term()}. + +-spec add(TarDescriptor, Name, Options) -> ok | {error, term()} when + TarDescriptor :: tar_descriptor(), + Name :: name_in_archive() | {name_in_archive(), file:filename_all()}, + Options :: [add_opt()]. add(Reader, {NameInArchive, Name}, Opts) when is_list(NameInArchive), is_list(Name) -> - do_add(Reader, Name, NameInArchive, undefined, Opts); + do_add(Reader, Name, NameInArchive, Opts); add(Reader, {NameInArchive, Bin}, Opts) when is_list(NameInArchive), is_binary(Bin) -> - do_add(Reader, Bin, NameInArchive, undefined, Opts); -add(Reader, {NameInArchive, Bin, Mode}, Opts) - when is_list(NameInArchive), is_binary(Bin), is_integer(Mode) -> - do_add(Reader, Bin, NameInArchive, Mode, Opts); + do_add(Reader, Bin, NameInArchive, Opts); add(Reader, Name, Opts) when is_list(Name) -> - do_add(Reader, Name, Name, undefined, Opts). + do_add(Reader, Name, Name, Opts). --spec add(reader(), string() | binary(), string(), [add_opt()]) -> - ok | {error, term()}. +-spec add(TarDescriptor, Filename, NameInArchive, Options) -> + ok | {error, term()} when + TarDescriptor :: tar_descriptor(), + Filename :: file:filename_all(), + NameInArchive :: name_in_archive(), + Options :: [add_opt()]. add(Reader, NameOrBin, NameInArchive, Options) when is_list(NameOrBin); is_binary(NameOrBin), is_list(NameInArchive), is_list(Options) -> - do_add(Reader, NameOrBin, NameInArchive, undefined, Options). - --spec add(reader(), string() | binary(), string(), integer(), [add_opt()]) -> - ok | {error, term()}. -add(Reader, NameOrBin, NameInArchive, Mode, Options) - when is_list(NameOrBin); is_binary(NameOrBin), - is_list(NameInArchive), is_integer(Mode), is_list(Options) -> - do_add(Reader, NameOrBin, NameInArchive, Mode, Options). + do_add(Reader, NameOrBin, NameInArchive, Options). -do_add(#reader{access=write}=Reader, Name, NameInArchive, Mode, Options) +do_add(#reader{access=write}=Reader, Name, NameInArchive, Options) when is_list(NameInArchive), is_list(Options) -> - RF = fun(F) -> apply_file_info_opts(Options, file:read_link_info(F, [{time, posix}])) end, + RF = apply_file_info_opts_fun(Options, read_link_info), Opts = #add_opts{read_info=RF}, - add1(Reader, Name, NameInArchive, Mode, add_opts(Options, Options, Opts)); -do_add(#reader{access=read},_,_,_,_) -> + add1(Reader, Name, NameInArchive, add_opts(Options, Options, Opts)); +do_add(#reader{access=read},_,_,_) -> {error, eacces}; -do_add(Reader,_,_,_,_) -> +do_add(Reader,_,_,_) -> {error, {badarg, Reader}}. add_opts([dereference|T], AllOptions, Opts) -> - RF = fun(F) -> apply_file_info_opts(AllOptions, file:read_file_info(F, [{time, posix}])) end, + RF = apply_file_info_opts_fun(AllOptions, read_file_info), add_opts(T, AllOptions, Opts#add_opts{read_info=RF}); add_opts([verbose|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{verbose=true}); @@ -510,6 +535,8 @@ add_opts([{mtime,Value}|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{mtime=Value}); add_opts([{ctime,Value}|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{ctime=Value}); +add_opts([{mode,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{mode=Value}); add_opts([{uid,Value}|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{uid=Value}); add_opts([{gid,Value}|T], AllOptions, Opts) -> @@ -530,6 +557,8 @@ do_apply_file_info_opts([{mtime,Value}|T], FileInfo) -> do_apply_file_info_opts(T, FileInfo#file_info{mtime=Value}); do_apply_file_info_opts([{ctime,Value}|T], FileInfo) -> do_apply_file_info_opts(T, FileInfo#file_info{ctime=Value}); +do_apply_file_info_opts([{mode,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{mode=Value}); do_apply_file_info_opts([{uid,Value}|T], FileInfo) -> do_apply_file_info_opts(T, FileInfo#file_info{uid=Value}); do_apply_file_info_opts([{gid,Value}|T], FileInfo) -> @@ -539,7 +568,12 @@ do_apply_file_info_opts([_|T], FileInfo) -> do_apply_file_info_opts([], FileInfo) -> FileInfo. -add1(#reader{}=Reader, Name, NameInArchive, undefined, #add_opts{read_info=ReadInfo}=Opts) +apply_file_info_opts_fun(Options, InfoFunction) -> + fun(F) -> + apply_file_info_opts(Options, file:InfoFunction(F, [{time, posix}])) + end. + +add1(#reader{}=Reader, Name, NameInArchive, #add_opts{read_info=ReadInfo}=Opts) when is_list(Name) -> Res = case ReadInfo(Name) of {error, Reason0} -> @@ -570,9 +604,9 @@ add1(#reader{}=Reader, Name, NameInArchive, undefined, #add_opts{read_info=ReadI {ok, _Reader} -> ok; {error, _Reason} = Err -> Err end; -add1(Reader, Bin, NameInArchive, Mode, Opts) when is_binary(Bin) -> +add1(Reader, Bin, NameInArchive, Opts) when is_binary(Bin) -> add_verbose(Opts, "a ~ts~n", [NameInArchive]), - Now = 0, + Now = os:system_time(seconds), Header = #tar_header{ name = NameInArchive, size = byte_size(Bin), @@ -582,7 +616,7 @@ add1(Reader, Bin, NameInArchive, Mode, Opts) when is_binary(Bin) -> ctime = add_opts_time(Opts#add_opts.ctime, Now), uid = Opts#add_opts.uid, gid = Opts#add_opts.gid, - mode = default_mode(Mode, 8#100644)}, + mode = Opts#add_opts.mode}, {ok, Reader2} = add_header(Reader, Header, Opts), Padding = skip_padding(byte_size(Bin)), Data = [Bin, <<0:Padding/unit:8>>], @@ -591,12 +625,9 @@ add1(Reader, Bin, NameInArchive, Mode, Opts) when is_binary(Bin) -> {error, Reason} -> {error, {NameInArchive, Reason}} end. -add_opts_time(undefined, _Now) -> 0; +add_opts_time(undefined, Now) -> Now; add_opts_time(Time, _Now) -> Time. -default_mode(undefined, Mode) -> Mode; -default_mode(Mode, _) -> Mode. - add_directory(Reader, DirName, NameInArchive, Info, Opts) -> case file:list_dir(DirName) of {ok, []} -> @@ -820,7 +851,7 @@ split_ustar_path(Path) -> false; true -> PathBin = binary:list_to_bin(Path), - case binary:split(PathBin, [<<$/>>], [global, trim_all]) of + case filename:split(PathBin) of [Part] when byte_size(Part) >= ?V7_NAME_LEN -> false; Parts -> @@ -1027,11 +1058,14 @@ do_get_format({error, _} = Err, _Bin) -> do_get_format(#header_v7{}=V7, Bin) when is_binary(Bin), byte_size(Bin) =:= ?BLOCK_SIZE -> Checksum = parse_octal(V7#header_v7.checksum), - Chk1 = compute_checksum(Bin), - Chk2 = compute_signed_checksum(Bin), - if Checksum =/= Chk1 andalso Checksum =/= Chk2 -> + IsBadChecksum = case compute_checksum(Bin) of + Checksum -> false; + _ -> compute_signed_checksum(Bin) =/= Checksum + end, + case IsBadChecksum of + true -> ?FORMAT_UNKNOWN; - true -> + false -> %% guess magic Ustar = to_ustar(V7, Bin), Star = to_star(V7, Bin), @@ -1170,8 +1204,8 @@ validate_sparse_entries([#sparse_entry{}=Entry|Rest], RealSize, I, LastOffset) - validate_sparse_entries(Rest, RealSize, I+1, Offset+NumBytes). --spec parse_sparse_map(header_gnu(), reader_type()) -> - {[sparse_entry()], reader_type()}. +-spec parse_sparse_map(header_gnu(), descriptor_type()) -> + {[sparse_entry()], descriptor_type()}. parse_sparse_map(#header_gnu{sparse=Sparse}, Reader) when Sparse#sparse_array.is_extended -> parse_sparse_map(Sparse, Reader, []); @@ -1211,6 +1245,9 @@ compute_signed_checksum(< checksum(Bin, 0). + +checksum(<>, Sum) -> + checksum(Rest, Sum+A+B+C+D); checksum(<>, Sum) -> checksum(Rest, Sum+A); checksum(<<>>, Sum) -> Sum. @@ -1259,39 +1296,40 @@ parse_numeric(<> = Bin) -> parse_octal(Bin) end. -parse_octal(Bin) when is_binary(Bin) -> +parse_octal(<>) -> %% skip leading/trailing zero bytes and spaces - do_parse_octal(Bin, <<>>). -do_parse_octal(<<>>, <<>>) -> - 0; -do_parse_octal(<<>>, Acc) -> - case io_lib:fread("~8u", binary:bin_to_list(Acc)) of - {error, _} -> throw({error, invalid_tar_checksum}); - {ok, [Octal], []} -> Octal; - {ok, _, _} -> throw({error, invalid_tar_checksum}) - end; -do_parse_octal(<<$\s,Rest/binary>>, Acc) -> + do_parse_octal(Bin, 0). + +do_parse_octal(<<$\s, Rest/binary>>, Acc) -> do_parse_octal(Rest, Acc); do_parse_octal(<<0, Rest/binary>>, Acc) -> do_parse_octal(Rest, Acc); do_parse_octal(<>, Acc) -> - do_parse_octal(Rest, <>). + Digit = C - $0, + case Digit band 7 of + Digit -> + do_parse_octal(Rest, Acc bsl 3 bor Digit); + _ -> + throw({error, invalid_tar_checksum}) + end; +do_parse_octal(<<>>, Acc) -> + Acc. parse_string(Bin) when is_binary(Bin) -> - do_parse_string(Bin, <<>>). -do_parse_string(<<>>, Acc) -> - case unicode:characters_to_list(Acc) of + N = strlen(Bin, 0), + <> = Bin, + case unicode:characters_to_list(Prefix) of Str when is_list(Str) -> Str; {incomplete, _Str, _Rest} -> - binary:bin_to_list(Acc); + binary_to_list(Bin); {error, _Str, _Rest} -> throw({error, {bad_header, invalid_string}}) - end; -do_parse_string(<<0, _/binary>>, Acc) -> - do_parse_string(<<>>, Acc); -do_parse_string(<>, Acc) -> - do_parse_string(Rest, <>). + end. + +strlen(<<>>, N) -> N; +strlen(<<0, _/binary>>, N) -> N; +strlen(<<_, Rest/binary>>, N) -> strlen(Rest, N + 1). convert_header(Bin, #reader{pos=Pos}=Reader) when byte_size(Bin) =:= ?BLOCK_SIZE, (Pos rem ?BLOCK_SIZE) =:= 0 -> @@ -1316,10 +1354,12 @@ convert_header(_Bin, _Reader) -> %% If the file is a directory, a slash is appended to the name. fileinfo_to_header(Name, #file_info{}=Fi, Link) when is_list(Name) -> BaseHeader = #tar_header{name=Name, - mtime=0, - atime=0, - ctime=0, + mtime=Fi#file_info.mtime, + atime=Fi#file_info.atime, + ctime=Fi#file_info.ctime, mode=Fi#file_info.mode, + uid=Fi#file_info.uid, + gid=Fi#file_info.gid, typeflag=?TYPE_REGULAR}, do_fileinfo_to_header(BaseHeader, Fi, Link). @@ -1537,9 +1577,9 @@ do_parse_pax(Reader, Bin, Headers) -> parse_pax_record(Bin) when is_binary(Bin) -> case binary:split(Bin, [<<$\n>>]) of [Record, Residual] -> - case [X || X <- binary:split(Record, [<<$\s>>], [global]), X =/= <<>>] of + case binary:split(Record, [<<$\s>>], [trim_all]) of [_Len, Record1] -> - case [X || X <- binary:split(Record1, [<<$=>>], [global]), X =/= <<>>] of + case binary:split(Record1, [<<$=>>], [trim_all]) of [AttrName, AttrValue] -> {AttrName, AttrValue, Residual}; _Other -> @@ -1639,7 +1679,8 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> create_extracted_dir(Name1, Opts); symlink -> read_verbose(Opts, "x ~ts~n", [Name0]), - create_symlink(Name1, Header#tar_header.linkname, Opts); + LinkName = safe_link_name(Header, Opts), + create_symlink(Name1, LinkName, Opts); Device when Device =:= char orelse Device =:= block -> %% char/block devices will be created as empty files %% and then have their major/minor device set later @@ -1659,51 +1700,18 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> make_safe_path([$/|Path], Opts) -> make_safe_path(Path, Opts); -make_safe_path(Path, #read_opts{cwd=Cwd}) -> - case safe_relative_path_links(Path, Cwd) of - unsafe -> - throw({error,{Path,unsafe_path}}); - SafePath -> - filename:absname(SafePath, Cwd) +make_safe_path(Path0, #read_opts{cwd=Cwd}) -> + case filelib:safe_relative_path(Path0, Cwd) of + unsafe -> throw({error,{Path0,unsafe_path}}); + Path -> filename:absname(Path, Cwd) end. -safe_relative_path_links(Path, Cwd) -> - case filename:pathtype(Path) of - relative -> safe_relative_path_links(filename:split(Path), Cwd, [], ""); - _ -> unsafe +safe_link_name(#tar_header{linkname=Path0},#read_opts{cwd=Cwd} ) -> + case filelib:safe_relative_path(Path0, Cwd) of + unsafe -> throw({error,{Path0,unsafe_symlink}}); + Path -> Path end. -safe_relative_path_links([], _Cwd, _PrevLinks, Acc) -> - Acc; - -safe_relative_path_links([Segment | Segments], Cwd, PrevLinks, Acc) -> - AccSegment = join(Acc, Segment), - - case hex_filename:safe_relative_path(AccSegment) of - unsafe -> - unsafe; - - SafeAccSegment -> - case file:read_link(join(Cwd, SafeAccSegment)) of - {ok, LinkPath} -> - case lists:member(LinkPath, PrevLinks) of - true -> - unsafe; - false -> - case safe_relative_path_links(filename:split(LinkPath), Cwd, [LinkPath | PrevLinks], Acc) of - unsafe -> unsafe; - NewAcc -> safe_relative_path_links(Segments, Cwd, [], NewAcc) - end - end; - - {error, _} -> - safe_relative_path_links(Segments, Cwd, PrevLinks, SafeAccSegment) - end - end. - -join([], Path) -> Path; -join(Left, Right) -> filename:join(Left, Right). - create_regular(Name, NameInArchive, Bin, Opts) -> case write_extracted_file(Name, Bin, Opts) of not_written -> @@ -1964,7 +1972,7 @@ read_sparse_hole(#sparse_file_reader{pos=Pos}=Reader, Offset, Len) -> num_bytes=NumBytes, pos=Pos+N2}}. --spec do_close(reader()) -> ok | {error, term()}. +-spec do_close(tar_descriptor()) -> ok | {error, term()}. do_close(#reader{handle=Handle,func=Fun}) when is_function(Fun,2) -> Fun(close,Handle). diff --git a/src/hex_erl_tar.hrl b/src/hex_erl_tar.hrl index d142270f..7cb6c63d 100644 --- a/src/hex_erl_tar.hrl +++ b/src/hex_erl_tar.hrl @@ -1,9 +1,12 @@ -% Copied from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/erl_tar.hrl - +%% This file is a copy of erl_tar.hrl from OTP with no modifications. +%% +%% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% SPDX-License-Identifier: Apache-2.0 +%% +%% Copyright Ericsson AB 1997-2025. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,14 +24,15 @@ %% Options used when adding files to a tar archive. -record(add_opts, { - read_info, %% Fun to use for read file/link info. - chunk_size = 0 :: integer(), %% For file reading when sending to sftp. 0=do not chunk - verbose = false :: boolean(), %% Verbose on/off. - atime = undefined :: undefined | integer(), - mtime = undefined :: undefined | integer(), - ctime = undefined :: undefined | integer(), - uid = 0 :: integer(), - gid = 0 :: integer()}). + read_info, %% Fun to use for read file/link info. + chunk_size = 0, %% For file reading when sending to sftp. 0=do not chunk + verbose = false, %% Verbose on/off. + atime = undefined, + mtime = undefined, + ctime = undefined, + mode = 8#100644, + uid = 0, + gid = 0}). -type add_opts() :: #add_opts{}. %% Options used when reading a tar archive. @@ -41,18 +45,20 @@ verbose = false :: boolean()}). %% Verbose on/off. -type read_opts() :: #read_opts{}. --type add_opt() :: dereference - | verbose - | {chunks, pos_integer()} - | {atime, integer()} - | {mtime, integer()} - | {ctime, integer()} - | {uid, integer()} - | {gid, integer()}. +-type add_opt() :: dereference | + verbose | + {chunks, pos_integer()} | + {atime, non_neg_integer()} | + {mtime, non_neg_integer()} | + {ctime, non_neg_integer()} | + {mode, non_neg_integer()} | + {uid, non_neg_integer()} | + {gid, non_neg_integer()}. +-type name_in_archive() :: string(). -type extract_opt() :: {cwd, string()} | - {files, [string()]} | + {files, [name_in_archive()]} | compressed | cooked | memory | @@ -65,21 +71,20 @@ verbose. -type filelist() :: [file:filename() | - {string(), binary()} | - {string(), file:filename()}]. + {name_in_archive(), file:filename_all()}]. -type tar_time() :: non_neg_integer(). %% The tar header, once fully parsed. -record(tar_header, { - name = "" :: string(), %% name of header file entry + name = "" :: name_in_archive(), %% name of header file entry mode = 8#100644 :: non_neg_integer(), %% permission and mode bits uid = 0 :: non_neg_integer(), %% user id of owner gid = 0 :: non_neg_integer(), %% group id of owner size = 0 :: non_neg_integer(), %% length in bytes mtime :: tar_time(), %% modified time typeflag :: char(), %% type of header entry - linkname = "" :: string(), %% target name of link + linkname = "" :: name_in_archive(), %% target name of link uname = "" :: string(), %% user name of owner gname = "" :: string(), %% group name of owner devmajor = 0 :: non_neg_integer(), %% major number of character or block device @@ -160,16 +165,18 @@ %% The overall tar reader, it holds the low-level file handle, %% its access, position, and the I/O primitives wrapper. -record(reader, { - handle :: file:io_device() | term(), + handle :: user_data(), access :: read | write | ram, pos = 0 :: non_neg_integer(), func :: file_op() }). --type reader() :: #reader{}. +-opaque tar_descriptor() :: #reader{}. +-export_type([tar_descriptor/0]). + %% A reader for a regular file within the tar archive, %% It tracks its current state relative to that file. -record(reg_file_reader, { - handle :: reader(), + handle :: tar_descriptor(), num_bytes = 0, pos = 0, size = 0 @@ -178,7 +185,7 @@ %% A reader for a sparse file within the tar archive, %% It tracks its current state relative to that file. -record(sparse_file_reader, { - handle :: reader(), + handle :: tar_descriptor(), num_bytes = 0, %% bytes remaining pos = 0, %% pos size = 0, %% total size of file @@ -187,13 +194,13 @@ -type sparse_file_reader() :: #sparse_file_reader{}. %% Types for the readers --type reader_type() :: reader() | reg_file_reader() | sparse_file_reader(). --type handle() :: file:io_device() | term(). +-type descriptor_type() :: tar_descriptor() | reg_file_reader() | sparse_file_reader(). +-type user_data() :: term(). %% Type for the I/O primitive wrapper function -type file_op() :: fun((write | close | read2 | position, - {handle(), iodata()} | handle() | {handle(), non_neg_integer()} - | {handle(), non_neg_integer()}) -> + {user_data(), iodata()} | user_data() | {user_data(), non_neg_integer()} + | {user_data(), non_neg_integer()}) -> ok | eof | {ok, string() | binary()} | {ok, non_neg_integer()} | {error, term()}). diff --git a/src/hex_filename.erl b/src/hex_filename.erl deleted file mode 100644 index 7ebd9d31..00000000 --- a/src/hex_filename.erl +++ /dev/null @@ -1,58 +0,0 @@ -% @private -% Excerpt from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/filename.erl#L761-L788 -% with modifications for changing local function calls to remote function calls -% to the `filename` module, for the functions `pathtype/1`, `split/1`, and `join/1` -% -% safe_relative_path/1 was not present in earlier OTP releases. - -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(hex_filename). --export([safe_relative_path/1]). - -safe_relative_path(Path) -> - case filename:pathtype(Path) of - relative -> - Cs0 = filename:split(Path), - safe_relative_path_1(Cs0, []); - _ -> - unsafe - end. - -safe_relative_path_1(["." | T], Acc) -> - safe_relative_path_1(T, Acc); -safe_relative_path_1([<<".">> | T], Acc) -> - safe_relative_path_1(T, Acc); -safe_relative_path_1([".." | T], Acc) -> - climb(T, Acc); -safe_relative_path_1([<<"..">> | T], Acc) -> - climb(T, Acc); -safe_relative_path_1([H | T], Acc) -> - safe_relative_path_1(T, [H | Acc]); -safe_relative_path_1([], []) -> - []; -safe_relative_path_1([], Acc) -> - filename:join(lists:reverse(Acc)). - -climb(_, []) -> - unsafe; -climb(T, [_ | Acc]) -> - safe_relative_path_1(T, Acc). diff --git a/src/hex_tarball.erl b/src/hex_tarball.erl index 51247b40..bbec61bc 100644 --- a/src/hex_tarball.erl +++ b/src/hex_tarball.erl @@ -565,7 +565,7 @@ add_file(Tar, {Filename, AbsFilename}) when is_list(Filename), is_list(AbsFilena _ -> Mode = FileInfo#file_info.mode, {ok, Contents} = file:read_file(AbsFilename), - ok = hex_erl_tar:add(Tar, Contents, Filename, Mode, tar_opts()) + ok = hex_erl_tar:add(Tar, Contents, Filename, [{mode, Mode} | tar_opts()]) end. %% @private diff --git a/test/hex_tarball_SUITE.erl b/test/hex_tarball_SUITE.erl index 5f7f3985..78abd432 100644 --- a/test/hex_tarball_SUITE.erl +++ b/test/hex_tarball_SUITE.erl @@ -95,7 +95,7 @@ disk_test(Config) -> Metadata, Files ), ?assertEqual( - <<"6D47908182CC721920B2F2C1D5777ED9E9EDBD86A29638448D97588AA0419C98">>, + <<"3DED88F7BAC738294907ACEED89FE63C9914713F988E608F2B17D7AE6EAC8446">>, hex_tarball:format_checksum(OuterChecksum) ), {ok, #{inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, metadata := Metadata}} = hex_tarball:unpack( From 48545682b1801adbda4070c29ccd9a04cba8e440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 7 Mar 2026 14:40:45 +0100 Subject: [PATCH 2/3] Fix erl_tar rejecting safe symlinks with .. in target Join symlink targets with the symlink's parent directory before validating. Previously the raw target was validated in isolation, causing safe symlinks like dir/link -> ../file to be incorrectly rejected as unsafe. --- src/hex_erl_tar.erl | 10 +++-- test/hex_tarball_SUITE.erl | 79 +++++++++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/hex_erl_tar.erl b/src/hex_erl_tar.erl index b77634c8..89efade7 100644 --- a/src/hex_erl_tar.erl +++ b/src/hex_erl_tar.erl @@ -2,6 +2,8 @@ %% 1. Module renamed from erl_tar to hex_erl_tar %% 2. -include changed from erl_tar.hrl to hex_erl_tar.hrl %% 3. -doc and -moduledoc attributes removed for OTP 24 compatibility +%% 4. safe_link_name/2 fixed to validate symlink targets relative to symlink's +%% parent directory instead of in isolation %% %% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f %% @@ -1706,10 +1708,12 @@ make_safe_path(Path0, #read_opts{cwd=Cwd}) -> Path -> filename:absname(Path, Cwd) end. -safe_link_name(#tar_header{linkname=Path0},#read_opts{cwd=Cwd} ) -> - case filelib:safe_relative_path(Path0, Cwd) of +safe_link_name(#tar_header{name=Name,linkname=Path0},#read_opts{cwd=Cwd} ) -> + ParentDir = filename:dirname(Name), + ResolvedTarget = filename:join(ParentDir, Path0), + case filelib:safe_relative_path(ResolvedTarget, Cwd) of unsafe -> throw({error,{Path0,unsafe_symlink}}); - Path -> Path + _Path -> Path0 end. create_regular(Name, NameInArchive, Bin, Opts) -> diff --git a/test/hex_tarball_SUITE.erl b/test/hex_tarball_SUITE.erl index 78abd432..6a4a0717 100644 --- a/test/hex_tarball_SUITE.erl +++ b/test/hex_tarball_SUITE.erl @@ -11,6 +11,7 @@ all() -> disk_test, timestamps_and_permissions_test, symlinks_test, + symlinks_parent_dir_test, memory_test, build_tools_test, requirements_test, @@ -116,12 +117,16 @@ disk_test(Config) -> timestamps_and_permissions_test(Config) -> BaseDir = ?config(priv_dir, Config), Foo = filename:join(BaseDir, "foo.sh"), + EmptyDir = filename:join(BaseDir, "timestamps_empty"), Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, ok = file:write_file(Foo, <<"">>), ok = file:change_mode(Foo, 8#100755), + ok = file:make_dir(EmptyDir), + ok = file:change_mode(EmptyDir, 8#100755), Files = [ + {"empty", EmptyDir}, {"foo.erl", <<"">>}, {"foo.sh", Foo} ], @@ -133,11 +138,12 @@ timestamps_and_permissions_test(Config) -> %% inside tarball {ok, Files2} = hex_erl_tar:extract({binary, Tarball}, [memory]), {_, ContentsBinary} = lists:keyfind("contents.tar.gz", 1, Files2), - {ok, [FooErlEntry, FooShEntry]} = hex_erl_tar:table({binary, ContentsBinary}, [ + {ok, [EmptyDirEntry, FooErlEntry, FooShEntry]} = hex_erl_tar:table({binary, ContentsBinary}, [ compressed, verbose ]), Epoch = epoch(), + {"empty", directory, _, Epoch, 8#40755, 0, 0} = EmptyDirEntry, {"foo.erl", regular, _, Epoch, 8#100644, 0, 0} = FooErlEntry, {"foo.sh", regular, _, Epoch, 8#100755, 0, 0} = FooShEntry, @@ -181,6 +187,77 @@ symlinks_test(Config) -> {ok, _} = file:read_link_info(filename:join([UnpackDir, "dir", "bar.sh"])), ok. +%% OTP's erl_tar validates symlink targets relative to the extraction directory (Cwd), +%% but symlink targets are relative to the symlink's parent directory. This causes +%% safe symlinks with ".." components to be incorrectly rejected. +%% +%% For example, a symlink "dir/link -> ../file" resolves to "file" which is inside the +%% extraction directory, but erl_tar rejects it because "../file" relative to Cwd is +%% considered unsafe. +%% +%% TODO: Fix safe_link_name in erl_tar to validate the resolved path and contribute +%% the fix back to OTP. +symlinks_parent_dir_test(Config) -> + BaseDir = ?config(priv_dir, Config), + + Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, + + %% dir/link -> ../file is safe (resolves to file within extraction dir) + Dir = filename:join(BaseDir, "dir2"), + FooSh = filename:join(BaseDir, "foo2.sh"), + LinkSh = filename:join(Dir, "link.sh"), + ok = file:make_dir(Dir), + ok = file:write_file(FooSh, <<"foo">>), + ok = file:make_symlink("../foo2.sh", LinkSh), + + Files = [ + {"foo.sh", FooSh}, + {"dir/link.sh", LinkSh} + ], + + {ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, Files), + {ok, _} = hex_tarball:unpack(Tarball, memory), + + UnpackDir = filename:join(BaseDir, "symlinks_parent_dir"), + {ok, _} = hex_tarball:unpack(Tarball, UnpackDir), + {ok, #file_info{type = symlink}} = + file:read_link_info(filename:join([UnpackDir, "dir", "link.sh"])), + {ok, "../foo2.sh"} = file:read_link(filename:join([UnpackDir, "dir", "link.sh"])), + + %% a/b/link -> ../../file is safe (resolves to file within extraction dir) + ABDir = filename:join([BaseDir, "a2", "b2"]), + FooSh2 = filename:join(BaseDir, "foo3.sh"), + LinkSh2 = filename:join(ABDir, "link.sh"), + ok = filelib:ensure_dir(LinkSh2), + ok = file:write_file(FooSh2, <<"foo">>), + ok = file:make_symlink("../../foo3.sh", LinkSh2), + + Files2 = [ + {"foo.sh", FooSh2}, + {"a/b/link.sh", LinkSh2} + ], + + {ok, #{tarball := Tarball2}} = hex_tarball:create(Metadata, Files2), + UnpackDir2 = filename:join(BaseDir, "symlinks_parent_dir2"), + {ok, _} = hex_tarball:unpack(Tarball2, UnpackDir2), + {ok, #file_info{type = symlink}} = + file:read_link_info(filename:join([UnpackDir2, "a", "b", "link.sh"])), + {ok, "../../foo3.sh"} = file:read_link(filename:join([UnpackDir2, "a", "b", "link.sh"])), + + %% dir/link -> ../../escape is unsafe (escapes extraction dir) + UnsafeDir = filename:join(BaseDir, "unsafe_dir"), + UnsafeLink = filename:join(UnsafeDir, "link.sh"), + ok = file:make_dir(UnsafeDir), + ok = file:make_symlink("../../escape", UnsafeLink), + + UnsafeFiles = [{"dir/link.sh", UnsafeLink}], + {ok, #{tarball := UnsafeTarball}} = hex_tarball:create(Metadata, UnsafeFiles), + UnpackDir3 = filename:join(BaseDir, "symlinks_parent_dir3"), + {error, {inner_tarball, {"../../escape", unsafe_symlink}}} = + hex_tarball:unpack(UnsafeTarball, UnpackDir3), + + ok. + build_tools_test(_Config) -> Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, Contents = [], From b6b52b83ad77770c91e3ad56bc69a2dc1f2a2ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 8 Mar 2026 16:05:45 +0100 Subject: [PATCH 3/3] Exclude generated files from formatting --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index ae03520e..451f9289 100644 --- a/rebar.config +++ b/rebar.config @@ -27,7 +27,7 @@ {erlfmt, [ write, {files, ["{src,test,examples}/*.{hrl,erl,app.src}", "rebar.config"]}, - {exclude_files, ["src/hex_pb_*.erl", "src/hex_erl_tar.{hrl,erl}"]} + {exclude_files, ["src/hex_pb_*.erl", "src/hex_erl_tar.{hrl,erl}", "src/safe_erl_term.erl"]} ]}. {profiles, [