diff --git a/rebar.config b/rebar.config index ae03520e..451f9289 100644 --- a/rebar.config +++ b/rebar.config @@ -27,7 +27,7 @@ {erlfmt, [ write, {files, ["{src,test,examples}/*.{hrl,erl,app.src}", "rebar.config"]}, - {exclude_files, ["src/hex_pb_*.erl", "src/hex_erl_tar.{hrl,erl}"]} + {exclude_files, ["src/hex_pb_*.erl", "src/hex_erl_tar.{hrl,erl}", "src/safe_erl_term.erl"]} ]}. {profiles, [ diff --git a/src/hex_erl_tar.erl b/src/hex_erl_tar.erl index 28ac0d29..89efade7 100644 --- a/src/hex_erl_tar.erl +++ b/src/hex_erl_tar.erl @@ -1,20 +1,17 @@ -%% @private -%% Copied from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/erl_tar.erl -%% with modifications: -%% - Change module name to `hex_erl_tar` -%% - Set tar mtimes to 0 and remove dependency on :os.system_time/1 -%% - Preserve modes when building tarball -%% - Do not crash if failing to write tar -%% - Allow setting file_info opts on :hex_erl_tar.add -%% - Add safe_relative_path_links/2 to check directory traversal vulnerability when extracting files, -%% it differs from OTP's current fix (2020-02-04) in that it checks regular files instead of -%% symlink targets. This allows creating symlinks with relative path targets such as `../tmp/log` -%% - Remove ram_file usage (backported from OTP master) - +%% This file is a copy of erl_tar.erl from OTP with the following modifications: +%% 1. Module renamed from erl_tar to hex_erl_tar +%% 2. -include changed from erl_tar.hrl to hex_erl_tar.hrl +%% 3. -doc and -moduledoc attributes removed for OTP 24 compatibility +%% 4. safe_link_name/2 fixed to validate symlink targets relative to symlink's +%% parent directory instead of in isolation +%% +%% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. +%% SPDX-License-Identifier: Apache-2.0 +%% +%% Copyright Ericsson AB 1997-2025. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,18 +39,20 @@ %% http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html -module(hex_erl_tar). + -export([init/3, create/2, create/3, extract/1, extract/2, table/1, table/2, t/1, tt/1, open/2, close/1, - add/3, add/4, add/5, + add/3, add/4, format_error/1]). -include_lib("kernel/include/file.hrl"). --include_lib("hex_erl_tar.hrl"). +-include("hex_erl_tar.hrl"). %% Converts the short error reason to a descriptive string. + -spec format_error(term()) -> string(). format_error(invalid_tar_checksum) -> "Checksum failed"; @@ -92,17 +91,21 @@ format_error(Term) -> lists:flatten(io_lib:format("~tp", [Term])). %% Initializes a new reader given a custom file handle and I/O wrappers --spec init(handle(), write | read, file_op()) -> {ok, reader()} | {error, badarg}. -init(Handle, AccessMode, Fun) when is_function(Fun, 2) -> - Reader = #reader{handle=Handle,access=AccessMode,func=Fun}, + + +-spec init(UserData :: user_data(), write | read, file_op()) -> + {ok, tar_descriptor()} | {error, badarg}. +init(UserData, AccessMode, Fun) when is_function(Fun, 2) -> + Reader = #reader{handle=UserData,access=AccessMode,func=Fun}, {ok, Pos, Reader2} = do_position(Reader, {cur, 0}), {ok, Reader2#reader{pos=Pos}}; -init(_Handle, _AccessMode, _Fun) -> +init(_UserData, _AccessMode, _Fun) -> {error, badarg}. %%%================================================================ %% Extracts all files from the tar file Name. --spec extract(open_handle()) -> ok | {error, term()}. + +-spec extract(Open :: open_type()) -> ok | {error, term()}. extract(Name) -> extract(Name, []). @@ -115,10 +118,11 @@ extract(Name) -> %% - {files, ListOfFilesToExtract}: Only extract ListOfFilesToExtract %% - verbose: Prints verbose information about the extraction, %% - {cwd, AbsoluteDir}: Sets the current working directory for the extraction --spec extract(open_handle(), [extract_opt()]) -> - ok - | {ok, [{string(), binary()}]} - | {error, term()}. + +-spec extract(Open :: open_type(), []) -> + ok | {error, term()}; + (Open :: open_type(), [extract_opt(), ...]) -> + ok | {ok, [{string(), binary()}]} | {error, term()}. extract({binary, Bin}, Opts) when is_list(Opts) -> do_extract({binary, Bin}, Opts); extract({file, Fd}, Opts) when is_list(Opts) -> @@ -178,7 +182,6 @@ check_extract(Name, #read_opts{files=Files}) -> %%%================================================================ %% The following table functions produce a list of information about %% the files contained in the archive. --type filename() :: string(). -type typeflag() :: regular | link | symlink | char | block | directory | fifo | reserved | unknown. @@ -186,23 +189,25 @@ check_extract(Name, #read_opts{files=Files}) -> -type uid() :: non_neg_integer(). -type gid() :: non_neg_integer(). --type tar_entry() :: {filename(), - typeflag(), - non_neg_integer(), - tar_time(), - mode(), - uid(), - gid()}. +-type tar_entry() :: {Name :: name_in_archive(), + Type :: typeflag(), + Size :: non_neg_integer(), + MTime :: tar_time(), + Mode :: mode(), + Uid :: uid(), + Gid :: gid()}. %% Returns a list of names of the files in the tar file Name. --spec table(open_handle()) -> {ok, [string()]} | {error, term()}. + +-spec table(Open :: open_type()) -> {ok, [name_in_archive()]} | {error, term()}. table(Name) -> table(Name, []). %% Returns a list of names of the files in the tar file Name. %% Options accepted: compressed, verbose, cooked. --spec table(open_handle(), [compressed | verbose | cooked]) -> - {ok, [tar_entry()]} | {error, term()}. + +-spec table(Open :: open_type(), [compressed | verbose | cooked]) -> + {ok, [name_in_archive() | tar_entry()]} | {error, term()}. table(Name, Opts) when is_list(Opts) -> foldl_read(Name, fun table1/4, [], table_opts(Opts)). @@ -242,6 +247,7 @@ typeflag(_) -> unknown. %% meant to be invoked from the shell. %% Prints each filename in the archive + -spec t(file:filename()) -> ok | {error, term()}. t(Name) when is_list(Name); is_binary(Name) -> case table(Name) of @@ -252,7 +258,8 @@ t(Name) when is_list(Name); is_binary(Name) -> end. %% Prints verbose information about each file in the archive --spec tt(open_handle()) -> ok | {error, term()}. + +-spec tt(open_type()) -> ok | {error, term()}. tt(Name) -> case table(Name, [verbose]) of {ok, List} -> @@ -314,11 +321,12 @@ month(12) -> "Dec". %%%================================================================ %% The open function with friends is to keep the file and binary api of this module --type open_handle() :: file:filename() +-type open_type() :: file:filename_all() | {binary, binary()} - | {file, term()}. --spec open(open_handle(), [write | compressed | cooked]) -> - {ok, reader()} | {error, term()}. + | {file, file:io_device()}. + +-spec open(Open :: open_type(), [write | compressed | cooked]) -> + {ok, tar_descriptor()} | {error, term()}. open({binary, Bin}, Mode) when is_binary(Bin) -> do_open({binary, Bin}, Mode); open({file, Fd}, Mode) -> @@ -334,32 +342,53 @@ do_open(Name, Mode) when is_list(Mode) -> {error, {Name, Reason}} end. -open1({binary,Bin0}, read, _Raw, Opts) when is_binary(Bin0) -> +open1({binary,Bin0}=Handle, read, _Raw, Opts) when is_binary(Bin0) -> Bin = case lists:member(compressed, Opts) of true -> + %% emulate file:open with Modes = [compressed_one ...] + Z = zlib:open(), + zlib:inflateInit(Z, 31, cut), try - zlib:gunzip(Bin0) + IoList = zlib:inflate(Z, Bin0), + zlib:inflateEnd(Z), + iolist_to_binary(IoList) catch - _:_ -> Bin0 + _:_ -> Bin0 + after + zlib:close(Z) end; false -> Bin0 end, + case file:open(Bin, [ram,binary,read]) of {ok,File} -> {ok, #reader{handle=File,access=read,func=fun file_op/2}}; - Error -> - Error + {error, Reason} -> + {error, {Handle, Reason}} end; -open1({file, Fd}, read, _Raw, _Opts) -> - Reader = #reader{handle=Fd,access=read,func=fun file_op/2}, - case do_position(Reader, {cur, 0}) of - {ok, Pos, Reader2} -> - {ok, Reader2#reader{pos=Pos}}; - {error, _} = Err -> - Err +open1({file, Fd}=Handle, read, [raw], Opts) -> + case not lists:member(compressed, Opts) of + true -> + Reader = #reader{handle=Fd,access=read,func=fun file_op/2}, + case do_position(Reader, {cur, 0}) of + {ok, Pos, Reader2} -> + {ok, Reader2#reader{pos=Pos}}; + {error, Reason} -> + {error, {Handle, Reason}} + end; + false -> + {error, {Handle, {incompatible_option, compressed}}} end; -open1(Name, Access, Raw, Opts) when is_list(Name) or is_binary(Name) -> +open1({file, _Fd}=Handle, read, [], _Opts) -> + {error, {Handle, {incompatible_option, cooked}}}; +open1(Name, Access, Raw, Opts0) when is_list(Name); is_binary(Name) -> + Opts = case lists:member(compressed, Opts0) andalso Access == read of + true -> + [compressed_one | (Opts0 -- [compressed])]; + false -> + Opts0 + end, case file:open(Name, Raw ++ [binary, Access|Opts]) of {ok, File} -> {ok, #reader{handle=File,access=Access,func=fun file_op/2}}; @@ -379,7 +408,7 @@ open_mode([read|Rest], false, Raw, Opts) -> open_mode([write|Rest], false, Raw, Opts) -> open_mode(Rest, write, Raw, Opts); open_mode([compressed|Rest], Access, Raw, Opts) -> - open_mode(Rest, Access, Raw, [compressed|Opts]); + open_mode(Rest, Access, Raw, [compressed,read_ahead|Opts]); open_mode([cooked|Rest], Access, _Raw, Opts) -> open_mode(Rest, Access, [], Opts); open_mode([], Access, Raw, Opts) -> @@ -397,7 +426,8 @@ file_op(close, Fd) -> file:close(Fd). %% Closes a tar archive. --spec close(reader()) -> ok | {error, term()}. + +-spec close(TarDescriptor :: tar_descriptor()) -> ok | {error, term()}. close(#reader{access=read}=Reader) -> ok = do_close(Reader); close(#reader{access=write}=Reader) -> @@ -418,16 +448,18 @@ pad_file(#reader{pos=Pos}=Reader) -> %% Creation/modification of tar archives %% Creates a tar file Name containing the given files. --spec create(file:filename(), filelist()) -> ok | {error, {string(), term()}}. + +-spec create(file:filename_all(), filelist()) -> ok | {error, {string(), term()}}. create(Name, FileList) when is_list(Name); is_binary(Name) -> create(Name, FileList, []). %% Creates a tar archive Name containing the given files. %% Accepted options: verbose, compressed, cooked --spec create(file:filename(), filelist(), [create_opt()]) -> + +-spec create(file:filename_all(), filelist(), [create_opt()]) -> ok | {error, term()} | {error, {string(), term()}}. create(Name, FileList, Options) when is_list(Name); is_binary(Name) -> - Mode = lists:filter(fun(X) -> (X=:=compressed) or (X=:=cooked) + Mode = lists:filter(fun(X) -> X =:= compressed orelse X =:= cooked end, Options), case open(Name, [write|Mode]) of {ok, TarFile} -> @@ -456,49 +488,44 @@ do_create(TarFile, [Name|Rest], Opts) -> end. %% Adds a file to a tape archive. --type add_type() :: string() - | {string(), string()} - | {string(), binary()}. --spec add(reader(), add_type(), [add_opt()]) -> ok | {error, term()}. + +-spec add(TarDescriptor, Name, Options) -> ok | {error, term()} when + TarDescriptor :: tar_descriptor(), + Name :: name_in_archive() | {name_in_archive(), file:filename_all()}, + Options :: [add_opt()]. add(Reader, {NameInArchive, Name}, Opts) when is_list(NameInArchive), is_list(Name) -> - do_add(Reader, Name, NameInArchive, undefined, Opts); + do_add(Reader, Name, NameInArchive, Opts); add(Reader, {NameInArchive, Bin}, Opts) when is_list(NameInArchive), is_binary(Bin) -> - do_add(Reader, Bin, NameInArchive, undefined, Opts); -add(Reader, {NameInArchive, Bin, Mode}, Opts) - when is_list(NameInArchive), is_binary(Bin), is_integer(Mode) -> - do_add(Reader, Bin, NameInArchive, Mode, Opts); + do_add(Reader, Bin, NameInArchive, Opts); add(Reader, Name, Opts) when is_list(Name) -> - do_add(Reader, Name, Name, undefined, Opts). + do_add(Reader, Name, Name, Opts). --spec add(reader(), string() | binary(), string(), [add_opt()]) -> - ok | {error, term()}. +-spec add(TarDescriptor, Filename, NameInArchive, Options) -> + ok | {error, term()} when + TarDescriptor :: tar_descriptor(), + Filename :: file:filename_all(), + NameInArchive :: name_in_archive(), + Options :: [add_opt()]. add(Reader, NameOrBin, NameInArchive, Options) when is_list(NameOrBin); is_binary(NameOrBin), is_list(NameInArchive), is_list(Options) -> - do_add(Reader, NameOrBin, NameInArchive, undefined, Options). - --spec add(reader(), string() | binary(), string(), integer(), [add_opt()]) -> - ok | {error, term()}. -add(Reader, NameOrBin, NameInArchive, Mode, Options) - when is_list(NameOrBin); is_binary(NameOrBin), - is_list(NameInArchive), is_integer(Mode), is_list(Options) -> - do_add(Reader, NameOrBin, NameInArchive, Mode, Options). + do_add(Reader, NameOrBin, NameInArchive, Options). -do_add(#reader{access=write}=Reader, Name, NameInArchive, Mode, Options) +do_add(#reader{access=write}=Reader, Name, NameInArchive, Options) when is_list(NameInArchive), is_list(Options) -> - RF = fun(F) -> apply_file_info_opts(Options, file:read_link_info(F, [{time, posix}])) end, + RF = apply_file_info_opts_fun(Options, read_link_info), Opts = #add_opts{read_info=RF}, - add1(Reader, Name, NameInArchive, Mode, add_opts(Options, Options, Opts)); -do_add(#reader{access=read},_,_,_,_) -> + add1(Reader, Name, NameInArchive, add_opts(Options, Options, Opts)); +do_add(#reader{access=read},_,_,_) -> {error, eacces}; -do_add(Reader,_,_,_,_) -> +do_add(Reader,_,_,_) -> {error, {badarg, Reader}}. add_opts([dereference|T], AllOptions, Opts) -> - RF = fun(F) -> apply_file_info_opts(AllOptions, file:read_file_info(F, [{time, posix}])) end, + RF = apply_file_info_opts_fun(AllOptions, read_file_info), add_opts(T, AllOptions, Opts#add_opts{read_info=RF}); add_opts([verbose|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{verbose=true}); @@ -510,6 +537,8 @@ add_opts([{mtime,Value}|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{mtime=Value}); add_opts([{ctime,Value}|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{ctime=Value}); +add_opts([{mode,Value}|T], AllOptions, Opts) -> + add_opts(T, AllOptions, Opts#add_opts{mode=Value}); add_opts([{uid,Value}|T], AllOptions, Opts) -> add_opts(T, AllOptions, Opts#add_opts{uid=Value}); add_opts([{gid,Value}|T], AllOptions, Opts) -> @@ -530,6 +559,8 @@ do_apply_file_info_opts([{mtime,Value}|T], FileInfo) -> do_apply_file_info_opts(T, FileInfo#file_info{mtime=Value}); do_apply_file_info_opts([{ctime,Value}|T], FileInfo) -> do_apply_file_info_opts(T, FileInfo#file_info{ctime=Value}); +do_apply_file_info_opts([{mode,Value}|T], FileInfo) -> + do_apply_file_info_opts(T, FileInfo#file_info{mode=Value}); do_apply_file_info_opts([{uid,Value}|T], FileInfo) -> do_apply_file_info_opts(T, FileInfo#file_info{uid=Value}); do_apply_file_info_opts([{gid,Value}|T], FileInfo) -> @@ -539,7 +570,12 @@ do_apply_file_info_opts([_|T], FileInfo) -> do_apply_file_info_opts([], FileInfo) -> FileInfo. -add1(#reader{}=Reader, Name, NameInArchive, undefined, #add_opts{read_info=ReadInfo}=Opts) +apply_file_info_opts_fun(Options, InfoFunction) -> + fun(F) -> + apply_file_info_opts(Options, file:InfoFunction(F, [{time, posix}])) + end. + +add1(#reader{}=Reader, Name, NameInArchive, #add_opts{read_info=ReadInfo}=Opts) when is_list(Name) -> Res = case ReadInfo(Name) of {error, Reason0} -> @@ -570,9 +606,9 @@ add1(#reader{}=Reader, Name, NameInArchive, undefined, #add_opts{read_info=ReadI {ok, _Reader} -> ok; {error, _Reason} = Err -> Err end; -add1(Reader, Bin, NameInArchive, Mode, Opts) when is_binary(Bin) -> +add1(Reader, Bin, NameInArchive, Opts) when is_binary(Bin) -> add_verbose(Opts, "a ~ts~n", [NameInArchive]), - Now = 0, + Now = os:system_time(seconds), Header = #tar_header{ name = NameInArchive, size = byte_size(Bin), @@ -582,7 +618,7 @@ add1(Reader, Bin, NameInArchive, Mode, Opts) when is_binary(Bin) -> ctime = add_opts_time(Opts#add_opts.ctime, Now), uid = Opts#add_opts.uid, gid = Opts#add_opts.gid, - mode = default_mode(Mode, 8#100644)}, + mode = Opts#add_opts.mode}, {ok, Reader2} = add_header(Reader, Header, Opts), Padding = skip_padding(byte_size(Bin)), Data = [Bin, <<0:Padding/unit:8>>], @@ -591,12 +627,9 @@ add1(Reader, Bin, NameInArchive, Mode, Opts) when is_binary(Bin) -> {error, Reason} -> {error, {NameInArchive, Reason}} end. -add_opts_time(undefined, _Now) -> 0; +add_opts_time(undefined, Now) -> Now; add_opts_time(Time, _Now) -> Time. -default_mode(undefined, Mode) -> Mode; -default_mode(Mode, _) -> Mode. - add_directory(Reader, DirName, NameInArchive, Info, Opts) -> case file:list_dir(DirName) of {ok, []} -> @@ -820,7 +853,7 @@ split_ustar_path(Path) -> false; true -> PathBin = binary:list_to_bin(Path), - case binary:split(PathBin, [<<$/>>], [global, trim_all]) of + case filename:split(PathBin) of [Part] when byte_size(Part) >= ?V7_NAME_LEN -> false; Parts -> @@ -1027,11 +1060,14 @@ do_get_format({error, _} = Err, _Bin) -> do_get_format(#header_v7{}=V7, Bin) when is_binary(Bin), byte_size(Bin) =:= ?BLOCK_SIZE -> Checksum = parse_octal(V7#header_v7.checksum), - Chk1 = compute_checksum(Bin), - Chk2 = compute_signed_checksum(Bin), - if Checksum =/= Chk1 andalso Checksum =/= Chk2 -> + IsBadChecksum = case compute_checksum(Bin) of + Checksum -> false; + _ -> compute_signed_checksum(Bin) =/= Checksum + end, + case IsBadChecksum of + true -> ?FORMAT_UNKNOWN; - true -> + false -> %% guess magic Ustar = to_ustar(V7, Bin), Star = to_star(V7, Bin), @@ -1170,8 +1206,8 @@ validate_sparse_entries([#sparse_entry{}=Entry|Rest], RealSize, I, LastOffset) - validate_sparse_entries(Rest, RealSize, I+1, Offset+NumBytes). --spec parse_sparse_map(header_gnu(), reader_type()) -> - {[sparse_entry()], reader_type()}. +-spec parse_sparse_map(header_gnu(), descriptor_type()) -> + {[sparse_entry()], descriptor_type()}. parse_sparse_map(#header_gnu{sparse=Sparse}, Reader) when Sparse#sparse_array.is_extended -> parse_sparse_map(Sparse, Reader, []); @@ -1211,6 +1247,9 @@ compute_signed_checksum(< checksum(Bin, 0). + +checksum(<>, Sum) -> + checksum(Rest, Sum+A+B+C+D); checksum(<>, Sum) -> checksum(Rest, Sum+A); checksum(<<>>, Sum) -> Sum. @@ -1259,39 +1298,40 @@ parse_numeric(<> = Bin) -> parse_octal(Bin) end. -parse_octal(Bin) when is_binary(Bin) -> +parse_octal(<>) -> %% skip leading/trailing zero bytes and spaces - do_parse_octal(Bin, <<>>). -do_parse_octal(<<>>, <<>>) -> - 0; -do_parse_octal(<<>>, Acc) -> - case io_lib:fread("~8u", binary:bin_to_list(Acc)) of - {error, _} -> throw({error, invalid_tar_checksum}); - {ok, [Octal], []} -> Octal; - {ok, _, _} -> throw({error, invalid_tar_checksum}) - end; -do_parse_octal(<<$\s,Rest/binary>>, Acc) -> + do_parse_octal(Bin, 0). + +do_parse_octal(<<$\s, Rest/binary>>, Acc) -> do_parse_octal(Rest, Acc); do_parse_octal(<<0, Rest/binary>>, Acc) -> do_parse_octal(Rest, Acc); do_parse_octal(<>, Acc) -> - do_parse_octal(Rest, <>). + Digit = C - $0, + case Digit band 7 of + Digit -> + do_parse_octal(Rest, Acc bsl 3 bor Digit); + _ -> + throw({error, invalid_tar_checksum}) + end; +do_parse_octal(<<>>, Acc) -> + Acc. parse_string(Bin) when is_binary(Bin) -> - do_parse_string(Bin, <<>>). -do_parse_string(<<>>, Acc) -> - case unicode:characters_to_list(Acc) of + N = strlen(Bin, 0), + <> = Bin, + case unicode:characters_to_list(Prefix) of Str when is_list(Str) -> Str; {incomplete, _Str, _Rest} -> - binary:bin_to_list(Acc); + binary_to_list(Bin); {error, _Str, _Rest} -> throw({error, {bad_header, invalid_string}}) - end; -do_parse_string(<<0, _/binary>>, Acc) -> - do_parse_string(<<>>, Acc); -do_parse_string(<>, Acc) -> - do_parse_string(Rest, <>). + end. + +strlen(<<>>, N) -> N; +strlen(<<0, _/binary>>, N) -> N; +strlen(<<_, Rest/binary>>, N) -> strlen(Rest, N + 1). convert_header(Bin, #reader{pos=Pos}=Reader) when byte_size(Bin) =:= ?BLOCK_SIZE, (Pos rem ?BLOCK_SIZE) =:= 0 -> @@ -1316,10 +1356,12 @@ convert_header(_Bin, _Reader) -> %% If the file is a directory, a slash is appended to the name. fileinfo_to_header(Name, #file_info{}=Fi, Link) when is_list(Name) -> BaseHeader = #tar_header{name=Name, - mtime=0, - atime=0, - ctime=0, + mtime=Fi#file_info.mtime, + atime=Fi#file_info.atime, + ctime=Fi#file_info.ctime, mode=Fi#file_info.mode, + uid=Fi#file_info.uid, + gid=Fi#file_info.gid, typeflag=?TYPE_REGULAR}, do_fileinfo_to_header(BaseHeader, Fi, Link). @@ -1537,9 +1579,9 @@ do_parse_pax(Reader, Bin, Headers) -> parse_pax_record(Bin) when is_binary(Bin) -> case binary:split(Bin, [<<$\n>>]) of [Record, Residual] -> - case [X || X <- binary:split(Record, [<<$\s>>], [global]), X =/= <<>>] of + case binary:split(Record, [<<$\s>>], [trim_all]) of [_Len, Record1] -> - case [X || X <- binary:split(Record1, [<<$=>>], [global]), X =/= <<>>] of + case binary:split(Record1, [<<$=>>], [trim_all]) of [AttrName, AttrValue] -> {AttrName, AttrValue, Residual}; _Other -> @@ -1639,7 +1681,8 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> create_extracted_dir(Name1, Opts); symlink -> read_verbose(Opts, "x ~ts~n", [Name0]), - create_symlink(Name1, Header#tar_header.linkname, Opts); + LinkName = safe_link_name(Header, Opts), + create_symlink(Name1, LinkName, Opts); Device when Device =:= char orelse Device =:= block -> %% char/block devices will be created as empty files %% and then have their major/minor device set later @@ -1659,51 +1702,20 @@ write_extracted_element(#tar_header{name=Name0}=Header, Bin, Opts) -> make_safe_path([$/|Path], Opts) -> make_safe_path(Path, Opts); -make_safe_path(Path, #read_opts{cwd=Cwd}) -> - case safe_relative_path_links(Path, Cwd) of - unsafe -> - throw({error,{Path,unsafe_path}}); - SafePath -> - filename:absname(SafePath, Cwd) +make_safe_path(Path0, #read_opts{cwd=Cwd}) -> + case filelib:safe_relative_path(Path0, Cwd) of + unsafe -> throw({error,{Path0,unsafe_path}}); + Path -> filename:absname(Path, Cwd) end. -safe_relative_path_links(Path, Cwd) -> - case filename:pathtype(Path) of - relative -> safe_relative_path_links(filename:split(Path), Cwd, [], ""); - _ -> unsafe +safe_link_name(#tar_header{name=Name,linkname=Path0},#read_opts{cwd=Cwd} ) -> + ParentDir = filename:dirname(Name), + ResolvedTarget = filename:join(ParentDir, Path0), + case filelib:safe_relative_path(ResolvedTarget, Cwd) of + unsafe -> throw({error,{Path0,unsafe_symlink}}); + _Path -> Path0 end. -safe_relative_path_links([], _Cwd, _PrevLinks, Acc) -> - Acc; - -safe_relative_path_links([Segment | Segments], Cwd, PrevLinks, Acc) -> - AccSegment = join(Acc, Segment), - - case hex_filename:safe_relative_path(AccSegment) of - unsafe -> - unsafe; - - SafeAccSegment -> - case file:read_link(join(Cwd, SafeAccSegment)) of - {ok, LinkPath} -> - case lists:member(LinkPath, PrevLinks) of - true -> - unsafe; - false -> - case safe_relative_path_links(filename:split(LinkPath), Cwd, [LinkPath | PrevLinks], Acc) of - unsafe -> unsafe; - NewAcc -> safe_relative_path_links(Segments, Cwd, [], NewAcc) - end - end; - - {error, _} -> - safe_relative_path_links(Segments, Cwd, PrevLinks, SafeAccSegment) - end - end. - -join([], Path) -> Path; -join(Left, Right) -> filename:join(Left, Right). - create_regular(Name, NameInArchive, Bin, Opts) -> case write_extracted_file(Name, Bin, Opts) of not_written -> @@ -1964,7 +1976,7 @@ read_sparse_hole(#sparse_file_reader{pos=Pos}=Reader, Offset, Len) -> num_bytes=NumBytes, pos=Pos+N2}}. --spec do_close(reader()) -> ok | {error, term()}. +-spec do_close(tar_descriptor()) -> ok | {error, term()}. do_close(#reader{handle=Handle,func=Fun}) when is_function(Fun,2) -> Fun(close,Handle). diff --git a/src/hex_erl_tar.hrl b/src/hex_erl_tar.hrl index d142270f..7cb6c63d 100644 --- a/src/hex_erl_tar.hrl +++ b/src/hex_erl_tar.hrl @@ -1,9 +1,12 @@ -% Copied from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/erl_tar.hrl - +%% This file is a copy of erl_tar.hrl from OTP with no modifications. +%% +%% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2017. All Rights Reserved. +%% SPDX-License-Identifier: Apache-2.0 +%% +%% Copyright Ericsson AB 1997-2025. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,14 +24,15 @@ %% Options used when adding files to a tar archive. -record(add_opts, { - read_info, %% Fun to use for read file/link info. - chunk_size = 0 :: integer(), %% For file reading when sending to sftp. 0=do not chunk - verbose = false :: boolean(), %% Verbose on/off. - atime = undefined :: undefined | integer(), - mtime = undefined :: undefined | integer(), - ctime = undefined :: undefined | integer(), - uid = 0 :: integer(), - gid = 0 :: integer()}). + read_info, %% Fun to use for read file/link info. + chunk_size = 0, %% For file reading when sending to sftp. 0=do not chunk + verbose = false, %% Verbose on/off. + atime = undefined, + mtime = undefined, + ctime = undefined, + mode = 8#100644, + uid = 0, + gid = 0}). -type add_opts() :: #add_opts{}. %% Options used when reading a tar archive. @@ -41,18 +45,20 @@ verbose = false :: boolean()}). %% Verbose on/off. -type read_opts() :: #read_opts{}. --type add_opt() :: dereference - | verbose - | {chunks, pos_integer()} - | {atime, integer()} - | {mtime, integer()} - | {ctime, integer()} - | {uid, integer()} - | {gid, integer()}. +-type add_opt() :: dereference | + verbose | + {chunks, pos_integer()} | + {atime, non_neg_integer()} | + {mtime, non_neg_integer()} | + {ctime, non_neg_integer()} | + {mode, non_neg_integer()} | + {uid, non_neg_integer()} | + {gid, non_neg_integer()}. +-type name_in_archive() :: string(). -type extract_opt() :: {cwd, string()} | - {files, [string()]} | + {files, [name_in_archive()]} | compressed | cooked | memory | @@ -65,21 +71,20 @@ verbose. -type filelist() :: [file:filename() | - {string(), binary()} | - {string(), file:filename()}]. + {name_in_archive(), file:filename_all()}]. -type tar_time() :: non_neg_integer(). %% The tar header, once fully parsed. -record(tar_header, { - name = "" :: string(), %% name of header file entry + name = "" :: name_in_archive(), %% name of header file entry mode = 8#100644 :: non_neg_integer(), %% permission and mode bits uid = 0 :: non_neg_integer(), %% user id of owner gid = 0 :: non_neg_integer(), %% group id of owner size = 0 :: non_neg_integer(), %% length in bytes mtime :: tar_time(), %% modified time typeflag :: char(), %% type of header entry - linkname = "" :: string(), %% target name of link + linkname = "" :: name_in_archive(), %% target name of link uname = "" :: string(), %% user name of owner gname = "" :: string(), %% group name of owner devmajor = 0 :: non_neg_integer(), %% major number of character or block device @@ -160,16 +165,18 @@ %% The overall tar reader, it holds the low-level file handle, %% its access, position, and the I/O primitives wrapper. -record(reader, { - handle :: file:io_device() | term(), + handle :: user_data(), access :: read | write | ram, pos = 0 :: non_neg_integer(), func :: file_op() }). --type reader() :: #reader{}. +-opaque tar_descriptor() :: #reader{}. +-export_type([tar_descriptor/0]). + %% A reader for a regular file within the tar archive, %% It tracks its current state relative to that file. -record(reg_file_reader, { - handle :: reader(), + handle :: tar_descriptor(), num_bytes = 0, pos = 0, size = 0 @@ -178,7 +185,7 @@ %% A reader for a sparse file within the tar archive, %% It tracks its current state relative to that file. -record(sparse_file_reader, { - handle :: reader(), + handle :: tar_descriptor(), num_bytes = 0, %% bytes remaining pos = 0, %% pos size = 0, %% total size of file @@ -187,13 +194,13 @@ -type sparse_file_reader() :: #sparse_file_reader{}. %% Types for the readers --type reader_type() :: reader() | reg_file_reader() | sparse_file_reader(). --type handle() :: file:io_device() | term(). +-type descriptor_type() :: tar_descriptor() | reg_file_reader() | sparse_file_reader(). +-type user_data() :: term(). %% Type for the I/O primitive wrapper function -type file_op() :: fun((write | close | read2 | position, - {handle(), iodata()} | handle() | {handle(), non_neg_integer()} - | {handle(), non_neg_integer()}) -> + {user_data(), iodata()} | user_data() | {user_data(), non_neg_integer()} + | {user_data(), non_neg_integer()}) -> ok | eof | {ok, string() | binary()} | {ok, non_neg_integer()} | {error, term()}). diff --git a/src/hex_filename.erl b/src/hex_filename.erl deleted file mode 100644 index 7ebd9d31..00000000 --- a/src/hex_filename.erl +++ /dev/null @@ -1,58 +0,0 @@ -% @private -% Excerpt from https://github.com/erlang/otp/blob/OTP-20.0.1/lib/stdlib/src/filename.erl#L761-L788 -% with modifications for changing local function calls to remote function calls -% to the `filename` module, for the functions `pathtype/1`, `split/1`, and `join/1` -% -% safe_relative_path/1 was not present in earlier OTP releases. - -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2017. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(hex_filename). --export([safe_relative_path/1]). - -safe_relative_path(Path) -> - case filename:pathtype(Path) of - relative -> - Cs0 = filename:split(Path), - safe_relative_path_1(Cs0, []); - _ -> - unsafe - end. - -safe_relative_path_1(["." | T], Acc) -> - safe_relative_path_1(T, Acc); -safe_relative_path_1([<<".">> | T], Acc) -> - safe_relative_path_1(T, Acc); -safe_relative_path_1([".." | T], Acc) -> - climb(T, Acc); -safe_relative_path_1([<<"..">> | T], Acc) -> - climb(T, Acc); -safe_relative_path_1([H | T], Acc) -> - safe_relative_path_1(T, [H | Acc]); -safe_relative_path_1([], []) -> - []; -safe_relative_path_1([], Acc) -> - filename:join(lists:reverse(Acc)). - -climb(_, []) -> - unsafe; -climb(T, [_ | Acc]) -> - safe_relative_path_1(T, Acc). diff --git a/src/hex_tarball.erl b/src/hex_tarball.erl index 51247b40..bbec61bc 100644 --- a/src/hex_tarball.erl +++ b/src/hex_tarball.erl @@ -565,7 +565,7 @@ add_file(Tar, {Filename, AbsFilename}) when is_list(Filename), is_list(AbsFilena _ -> Mode = FileInfo#file_info.mode, {ok, Contents} = file:read_file(AbsFilename), - ok = hex_erl_tar:add(Tar, Contents, Filename, Mode, tar_opts()) + ok = hex_erl_tar:add(Tar, Contents, Filename, [{mode, Mode} | tar_opts()]) end. %% @private diff --git a/test/hex_tarball_SUITE.erl b/test/hex_tarball_SUITE.erl index 5f7f3985..6a4a0717 100644 --- a/test/hex_tarball_SUITE.erl +++ b/test/hex_tarball_SUITE.erl @@ -11,6 +11,7 @@ all() -> disk_test, timestamps_and_permissions_test, symlinks_test, + symlinks_parent_dir_test, memory_test, build_tools_test, requirements_test, @@ -95,7 +96,7 @@ disk_test(Config) -> Metadata, Files ), ?assertEqual( - <<"6D47908182CC721920B2F2C1D5777ED9E9EDBD86A29638448D97588AA0419C98">>, + <<"3DED88F7BAC738294907ACEED89FE63C9914713F988E608F2B17D7AE6EAC8446">>, hex_tarball:format_checksum(OuterChecksum) ), {ok, #{inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, metadata := Metadata}} = hex_tarball:unpack( @@ -116,12 +117,16 @@ disk_test(Config) -> timestamps_and_permissions_test(Config) -> BaseDir = ?config(priv_dir, Config), Foo = filename:join(BaseDir, "foo.sh"), + EmptyDir = filename:join(BaseDir, "timestamps_empty"), Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, ok = file:write_file(Foo, <<"">>), ok = file:change_mode(Foo, 8#100755), + ok = file:make_dir(EmptyDir), + ok = file:change_mode(EmptyDir, 8#100755), Files = [ + {"empty", EmptyDir}, {"foo.erl", <<"">>}, {"foo.sh", Foo} ], @@ -133,11 +138,12 @@ timestamps_and_permissions_test(Config) -> %% inside tarball {ok, Files2} = hex_erl_tar:extract({binary, Tarball}, [memory]), {_, ContentsBinary} = lists:keyfind("contents.tar.gz", 1, Files2), - {ok, [FooErlEntry, FooShEntry]} = hex_erl_tar:table({binary, ContentsBinary}, [ + {ok, [EmptyDirEntry, FooErlEntry, FooShEntry]} = hex_erl_tar:table({binary, ContentsBinary}, [ compressed, verbose ]), Epoch = epoch(), + {"empty", directory, _, Epoch, 8#40755, 0, 0} = EmptyDirEntry, {"foo.erl", regular, _, Epoch, 8#100644, 0, 0} = FooErlEntry, {"foo.sh", regular, _, Epoch, 8#100755, 0, 0} = FooShEntry, @@ -181,6 +187,77 @@ symlinks_test(Config) -> {ok, _} = file:read_link_info(filename:join([UnpackDir, "dir", "bar.sh"])), ok. +%% OTP's erl_tar validates symlink targets relative to the extraction directory (Cwd), +%% but symlink targets are relative to the symlink's parent directory. This causes +%% safe symlinks with ".." components to be incorrectly rejected. +%% +%% For example, a symlink "dir/link -> ../file" resolves to "file" which is inside the +%% extraction directory, but erl_tar rejects it because "../file" relative to Cwd is +%% considered unsafe. +%% +%% TODO: Fix safe_link_name in erl_tar to validate the resolved path and contribute +%% the fix back to OTP. +symlinks_parent_dir_test(Config) -> + BaseDir = ?config(priv_dir, Config), + + Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, + + %% dir/link -> ../file is safe (resolves to file within extraction dir) + Dir = filename:join(BaseDir, "dir2"), + FooSh = filename:join(BaseDir, "foo2.sh"), + LinkSh = filename:join(Dir, "link.sh"), + ok = file:make_dir(Dir), + ok = file:write_file(FooSh, <<"foo">>), + ok = file:make_symlink("../foo2.sh", LinkSh), + + Files = [ + {"foo.sh", FooSh}, + {"dir/link.sh", LinkSh} + ], + + {ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, Files), + {ok, _} = hex_tarball:unpack(Tarball, memory), + + UnpackDir = filename:join(BaseDir, "symlinks_parent_dir"), + {ok, _} = hex_tarball:unpack(Tarball, UnpackDir), + {ok, #file_info{type = symlink}} = + file:read_link_info(filename:join([UnpackDir, "dir", "link.sh"])), + {ok, "../foo2.sh"} = file:read_link(filename:join([UnpackDir, "dir", "link.sh"])), + + %% a/b/link -> ../../file is safe (resolves to file within extraction dir) + ABDir = filename:join([BaseDir, "a2", "b2"]), + FooSh2 = filename:join(BaseDir, "foo3.sh"), + LinkSh2 = filename:join(ABDir, "link.sh"), + ok = filelib:ensure_dir(LinkSh2), + ok = file:write_file(FooSh2, <<"foo">>), + ok = file:make_symlink("../../foo3.sh", LinkSh2), + + Files2 = [ + {"foo.sh", FooSh2}, + {"a/b/link.sh", LinkSh2} + ], + + {ok, #{tarball := Tarball2}} = hex_tarball:create(Metadata, Files2), + UnpackDir2 = filename:join(BaseDir, "symlinks_parent_dir2"), + {ok, _} = hex_tarball:unpack(Tarball2, UnpackDir2), + {ok, #file_info{type = symlink}} = + file:read_link_info(filename:join([UnpackDir2, "a", "b", "link.sh"])), + {ok, "../../foo3.sh"} = file:read_link(filename:join([UnpackDir2, "a", "b", "link.sh"])), + + %% dir/link -> ../../escape is unsafe (escapes extraction dir) + UnsafeDir = filename:join(BaseDir, "unsafe_dir"), + UnsafeLink = filename:join(UnsafeDir, "link.sh"), + ok = file:make_dir(UnsafeDir), + ok = file:make_symlink("../../escape", UnsafeLink), + + UnsafeFiles = [{"dir/link.sh", UnsafeLink}], + {ok, #{tarball := UnsafeTarball}} = hex_tarball:create(Metadata, UnsafeFiles), + UnpackDir3 = filename:join(BaseDir, "symlinks_parent_dir3"), + {error, {inner_tarball, {"../../escape", unsafe_symlink}}} = + hex_tarball:unpack(UnsafeTarball, UnpackDir3), + + ok. + build_tools_test(_Config) -> Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, Contents = [],