diff --git a/Build/Download.pm b/Build/Download.pm index 37014300..e67f7a52 100644 --- a/Build/Download.pm +++ b/Build/Download.pm @@ -25,6 +25,7 @@ use strict; use LWP::UserAgent; use Digest::MD5 (); use Digest::SHA (); +use IO::Compress::Gzip qw(gzip $GzipError); our $debug; @@ -33,10 +34,13 @@ our $debug; # sub create_ua { my (%opt) = @_; - my $agent = $opt{'agent'} || 'openSUSE build script'; + my $agent = $opt{'agent'} || 'Wget/1.21.1'; my $timeout = $opt{'timeout'} || 60; my $ssl_opts = $opt{'ssl_opts'} || { verify_hostname => 1 }; my $ua = LWP::UserAgent->new(agent => $agent, timeout => $timeout, ssl_opts => $ssl_opts, keep_alive => 3); + $ua->default_header('Accept' => '*/*'); + $ua->default_header('Accept-Encoding' => 'identity'); + $ua->default_header('Connection' => 'Keep-Alive'); $ua->env_proxy; $ua->cookie_jar($opt{'cookie_jar'}) if $opt{'cookie_jar'}; return $ua; @@ -197,7 +201,22 @@ sub download { die("download of $url failed: $status\n") unless $retry-- > 0 && $res->previous; warn("retrying $url\n"); } - checkfiledigest($dest, $opt{'digest'}) if $opt{'digest'}; + if ($opt{'digest'}) { + eval { checkfiledigest($dest, $opt{'digest'}) }; + if ($@) { + # Digest failed, try again with gzip encoding + my $gz_dest = "$dest.gz"; + my @headers = @{$opt{'headers'} || []}; + push @headers, 'Accept-Encoding', 'gzip'; + my $gz_res = eval { ua_mirror($ua, $url, $gz_dest, $opt{'maxsize'}, @headers) }; + if ($@ || !$gz_res->is_success) { + unlink($gz_dest); + die($@ || "download of gzipped $url failed: " . $gz_res->status_line); + } + eval { checkfiledigest($gz_dest, $opt{'digest'}) }; + die($@) if $@; # If gzipped also fails, die with the new error + } + } if ($destfinal) { rename($dest, $destfinal) || die("rename $dest $destfinal: $!\n"); } diff --git a/PBuild/RemoteAssets.pm b/PBuild/RemoteAssets.pm index 63fe33d4..40fe3398 100644 --- a/PBuild/RemoteAssets.pm +++ b/PBuild/RemoteAssets.pm @@ -249,7 +249,7 @@ sub fedpkg_fetch { my @assets = grep {$_->{'digest'}} @$assets; return unless @assets; die("need a parsed name to download fedpkg assets\n") unless $p->{'name'}; - print "fetching ".PBuild::Util::plural(scalar(@assets), 'asset')." from $url\n"; + #print "fetching ".PBuild::Util::plural(scalar(@assets), 'asset')." from $url\n"; for my $asset (@assets) { my $assetid = $asset->{'assetid'}; my $adir = "$assetdir/".substr($assetid, 0, 2); @@ -261,6 +261,7 @@ sub fedpkg_fetch { $path = PBuild::Util::urlencode($path); my $fedpkg_url = $url; $fedpkg_url =~ s/\/?$/\/$path/; + print "fetching ".PBuild::Util::plural(scalar(@assets), 'asset')." from $fedpkg_url\n"; if (Build::Download::download($fedpkg_url, "$adir/.$assetid.$$", undef, 'retry' => 3, 'digest' => $asset->{'digest'}, 'missingok' => 1, 'headers' => [ 'Accept-Encoding' => 'identity' ])) { rename_unless_present("$adir/.$assetid.$$", "$adir/$assetid"); }