From db4099dc20089159bde11fd6293e385411d2438a Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 12:43:51 +0530 Subject: [PATCH 01/12] Hadle exception thrown when src is missing in artifactory Signed-off-by: poorndm --- spec/functional/fetchers/net_fetcher_spec.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/functional/fetchers/net_fetcher_spec.rb b/spec/functional/fetchers/net_fetcher_spec.rb index 73c278275..960b5fb2a 100644 --- a/spec/functional/fetchers/net_fetcher_spec.rb +++ b/spec/functional/fetchers/net_fetcher_spec.rb @@ -176,8 +176,10 @@ def stub_ohai(options = {}, &block); end context "when use_internal_sources is true and no internal source url" do before { Omnibus::Config.use_internal_sources(true) } - it "raises an exception" do - expect { fetch! }.to raise_error(InternalSourceMissing) + it "logs a message and fetches from the source URL" do + expect(fetcher).to receive(:log).with("Internal source missing for #{fetcher}. Fetching from source URL instead.") + expect(fetcher).to receive(:fetch_from_source_url) + fetcher.fetch! end end @@ -243,7 +245,6 @@ def stub_ohai(options = {}, &block); end context "when the checksum is invalid" do let(:source_sha512) { "bad01234checksum" } - it "raises an exception" do expect { fetch! }.to raise_error(ChecksumMismatch) end From d2402f85725781359068548aad9a73d9433e1023 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 13:02:40 +0530 Subject: [PATCH 02/12] Fix syntax erro Signed-off-by: poorndm --- spec/functional/fetchers/net_fetcher_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/functional/fetchers/net_fetcher_spec.rb b/spec/functional/fetchers/net_fetcher_spec.rb index 960b5fb2a..461ff0120 100644 --- a/spec/functional/fetchers/net_fetcher_spec.rb +++ b/spec/functional/fetchers/net_fetcher_spec.rb @@ -177,8 +177,8 @@ def stub_ohai(options = {}, &block); end before { Omnibus::Config.use_internal_sources(true) } it "logs a message and fetches from the source URL" do - expect(fetcher).to receive(:log).with("Internal source missing for #{fetcher}. Fetching from source URL instead.") - expect(fetcher).to receive(:fetch_from_source_url) + expect(fetch).to receive(:log).with("****** Internal source missing for : #{fetch}. Fetching from source URL instead. ******") + expect(fetch).to receive(:fetch_from_source_url) fetcher.fetch! end end From fad153645497dd961eabfe3425e92cf9f8567cf6 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 13:11:49 +0530 Subject: [PATCH 03/12] modified the code to fix errors Signed-off-by: poorndm --- spec/functional/fetchers/net_fetcher_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/functional/fetchers/net_fetcher_spec.rb b/spec/functional/fetchers/net_fetcher_spec.rb index 461ff0120..e84f16a10 100644 --- a/spec/functional/fetchers/net_fetcher_spec.rb +++ b/spec/functional/fetchers/net_fetcher_spec.rb @@ -177,8 +177,8 @@ def stub_ohai(options = {}, &block); end before { Omnibus::Config.use_internal_sources(true) } it "logs a message and fetches from the source URL" do - expect(fetch).to receive(:log).with("****** Internal source missing for : #{fetch}. Fetching from source URL instead. ******") - expect(fetch).to receive(:fetch_from_source_url) + expect(subject).to receive(:log).with("****** Internal source missing for : #{subject}. Fetching from source URL instead. ******") + expect(subject).to receive(:fetch_from_source_url) fetcher.fetch! end end From a44cbf0cb0b370b6152c017578c345c5e18f2233 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 13:21:46 +0530 Subject: [PATCH 04/12] modified the code to fix errors Signed-off-by: poorndm --- spec/functional/fetchers/net_fetcher_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/functional/fetchers/net_fetcher_spec.rb b/spec/functional/fetchers/net_fetcher_spec.rb index e84f16a10..9964debc0 100644 --- a/spec/functional/fetchers/net_fetcher_spec.rb +++ b/spec/functional/fetchers/net_fetcher_spec.rb @@ -179,7 +179,7 @@ def stub_ohai(options = {}, &block); end it "logs a message and fetches from the source URL" do expect(subject).to receive(:log).with("****** Internal source missing for : #{subject}. Fetching from source URL instead. ******") expect(subject).to receive(:fetch_from_source_url) - fetcher.fetch! + subject.fetch! end end From 60377682cc773836f685b4e5b8a98a7c031cc65a Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 14:04:56 +0530 Subject: [PATCH 05/12] Modified netfetcher to handle the exceptions Signed-off-by: poorndm --- lib/omnibus/fetchers/net_fetcher.rb | 4 +++- spec/functional/fetchers/net_fetcher_spec.rb | 16 ++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/omnibus/fetchers/net_fetcher.rb b/lib/omnibus/fetchers/net_fetcher.rb index eec94dcd3..f8c60b764 100644 --- a/lib/omnibus/fetchers/net_fetcher.rb +++ b/lib/omnibus/fetchers/net_fetcher.rb @@ -148,12 +148,14 @@ def download_url if Config.use_s3_caching S3Cache.url_for(self) elsif Config.use_internal_sources && !source[:internal] - raise InternalSourceMissing.new(self) + log.warn(log_key) { "Internal source missing for #{name}; falling back to external source URL." } + source[:url] else source[:url] end end + # # Download the given file using Ruby's +OpenURI+ implementation. This method # may emit warnings as defined in software definitions using the +:warning+ diff --git a/spec/functional/fetchers/net_fetcher_spec.rb b/spec/functional/fetchers/net_fetcher_spec.rb index 9964debc0..0b4ab92d8 100644 --- a/spec/functional/fetchers/net_fetcher_spec.rb +++ b/spec/functional/fetchers/net_fetcher_spec.rb @@ -173,16 +173,19 @@ def stub_ohai(options = {}, &block); end end end - context "when use_internal_sources is true and no internal source url" do - before { Omnibus::Config.use_internal_sources(true) } + context "when use_internal_sources is true but internal source is missing" do + before do + Config.use_internal_sources(true) + end - it "logs a message and fetches from the source URL" do - expect(subject).to receive(:log).with("****** Internal source missing for : #{subject}. Fetching from source URL instead. ******") - expect(subject).to receive(:fetch_from_source_url) - subject.fetch! + it "logs a warning and falls back to the external url" do + expect(subject).to receive(:log).and_call_original + url = subject.send(:download_url) + expect(url).to eq(source[:url]) end end + context "when use_internal_sources is true and internal source url given" do before { Omnibus::Config.use_internal_sources(true) } let(:source) do @@ -245,6 +248,7 @@ def stub_ohai(options = {}, &block); end context "when the checksum is invalid" do let(:source_sha512) { "bad01234checksum" } + it "raises an exception" do expect { fetch! }.to raise_error(ChecksumMismatch) end From a5759b016bf635c9fcf98e826ae95d6c96ca6cf4 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 14:15:33 +0530 Subject: [PATCH 06/12] Modified netfetcher to handle the exceptions Signed-off-by: poorndm --- lib/omnibus/fetchers/net_fetcher.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/omnibus/fetchers/net_fetcher.rb b/lib/omnibus/fetchers/net_fetcher.rb index f8c60b764..3b9dfba78 100644 --- a/lib/omnibus/fetchers/net_fetcher.rb +++ b/lib/omnibus/fetchers/net_fetcher.rb @@ -155,7 +155,6 @@ def download_url end end - # # Download the given file using Ruby's +OpenURI+ implementation. This method # may emit warnings as defined in software definitions using the +:warning+ From 0bc391e356137c82294e59b36fc156890db62f6d Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 14:23:20 +0530 Subject: [PATCH 07/12] Modified netfetcher to handle the exceptions Signed-off-by: poorndm --- spec/functional/fetchers/net_fetcher_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/functional/fetchers/net_fetcher_spec.rb b/spec/functional/fetchers/net_fetcher_spec.rb index 0b4ab92d8..9215e879c 100644 --- a/spec/functional/fetchers/net_fetcher_spec.rb +++ b/spec/functional/fetchers/net_fetcher_spec.rb @@ -185,7 +185,6 @@ def stub_ohai(options = {}, &block); end end end - context "when use_internal_sources is true and internal source url given" do before { Omnibus::Config.use_internal_sources(true) } let(:source) do From 1d6dc91777b10be47c731506d2a69b76c179a981 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 16:24:42 +0530 Subject: [PATCH 08/12] Modified to handle missing iternal-source Signed-off-by: poorndm --- lib/omnibus/exceptions.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/omnibus/exceptions.rb b/lib/omnibus/exceptions.rb index 180d2b47c..9bed89cda 100644 --- a/lib/omnibus/exceptions.rb +++ b/lib/omnibus/exceptions.rb @@ -354,12 +354,6 @@ def to_s class InternalSourceMissing < Error def initialize(software) - super <<~EOH - Internal source missing for #{software.name}. - - When :use_internal_sources is set in the configuration you must specify an - internal_source for sources fetched from a url. - EOH + Omnibus.logger.warn("Internal source missing for #{software.name}; falling back to external source.") end - end -end + end \ No newline at end of file From 66176d3841f8b764fe07398690e6b49bae4f38b1 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 16:34:14 +0530 Subject: [PATCH 09/12] Modified exception Signed-off-by: poorndm --- lib/omnibus/exceptions.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/omnibus/exceptions.rb b/lib/omnibus/exceptions.rb index 9bed89cda..180d2b47c 100644 --- a/lib/omnibus/exceptions.rb +++ b/lib/omnibus/exceptions.rb @@ -354,6 +354,12 @@ def to_s class InternalSourceMissing < Error def initialize(software) - Omnibus.logger.warn("Internal source missing for #{software.name}; falling back to external source.") + super <<~EOH + Internal source missing for #{software.name}. + + When :use_internal_sources is set in the configuration you must specify an + internal_source for sources fetched from a url. + EOH end - end \ No newline at end of file + end +end From 014db634eb8f12680064b6356dc6810042478ddf Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 17:33:47 +0530 Subject: [PATCH 10/12] Download - to hadle 404 error Signed-off-by: poorndm --- lib/omnibus/fetchers/net_fetcher.rb | 48 +++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/lib/omnibus/fetchers/net_fetcher.rb b/lib/omnibus/fetchers/net_fetcher.rb index 3b9dfba78..d11dc13fc 100644 --- a/lib/omnibus/fetchers/net_fetcher.rb +++ b/lib/omnibus/fetchers/net_fetcher.rb @@ -162,23 +162,53 @@ def download_url # # @return [void] # - def download - log.warn(log_key) { source[:warning] } if source.key?(:warning) + # def download + # log.warn(log_key) { source[:warning] } if source.key?(:warning) - options = {} + # options = {} - if source[:unsafe] - log.warn(log_key) { "Permitting unsafe redirects!" } - options[:allow_unsafe_redirects] = true - end + # if source[:unsafe] + # log.warn(log_key) { "Permitting unsafe redirects!" } + # options[:allow_unsafe_redirects] = true + # end + + # # Set the cookie if one was given + # options["Cookie"] = source[:cookie] if source[:cookie] + # options["Authorization"] = source[:authorization] if source[:authorization] + + # download_file!(download_url, downloaded_file, options) + # end + def download + options = {} - # Set the cookie if one was given options["Cookie"] = source[:cookie] if source[:cookie] options["Authorization"] = source[:authorization] if source[:authorization] - download_file!(download_url, downloaded_file, options) + begin + # Attempt the original download which tries internal source if configured + super + rescue OpenURI::HTTPError => e + # Check if the error is a 404 from internal source while internal sources are enabled + if e.io.status[0] == "404" && Config.use_internal_sources && source[:internal] + log.warn(log_key) { "Internal source returned 404 Not Found for #{name}, falling back to external source URL." } + + # Temporarily disable internal sources to force external URL usage + Config.use_internal_sources(false) + begin + # Retry download explicitly from external source URL + download_file!(source[:url], downloaded_file, options) + ensure + # Restore the original internal sources config + Config.use_internal_sources(true) + end + else + # Raise any other HTTP errors as usual + raise + end + end end + # # Extract the downloaded file, using the magical logic based off of the # ending file extension. In the rare event the file cannot be extracted, it From 9555048af57589bdd488116e9be005964c65fcbc Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 17:51:28 +0530 Subject: [PATCH 11/12] Download - to hadle 404 error Signed-off-by: poorndm --- lib/omnibus/fetchers/net_fetcher.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/omnibus/fetchers/net_fetcher.rb b/lib/omnibus/fetchers/net_fetcher.rb index d11dc13fc..02e86e54c 100644 --- a/lib/omnibus/fetchers/net_fetcher.rb +++ b/lib/omnibus/fetchers/net_fetcher.rb @@ -208,7 +208,6 @@ def download end end - # # Extract the downloaded file, using the magical logic based off of the # ending file extension. In the rare event the file cannot be extracted, it From 63eb75d6d3c5235026a478d22e5fae0e95f9cd91 Mon Sep 17 00:00:00 2001 From: poorndm Date: Wed, 13 Aug 2025 18:46:17 +0530 Subject: [PATCH 12/12] Modified download to handle 404 error while downloading from internal_src Signed-off-by: poorndm --- lib/omnibus/fetchers/net_fetcher.rb | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/omnibus/fetchers/net_fetcher.rb b/lib/omnibus/fetchers/net_fetcher.rb index 02e86e54c..a892bb30b 100644 --- a/lib/omnibus/fetchers/net_fetcher.rb +++ b/lib/omnibus/fetchers/net_fetcher.rb @@ -179,30 +179,29 @@ def download_url # download_file!(download_url, downloaded_file, options) # end def download + log.warn(log_key) { source[:warning] } if source.key?(:warning) + options = {} + if source[:unsafe] + log.warn(log_key) { "Permitting unsafe redirects!" } + options[:allow_unsafe_redirects] = true + end + + # Set the cookie if one was given options["Cookie"] = source[:cookie] if source[:cookie] options["Authorization"] = source[:authorization] if source[:authorization] begin - # Attempt the original download which tries internal source if configured - super + # Attempt download from internal or S3 source URL + download_file!(download_url, downloaded_file, options) rescue OpenURI::HTTPError => e - # Check if the error is a 404 from internal source while internal sources are enabled + # If 404 comes from internal source, fallback to external URL if e.io.status[0] == "404" && Config.use_internal_sources && source[:internal] log.warn(log_key) { "Internal source returned 404 Not Found for #{name}, falling back to external source URL." } - - # Temporarily disable internal sources to force external URL usage - Config.use_internal_sources(false) - begin - # Retry download explicitly from external source URL - download_file!(source[:url], downloaded_file, options) - ensure - # Restore the original internal sources config - Config.use_internal_sources(true) - end + download_file!(source[:url], downloaded_file, options) else - # Raise any other HTTP errors as usual + # Re-raise other errors raise end end