Skip to content

Commit b823442

Browse files
samueltaripinsamueltaripin
committed
optimize bypass cert logic
1 parent d6aefd1 commit b823442

3 files changed

Lines changed: 175 additions & 42 deletions

File tree

‎internal/ospackage/debutils/download.go‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -500,13 +500,14 @@ func DownloadPackagesComplete(pkgList []string, destDir, dotFile string, pkgSour
500500
downloadPkgList = append(downloadPkgList, filepath.Base(pkg.URL))
501501
}
502502

503-
// Determine if we should skip TLS verification
504-
// Use insecure mode if ANY repository has it enabled
505-
insecureSkipVerify := RepoCfg.InsecureSkipVerify
506-
for _, cfg := range RepoCfgs {
507-
if cfg.InsecureSkipVerify {
508-
insecureSkipVerify = true
509-
break
503+
// Check if any repository requires insecure mode
504+
hasInsecureRepo := RepoCfg.InsecureSkipVerify
505+
if !hasInsecureRepo {
506+
for _, cfg := range RepoCfgs {
507+
if cfg.InsecureSkipVerify {
508+
hasInsecureRepo = true
509+
break
510+
}
510511
}
511512
}
512513

@@ -519,10 +520,41 @@ func DownloadPackagesComplete(pkgList []string, destDir, dotFile string, pkgSour
519520
return downloadPkgList, nil, fmt.Errorf("creating cache directory %s: %w", absDestDir, err)
520521
}
521522

522-
// Download packages using configured workers and cache directory
523+
// Download packages with appropriate security settings
523524
log.Infof("downloading %d packages to %s using %d workers", len(urls), absDestDir, config.Workers())
524-
if err := pkgfetcher.FetchPackages(urls, absDestDir, config.Workers(), insecureSkipVerify); err != nil {
525-
return downloadPkgList, nil, fmt.Errorf("fetch failed: %w", err)
525+
if !hasInsecureRepo {
526+
// All repositories are secure - use secure client for all downloads (no per-URL checking)
527+
log.Debugf("all repositories use secure connections, using secure client for all downloads")
528+
if err := pkgfetcher.FetchPackages(urls, absDestDir, config.Workers(), false); err != nil {
529+
return downloadPkgList, nil, fmt.Errorf("fetch failed: %w", err)
530+
}
531+
} else {
532+
// Build security configurations for repositories with insecure mode
533+
var securityConfigs []pkgfetcher.URLSecurityConfig
534+
535+
// Add base repository config if insecure
536+
if RepoCfg.InsecureSkipVerify {
537+
securityConfigs = append(securityConfigs, pkgfetcher.URLSecurityConfig{
538+
URLPrefixes: []string{RepoCfg.PkgPrefix},
539+
InsecureSkipVerify: true,
540+
})
541+
}
542+
543+
// Add multiple repository configs if insecure
544+
for _, cfg := range RepoCfgs {
545+
if cfg.InsecureSkipVerify {
546+
securityConfigs = append(securityConfigs, pkgfetcher.URLSecurityConfig{
547+
URLPrefixes: []string{cfg.PkgPrefix},
548+
InsecureSkipVerify: true,
549+
})
550+
}
551+
}
552+
553+
// Download with per-repository security settings (URL checking enabled)
554+
log.Debugf("mixed repository security settings, using per-URL security checking")
555+
if err := pkgfetcher.FetchPackagesWithSecurityConfig(urls, absDestDir, config.Workers(), securityConfigs); err != nil {
556+
return downloadPkgList, nil, fmt.Errorf("fetch failed: %w", err)
557+
}
526558
}
527559
log.Info("all downloads complete")
528560

‎internal/ospackage/pkgfetcher/pkgfetcher.go‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"os"
88
"path"
99
"path/filepath"
10+
"strings"
1011
"sync"
1112
"time"
1213

@@ -15,15 +16,67 @@ import (
1516
"github.com/schollz/progressbar/v3"
1617
)
1718

19+
// URLSecurityConfig holds URL patterns and their security settings
20+
type URLSecurityConfig struct {
21+
URLPrefixes []string // List of URL prefixes (e.g., "https://repo1.com", "http://repo2.com")
22+
InsecureSkipVerify bool // Whether to skip TLS verification for these URLs
23+
}
24+
1825
// FetchPackages downloads the given URLs into destDir using a pool of workers.
1926
// It shows a single progress bar tracking files completed vs total.
27+
// Deprecated: Use FetchPackagesWithSecurityConfig for per-repository security settings
2028
func FetchPackages(urls []string, destDir string, workers int, insecureSkipVerify bool) error {
29+
// Create a single security config for all URLs (legacy behavior)
30+
var securityConfigs []URLSecurityConfig
31+
if insecureSkipVerify {
32+
securityConfigs = []URLSecurityConfig{{
33+
URLPrefixes: []string{""}, // Empty prefix matches all URLs
34+
InsecureSkipVerify: true,
35+
}}
36+
}
37+
return FetchPackagesWithSecurityConfig(urls, destDir, workers, securityConfigs)
38+
}
39+
40+
// FetchPackagesWithSecurityConfig downloads URLs with per-repository security settings.
41+
// It creates secure and insecure HTTP clients and selects the appropriate one based on URL matching.
42+
func FetchPackagesWithSecurityConfig(urls []string, destDir string, workers int, securityConfigs []URLSecurityConfig) error {
2143
log := logger.Logger()
2244

2345
total := len(urls)
2446
jobs := make(chan string, total)
2547
var wg sync.WaitGroup
2648

49+
// Create two HTTP clients: one secure, one insecure
50+
secureClient := network.NewHTTPClientWithOptions(network.HTTPClientOptions{
51+
InsecureSkipVerify: false,
52+
})
53+
insecureClient := network.NewHTTPClientWithOptions(network.HTTPClientOptions{
54+
InsecureSkipVerify: true,
55+
})
56+
57+
// Helper function to determine if a URL should use insecure client
58+
shouldUseInsecure := func(downloadURL string) bool {
59+
for _, config := range securityConfigs {
60+
if !config.InsecureSkipVerify {
61+
continue
62+
}
63+
// If URLPrefixes is empty or contains empty string, apply to all URLs (legacy behavior)
64+
if len(config.URLPrefixes) == 0 || (len(config.URLPrefixes) == 1 && config.URLPrefixes[0] == "") {
65+
return true
66+
}
67+
// Check if the URL matches any of the prefixes
68+
for _, prefix := range config.URLPrefixes {
69+
// Normalize URLs for comparison (handle trailing slashes)
70+
normalizedURL := strings.TrimSuffix(downloadURL, "/")
71+
normalizedPrefix := strings.TrimSuffix(prefix, "/")
72+
if strings.HasPrefix(normalizedURL, normalizedPrefix) {
73+
return true
74+
}
75+
}
76+
}
77+
return false
78+
}
79+
2780
// create a single progress bar for total files
2881
bar := progressbar.NewOptions(total,
2982
progressbar.OptionEnableColorCodes(true),
@@ -49,8 +102,8 @@ func FetchPackages(urls []string, destDir string, workers int, insecureSkipVerif
49102
wg.Add(1)
50103
go func() {
51104
defer wg.Done()
52-
for url := range jobs {
53-
name := path.Base(url)
105+
for downloadURL := range jobs {
106+
name := path.Base(downloadURL)
54107

55108
// update description to current file
56109
bar.Describe(name)
@@ -78,11 +131,16 @@ func FetchPackages(urls []string, destDir string, workers int, insecureSkipVerif
78131
}
79132
err := func() error {
80133

81-
// Create HTTP client with appropriate TLS settings
82-
client := network.NewHTTPClientWithOptions(network.HTTPClientOptions{
83-
InsecureSkipVerify: insecureSkipVerify,
84-
})
85-
resp, err := client.Get(url)
134+
// Select the appropriate HTTP client based on URL
135+
var client *http.Client
136+
if shouldUseInsecure(downloadURL) {
137+
client = insecureClient
138+
log.Debugf("using insecure client for %s", downloadURL)
139+
} else {
140+
client = secureClient
141+
}
142+
143+
resp, err := client.Get(downloadURL)
86144
if err != nil {
87145
return err
88146
}
@@ -105,7 +163,7 @@ func FetchPackages(urls []string, destDir string, workers int, insecureSkipVerif
105163
}()
106164

107165
if err != nil {
108-
log.Errorf("downloading %s failed: %v", url, err)
166+
log.Errorf("downloading %s failed: %v", downloadURL, err)
109167
downloadError = true
110168
}
111169
// increment progress bar

‎internal/ospackage/rpmutils/download.go‎

Lines changed: 67 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -59,22 +59,25 @@ func UserPackages() ([]ospackage.PackageInfo, error) {
5959
log.Infof("fetching packages from %s", "user package list")
6060

6161
repoList := make([]struct {
62-
id string
63-
codename string
64-
url string
65-
pkey string
62+
id string
63+
codename string
64+
url string
65+
pkey string
66+
insecureSkipVerify bool
6667
}, len(UserRepo))
6768
for i, repo := range UserRepo {
6869
repoList[i] = struct {
69-
id string
70-
codename string
71-
url string
72-
pkey string
70+
id string
71+
codename string
72+
url string
73+
pkey string
74+
insecureSkipVerify bool
7375
}{
74-
id: fmt.Sprintf("rpmcustrepo%d", i+1),
75-
codename: repo.Codename,
76-
url: repo.URL,
77-
pkey: repo.PKey,
76+
id: fmt.Sprintf("rpmcustrepo%d", i+1),
77+
codename: repo.Codename,
78+
url: repo.URL,
79+
pkey: repo.PKey,
80+
insecureSkipVerify: repo.InsecureSkipVerify,
7881
}
7982
}
8083

@@ -86,13 +89,14 @@ func UserPackages() ([]ospackage.PackageInfo, error) {
8689
pkey := repoItem.pkey
8790

8891
repo := RepoConfig{
89-
Name: id,
90-
GPGCheck: true,
91-
RepoGPGCheck: true,
92-
Enabled: true,
93-
GPGKey: pkey,
94-
URL: baseURL,
95-
Section: fmt.Sprintf("[%s]", codename),
92+
Name: id,
93+
GPGCheck: true,
94+
RepoGPGCheck: true,
95+
Enabled: true,
96+
GPGKey: pkey,
97+
URL: baseURL,
98+
Section: fmt.Sprintf("[%s]", codename),
99+
InsecureSkipVerify: repoItem.insecureSkipVerify,
96100
}
97101

98102
userRepo = append(userRepo, repo)
@@ -459,8 +463,16 @@ func DownloadPackagesComplete(pkgList []string, destDir, dotFile string, pkgSour
459463
downloadPkgList = append(downloadPkgList, pkg.Name)
460464
}
461465

462-
// Use insecure mode if the repository has it enabled
463-
insecureSkipVerify := RepoCfg.InsecureSkipVerify
466+
// Check if any repository requires insecure mode
467+
hasInsecureRepo := RepoCfg.InsecureSkipVerify
468+
if !hasInsecureRepo {
469+
for _, userRepoItem := range UserRepo {
470+
if userRepoItem.InsecureSkipVerify {
471+
hasInsecureRepo = true
472+
break
473+
}
474+
}
475+
}
464476

465477
// Ensure dest directory exists
466478
absDestDir, err := filepath.Abs(destDir)
@@ -471,10 +483,41 @@ func DownloadPackagesComplete(pkgList []string, destDir, dotFile string, pkgSour
471483
return downloadPkgList, nil, fmt.Errorf("creating cache directory %s: %v", absDestDir, err)
472484
}
473485

474-
// Download packages using configured workers and cache directory
486+
// Download packages with appropriate security settings
475487
log.Infof("Downloading %d packages to %s using %d workers", len(urls), absDestDir, config.Workers())
476-
if err := pkgfetcher.FetchPackages(urls, absDestDir, config.Workers(), insecureSkipVerify); err != nil {
477-
return downloadPkgList, nil, fmt.Errorf("fetch failed: %v", err)
488+
if !hasInsecureRepo {
489+
// All repositories are secure - use secure client for all downloads (no per-URL checking)
490+
log.Debugf("all repositories use secure connections, using secure client for all downloads")
491+
if err := pkgfetcher.FetchPackages(urls, absDestDir, config.Workers(), false); err != nil {
492+
return downloadPkgList, nil, fmt.Errorf("fetch failed: %v", err)
493+
}
494+
} else {
495+
// Build security configurations for repositories with insecure mode
496+
var securityConfigs []pkgfetcher.URLSecurityConfig
497+
498+
// Add base repository config if insecure
499+
if RepoCfg.InsecureSkipVerify {
500+
securityConfigs = append(securityConfigs, pkgfetcher.URLSecurityConfig{
501+
URLPrefixes: []string{RepoCfg.URL},
502+
InsecureSkipVerify: true,
503+
})
504+
}
505+
506+
// Add user repository configs if insecure
507+
for _, userRepoItem := range UserRepo {
508+
if userRepoItem.InsecureSkipVerify {
509+
securityConfigs = append(securityConfigs, pkgfetcher.URLSecurityConfig{
510+
URLPrefixes: []string{userRepoItem.URL},
511+
InsecureSkipVerify: true,
512+
})
513+
}
514+
}
515+
516+
// Download with per-repository security settings (URL checking enabled)
517+
log.Debugf("mixed repository security settings, using per-URL security checking")
518+
if err := pkgfetcher.FetchPackagesWithSecurityConfig(urls, absDestDir, config.Workers(), securityConfigs); err != nil {
519+
return downloadPkgList, nil, fmt.Errorf("fetch failed: %v", err)
520+
}
478521
}
479522
log.Info("All downloads complete")
480523

0 commit comments

Comments
 (0)