fix cache on deb multirepo

Author: samueltaripin

internal/ospackage/debutils/download.go

@@ -59,6 +59,7 @@ type pkgChecksum struct {
 var (
 	RepoCfg      RepoConfig
 	RepoCfgs     []RepoConfig // Support for multiple repositories
+	UserRepoCfgs []RepoConfig // Populated from UserRepo packageRepositories
 	PkgChecksum  []pkgChecksum
 	GzHref       string
 	Architecture string
@@ -68,14 +69,32 @@ var (
 	urlExistenceCacheMu     sync.Mutex
 	urlExistenceCache       map[string]bool
 	urlExistenceCacheLoaded bool
+
+	packageListURLCacheMu     sync.Mutex
+	packageListURLCache       map[string]string
+	packageListURLCacheLoaded bool
 )
 
 const urlExistenceCacheFileName = "url_exists_cache.json"
+const packageListURLCacheFileName = "package_list_url_cache.json"
 
 func urlExistenceCacheFilePath() string {
 	return filepath.Join(config.TempDir(), "builds", urlExistenceCacheFileName)
 }
 
+func packageListURLCacheFilePath() string {
+	return filepath.Join(config.TempDir(), "builds", packageListURLCacheFileName)
+}
+
+func packageListURLCacheKey(baseURL, codename, arch, component string) string {
+	return strings.Join([]string{
+		strings.TrimSpace(baseURL),
+		strings.TrimSpace(codename),
+		strings.TrimSpace(arch),
+		strings.TrimSpace(component),
+	}, "|")
+}
+
 func loadURLExistenceCacheLocked() {
 	if urlExistenceCacheLoaded {
 		return
@@ -129,6 +148,63 @@ func saveURLExistenceToCache(url string, exists bool) {
 	}
 }
 
+func loadPackageListURLCacheLocked() {
+	if packageListURLCacheLoaded {
+		return
+	}
+	packageListURLCacheLoaded = true
+	packageListURLCache = make(map[string]string)
+
+	cacheFile := packageListURLCacheFilePath()
+	data, err := os.ReadFile(cacheFile)
+	if err != nil {
+		return
+	}
+
+	if err := json.Unmarshal(data, &packageListURLCache); err != nil {
+		packageListURLCache = make(map[string]string)
+	}
+}
+
+func getPackageListURLFromCache(baseURL, codename, arch, component string) (string, bool) {
+	packageListURLCacheMu.Lock()
+	defer packageListURLCacheMu.Unlock()
+
+	loadPackageListURLCacheLocked()
+	url, ok := packageListURLCache[packageListURLCacheKey(baseURL, codename, arch, component)]
+	if !ok || strings.TrimSpace(url) == "" {
+		return "", false
+	}
+	return url, true
+}
+
+func savePackageListURLToCache(baseURL, codename, arch, component, packageListURL string) {
+	log := logger.Logger()
+
+	packageListURLCacheMu.Lock()
+	defer packageListURLCacheMu.Unlock()
+
+	loadPackageListURLCacheLocked()
+	key := packageListURLCacheKey(baseURL, codename, arch, component)
+	packageListURLCache[key] = packageListURL
+
+	cacheFile := packageListURLCacheFilePath()
+	if err := os.MkdirAll(filepath.Dir(cacheFile), 0755); err != nil {
+		log.Warnf("failed to create package-list URL cache directory: %v", err)
+		return
+	}
+
+	data, err := json.Marshal(packageListURLCache)
+	if err != nil {
+		log.Warnf("failed to marshal package-list URL cache: %v", err)
+		return
+	}
+
+	if err := os.WriteFile(cacheFile, data, 0644); err != nil {
+		log.Warnf("failed to persist package-list URL cache: %v", err)
+	}
+}
+
 func extractDebPackageNameFromFile(fileName string) string {
 	base := strings.TrimSuffix(fileName, ".deb")
 	if idx := strings.Index(base, "_"); idx > 0 {
@@ -244,7 +320,7 @@ func isDebRequirementInCache(
 }
 
 func debMetadataBuildPaths() []string {
-	buildPaths := make([]string, 0, 1+len(RepoCfgs))
+	buildPaths := make([]string, 0, 1+len(RepoCfgs)+len(UserRepoCfgs))
 	seen := make(map[string]struct{})
 	add := func(path string) {
 		path = strings.TrimSpace(path)
@@ -262,6 +338,9 @@ func debMetadataBuildPaths() []string {
 	for _, rc := range RepoCfgs {
 		add(rc.BuildPath)
 	}
+	for _, rc := range UserRepoCfgs {
+		add(rc.BuildPath)
+	}
 
 	return buildPaths
 }
@@ -504,10 +583,13 @@ func BuildRepoConfigs(userRepoList []Repository, arch string) ([]RepoConfig, err
 	return userRepo, nil
 }
 
-func UserPackages() ([]ospackage.PackageInfo, error) {
-
-	log := logger.Logger()
-	log.Infof("fetching packages from %s", "user package list")
+// initializeUserRepoCfgs builds the UserRepoCfgs from UserRepo without fetching package metadata.
+// This is called early to ensure UserRepoCfgs is available before cache checks.
+func initializeUserRepoCfgs() error {
+	// UserRepoCfgs is already initialized, skip
+	if len(UserRepoCfgs) > 0 {
+		return nil
+	}
 
 	var repoList []Repository
 	repoGroup := "custrepo"
@@ -528,18 +610,32 @@ func UserPackages() ([]ospackage.PackageInfo, error) {
 		})
 	}
 
-	// If no valid repositories were found (all were placeholders), return empty package list
+	// If no valid repositories were found (all were placeholders), nothing to do
 	if len(repoList) == 0 {
-		return []ospackage.PackageInfo{}, nil
+		return nil
 	}
 
 	userRepo, err := BuildRepoConfigs(repoList, Architecture)
 	if err != nil {
-		return nil, fmt.Errorf("building user repo configs failed: %w", err)
+		return fmt.Errorf("building user repo configs failed: %w", err)
+	}
+	UserRepoCfgs = userRepo
+
+	return nil
+}
+
+func UserPackages() ([]ospackage.PackageInfo, error) {
+
+	log := logger.Logger()
+	log.Infof("fetching packages from %s", "user package list")
+
+	// Initialize UserRepoCfgs early if not already done
+	if err := initializeUserRepoCfgs(); err != nil {
+		return nil, err
 	}
 
 	var allUserPackages []ospackage.PackageInfo
-	for _, rpItx := range userRepo {
+	for _, rpItx := range UserRepoCfgs {
 
 		userPkgs, err := ParseRepositoryMetadata(rpItx.PkgPrefix, rpItx.PkgList, rpItx.ReleaseFile, rpItx.ReleaseSign, rpItx.PbGPGKey, rpItx.BuildPath, rpItx.Arch, rpItx.AllowPackages)
 		if err != nil {
@@ -777,6 +873,13 @@ func DownloadPackagesComplete(pkgList []string, destDir, dotFile string, pkgSour
 	var downloadPkgList []string
 
 	log := logger.Logger()
+
+	// Initialize user repo configs early so they are available for cache metadata lookup
+	if err := initializeUserRepoCfgs(); err != nil {
+		log.Warnf("Failed to initialize user repo configs: %v", err)
+		// Continue anyway; user repos are optional
+	}
+
 	absDestDir, err := filepath.Abs(destDir)
 	if err != nil {
 		return downloadPkgList, nil, fmt.Errorf("resolving cache directory: %w", err)

internal/ospackage/debutils/download_test.go

@@ -22,6 +22,13 @@ func resetURLExistenceCacheForTest(t *testing.T) {
 	urlExistenceCacheMu.Unlock()
 
 	_ = os.Remove(urlExistenceCacheFilePath())
+
+	packageListURLCacheMu.Lock()
+	packageListURLCache = nil
+	packageListURLCacheLoaded = false
+	packageListURLCacheMu.Unlock()
+
+	_ = os.Remove(packageListURLCacheFilePath())
 }
 
 // TestPackages tests the Packages function
@@ -658,6 +665,40 @@ func TestCheckFileExists_UsesCacheOffline(t *testing.T) {
 	}
 }
 
+func TestGetPackagesNames_UsesCachedURLOffline(t *testing.T) {
+	resetURLExistenceCacheForTest(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/dists/stable/main/binary-amd64/Packages.gz" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	}))
+
+	baseURL := server.URL
+	first, err := GetPackagesNames(baseURL, "stable", "amd64", "main")
+	if err != nil {
+		t.Fatalf("first GetPackagesNames failed: %v", err)
+	}
+	if first == "" {
+		t.Fatalf("expected first GetPackagesNames to return URL")
+	}
+
+	server.Close()
+
+	second, err := GetPackagesNames(baseURL, "stable", "amd64", "main")
+	if err != nil {
+		t.Fatalf("second GetPackagesNames should use cache and work offline: %v", err)
+	}
+	if second == "" {
+		t.Fatalf("expected cached GetPackagesNames to return URL")
+	}
+	if second != first {
+		t.Fatalf("expected cached URL %q, got %q", first, second)
+	}
+}
+
 // TestValidate tests the Validate function
 func TestValidate(t *testing.T) {
 	// Save original PkgChecksum

internal/ospackage/debutils/resolver.go

@@ -195,27 +195,45 @@ func ParseRepositoryMetadata(baseURL string, pkggz string, releaseFile string, r
 		metaURLList = []string{releaseFile, releaseSign}
 	}
 
-	// Always refresh metadata files (Release, signature, key) to detect upstream changes.
+	// Only skip Release file refresh if local files exist; allow network check for freshness.
+	// This balances offline resilience with metadata staleness detection.
+	refreshNeeded := false
 	for _, f := range metaLocalFiles {
-		if _, err := os.Stat(f); err == nil {
-			if remErr := os.Remove(f); remErr != nil {
-				return nil, fmt.Errorf("failed to remove old file %s: %w", f, remErr)
-			}
+		if _, err := os.Stat(f); err != nil {
+			refreshNeeded = true
+			break
 		}
 	}
 
-	// Download the metadata files
-	err := pkgfetcher.FetchPackages(metaURLList, pkgMetaDir, 1)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch critical repo config packages: %w", err)
-	}
-	// Verify the release file
-	relVryResult, err := VerifyRelease(localReleaseFile, localReleaseSign, localPBGPGKey)
-	if err != nil {
-		return nil, fmt.Errorf("failed to verify release file: %w", err)
+	// Download the metadata files only if missing
+	if refreshNeeded {
+		log.Infof("Refreshing metadata files for %s", baseURL)
+		for _, f := range metaLocalFiles {
+			if _, err := os.Stat(f); err == nil {
+				if remErr := os.Remove(f); remErr != nil {
+					return nil, fmt.Errorf("failed to remove old file %s: %w", f, remErr)
+				}
+			}
+		}
+		err := pkgfetcher.FetchPackages(metaURLList, pkgMetaDir, 1)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch critical repo config packages: %w", err)
+		}
+	} else {
+		log.Debugf("Using existing metadata files for %s (offline mode)", baseURL)
 	}
-	if !relVryResult {
-		return nil, fmt.Errorf("release file verification failed")
+
+	// Verify the release file if it was refreshed online; offline cached files are trusted.
+	if refreshNeeded {
+		relVryResult, err := VerifyRelease(localReleaseFile, localReleaseSign, localPBGPGKey)
+		if err != nil {
+			return nil, fmt.Errorf("failed to verify release file: %w", err)
+		}
+		if !relVryResult {
+			return nil, fmt.Errorf("release file verification failed")
+		}
+	} else {
+		log.Debugf("Skipping release file verification (using cached offline files)")
 	}
 
 	// verify the sham256 checksum of the Packages.gz file

internal/ospackage/debutils/zip.go

@@ -84,6 +84,10 @@ func GetPackagesNames(baseURL string, codename string, arch string, component st
 	if baseURL == "<URL>" || baseURL == "" {
 		return "", nil
 	}
+	if cachedURL, ok := getPackageListURLFromCache(baseURL, codename, arch, component); ok {
+		logger.Logger().Debugf("Using cached package list URL for %s/%s/%s/%s: %s", baseURL, codename, arch, component, cachedURL)
+		return cachedURL, nil
+	}
 	possibleFiles := []string{"Packages.gz", "Packages.xz"}
 	var foundFile string
 	for _, fname := range possibleFiles {
@@ -94,6 +98,7 @@ func GetPackagesNames(baseURL string, codename string, arch string, component st
 		}
 		if fileExist {
 			foundFile = packageListURL
+			savePackageListURLToCache(baseURL, codename, arch, component, packageListURL)
 			break
 		} else {
 			logger.Logger().Debugf("Searching package list at: %s", packageListURL)