Full cache for offline mode

docs/release-notes.md

@@ -13,6 +13,8 @@
 - Ability to set priority for repositories to manage conflicts.
 - Ability to prioritize specific packages to manage conflicts.
 - Caching for consistent and faster composition.
+- Debian repository GPG keys are now cached in `cache_dir/gpg-keys` and reused on rebuilds to avoid re-downloading.
+- RPM repository metadata is now cached in `cache_dir/rpm-metadata` and reused on rebuilds to avoid network fetches.
 - Native support for Debian and RPM based distributions.
 - Support for building immutable OS images with DM-Verity and read-only file
   system support.

internal/chroot/chrootbuild/chrootbuild.go

@@ -230,6 +230,14 @@ func (chrootBuilder *ChrootBuilder) GetChrootEnvPackageList() ([]string, error)
 	return pkgList, nil
 }
 
+// chrootenvPkgCacheDir returns the isolated subdirectory used to download and install
+// the chroot-build packages (e.g. mmdebstrap, grub). Keeping it separate from
+// ChrootPkgCacheDir (the image-package cache) prevents the stale-cache check from
+// wiping the image packages when the two package sets do not overlap.
+func (chrootBuilder *ChrootBuilder) chrootenvPkgCacheDir() string {
+	return filepath.Join(chrootBuilder.ChrootPkgCacheDir, "chrootenv")
+}
+
 func (chrootBuilder *ChrootBuilder) downloadChrootEnvPackages() ([]string, []string, error) {
 	var pkgsList []string
 	var allPkgsList []string
@@ -245,23 +253,24 @@ func (chrootBuilder *ChrootBuilder) downloadChrootEnvPackages() ([]string, []str
 	}
 	pkgsList = append(essentialPkgsList, pkgsList...)
 
-	if _, err := os.Stat(chrootBuilder.ChrootPkgCacheDir); os.IsNotExist(err) {
-		if err := os.MkdirAll(chrootBuilder.ChrootPkgCacheDir, 0700); err != nil {
+	downloadDir := chrootBuilder.chrootenvPkgCacheDir()
+	if _, err := os.Stat(downloadDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(downloadDir, 0700); err != nil {
 			log.Errorf("Failed to create chroot package cache directory: %v", err)
 			return pkgsList, allPkgsList, fmt.Errorf("failed to create chroot package cache directory: %w", err)
 		}
 	}
 
-	dotFilePath := filepath.Join(chrootBuilder.ChrootPkgCacheDir, "chrootpkgs.dot")
+	dotFilePath := filepath.Join(downloadDir, "chrootpkgs.dot")
 
 	if pkgType == "rpm" {
-		allPkgsList, err = rpmutils.DownloadPackages(pkgsList, chrootBuilder.ChrootPkgCacheDir, dotFilePath, nil, false)
+		allPkgsList, err = rpmutils.DownloadPackages(pkgsList, downloadDir, dotFilePath, nil, false)
 		if err != nil {
 			return pkgsList, allPkgsList, fmt.Errorf("failed to download chroot environment packages: %w", err)
 		}
 		return pkgsList, allPkgsList, nil
 	} else if pkgType == "deb" {
-		allPkgsList, err = debutils.DownloadPackages(pkgsList, chrootBuilder.ChrootPkgCacheDir, dotFilePath, nil, false)
+		allPkgsList, err = debutils.DownloadPackages(pkgsList, downloadDir, dotFilePath, nil, false)
 		if err != nil {
 			return pkgsList, allPkgsList, fmt.Errorf("failed to download chroot environment packages: %w", err)
 		}
@@ -294,19 +303,21 @@ func (chrootBuilder *ChrootBuilder) BuildChrootEnv(targetOs string, targetDist s
 	}
 	log.Infof("Downloaded %d packages for chroot environment", len(allPkgsList))
 
-	chrootPkgCacheDir := chrootBuilder.GetChrootPkgCacheDir()
+	// Use the isolated chrootenv download dir (not the shared image-package cache dir)
+	// so that UpdateLocalDebRepo and InstallDebPkg operate on the chrootenv-specific packages.
+	chrootenvDir := chrootBuilder.chrootenvPkgCacheDir()
 	if pkgType == "rpm" {
 		if err := chrootBuilder.RpmInstaller.InstallRpmPkg(targetOs, chrootEnvPath,
-			chrootPkgCacheDir, allPkgsList); err != nil {
+			chrootenvDir, allPkgsList); err != nil {
 			return fmt.Errorf("failed to install packages in chroot environment: %w", err)
 		}
 	} else if pkgType == "deb" {
-		if err = chrootBuilder.DebInstaller.UpdateLocalDebRepo(chrootPkgCacheDir, targetArch, false); err != nil {
+		if err = chrootBuilder.DebInstaller.UpdateLocalDebRepo(chrootenvDir, targetArch, false); err != nil {
 			return fmt.Errorf("failed to create debian local repository: %w", err)
 		}
 
 		if err := chrootBuilder.DebInstaller.InstallDebPkg(chrootBuilder.TargetOsConfigDir,
-			chrootEnvPath, chrootPkgCacheDir, pkgsList); err != nil {
+			chrootEnvPath, chrootenvDir, pkgsList); err != nil {
 			return fmt.Errorf("failed to install packages in chroot environment: %w", err)
 		}
 	} else {

internal/config/apt_sources.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
@@ -391,9 +393,8 @@ func (t *ImageTemplate) downloadAndAddGPGKeys(repos []PackageRepository) error {
 				return fmt.Errorf("failed to read local GPG key from %s: %w", repo.PKey, err)
 			}
 		} else {
-			// For URLs, download the GPG key
-			log.Infof("Downloading GPG key for repository %s from %s", getRepositoryName(repo), repo.PKey)
-			keyData, err = downloadGPGKey(repo.PKey)
+			// For URLs, prefer persistent cache and download only on cache miss
+			keyData, err = getCachedOrDownloadGPGKey(repo.PKey)
 			if err != nil {
 				return fmt.Errorf("failed to download GPG key from %s: %w", repo.PKey, err)
 			}
@@ -431,6 +432,58 @@ func (t *ImageTemplate) downloadAndAddGPGKeys(repos []PackageRepository) error {
 	return nil
 }
 
+// gpgKeyCacheFilePath returns the persistent cache path for a repository GPG key URL.
+func gpgKeyCacheFilePath(keyURL string) (string, error) {
+	cacheDir, err := CacheDir()
+	if err != nil {
+		return "", fmt.Errorf("resolve cache directory: %w", err)
+	}
+
+	gpgCacheDir := filepath.Join(cacheDir, "gpg-keys")
+	if err := os.MkdirAll(gpgCacheDir, 0755); err != nil {
+		return "", fmt.Errorf("create GPG cache directory %s: %w", gpgCacheDir, err)
+	}
+
+	hash := sha256.Sum256([]byte(keyURL))
+	hashHex := hex.EncodeToString(hash[:])
+
+	return filepath.Join(gpgCacheDir, fmt.Sprintf("%s.gpg", hashHex)), nil
+}
+
+// getCachedOrDownloadGPGKey loads a key from persistent cache, downloading only on cache miss.
+func getCachedOrDownloadGPGKey(keyURL string) ([]byte, error) {
+	log := logger.Logger()
+
+	cacheFilePath, err := gpgKeyCacheFilePath(keyURL)
+	if err == nil {
+		if cachedData, readErr := os.ReadFile(cacheFilePath); readErr == nil {
+			log.Infof("Using cached GPG key (%d bytes) for %s", len(cachedData), keyURL)
+			return cachedData, nil
+		}
+	} else {
+		// Cache path failures should not block build; fall back to direct download.
+		log.Warnf("Failed to initialize GPG key cache for %s: %v", keyURL, err)
+	}
+
+	log.Infof("Downloading GPG key from %s", keyURL)
+	keyData, err := downloadGPGKey(keyURL)
+	if err != nil {
+		return nil, err
+	}
+
+	if cacheFilePath == "" {
+		return keyData, nil
+	}
+
+	if writeErr := os.WriteFile(cacheFilePath, keyData, 0644); writeErr != nil {
+		log.Warnf("Failed to persist GPG key cache for %s at %s: %v", keyURL, cacheFilePath, writeErr)
+		return keyData, nil
+	}
+
+	log.Infof("Cached GPG key at %s", cacheFilePath)
+	return keyData, nil
+}
+
 // downloadGPGKey downloads a GPG key from the given URL
 func downloadGPGKey(keyURL string) ([]byte, error) {
 	log := logger.Logger()

internal/config/apt_sources_integration_test.go

@@ -7,6 +7,32 @@ import (
 	"testing"
 )
 
+func createLocalTestGPGKey(t *testing.T, pattern string) string {
+	t.Helper()
+
+	tempFile, err := os.CreateTemp("", pattern)
+	if err != nil {
+		t.Fatalf("Failed to create local test GPG key file: %v", err)
+	}
+
+	if _, err := tempFile.WriteString("dummy-gpg-key-content"); err != nil {
+		tempFile.Close()
+		os.Remove(tempFile.Name())
+		t.Fatalf("Failed to write local test GPG key file: %v", err)
+	}
+
+	if err := tempFile.Close(); err != nil {
+		os.Remove(tempFile.Name())
+		t.Fatalf("Failed to close local test GPG key file: %v", err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.Remove(tempFile.Name())
+	})
+
+	return tempFile.Name()
+}
+
 // resolveAdditionalFilePath converts relative paths (like ../../../../../../tmp/file.gpg)
 // to absolute paths by joining with working directory or config root
 func resolveAdditionalFilePath(relativePath string) (string, error) {
@@ -38,6 +64,9 @@ func resolveAdditionalFilePath(relativePath string) (string, error) {
 
 // TestIntegrationAptSourcesGeneration tests the complete flow
 func TestIntegrationAptSourcesGeneration(t *testing.T) {
+	sedKeyPath := createLocalTestGPGKey(t, "sed-test-key-*.gpg")
+	openvinoKeyPath := createLocalTestGPGKey(t, "openvino-test-key-*.gpg")
+
 	// Create a realistic test template similar to the example
 	template := &ImageTemplate{
 		Image: ImageInfo{
@@ -54,14 +83,14 @@ func TestIntegrationAptSourcesGeneration(t *testing.T) {
 			{
 				Codename:  "sed",
 				URL:       "https://eci.intel.com/sed-repos/noble",
-				PKey:      "https://eci.intel.com/sed-repos/gpg-keys/GPG-PUB-KEY-INTEL-SED.gpg",
+				PKey:      sedKeyPath,
 				Priority:  1000,
 				Component: "",
 			},
 			{
 				Codename:  "ubuntu24",
 				URL:       "https://apt.repos.intel.com/openvino/2025",
-				PKey:      "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB",
+				PKey:      openvinoKeyPath,
 				Component: "main contrib",
 			},
 		},
@@ -132,6 +161,9 @@ func TestIntegrationAptSourcesGeneration(t *testing.T) {
 
 // TestIntegrationAptPreferencesGeneration tests the complete flow including preferences
 func TestIntegrationAptPreferencesGeneration(t *testing.T) {
+	sedKeyPath := createLocalTestGPGKey(t, "sed-test-key-*.gpg")
+	openvinoKeyPath := createLocalTestGPGKey(t, "openvino-test-key-*.gpg")
+
 	// Create a realistic test template with priorities
 	template := &ImageTemplate{
 		Image: ImageInfo{
@@ -149,14 +181,14 @@ func TestIntegrationAptPreferencesGeneration(t *testing.T) {
 				ID:       "sed-repo",
 				Codename: "sed",
 				URL:      "https://eci.intel.com/sed-repos/noble",
-				PKey:     "https://eci.intel.com/sed-repos/gpg-keys/GPG-PUB-KEY-INTEL-SED.gpg",
+				PKey:     sedKeyPath,
 				Priority: 1000,
 			},
 			{
 				ID:        "openvino-repo",
 				Codename:  "ubuntu24",
 				URL:       "https://apt.repos.intel.com/openvino/2025",
-				PKey:      "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB",
+				PKey:      openvinoKeyPath,
 				Component: "main contrib",
 				Priority:  500,
 			},

internal/config/apt_sources_test.go

@@ -729,3 +729,60 @@ func TestCreateTempAptPreferencesFile(t *testing.T) {
 		t.Errorf("File content mismatch. Expected:\n%s\nGot:\n%s", content, string(fileContent))
 	}
 }
+
+func TestGPGKeyCacheFilePath_StableForSameURL(t *testing.T) {
+	oldCacheDir := Global().CacheDir
+	Global().CacheDir = t.TempDir()
+	t.Cleanup(func() {
+		Global().CacheDir = oldCacheDir
+	})
+
+	keyURL := "https://example.com/keys/repo-key.gpg"
+
+	path1, err := gpgKeyCacheFilePath(keyURL)
+	if err != nil {
+		t.Fatalf("gpgKeyCacheFilePath failed: %v", err)
+	}
+
+	path2, err := gpgKeyCacheFilePath(keyURL)
+	if err != nil {
+		t.Fatalf("gpgKeyCacheFilePath failed on second call: %v", err)
+	}
+
+	if path1 != path2 {
+		t.Errorf("Expected stable cache path, got %q and %q", path1, path2)
+	}
+
+	if !strings.HasSuffix(path1, ".gpg") {
+		t.Errorf("Expected cache path to end with .gpg, got %q", path1)
+	}
+}
+
+func TestGetCachedOrDownloadGPGKey_UsesCacheOnHit(t *testing.T) {
+	oldCacheDir := Global().CacheDir
+	Global().CacheDir = t.TempDir()
+	t.Cleanup(func() {
+		Global().CacheDir = oldCacheDir
+	})
+
+	keyURL := "https://invalid.example.test/non-routable-key.gpg"
+	want := []byte("cached-key-data")
+
+	cachePath, err := gpgKeyCacheFilePath(keyURL)
+	if err != nil {
+		t.Fatalf("gpgKeyCacheFilePath failed: %v", err)
+	}
+
+	if err := os.WriteFile(cachePath, want, 0644); err != nil {
+		t.Fatalf("failed to seed cached key: %v", err)
+	}
+
+	got, err := getCachedOrDownloadGPGKey(keyURL)
+	if err != nil {
+		t.Fatalf("getCachedOrDownloadGPGKey should use cache hit, got error: %v", err)
+	}
+
+	if string(got) != string(want) {
+		t.Errorf("cache hit returned wrong key data: got %q want %q", string(got), string(want))
+	}
+}

internal/image/imageos/imageos.go

@@ -464,6 +464,14 @@ func (imageOs *ImageOs) initDebLocalRepoWithinInstallRoot(installRoot string) er
 		return fmt.Errorf("failed to get chroot environment path for install root %s: %w", installRoot, err)
 	}
 
+	if err := imageOs.chrootEnv.UpdateChrootLocalRepoMetadata(
+		chroot.ChrootRepoDir,
+		imageOs.template.Target.Arch,
+		false,
+	); err != nil {
+		return fmt.Errorf("failed to refresh local debian repository metadata: %w", err)
+	}
+
 	// from local.list
 	repoPath := filepath.Join(chrootInstallRoot, "/cdrom/cache-repo")
 	chrootPkgCacheDir := imageOs.chrootEnv.GetChrootPkgCacheDir()