feat: bump go-conainerregistry

Add options to configure debugging and setting default parallel jobs.

See: siderolabs/talos#13462

Signed-off-by: Noel Georgi <git@frezbo.dev>

Author: frezbo

cmd/image-factory/cmd/options.go

@@ -9,6 +9,8 @@ import (
 	"net/url"
 	"strings"
 	"time"
+
+	"github.com/siderolabs/image-factory/internal/remotewrap"
 )
 
 // Validate checks Options for inconsistencies that would otherwise produce
@@ -46,6 +48,12 @@ func (o *Options) Validate() error {
 		}
 	}
 
+	// 0 means "unset" (falls back to remotewrap.DefaultJobs); any explicit value must not be
+	// below the default, as lower concurrency can deadlock the limiter under load.
+	if o.Registry.Jobs != 0 && o.Registry.Jobs < remotewrap.DefaultJobs {
+		return fmt.Errorf("registry.jobs must be >= %d if set, got %d", remotewrap.DefaultJobs, o.Registry.Jobs)
+	}
+
 	return nil
 }
 
@@ -79,6 +87,26 @@ type Options struct { //nolint:govet // keeping order for semantic clarity
 
 	// Enterprise contains configuration for enterprise-specific features.
 	Enterprise EnterpriseOptions `koanf:"enterprise"`
+
+	// Registry contains low-level tuning for the registry client (pull/push concurrency, debugging).
+	Registry RegistryOptions `koanf:"registry"`
+}
+
+// RegistryOptions tunes the shared registry client used for all pull/push operations.
+type RegistryOptions struct {
+	// Jobs is the maximum number of concurrent blob pull/push operations per registry client.
+	//
+	// go-containerregistry gates concurrent blob fetches on this value; too low a value can
+	// deadlock under Image Factory's concurrent, multiplexed fetch pattern.
+	// Defaults to remotewrap.DefaultJobs.
+	Jobs int `koanf:"jobs"`
+
+	// Debug tracks registry response bodies to help diagnose pull-limiter token leaks/stalls:
+	// it periodically logs how many bodies are open and dumps any body that stays open too long
+	// together with the stack that opened it.
+	//
+	// Set via config or the IF_REGISTRY_DEBUG environment variable.
+	Debug bool `koanf:"debug"`
 }
 
 // AssetBuilderOptions contains settings for building assets.
@@ -606,4 +634,8 @@ var DefaultOptions = Options{
 			},
 		},
 	},
+
+	Registry: RegistryOptions{
+		Jobs: remotewrap.DefaultJobs,
+	},
 }

cmd/image-factory/cmd/service.go

@@ -66,6 +66,10 @@ func RunFactory(ctx context.Context, logger *zap.Logger, opts Options) error {
 	)
 	defer logger.Info("shutting down", zap.String("name", version.Name))
 
+	// apply registry client tuning before any puller/pusher is constructed
+	remotewrap.SetJobs(opts.Registry.Jobs)
+	remotewrap.SetDebug(opts.Registry.Debug)
+
 	// many image generation steps rely on SOURCE_DATE_EPOCH
 	// to ensure reproducibility, set it to a fixed value
 	if err := os.Setenv("SOURCE_DATE_EPOCH", "1559424892"); err != nil { // this value matches `pkgs` SOURCE_DATE_EPOCH
@@ -605,7 +609,7 @@ func buildSchematicFactory(ctx context.Context, logger *zap.Logger, eg *errgroup
 //
 // Enable registry auth from the standard Docker config, and from GitHub via the token.
 func remoteOptions() []remote.Option {
-	return []remote.Option{
+	opts := []remote.Option{
 		remote.WithAuthFromKeychain(
 			authn.NewMultiKeychain(
 				authn.DefaultKeychain,
@@ -614,6 +618,15 @@ func remoteOptions() []remote.Option {
 			),
 		),
 	}
+
+	// When registry debugging is enabled, route cosign's remote operations through
+	// the same instrumented transport so leaked blob/manifest bodies (i.e. leaked
+	// pull-limiter tokens) in the sign/verify paths are tracked too.
+	if remotewrap.DebugEnabled() {
+		opts = append(opts, remote.WithTransport(remotewrap.RoundTripper()))
+	}
+
+	return opts
 }
 
 // buildCacheSigner constructs the image signer from options.

deploy/helm/image-factory/README.md

@@ -326,7 +326,7 @@ extraVolumeMounts:
 | cacheSigningKey | object | `{"existingSecret":"","key":""}` | Image Cache Signing Key Configuration # This secret contains the ECDSA private key used to sign cached Talos image artifacts. # This ensures that nodes can verify the integrity of images served by the Image Factory. # If you are running a self-hosted Image Factory, this key is required. |
 | cacheSigningKey.existingSecret | string | `""` | Name of an existing Secret containing the ECDSA private key. IMPORTANT: The existing secret MUST contain a data key named exactly "cache-signing.key". If your secret uses a different key, Image Factory will not find the file. Example creation: kubectl create secret generic image-factory-cache-signing-key --from-file=cache-signing.key=./signing-key.key |
 | cacheSigningKey.key | string | `""` | If 'existingSecret' is empty, a new Secret will be created. The ECDSA private key content (multiline string). Generate using: openssl ecparam -name prime256v1 -genkey -noout -out cache-signing.key |
-| config | object | `{"artifacts":{"schematic":{"insecure":false,"namespace":"siderolabs/image-factory","registry":"registry.example.com","repository":"schematics"}},"authentication":{"enabled":false,"htpasswdPath":"/etc/image-factory/auth/htpasswd"},"cache":{"signingKeyPath":"/etc/image-factory/keys/cache-signing.key"}}` | Sidero Image-Factory Configuration |
+| config | object | `{"artifacts":{"schematic":{"insecure":false,"namespace":"siderolabs/image-factory","registry":"registry.example.com","repository":"schematics"}},"authentication":{"enabled":false,"htpasswdPath":"/etc/image-factory/auth/htpasswd"},"cache":{"signingKeyPath":"/etc/image-factory/keys/cache-signing.key"},"registry":{"debug":false,"jobs":64}}` | Sidero Image-Factory Configuration |
 | env | list | `[]` | Environment variables to pass to Image Factory |
 | envFrom | list | `[]` | envFrom to pass to Image Factory |
 | extraObjects | list | `[]` |  |

deploy/helm/image-factory/values.schema.json

@@ -86,6 +86,18 @@
               "type": "string"
             }
           }
+        },
+        "registry": {
+          "type": "object",
+          "properties": {
+            "debug": {
+              "type": "boolean"
+            },
+            "jobs": {
+              "type": "integer",
+              "minimum": 64
+            }
+          }
         }
       },
       "additionalProperties": true

deploy/helm/image-factory/values.yaml

@@ -237,6 +237,19 @@ config:
         ## Repository name for VEX data
         # repository: talos-vex-data
 
+  ## Registry Client Tuning
+  ## Low-level tuning for the shared registry client used for all pull/push operations.
+  registry:
+    ## Maximum number of concurrent blob pull/push operations per registry client.
+    ## Must be >= 64; lower values can deadlock the go-containerregistry pull limiter
+    ## under Image Factory's concurrent, multiplexed fetch pattern.
+    # @schema minimum:64
+    jobs: 64
+    ## Track registry response bodies to diagnose pull-limiter token leaks/stalls: periodically
+    ## logs how many bodies are open and dumps any body that stays open too long together with the
+    ## stack that opened it. Can also be set via the IF_REGISTRY_DEBUG environment variable.
+    debug: false
+
 # -- Environment variables to pass to Image Factory
 env: []
 

docs/configuration.md

@@ -1017,6 +1017,38 @@ Capacity caps the number of cached objects before LRU eviction.
 
 ---
 
+### `registry`
+
+Registry contains low-level tuning for the registry client (pull/push concurrency, debugging).
+
+---
+
+### `registry.jobs`
+
+- **Type:** `int`
+- **Env:** `REGISTRY_JOBS`
+
+Jobs is the maximum number of concurrent blob pull/push operations per registry client.
+
+go-containerregistry gates concurrent blob fetches on this value; too low a value can
+deadlock under Image Factory's concurrent, multiplexed fetch pattern.
+Defaults to remotewrap.DefaultJobs.
+
+---
+
+### `registry.debug`
+
+- **Type:** `bool`
+- **Env:** `REGISTRY_DEBUG`
+
+Debug tracks registry response bodies to help diagnose pull-limiter token leaks/stalls:
+it periodically logs how many bodies are open and dumps any body that stays open too long
+together with the stack that opened it.
+
+Set via config or the IF_REGISTRY_DEBUG environment variable.
+
+---
+
 ## Default Configuration
 
 ### YAML
@@ -1122,6 +1154,9 @@ http:
     keyFile: ""
 metrics:
     addr: :2122
+registry:
+    debug: false
+    jobs: 64
 secureBoot:
     awsKMS:
         certARN: ""
@@ -1216,6 +1251,8 @@ IF_HTTP_EXTERNALURL=https://localhost/
 IF_HTTP_HTTPLISTENADDR=:8080
 IF_HTTP_KEYFILE=
 IF_METRICS_ADDR=:2122
+IF_REGISTRY_DEBUG=false
+IF_REGISTRY_JOBS=64
 IF_SECUREBOOT_AWSKMS_CERTARN=
 IF_SECUREBOOT_AWSKMS_CERTPATH=
 IF_SECUREBOOT_AWSKMS_KEYID=

go.mod

@@ -2,11 +2,6 @@ module github.com/siderolabs/image-factory
 
 go 1.26.3
 
-// downgrade the go-containerregistry to workaround the security changes in
-// https://github.com/google/go-containerregistry/pull/2227/changes: it breaks
-// our extensions
-replace github.com/google/go-containerregistry => github.com/google/go-containerregistry v0.21.2
-
 require (
 	cloud.google.com/go/auth v0.20.0
 	github.com/CalebQ42/squashfs v1.4.1
@@ -179,7 +174,6 @@ require (
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/platforms v1.0.0-rc.4 // indirect
 	github.com/containerd/plugin v1.1.0 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect
 	github.com/containerd/ttrpc v1.2.8 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/containernetworking/cni v1.3.0 // indirect
@@ -194,7 +188,6 @@ require (
 	github.com/diskfs/go-diskfs v1.7.0 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/cli v29.4.3+incompatible // indirect
-	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.9.5 // indirect
 	github.com/docker/go-connections v0.7.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -433,7 +426,6 @@ require (
 	github.com/transparency-dev/merkle v0.0.2 // indirect
 	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
 	github.com/vbatts/go-mtree v0.7.0 // indirect
-	github.com/vbatts/tar-split v0.12.2 // indirect
 	github.com/vifraa/gopom v1.0.0 // indirect
 	github.com/vultr/metadata v1.1.0 // indirect
 	github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 // indirect

go.sum

@@ -378,8 +378,6 @@ github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0
 github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A=
 github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U=
 github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc=
-github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw=
-github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY=
 github.com/containerd/ttrpc v1.2.8 h1:xbVu6D4qF2jihdh9rDVOKqUMiFBQk6YctTdo1zk087Y=
 github.com/containerd/ttrpc v1.2.8/go.mod h1:wyZW2K79t4Hfcxl+GUvkZqRBzJlqFFvgEeeWXa42tyE=
 github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40=
@@ -424,8 +422,6 @@ github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz
 github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/docker/cli v29.4.3+incompatible h1:u+UliYm2J/rYrIh2FqHQg32neRG8GjbvNuwQRTzGspU=
 github.com/docker/cli v29.4.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
-github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY=
 github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
 github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
@@ -694,8 +690,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/google/go-containerregistry v0.21.2 h1:vYaMU4nU55JJGFC9JR/s8NZcTjbE9DBBbvusTW9NeS0=
-github.com/google/go-containerregistry v0.21.2/go.mod h1:ctO5aCaewH4AK1AumSF5DPW+0+R+d2FmylMJdp5G7p0=
+github.com/google/go-containerregistry v0.21.6 h1:T+yqQIlJXKrM98Om4DlW3GoWQAmhZuLMwoDOvVrtiUM=
+github.com/google/go-containerregistry v0.21.6/go.mod h1:U7MMSBIJynke2MVQrQk19NP9k/uQsGz/h0amIFSHMbo=
 github.com/google/go-github/v73 v73.0.0 h1:aR+Utnh+Y4mMkS+2qLQwcQ/cF9mOTpdwnzlaw//rG24=
 github.com/google/go-github/v73 v73.0.0/go.mod h1:fa6w8+/V+edSU0muqdhCVY7Beh1M8F1IlQPZIANKIYw=
 github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
@@ -1423,8 +1419,6 @@ github.com/umisama/go-cpe v0.0.0-20190323060751-cdd6c3c28a23 h1:+168JmE638t0Oxro
 github.com/umisama/go-cpe v0.0.0-20190323060751-cdd6c3c28a23/go.mod h1:Jv/KoYWD3+46wW8r3pEwISwtgv5Q8NTfFto2wFRKvoA=
 github.com/vbatts/go-mtree v0.7.0 h1:ytmOc3MTRidZiBi9VBCyZ2BHe4fZS47L5v7BVXDWW4E=
 github.com/vbatts/go-mtree v0.7.0/go.mod h1:EjdpFC+LZy1TXbRGNa1MKKgjQ+7ew3foMFJK8o4/TdY=
-github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4=
-github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
 github.com/vifraa/gopom v1.0.0 h1:L9XlKbyvid8PAIK8nr0lihMApJQg/12OBvMA28BcWh0=
 github.com/vifraa/gopom v1.0.0/go.mod h1:oPa1dcrGrtlO37WPDBm5SqHAT+wTgF8An1Q71Z6Vv4o=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=

internal/asset/cache/s3/reference.go

@@ -22,6 +22,8 @@ func newObjectReference(asset cache.BootAsset) (*objectReference, error) {
 		return nil, err
 	}
 
+	defer r.Close() //nolint:errcheck
+
 	obj := &objectReference{}
 
 	err = json.NewDecoder(r).Decode(obj)