feat: add parallel_pull_as_fallback config option

Add a new config option parallel_pull_as_fallback under
[pull_modes.parallel_pull_unpack] that enables parallel-pull as an
automatic fallback when lazy-load is the primary mode but no SOCI
index is found for an image.

Today, lazy-load and parallel-pull are mutually exclusive at the daemon
level. When lazy-load is enabled and no SOCI index exists, the
snapshotter defers to the container runtime's sequential pull, which is
5-40% slower than Docker. This forces operators to choose between
optimal performance for indexed images (lazy-load) or non-indexed
images (parallel-pull), but not both.

With parallel_pull_as_fallback = true (and enable = false), the
snapshotter first attempts lazy-load for every image. Only when no
SOCI index is found does it fall back to parallel-pull instead of the
slow sequential path. This gives optimal performance for both cases:
- Images WITH a SOCI index: lazy-load (83-96% faster than Docker)
- Images WITHOUT a SOCI index: parallel-pull (14-50% faster than Docker)

The option defaults to false, preserving existing behavior for all
current users. When enable = true, the fallback is a no-op since
parallel-pull is already the primary mode.

Tested on AL2 and AL2023 instances with both small (nginx) and large
(20GB+) container images. All existing unit tests pass.

Signed-off-by: deeppcs <deeppcs@amazon.com>

Author: deeppcs

config/config.toml

@@ -73,6 +73,7 @@ skip_check_snapshotter_supported = false
     max_concurrent_unpacks_per_image = 1
     discard_unpacked_layers = false
     enable = false
+    experimental_parallel_pull_as_fallback = false
 
 [kubeconfig_keychain]
   enable_keychain = false

config/config_test.go

@@ -46,6 +46,11 @@ func TestConfigDefaults(t *testing.T) {
 			expected: DefaultParallelPullUnpackEnable,
 			actual:   cfg.PullModes.Parallel.Enable,
 		},
+		{
+			name:     "parallel pull as fallback",
+			expected: DefaultExperimentalParallelPullAsFallback,
+			actual:   cfg.PullModes.Parallel.ExperimentalParallelPullAsFallback,
+		},
 		{
 			name:     "metrics network",
 			expected: defaultMetricsNetwork,
@@ -287,6 +292,39 @@ concurrent_download_chunk_size = "badchunksize"
 				}
 			},
 		},
+		{
+			name: "ParallelPullAsFallback",
+			config: []byte(`
+[pull_modes.soci_v2]
+enable = true
+
+[pull_modes.parallel_pull_unpack]
+enable = false
+experimental_parallel_pull_as_fallback = true
+max_concurrent_downloads_per_image = 10
+concurrent_download_chunk_size = "16mb"
+`),
+			assert: func(t *testing.T, actual *Config, err error) {
+				if err != nil {
+					t.Errorf("Expected no error, got %v", err)
+				}
+				if !actual.PullModes.SOCIv2.Enable {
+					t.Error("Expected soci_v2 to be enabled")
+				}
+				if actual.PullModes.Parallel.Enable {
+					t.Error("Expected parallel_pull_unpack.enable to be false")
+				}
+				if !actual.PullModes.Parallel.ExperimentalParallelPullAsFallback {
+					t.Error("Expected experimental_parallel_pull_as_fallback to be true")
+				}
+				if actual.PullModes.Parallel.MaxConcurrentDownloadsPerImage != 10 {
+					t.Errorf("Expected max_concurrent_downloads_per_image to be 10, got %d", actual.PullModes.Parallel.MaxConcurrentDownloadsPerImage)
+				}
+				if actual.PullModes.Parallel.ConcurrentDownloadChunkSize != 16*1024*1024 {
+					t.Errorf("Expected concurrent_download_chunk_size to be %d, got %d", 16*1024*1024, actual.PullModes.Parallel.ConcurrentDownloadChunkSize)
+				}
+			},
+		},
 	}
 
 	for _, test := range tests {

config/defaults.go

@@ -123,6 +123,12 @@ const (
 	// DefaultParallelPullEnable is the default value for whether parallel pull and unpack is enabled
 	DefaultParallelPullUnpackEnable = false
 
+	// DefaultParallelPullAsFallback is the default value for whether parallel pull is used
+	// as a fallback when lazy-load finds no SOCI index.
+	// This is EXPERIMENTAL — lazy-load with containerd content store may have
+	// garbage collection edge cases. See https://github.com/awslabs/soci-snapshotter/issues/1843
+	DefaultExperimentalParallelPullAsFallback = false
+
 	// Defaults for ParallelPullUnpack.
 	// The default values should mirror default containerd values.
 

config/pull_modes.go

@@ -44,6 +44,19 @@ type V2 struct {
 type Parallel struct {
 	ParallelConfig
 	Enable bool `toml:"enable"`
+
+	// ParallelPullAsFallback enables parallel-pull as an automatic fallback
+	// when lazy-load is the primary mode but no SOCI index is found for an image.
+	// When true (and Enable is false), the snapshotter will first attempt lazy-load;
+	// if no SOCI index exists, it falls back to parallel-pull instead of deferring
+	// to the container runtime's slower sequential pull.
+	// If Enable is true, this option is a no-op (parallel-pull is already the primary mode).
+	//
+	// EXPERIMENTAL: This requires the containerd content store for both lazy-load
+	// and parallel-pull (unless discard_unpacked_layers = true).
+	// Lazy-load with the containerd content store may have
+	// garbage collection edge cases. See https://github.com/awslabs/soci-snapshotter/issues/1843
+	ExperimentalParallelPullAsFallback bool `toml:"experimental_parallel_pull_as_fallback"`
 }
 
 func defaultPullModes(cfg *Config) error {

docs/config.md

@@ -97,6 +97,24 @@ This set of variables must be at the top of your TOML file due to not belonging
 
 ## config/service.go
 
+## config/pull_modes.go
+
+### [pull_modes.soci_v1]
+- `enable` (bool) — Enables SOCI v1 index discovery via the OCI Referrers API. Default: false.
+
+### [pull_modes.soci_v2]
+- `enable` (bool) — Enables SOCI v2 index discovery via image manifest annotations. Default: true.
+
+### [pull_modes.parallel_pull_unpack]
+- `enable` (bool) — Enables parallel pull and unpack as the primary pull mode. When true, lazy-load is skipped entirely. Default: false.
+- `experimental_parallel_pull_as_fallback` (bool) — **[EXPERIMENTAL]** When true (and `enable` is false), uses parallel-pull as an automatic fallback when lazy-load is the primary mode but no SOCI index is found for an image. Requires containerd content store (unless `discard_unpacked_layers = true`). Lazy-load with the containerd content store may have garbage collection edge cases. See [#1843](https://github.com/awslabs/soci-snapshotter/issues/1843). Ignored when `enable` is true. Default: false.
+- `max_concurrent_downloads` (int) — Max concurrent downloads across all images. -1 for unlimited. Default: -1.
+- `max_concurrent_downloads_per_image` (int) — Max concurrent downloads per image. Default: 3.
+- `concurrent_download_chunk_size` (string) — Size of each download chunk (e.g. "8mb", "16mb"). Empty means full layer. Default: "".
+- `max_concurrent_unpacks` (int) — Max concurrent unpacks across all images. -1 for unlimited. Default: -1.
+- `max_concurrent_unpacks_per_image` (int) — Max concurrent unpacks per image. Default: 1.
+- `discard_unpacked_layers` (bool) — Discard layer blobs after unpacking to save disk space. Default: false.
+
 ### [snapshotter]
 - `min_layer_size` (int) — Sets the minimum threshold for lazy loading a layer. Any layer smaller than this value will ignore the zTOC for the layer and pull the entire layer ahead of time. We generally recommend setting it to 10MiB (10000000). Default: 0.
 - `allow_invalid_mounts_on_restart` (bool) — Allows the snapshotter to start even if preexisting snapshots cannot connect to their data source on startup. Useful on unexpected daemon crashes/corruption. Default: false.

docs/parallel-mode.md

@@ -94,6 +94,32 @@ If you have any questions or need further assistance, please don't hesitate to r
 
 ## Known Limitations
 
+### Parallel Pull as Fallback for Lazy-Load
+
+When using lazy-load as the primary pull mode (`soci_v2.enable = true` or `soci_v1.enable = true`), images without a SOCI index normally fall back to the container runtime's sequential pull, which can be slower than a standard image pull. To avoid this behavior, you can enable `experimental_parallel_pull_as_fallback`:
+
+```toml
+[pull_modes.soci_v2]
+  enable = true
+
+[pull_modes.parallel_pull_unpack]
+  enable = false
+  experimental_parallel_pull_as_fallback = true
+  max_concurrent_downloads_per_image = 10
+  concurrent_download_chunk_size = "16mb"
+  max_concurrent_unpacks_per_image = 10
+  discard_unpacked_layers = true
+```
+
+With this configuration:
+- Images **with** a SOCI index use lazy-load
+- Images **without** a SOCI index use parallel-pull
+- No image falls through to the slow sequential containerd pull
+
+> **EXPERIMENTAL**: This option requires the containerd content store (`type = "containerd"` under `[content_store]`) for both lazy-load and parallel-pull. Lazy-load with the containerd content store may have garbage collection edge cases and does not carry the same stability guarantees as using either mode independently. See [#1843](https://github.com/awslabs/soci-snapshotter/issues/1843) for details.
+
+Note: `experimental_parallel_pull_as_fallback` is ignored when `enable = true`, since parallel-pull is already the primary mode in that case.
+
 ### Registries
 
 Any registry that supports ranged GET requests and has sufficient request limits should work with parallel pull mode. If a registry is rate limiting image pull requests, users can attempt to lower `max_concurrent_downloads` or `max_concurrent_downloads_per_image` and see if it alleviates the issue, however this will result in less of a performance benefit compared to regular pulling.

fs/adaptive_fetch_image_layers.go

@@ -162,7 +162,7 @@ func checkParallelPullUnpack(cfg *config.Parallel) error {
 	if cfg == nil {
 		return errors.New("parallel pull config is nil")
 	}
-	if !cfg.Enable {
+	if !cfg.Enable && !cfg.ExperimentalParallelPullAsFallback {
 		return ErrParallelPullIsDisabled
 	}
 	// If global concurrent downloads/unpacks are unlimited, any value for per-image concurrent downloads/unpacks are valid

fs/fs.go

@@ -242,7 +242,12 @@ func NewFilesystem(ctx context.Context, root string, cfg config.FSConfig, opts .
 	if pullModes.Parallel.Enable &&
 		cfg.ContentStoreConfig.Type != config.ContainerdContentStoreType &&
 		!pullModes.Parallel.DiscardUnpackedLayers {
-		return nil, errors.New("parallel_pull_unpack mode requires containerd content store (type=\"containerd\" under [content_store])")
+		return nil, errors.New("parallel_pull_unpack mode requires containerd content store (type=\"containerd\" under [content_store] or discard_unpacked_layers = true)")
+	}
+	if pullModes.Parallel.ExperimentalParallelPullAsFallback &&
+		cfg.ContentStoreConfig.Type != config.ContainerdContentStoreType &&
+		!pullModes.Parallel.DiscardUnpackedLayers {
+		return nil, errors.New("experimental_parallel_pull_as_fallback requires containerd content store (type=\"containerd\" under [content_store] or discard_unpacked_layers = true)")
 	}
 	client := store.NewContainerdClient(cfg.ContentStoreConfig.ContainerdAddress)
 
@@ -339,7 +344,7 @@ func NewFilesystem(ctx context.Context, root string, cfg config.FSConfig, opts .
 }
 
 func createParallelPullStructs(ctx context.Context, storage LayerUnpackJobStorage, parallelConfig *config.Parallel) (*unpackJobs, error) {
-	if !parallelConfig.Enable {
+	if !parallelConfig.Enable && !parallelConfig.ExperimentalParallelPullAsFallback {
 		return nil, nil
 	}
 
@@ -430,7 +435,7 @@ type filesystem struct {
 }
 
 func (fs *filesystem) MountParallel(ctx context.Context, mountpoint string, labels map[string]string, mounts []mount.Mount) error {
-	if !fs.pullModes.Parallel.Enable {
+	if !fs.pullModes.Parallel.Enable && !fs.pullModes.Parallel.ExperimentalParallelPullAsFallback {
 		return ErrParallelPullIsDisabled
 	}
 
@@ -727,7 +732,7 @@ func (fs *filesystem) getImageManifest(ctx context.Context, dgst string) (*ocisp
 // CleanImage stops all parallel operations for the specific image.
 // Generally this will be called when removing a snapshot for an image.
 func (fs *filesystem) CleanImage(ctx context.Context, imgDigest string) error {
-	if !fs.pullModes.Parallel.Enable {
+	if !fs.pullModes.Parallel.Enable && !fs.pullModes.Parallel.ExperimentalParallelPullAsFallback {
 		return nil
 	}
 

integration/pull_modes_test.go

@@ -364,3 +364,159 @@ func testDanglingV2Annotation(t *testing.T, imgName string) {
 		t.Fatalf("expected v2 index digest %s, got %s", v2IndexDigest, indexDigestUsed)
 	}
 }
+
+func TestExperimentalParallelPullAsFallback(t *testing.T) {
+	for _, imgName := range pullModesImages {
+		t.Run(imgName, func(t *testing.T) {
+			testExperimentalParallelPullAsFallback(t, imgName)
+		})
+	}
+}
+
+func testExperimentalParallelPullAsFallback(t *testing.T, imgName string) {
+	regConfig := newRegistryConfig()
+	sh, done := newShellWithRegistry(t, regConfig)
+	defer done()
+
+	srcInfo := dockerhub(imgName)
+	dstInfo := regConfig.mirror(imgName)
+	sh.X("nerdctl", "login", "-u", regConfig.user, "-p", regConfig.pass, dstInfo.ref)
+
+	// Create a SOCI-indexed version of the image for lazy-load tests
+	rebootContainerd(t, sh, "", "")
+	v2IndexDigest := createAndPushV2Index(t, sh, srcInfo, dstInfo)
+
+	// Also push the original (non-indexed) image to a separate tag for fallback tests
+	rebootContainerd(t, sh, "", "")
+	noIndexInfo := regConfig.mirror(imgName)
+	noIndexRef, err := reference.Parse(noIndexInfo.ref)
+	if err != nil {
+		t.Fatal(err)
+	}
+	noIndexRef.Object = "no-soci-index"
+	sh.X("nerdctl", "pull", "-q", srcInfo.ref)
+	sh.X("nerdctl", "image", "tag", srcInfo.ref, noIndexRef.String())
+	sh.X("nerdctl", "push", noIndexRef.String())
+
+	tests := []struct {
+		name             string
+		pullModes        config.PullModes
+		contentStoreType store.ContentStoreType
+		imageRef         string
+		expectedDigest   string
+		// checkLocal means we expect local/parallel snapshots (not remote/lazy)
+		checkLocal bool
+		// checkDeferred means we expect deferred-to-runtime snapshots
+		checkDeferred bool
+	}{
+		{
+			// Fallback enabled + image WITH SOCI index → lazy-load should be used
+			name: "fallback enabled with indexed image uses lazy-load",
+			pullModes: config.PullModes{
+				SOCIv1: config.V1{Enable: false},
+				SOCIv2: config.V2{Enable: true},
+				Parallel: config.Parallel{
+					ExperimentalParallelPullAsFallback: true,
+				},
+			},
+			contentStoreType: store.ContainerdContentStoreType,
+			imageRef:         dstInfo.ref,
+			expectedDigest:   v2IndexDigest,
+		},
+		{
+			// Fallback enabled + image WITHOUT SOCI index → parallel-pull fallback
+			name: "fallback enabled with non-indexed image uses parallel pull",
+			pullModes: config.PullModes{
+				SOCIv1: config.V1{Enable: false},
+				SOCIv2: config.V2{Enable: true},
+				Parallel: config.Parallel{
+					ExperimentalParallelPullAsFallback: true,
+				},
+			},
+			contentStoreType: store.ContainerdContentStoreType,
+			imageRef:         noIndexRef.String(),
+			expectedDigest:   "",
+			checkLocal:       true,
+		},
+		{
+			// Fallback disabled + image WITHOUT SOCI index → defers to container runtime
+			name: "fallback disabled with non-indexed image defers to runtime",
+			pullModes: config.PullModes{
+				SOCIv1: config.V1{Enable: false},
+				SOCIv2: config.V2{Enable: true},
+			},
+			imageRef:       noIndexRef.String(),
+			expectedDigest: "",
+			checkDeferred:  true,
+		},
+		{
+			// enable=true takes precedence over fallback — parallel pull is primary
+			name: "enable true takes precedence over fallback",
+			pullModes: config.PullModes{
+				SOCIv1: config.V1{Enable: false},
+				SOCIv2: config.V2{Enable: true},
+				Parallel: config.Parallel{
+					Enable:                             true,
+					ExperimentalParallelPullAsFallback: true,
+				},
+			},
+			contentStoreType: store.ContainerdContentStoreType,
+			imageRef:         dstInfo.ref,
+			expectedDigest:   "",
+			checkLocal:       true,
+		},
+		{
+			// Fallback with SOCI content store + discard_unpacked_layers works
+			name: "fallback with soci content store and discard unpacked layers",
+			pullModes: config.PullModes{
+				SOCIv1: config.V1{Enable: false},
+				SOCIv2: config.V2{Enable: true},
+				Parallel: config.Parallel{
+					ExperimentalParallelPullAsFallback: true,
+					ParallelConfig: config.ParallelConfig{
+						DiscardUnpackedLayers: true,
+					},
+				},
+			},
+			// No contentStoreType set — defaults to SOCI content store
+			imageRef:       noIndexRef.String(),
+			expectedDigest: "",
+			checkLocal:     true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			opts := []snapshotterConfigOpt{withPullModes(test.pullModes)}
+			if test.contentStoreType == store.ContainerdContentStoreType {
+				opts = append(opts, withContainerdContentStore())
+			}
+			var indexDigestUsed string
+			monitorFunc := func(s string) {
+				structuredLog := make(map[string]string)
+				err := json.Unmarshal([]byte(s), &structuredLog)
+				if err != nil {
+					return
+				}
+				if structuredLog["msg"] == "fetching SOCI artifacts using index descriptor" {
+					indexDigestUsed = structuredLog["digest"]
+				}
+			}
+			rsm := testutil.NewRemoteSnapshotMonitor()
+			m := rebootContainerd(t, sh, "", getSnapshotterConfigToml(t, opts...), rsm.MonitorFunc, monitorFunc)
+			defer m.Cleanup(t)
+			sh.X(append(imagePullCmd, test.imageRef)...)
+
+			if test.expectedDigest != "" {
+				rsm.CheckAllRemoteSnapshots(t)
+			} else if test.checkLocal {
+				rsm.CheckAllLocalSnapshots(t)
+			} else if test.checkDeferred {
+				rsm.CheckAllDeferredSnapshots(t)
+			}
+			if indexDigestUsed != test.expectedDigest {
+				t.Fatalf("expected digest %s, got %s", test.expectedDigest, indexDigestUsed)
+			}
+		})
+	}
+}

service/service.go

@@ -124,6 +124,15 @@ func NewSociSnapshotterService(ctx context.Context, root string, serviceCfg *con
 	if serviceCfg.PullModes.Parallel.Enable {
 		snOpts = append(snOpts, snbase.ParallelPullUnpack)
 	}
+	if serviceCfg.PullModes.Parallel.ExperimentalParallelPullAsFallback && !serviceCfg.PullModes.Parallel.Enable {
+		log.G(ctx).Warn("EXPERIMENTAL: experimental_parallel_pull_as_fallback is enabled. " +
+			"Lazy-load and parallel-pull will coexist using the same content store. " +
+			"When using the containerd content store, this combination may have " +
+			"garbage collection edge cases and does not carry the same stability " +
+			"guarantees as using either mode independently. " +
+			"See https://github.com/awslabs/soci-snapshotter/issues/1843")
+		snOpts = append(snOpts, snbase.ParallelPullAsFallback)
+	}
 
 	snapshotter, err = snbase.NewSnapshotter(ctx, snapshotterRoot(root), fs, snOpts...)
 	if err != nil {