feat: nydusify: add backend configure support of optimize subcommand

When localfs-backend is true (default is false), still pull whole image
to build optimized image; otherwise, use registry backend to fetch
needed chunk during building process.

Signed-off-by: maxing.lan <[email protected]>

Author: maxing.lan

contrib/nydusify/cmd/nydusify.go

@@ -200,7 +200,8 @@ func main() {
 			Required: false,
 			Value:    false,
 			Usage:    "Enable debug log level, overwrites the 'log-level' option",
-			EnvVars:  []string{"DEBUG_LOG_LEVEL"}},
+			EnvVars:  []string{"DEBUG_LOG_LEVEL"},
+		},
 		&cli.StringFlag{
 			Name:    "log-level",
 			Aliases: []string{"l"},
@@ -1223,10 +1224,54 @@ func main() {
 					Value: "0MB",
 					Usage: "Chunk size for pushing a blob layer in chunked",
 				},
+
+				&cli.StringFlag{
+					Name:    "source-backend-type",
+					Value:   "",
+					Usage:   "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3', 'localfs'",
+					EnvVars: []string{"BACKEND_TYPE"},
+				},
+				&cli.StringFlag{
+					Name:    "source-backend-config",
+					Value:   "",
+					Usage:   "Json string for storage backend configuration",
+					EnvVars: []string{"BACKEND_CONFIG"},
+				},
+				&cli.PathFlag{
+					Name:      "source-backend-config-file",
+					Value:     "",
+					TakesFile: true,
+					Usage:     "Json configuration file for storage backend",
+					EnvVars:   []string{"BACKEND_CONFIG_FILE"},
+				},
 			},
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
 
+				backendType, backendConfig, err := getBackendConfig(c, "", false)
+				if err != nil {
+					return err
+				} else if backendConfig == "" {
+
+					backendType = "registry"
+					parsed, err := reference.ParseNormalizedNamed(c.String("target"))
+					if err != nil {
+						return err
+					}
+
+					backendConfigStruct, err := utils.NewRegistryBackendConfig(parsed, c.Bool("target-insecure"))
+					if err != nil {
+						return errors.Wrap(err, "parse registry backend configuration")
+					}
+
+					bytes, err := json.Marshal(backendConfigStruct)
+					if err != nil {
+						return errors.Wrap(err, "marshal registry backend configuration")
+					}
+					backendConfig = string(bytes)
+
+				}
+
 				pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size"))
 				if err != nil {
 					return errors.Wrap(err, "invalid --push-chunk-size option")
@@ -1248,6 +1293,9 @@ func main() {
 
 					PushChunkSize:     int64(pushChunkSize),
 					PrefetchFilesPath: c.String("prefetch-files"),
+
+					BackendType:   backendType,
+					BackendConfig: backendConfig,
 				}
 
 				return optimizer.Optimize(context.Background(), opt)

contrib/nydusify/pkg/optimizer/builder.go

@@ -19,9 +19,13 @@ func isSignalKilled(err error) bool {
 }
 
 type BuildOption struct {
-	BuilderPath         string
-	PrefetchFilesPath   string
-	BootstrapPath       string
+	BuilderPath       string
+	PrefetchFilesPath string
+	BootstrapPath     string
+	BackendType       string
+	BackendConfig     string
+	// `BlobDir` is used to store optimized blob,
+	// Beside, `BlobDir` is also used to store the original blobs when backend is localfs
 	BlobDir             string
 	OutputBootstrapPath string
 	OutputJSONPath      string
@@ -42,14 +46,21 @@ func Build(option BuildOption) (string, error) {
 		option.PrefetchFilesPath,
 		"--bootstrap",
 		option.BootstrapPath,
-		"--blob-dir",
+		"--output-blob-dir",
 		option.BlobDir,
 		"--output-bootstrap",
 		option.OutputBootstrapPath,
 		"--output-json",
 		outputJSONPath,
 	}
 
+	if option.BackendType == "localfs" {
+		args = append(args, "--blob-dir", option.BlobDir)
+	} else {
+		args = append(args, "--backend-type", option.BackendType)
+		args = append(args, "--backend-config", option.BackendConfig)
+	}
+
 	ctx := context.Background()
 	var cancel context.CancelFunc
 	if option.Timeout != nil {

contrib/nydusify/pkg/optimizer/optimizer.go

@@ -61,6 +61,9 @@ type Opt struct {
 	Platforms    string
 
 	PushChunkSize int64
+
+	BackendType   string
+	BackendConfig string
 }
 
 // the information generated during building
@@ -269,8 +272,10 @@ func Optimize(ctx context.Context, opt Opt) error {
 	}
 	defer os.RemoveAll(buildDir)
 
-	if err := fetchBlobs(ctx, opt, buildDir); err != nil {
-		return errors.Wrap(err, "prepare nydus blobs")
+	if opt.BackendType == "localfs" {
+		if err := fetchBlobs(ctx, opt, buildDir); err != nil {
+			return errors.Wrap(err, "prepare nydus blobs")
+		}
 	}
 
 	originalBootstrap := filepath.Join(buildDir, "nydus_bootstrap")
@@ -289,12 +294,17 @@ func Optimize(ctx context.Context, opt Opt) error {
 
 	compressAlgo := bootstrapDesc.Digest.Algorithm().String()
 	blobDir := filepath.Join(buildDir + "/content/blobs/" + compressAlgo)
+	if err := os.MkdirAll(blobDir, 0755); err != nil {
+		return errors.Wrap(err, "create blob directory")
+	}
 	outPutJSONPath := filepath.Join(buildDir, "output.json")
 	newBootstrapPath := filepath.Join(buildDir, "optimized_bootstrap")
 	builderOpt := BuildOption{
 		BuilderPath:         opt.NydusImagePath,
 		PrefetchFilesPath:   opt.PrefetchFilesPath,
 		BootstrapPath:       originalBootstrap,
+		BackendType:         opt.BackendType,
+		BackendConfig:       opt.BackendConfig,
 		BlobDir:             blobDir,
 		OutputBootstrapPath: newBootstrapPath,
 		OutputJSONPath:      outPutJSONPath,

docs/nydusify.md

@@ -262,6 +262,24 @@ nerdctl --snapshotter nydus run \
 
 The original container ID need to be a full container ID rather than an abbreviation.
 
+## Optimize nydus image from prefetch files
+
+The nydusify optimize command can optimize a nydus image from prefetch files, prefetch files are file access patterns during container startup. This will generate a new bootstrap and a new blob wich contains all data indicated by prefetch files.
+
+The content of prefetch files likes this:
+```
+/path/to/file1 start_offset1-end_offset1, start_offset2-end_offset2, ...
+/path/to/file2 start_offset1-end_offset1, start_offset2-end_offset2, ...
+```
+
+``` shell
+nydusify optimize \
+  --nydus-image  /path/to/nydus-image \
+  --source myregistry/repo:tag-nydus \
+  --target myregistry/repo:tag-nydus-optimized \
+  --prefetch-files /path/to/prefetch-files \
+```
+
 ## More Nydusify Options
 
 See `nydusify convert/check/mount --help`