Support hidden --include-pypi flag for catalog created from community registry.

Author: cmrigney

cmd/docker-mcp/commands/catalog_next.go

@@ -40,6 +40,7 @@ func createCatalogNextCommand() *cobra.Command {
 		FromLegacyCatalog     string
 		FromCommunityRegistry string
 		Servers               []string
+		IncludePyPI           bool
 	}
 
 	cmd := &cobra.Command{
@@ -61,13 +62,17 @@ func createCatalogNextCommand() *cobra.Command {
 				return fmt.Errorf("only one of --from-profile, --from-legacy-catalog, or --from-community-registry can be specified")
 			}
 
+			if opts.IncludePyPI && opts.FromCommunityRegistry == "" {
+				return fmt.Errorf("--include-pypi can only be used when creating a catalog from a community registry")
+			}
+
 			dao, err := db.New()
 			if err != nil {
 				return err
 			}
 			registryClient := registryapi.NewClient()
 			ociService := oci.NewService()
-			return catalognext.Create(cmd.Context(), dao, registryClient, ociService, args[0], opts.Servers, opts.FromWorkingSet, opts.FromLegacyCatalog, opts.FromCommunityRegistry, opts.Title)
+			return catalognext.Create(cmd.Context(), dao, registryClient, ociService, args[0], opts.Servers, opts.FromWorkingSet, opts.FromLegacyCatalog, opts.FromCommunityRegistry, opts.Title, opts.IncludePyPI)
 		},
 	}
 
@@ -78,6 +83,9 @@ func createCatalogNextCommand() *cobra.Command {
 	flags.StringVar(&opts.FromCommunityRegistry, "from-community-registry", "", "Community registry hostname to fetch servers from (e.g. registry.modelcontextprotocol.io)")
 	flags.StringVar(&opts.Title, "title", "", "Title of the catalog")
 
+	flags.BoolVar(&opts.IncludePyPI, "include-pypi", false, "Include PyPI servers when creating a catalog from a community registry")
+	cmd.Flags().MarkHidden("include-pypi") //nolint:errcheck
+
 	return cmd
 }
 

pkg/catalog/registry_to_catalog.go

@@ -17,6 +17,15 @@ import (
 // no compatible package type (e.g. no OCI+stdio package and no remote).
 var ErrIncompatibleServer = errors.New("incompatible server")
 
+// TransformSource describes the package type that TransformToDocker resolved.
+type TransformSource string
+
+const (
+	TransformSourceOCI    TransformSource = "oci"
+	TransformSourcePyPI   TransformSource = "pypi"
+	TransformSourceRemote TransformSource = "remote"
+)
+
 // TransformOption configures the behavior of TransformToDocker.
 type TransformOption func(*transformOptions)
 
@@ -397,8 +406,9 @@ func getPublisherProvidedMeta(meta *v0.ServerMeta) map[string]any {
 	return meta.PublisherProvided
 }
 
-// TransformToDocker transforms a ServerDetail (community format) to Server (catalog format)
-func TransformToDocker(ctx context.Context, serverDetail ServerDetail, opts ...TransformOption) (*Server, error) {
+// TransformToDocker transforms a ServerDetail (community format) to Server (catalog format).
+// The returned TransformSource indicates which package type was used (oci, pypi, or remote).
+func TransformToDocker(ctx context.Context, serverDetail ServerDetail, opts ...TransformOption) (*Server, TransformSource, error) {
 	options := transformOptions{
 		allowPyPI: true,
 	}
@@ -441,20 +451,23 @@ func TransformToDocker(ctx context.Context, serverDetail ServerDetail, opts ...T
 		Description: serverDetail.Description,
 	}
 
+	var source TransformSource
+
 	// Add image and command for OCI or PyPI package
 	if pkg != nil {
 		switch pkg.RegistryType {
 		case "oci":
 			if image := extractImageInfo(*pkg); image != "" {
 				server.Image = image
 				server.Type = "server"
+				source = TransformSourceOCI
 			}
 		case "pypi":
 			var pythonVersion string
 			if options.pypiResolver != nil {
 				pv, found := options.pypiResolver(ctx, pkg.Identifier, pkg.Version, pkg.RegistryBaseURL)
 				if !found {
-					return nil, fmt.Errorf("pypi package %s@%s was not found", pkg.Identifier, pkg.Version)
+					return nil, "", fmt.Errorf("pypi package %s@%s was not found", pkg.Identifier, pkg.Version)
 				}
 				pythonVersion = pv
 			}
@@ -465,9 +478,10 @@ func TransformToDocker(ctx context.Context, serverDetail ServerDetail, opts ...T
 				server.Volumes = volumes
 				server.Type = "server"
 				server.LongLived = true
+				source = TransformSourcePyPI
 			}
 		default:
-			return nil, fmt.Errorf("unsupported registry type: %s", pkg.RegistryType)
+			return nil, "", fmt.Errorf("unsupported registry type: %s", pkg.RegistryType)
 		}
 	}
 
@@ -476,11 +490,12 @@ func TransformToDocker(ctx context.Context, serverDetail ServerDetail, opts ...T
 		remoteVal := convertRemote(*remote, serverName)
 		server.Remote = remoteVal
 		server.Type = "remote"
+		source = TransformSourceRemote
 	}
 
 	// Validate that we have at least one way to run the server
 	if server.Image == "" && server.Remote.URL == "" {
-		return nil, fmt.Errorf("%w: no compatible packages for %s", ErrIncompatibleServer, serverDetail.Name)
+		return nil, "", fmt.Errorf("%w: no compatible packages for %s", ErrIncompatibleServer, serverDetail.Name)
 	}
 
 	// Add config schema if we have config variables
@@ -548,5 +563,5 @@ func TransformToDocker(ctx context.Context, serverDetail ServerDetail, opts ...T
 		}
 	}
 
-	return server, nil
+	return server, source, nil
 }

pkg/catalog/registry_to_catalog_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 // transformTestJSON is a test helper that unmarshals registry JSON, calls TransformToDocker,
-// and returns both the Server and a pretty-printed JSON string of the result.
+// and returns the Server, the TransformSource, and a pretty-printed JSON string of the result.
 func transformTestJSON(t *testing.T, registryJSON string, resolver PyPIVersionResolver) (Server, string) {
 	t.Helper()
 	var serverResponse v0.ServerResponse
@@ -22,7 +22,7 @@ func transformTestJSON(t *testing.T, registryJSON string, resolver PyPIVersionRe
 	if resolver != nil {
 		opts = append(opts, WithPyPIResolver(resolver))
 	}
-	result, err := TransformToDocker(t.Context(), serverResponse.Server, opts...)
+	result, _, err := TransformToDocker(t.Context(), serverResponse.Server, opts...)
 	if err != nil {
 		t.Fatalf("TransformToDocker failed: %v", err)
 	}
@@ -1137,7 +1137,7 @@ func TestTransformPyPIDisallowed(t *testing.T) {
 		t.Fatalf("Failed to parse registry JSON: %v", err)
 	}
 
-	_, err := TransformToDocker(t.Context(), serverResponse.Server, WithAllowPyPI(false))
+	_, _, err := TransformToDocker(t.Context(), serverResponse.Server, WithAllowPyPI(false))
 	if err == nil {
 		t.Fatal("Expected error when PyPI is disallowed, got nil")
 	}
@@ -1169,10 +1169,13 @@ func TestTransformPyPIAllowedByDefault(t *testing.T) {
 		t.Fatalf("Failed to parse registry JSON: %v", err)
 	}
 
-	result, err := TransformToDocker(t.Context(), serverResponse.Server)
+	result, source, err := TransformToDocker(t.Context(), serverResponse.Server)
 	if err != nil {
 		t.Fatalf("Expected success for PyPI with default options, got: %v", err)
 	}
+	if source != TransformSourcePyPI {
+		t.Errorf("Expected source %q, got %q", TransformSourcePyPI, source)
+	}
 	if result.Type != "server" {
 		t.Errorf("Expected type 'server', got '%s'", result.Type)
 	}
@@ -1207,7 +1210,7 @@ func TestTransformPyPIPackageNotFound(t *testing.T) {
 		t.Fatalf("Failed to parse registry JSON: %v", err)
 	}
 
-	_, err := TransformToDocker(t.Context(), serverResponse.Server, WithPyPIResolver(notFoundResolver))
+	_, _, err := TransformToDocker(t.Context(), serverResponse.Server, WithPyPIResolver(notFoundResolver))
 	if err == nil {
 		t.Fatal("Expected error when PyPI package is not found, got nil")
 	}

pkg/catalog_next/create.go

@@ -21,7 +21,7 @@ import (
 	"github.com/docker/mcp-gateway/pkg/workingset"
 )
 
-func Create(ctx context.Context, dao db.DAO, registryClient registryapi.Client, ociService oci.Service, refStr string, servers []string, workingSetID string, legacyCatalogURL string, communityRegistryRef string, title string) error {
+func Create(ctx context.Context, dao db.DAO, registryClient registryapi.Client, ociService oci.Service, refStr string, servers []string, workingSetID string, legacyCatalogURL string, communityRegistryRef string, title string, includePyPI bool) error {
 	telemetry.Init()
 	start := time.Now()
 	var success bool
@@ -49,7 +49,7 @@ func Create(ctx context.Context, dao db.DAO, registryClient registryapi.Client,
 			return fmt.Errorf("failed to create catalog from legacy catalog: %w", err)
 		}
 	} else if communityRegistryRef != "" {
-		catalog, err = createCatalogFromCommunityRegistry(ctx, registryClient, communityRegistryRef)
+		catalog, err = createCatalogFromCommunityRegistry(ctx, registryClient, communityRegistryRef, includePyPI)
 		if err != nil {
 			return fmt.Errorf("failed to create catalog from community registry: %w", err)
 		}
@@ -202,13 +202,14 @@ func workingSetServerToCatalogServer(server workingset.Server) Server {
 type communityRegistryResult struct {
 	serversAdded   int
 	serversOCI     int
+	serversPyPI    int
 	serversRemote  int
 	serversSkipped int
 	totalServers   int
 	skippedByType  map[string]int
 }
 
-func createCatalogFromCommunityRegistry(ctx context.Context, registryClient registryapi.Client, registryRef string) (Catalog, error) {
+func createCatalogFromCommunityRegistry(ctx context.Context, registryClient registryapi.Client, registryRef string, includePyPI bool) (Catalog, error) {
 	baseURL := "https://" + registryRef
 	servers, err := registryClient.ListServers(ctx, baseURL, "")
 	if err != nil {
@@ -217,10 +218,10 @@ func createCatalogFromCommunityRegistry(ctx context.Context, registryClient regi
 
 	catalogServers := make([]Server, 0)
 	skippedByType := make(map[string]int)
-	var ociCount, remoteCount int
+	var ociCount, remoteCount, pypiCount int
 
 	for _, serverResp := range servers {
-		catalogServer, err := legacycatalog.TransformToDocker(ctx, serverResp.Server, legacycatalog.WithAllowPyPI(false))
+		catalogServer, transformSource, err := legacycatalog.TransformToDocker(ctx, serverResp.Server, legacycatalog.WithAllowPyPI(includePyPI), legacycatalog.WithPyPIResolver(legacycatalog.DefaultPyPIVersionResolver()))
 		if err != nil {
 			if !errors.Is(err, legacycatalog.ErrIncompatibleServer) {
 				fmt.Fprintf(os.Stderr, "Warning: failed to transform server %q: %v\n", serverResp.Server.Name, err)
@@ -242,7 +243,12 @@ func createCatalogFromCommunityRegistry(ctx context.Context, registryClient regi
 		var s Server
 		switch catalogServer.Type {
 		case "server":
-			ociCount++
+			switch transformSource {
+			case legacycatalog.TransformSourcePyPI:
+				pypiCount++
+			default:
+				ociCount++
+			}
 			s = Server{
 				Type:  workingset.ServerTypeImage,
 				Image: catalogServer.Image,
@@ -272,12 +278,13 @@ func createCatalogFromCommunityRegistry(ctx context.Context, registryClient regi
 	result := communityRegistryResult{
 		serversAdded:   len(catalogServers),
 		serversOCI:     ociCount,
+		serversPyPI:    pypiCount,
 		serversRemote:  remoteCount,
 		serversSkipped: totalSkipped(skippedByType),
 		totalServers:   len(servers),
 		skippedByType:  skippedByType,
 	}
-	printCommunityRegistryResult(registryRef, result)
+	printCommunityRegistryResult(registryRef, result, includePyPI)
 
 	return Catalog{
 		CatalogArtifact: CatalogArtifact{
@@ -296,11 +303,14 @@ func totalSkipped(skippedByType map[string]int) int {
 	return total
 }
 
-func printCommunityRegistryResult(refStr string, result communityRegistryResult) {
+func printCommunityRegistryResult(refStr string, result communityRegistryResult, includePyPI bool) {
 	fmt.Fprintf(os.Stderr, "Fetched %d servers from %s\n", result.serversAdded, refStr)
 	fmt.Fprintf(os.Stderr, "  Total in registry: %d\n", result.totalServers)
 	fmt.Fprintf(os.Stderr, "  Imported:          %d\n", result.serversAdded)
 	fmt.Fprintf(os.Stderr, "    OCI (stdio):     %d\n", result.serversOCI)
+	if includePyPI {
+		fmt.Fprintf(os.Stderr, "    PyPI (stdio):    %d\n", result.serversPyPI)
+	}
 	fmt.Fprintf(os.Stderr, "    Remote:          %d\n", result.serversRemote)
 	fmt.Fprintf(os.Stderr, "  Skipped:           %d\n", result.serversSkipped)