experiment: introduce a sandbox

The sandbox executor will be useful for running untrusted / semi-trusted code.

It runs commands in a kubernetes pod, using an agent for efficiency.

Author: justinsb

experimental/README.md

@@ -0,0 +1,2 @@
+This directory is a gathering place for experimentation and development
+of tools to help maintainers do their everyday tasks.

experimental/ksandbox/README.md

@@ -0,0 +1,8 @@
+# ksandbox
+
+ksandbox is a project to easily run commands in a kubernetes Pod,
+acting as a sort of simple sandbox for executing tasks or code,
+particularly where those tasks are not trusted with a github token
+or other security credential.
+
+This project is experimental.

experimental/ksandbox/cmd/ksandbox-agent/main.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/kubernetes-sigs/maintainers/experimental/ksandbox/pkg/server"
+	"google.golang.org/grpc/credentials"
+	"k8s.io/klog/v2"
+)
+
+func main() {
+	ctx := context.Background()
+	err := run(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context) error {
+	klog.InitFlags(nil)
+
+	listen := ":7007"
+	flag.StringVar(&listen, "listen", listen, "port on which to listen for requests")
+	tlsDir := "tls"
+	flag.StringVar(&tlsDir, "tls-dir", tlsDir, "directory for tls credentials")
+	deleteTLS := true
+	flag.BoolVar(&deleteTLS, "delete-tls", deleteTLS, "automatically delete tls credentials after reading")
+
+	installDir := ""
+	flag.StringVar(&installDir, "install", installDir, "copy into this directory and exit")
+
+	flag.Parse()
+
+	if installDir != "" {
+		return installTo(installDir)
+	}
+
+	// TODO: Auto-shutdown after 1 hour?
+
+	s, err := server.NewAgentServer()
+	if err != nil {
+		return err
+	}
+
+	var creds credentials.TransportCredentials
+
+	if tlsDir != "" {
+		// Load our serving certificate and enforce client certificates
+		certFile := filepath.Join(tlsDir, "server.crt")
+		keyFile := filepath.Join(tlsDir, "server.key")
+
+		serverKeypair, err := tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			return fmt.Errorf("failed to load TLS credentials from %q: %w", tlsDir, err)
+		}
+
+		clientCACertPath := filepath.Join(tlsDir, "client-ca.crt")
+		clientCACertBytes, err := os.ReadFile(clientCACertPath)
+		if err != nil {
+			return fmt.Errorf("failed to read %q: %w", clientCACertPath, err)
+		}
+		clientCACertPool := x509.NewCertPool()
+		if !clientCACertPool.AppendCertsFromPEM(clientCACertBytes) {
+			return fmt.Errorf("failed to parse any certificates from %q: %w", clientCACertPath, err)
+		}
+
+		creds = credentials.NewTLS(&tls.Config{
+			Certificates: []tls.Certificate{serverKeypair},
+			ClientCAs:    clientCACertPool,
+			ClientAuth:   tls.RequireAndVerifyClientCert,
+		})
+
+		if deleteTLS {
+			// We delete TLS certificates so that they aren't sitting on disk.
+			// This isn't perfect, but it prevents trival and accidental leakage.
+			// The credentials aren't particular high-value anyway - they are single-use (and we connect _to_ the pod)
+			// TODO: Should we delete the ksandbox-agent binary, just so we don't have anything else obviously on the disk?
+			if err := os.RemoveAll(tlsDir); err != nil {
+				return fmt.Errorf("unable to delete tls credentials: %w", err)
+			}
+		}
+	}
+
+	if err := s.ListenAndServe(listen, creds); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// copyFile copies the file from src to dest, setting the mode of the created file
+func copyFile(src, dest string, mode os.FileMode) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("unable to open %q: %w", src, err)
+	}
+	defer f.Close()
+
+	out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
+	if err != nil {
+		return fmt.Errorf("unable to create %q: %w", dest, err)
+	}
+	if _, err := io.Copy(out, f); err != nil {
+		out.Close()
+		return fmt.Errorf("error writing %q: %w", dest, err)
+	}
+	if err := out.Close(); err != nil {
+		return fmt.Errorf("error closing %q: %w", dest, err)
+	}
+	return nil
+}
+
+// installTo copies the agent and PKI keys to the specified
+// directory.
+//
+// installDir will normally be a shared volume mount which is then
+// used as the entrypoint for the main container.
+func installTo(installDir string) error {
+	installBin := filepath.Join(installDir, "ksandbox-agent")
+
+	if err := copyFile(os.Args[0], installBin, os.FileMode(0755)); err != nil {
+		return fmt.Errorf("error copying file %q: %w", os.Args[0], err)
+	}
+
+	// Also copy TLS material (so we can delete it, since https://github.com/kubernetes/kubernetes/pull/58720)
+	{
+		copySrcDir := "/tls"
+		copyDestDir := filepath.Join(installDir, "tls")
+		if err := os.MkdirAll(copyDestDir, 0700); err != nil {
+			return fmt.Errorf("error creating %q: %w", copyDestDir, err)
+		}
+
+		files, err := os.ReadDir(copySrcDir)
+		if err != nil {
+			return fmt.Errorf("error reading %q directory: %w", copySrcDir, err)
+		}
+
+		for _, f := range files {
+			if f.IsDir() {
+				continue
+			}
+
+			src := filepath.Join(copySrcDir, f.Name())
+			if filepath.Ext(src) != ".crt" && filepath.Ext(src) != ".key" {
+				klog.Infof("skipping copy of %q; isn't .crt or .key", src)
+				continue
+			}
+
+			out := filepath.Join(copyDestDir, f.Name())
+			if err := copyFile(src, out, os.FileMode(0600)); err != nil {
+				return fmt.Errorf("error copying file %q: %w", src, err)
+			}
+		}
+	}
+
+	return nil
+}

experimental/ksandbox/cmd/ksandbox-testclient/README.md

@@ -0,0 +1,2 @@
+The ksandbox-testclient is a simple client of ksandbox, used to verify execution of
+the agent.

experimental/ksandbox/cmd/ksandbox-testclient/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/kubernetes-sigs/maintainers/experimental/ksandbox/pkg/client"
+	protocol "github.com/kubernetes-sigs/maintainers/experimental/ksandbox/protocol/ksandbox/v1alpha1"
+	"google.golang.org/protobuf/encoding/prototext"
+)
+
+func main() {
+	ctx := context.Background()
+	err := run(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context) error {
+	namespace := "default"
+	image := ""
+	flag.StringVar(&image, "image", image, "image to execute")
+	buildAgentImage := ""
+	flag.StringVar(&buildAgentImage, "agent", buildAgentImage, "name of build agent image")
+
+	flag.Parse()
+
+	if image == "" {
+		return fmt.Errorf("must specify --image")
+	}
+	if buildAgentImage == "" {
+		return fmt.Errorf("must specify --agent")
+	}
+	command := flag.Args()
+	if len(command) == 0 {
+		return fmt.Errorf("must specify command after flags")
+	}
+
+	// We assume this is being run on a developer machine (it's a test program),
+	// rather than in-cluster.
+	usePortForward := true
+	c, err := client.NewAgentClient(ctx, namespace, buildAgentImage, image, usePortForward)
+	if err != nil {
+		return fmt.Errorf("error building agent client: %w", err)
+	}
+	defer c.Close()
+
+	request := &protocol.ExecuteCommandRequest{
+		Command: command,
+	}
+	response, err := c.ExecuteCommand(ctx, request)
+	if err != nil {
+		return fmt.Errorf("error executing in buildagent: %w", err)
+	}
+
+	fmt.Printf("response: %s", prototext.Format(response))
+
+	return nil
+}

experimental/ksandbox/dev/images/protoc/Dockerfile

@@ -0,0 +1,31 @@
+# Compile protoc ourselves, from source
+FROM debian:latest AS protoc-builder
+
+RUN apt-get update
+RUN apt-get install -y g++ git cmake
+
+WORKDIR /src
+RUN git clone https://github.com/protocolbuffers/protobuf.git
+
+WORKDIR /src/protobuf
+RUN git checkout v3.12.4
+RUN git submodule update --init --recursive
+
+WORKDIR /src/protobuf/cmake
+RUN cmake .
+RUN cmake --build . -j8
+
+RUN cp protoc /usr/local/bin
+
+RUN /usr/local/bin/protoc --version
+
+FROM golang:1.23.5
+
+RUN apt-get update; apt-get install --yes unzip
+
+RUN go install google.golang.org/protobuf/cmd/[email protected]
+RUN go install google.golang.org/grpc/cmd/[email protected]
+
+COPY --from=protoc-builder /usr/local/bin/protoc /usr/local/bin/protoc
+
+ENTRYPOINT [ "/usr/local/bin/protoc" ]

experimental/ksandbox/dev/tasks/build-images

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+SRC_DIR=${REPO_ROOT}/experimental/ksandbox
+cd "${SRC_DIR}"
+
+KO="go run github.com/google/[email protected]"
+
+export KO_DOCKER_REPO="${IMAGE_PREFIX:-}ksandbox-agent"
+
+if [[ -z "${TAG:-}" ]]; then
+  TAG=latest
+fi
+
+${KO} build ${PUSH_FLAGS:-} --tags ${TAG} --platform=linux/amd64,linux/arm64 --bare ./cmd/ksandbox-agent/

experimental/ksandbox/dev/tasks/generate-proto

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+SRC_DIR=${REPO_ROOT}/experimental/ksandbox
+cd "${SRC_DIR}"
+
+# Would be cool to use ksandbox for this ... but a circular dependency!
+docker buildx build --load -t protoc dev/images/protoc --progress=plain
+
+docker run -it -v `pwd`:/src -w /src protoc \
+  --go_out . \
+  --go_opt paths=source_relative \
+  --go-grpc_out . \
+  --go-grpc_opt paths=source_relative \
+  protocol/ksandbox/v1alpha1/agent.proto

experimental/ksandbox/dev/tasks/run-e2e

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+SRC_DIR=${REPO_ROOT}/experimental/ksandbox
+cd "${SRC_DIR}"
+
+# Pick a probably-unique tag
+export TAG=`date +%Y%m%d%H%M%S`
+
+# Build the image
+echo "Building image"
+export IMAGE_PREFIX=fake.registry/
+PUSH_FLAGS=--local dev/tasks/build-images
+
+AGENT_IMAGE="${IMAGE_PREFIX:-}ksandbox-agent:${TAG}"
+
+# Load the image into kind
+echo "Loading image into kind"
+kind load docker-image ${AGENT_IMAGE}
+
+# Run the test client
+echo "Running test client"
+go run ./cmd/ksandbox-testclient/ --agent ${AGENT_IMAGE} --image golang:1.23.5 go version