Skip to content

Commit e18ea02

Browse files
committed
experiment: introduce a sandbox
The sandbox executor will be useful for running untrusted / semi-trusted code. It runs commands in a kubernetes pod, using an agent for efficiency.
1 parent d37e758 commit e18ea02

File tree

15 files changed

+1324
-0
lines changed

15 files changed

+1324
-0
lines changed

experimental/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
This directory is a gathering place for experimentation and development
2+
of tools to help maintainers do their everyday tasks.

experimental/ksandbox/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# ksandbox
2+
3+
ksandbox is a project to easily run commands in a kubernetes Pod,
4+
acting as a sort of simple sandbox for executing tasks or code,
5+
particularly where those tasks are not trusted with a github token
6+
or other security credential.
7+
8+
This project is experimental.
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"crypto/tls"
6+
"crypto/x509"
7+
"flag"
8+
"fmt"
9+
"io"
10+
"os"
11+
"path/filepath"
12+
13+
"github.com/kubernetes-sigs/maintainers/experimental/ksandbox/pkg/server"
14+
"google.golang.org/grpc/credentials"
15+
"k8s.io/klog/v2"
16+
)
17+
18+
func main() {
19+
ctx := context.Background()
20+
err := run(ctx)
21+
if err != nil {
22+
fmt.Fprintf(os.Stderr, "%v\n", err)
23+
os.Exit(1)
24+
}
25+
}
26+
27+
func run(ctx context.Context) error {
28+
klog.InitFlags(nil)
29+
30+
listen := ":7007"
31+
flag.StringVar(&listen, "listen", listen, "port on which to listen for requests")
32+
tlsDir := "tls"
33+
flag.StringVar(&tlsDir, "tls-dir", tlsDir, "directory for tls credentials")
34+
deleteTLS := true
35+
flag.BoolVar(&deleteTLS, "delete-tls", deleteTLS, "automatically delete tls credentials after reading")
36+
37+
installDir := ""
38+
flag.StringVar(&installDir, "install", installDir, "copy into this directory and exit")
39+
40+
flag.Parse()
41+
42+
if installDir != "" {
43+
return installTo(installDir)
44+
}
45+
46+
// TODO: Auto-shutdown after 1 hour?
47+
48+
s, err := server.NewAgentServer()
49+
if err != nil {
50+
return err
51+
}
52+
53+
var creds credentials.TransportCredentials
54+
55+
if tlsDir != "" {
56+
// Load our serving certificate and enforce client certificates
57+
certFile := filepath.Join(tlsDir, "server.crt")
58+
keyFile := filepath.Join(tlsDir, "server.key")
59+
60+
serverKeypair, err := tls.LoadX509KeyPair(certFile, keyFile)
61+
if err != nil {
62+
return fmt.Errorf("failed to load TLS credentials from %q: %w", tlsDir, err)
63+
}
64+
65+
clientCACertPath := filepath.Join(tlsDir, "client-ca.crt")
66+
clientCACertBytes, err := os.ReadFile(clientCACertPath)
67+
if err != nil {
68+
return fmt.Errorf("failed to read %q: %w", clientCACertPath, err)
69+
}
70+
clientCACertPool := x509.NewCertPool()
71+
if !clientCACertPool.AppendCertsFromPEM(clientCACertBytes) {
72+
return fmt.Errorf("failed to parse any certificates from %q: %w", clientCACertPath, err)
73+
}
74+
75+
creds = credentials.NewTLS(&tls.Config{
76+
Certificates: []tls.Certificate{serverKeypair},
77+
ClientCAs: clientCACertPool,
78+
ClientAuth: tls.RequireAndVerifyClientCert,
79+
})
80+
81+
if deleteTLS {
82+
// We delete TLS certificates so that they aren't sitting on disk.
83+
// This isn't perfect, but it prevents trival and accidental leakage.
84+
// The credentials aren't particular high-value anyway - they are single-use (and we connect _to_ the pod)
85+
// TODO: Should we delete the ksandbox-agent binary, just so we don't have anything else obviously on the disk?
86+
if err := os.RemoveAll(tlsDir); err != nil {
87+
return fmt.Errorf("unable to delete tls credentials: %w", err)
88+
}
89+
}
90+
}
91+
92+
if err := s.ListenAndServe(listen, creds); err != nil {
93+
return err
94+
}
95+
96+
return nil
97+
}
98+
99+
// copyFile copies the file from src to dest, setting the mode of the created file
100+
func copyFile(src, dest string, mode os.FileMode) error {
101+
f, err := os.Open(src)
102+
if err != nil {
103+
return fmt.Errorf("unable to open %q: %w", src, err)
104+
}
105+
defer f.Close()
106+
107+
out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
108+
if err != nil {
109+
return fmt.Errorf("unable to create %q: %w", dest, err)
110+
}
111+
if _, err := io.Copy(out, f); err != nil {
112+
out.Close()
113+
return fmt.Errorf("error writing %q: %w", dest, err)
114+
}
115+
if err := out.Close(); err != nil {
116+
return fmt.Errorf("error closing %q: %w", dest, err)
117+
}
118+
return nil
119+
}
120+
121+
// installTo copies the agent and PKI keys to the specified
122+
// directory.
123+
//
124+
// installDir will normally be a shared volume mount which is then
125+
// used as the entrypoint for the main container.
126+
func installTo(installDir string) error {
127+
installBin := filepath.Join(installDir, "ksandbox-agent")
128+
129+
if err := copyFile(os.Args[0], installBin, os.FileMode(0755)); err != nil {
130+
return fmt.Errorf("error copying file %q: %w", os.Args[0], err)
131+
}
132+
133+
// Also copy TLS material (so we can delete it, since https://github.com/kubernetes/kubernetes/pull/58720)
134+
{
135+
copySrcDir := "/tls"
136+
copyDestDir := filepath.Join(installDir, "tls")
137+
if err := os.MkdirAll(copyDestDir, 0700); err != nil {
138+
return fmt.Errorf("error creating %q: %w", copyDestDir, err)
139+
}
140+
141+
files, err := os.ReadDir(copySrcDir)
142+
if err != nil {
143+
return fmt.Errorf("error reading %q directory: %w", copySrcDir, err)
144+
}
145+
146+
for _, f := range files {
147+
if f.IsDir() {
148+
continue
149+
}
150+
151+
src := filepath.Join(copySrcDir, f.Name())
152+
if filepath.Ext(src) != ".crt" && filepath.Ext(src) != ".key" {
153+
klog.Infof("skipping copy of %q; isn't .crt or .key", src)
154+
continue
155+
}
156+
157+
out := filepath.Join(copyDestDir, f.Name())
158+
if err := copyFile(src, out, os.FileMode(0600)); err != nil {
159+
return fmt.Errorf("error copying file %q: %w", src, err)
160+
}
161+
}
162+
}
163+
164+
return nil
165+
}
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
The ksandbox-testclient is a simple client of ksandbox, used to verify execution of
2+
the agent.
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"os"
8+
9+
"github.com/kubernetes-sigs/maintainers/experimental/ksandbox/pkg/client"
10+
protocol "github.com/kubernetes-sigs/maintainers/experimental/ksandbox/protocol/ksandbox/v1alpha1"
11+
"google.golang.org/protobuf/encoding/prototext"
12+
)
13+
14+
func main() {
15+
ctx := context.Background()
16+
err := run(ctx)
17+
if err != nil {
18+
fmt.Fprintf(os.Stderr, "%v\n", err)
19+
os.Exit(1)
20+
}
21+
}
22+
23+
func run(ctx context.Context) error {
24+
namespace := "default"
25+
image := ""
26+
flag.StringVar(&image, "image", image, "image to execute")
27+
buildAgentImage := ""
28+
flag.StringVar(&buildAgentImage, "agent", buildAgentImage, "name of build agent image")
29+
30+
flag.Parse()
31+
32+
if image == "" {
33+
return fmt.Errorf("must specify --image")
34+
}
35+
if buildAgentImage == "" {
36+
return fmt.Errorf("must specify --agent")
37+
}
38+
command := flag.Args()
39+
if len(command) == 0 {
40+
return fmt.Errorf("must specify command after flags")
41+
}
42+
43+
// We assume this is being run on a developer machine (it's a test program),
44+
// rather than in-cluster.
45+
usePortForward := true
46+
c, err := client.NewAgentClient(ctx, namespace, buildAgentImage, image, usePortForward)
47+
if err != nil {
48+
return fmt.Errorf("error building agent client: %w", err)
49+
}
50+
defer c.Close()
51+
52+
request := &protocol.ExecuteCommandRequest{
53+
Command: command,
54+
}
55+
response, err := c.ExecuteCommand(ctx, request)
56+
if err != nil {
57+
return fmt.Errorf("error executing in buildagent: %w", err)
58+
}
59+
60+
fmt.Printf("response: %s", prototext.Format(response))
61+
62+
return nil
63+
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Compile protoc ourselves, from source
2+
FROM debian:latest AS protoc-builder
3+
4+
RUN apt-get update
5+
RUN apt-get install -y g++ git cmake
6+
7+
WORKDIR /src
8+
RUN git clone https://github.com/protocolbuffers/protobuf.git
9+
10+
WORKDIR /src/protobuf
11+
RUN git checkout v3.12.4
12+
RUN git submodule update --init --recursive
13+
14+
WORKDIR /src/protobuf/cmake
15+
RUN cmake .
16+
RUN cmake --build . -j8
17+
18+
RUN cp protoc /usr/local/bin
19+
20+
RUN /usr/local/bin/protoc --version
21+
22+
FROM golang:1.23.5
23+
24+
RUN apt-get update; apt-get install --yes unzip
25+
26+
RUN go install google.golang.org/protobuf/cmd/[email protected]
27+
RUN go install google.golang.org/grpc/cmd/[email protected]
28+
29+
COPY --from=protoc-builder /usr/local/bin/protoc /usr/local/bin/protoc
30+
31+
ENTRYPOINT [ "/usr/local/bin/protoc" ]
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2025 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
REPO_ROOT="$(git rev-parse --show-toplevel)"
22+
SRC_DIR=${REPO_ROOT}/experimental/ksandbox
23+
cd "${SRC_DIR}"
24+
25+
KO="go run github.com/google/[email protected]"
26+
27+
export KO_DOCKER_REPO="${IMAGE_PREFIX:-}ksandbox-agent"
28+
29+
if [[ -z "${TAG:-}" ]]; then
30+
TAG=latest
31+
fi
32+
33+
${KO} build ${PUSH_FLAGS:-} --tags ${TAG} --platform=linux/amd64,linux/arm64 --bare ./cmd/ksandbox-agent/
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2025 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
18+
set -o errexit
19+
set -o nounset
20+
set -o pipefail
21+
22+
REPO_ROOT="$(git rev-parse --show-toplevel)"
23+
SRC_DIR=${REPO_ROOT}/experimental/ksandbox
24+
cd "${SRC_DIR}"
25+
26+
# Would be cool to use ksandbox for this ... but a circular dependency!
27+
docker buildx build --load -t protoc dev/images/protoc --progress=plain
28+
29+
docker run -it -v `pwd`:/src -w /src protoc \
30+
--go_out . \
31+
--go_opt paths=source_relative \
32+
--go-grpc_out . \
33+
--go-grpc_opt paths=source_relative \
34+
protocol/ksandbox/v1alpha1/agent.proto
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2025 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
REPO_ROOT="$(git rev-parse --show-toplevel)"
22+
SRC_DIR=${REPO_ROOT}/experimental/ksandbox
23+
cd "${SRC_DIR}"
24+
25+
# Pick a probably-unique tag
26+
export TAG=`date +%Y%m%d%H%M%S`
27+
28+
# Build the image
29+
echo "Building image"
30+
export IMAGE_PREFIX=fake.registry/
31+
PUSH_FLAGS=--local dev/tasks/build-images
32+
33+
AGENT_IMAGE="${IMAGE_PREFIX:-}ksandbox-agent:${TAG}"
34+
35+
# Load the image into kind
36+
echo "Loading image into kind"
37+
kind load docker-image ${AGENT_IMAGE}
38+
39+
# Run the test client
40+
echo "Running test client"
41+
go run ./cmd/ksandbox-testclient/ --agent ${AGENT_IMAGE} --image golang:1.23.5 go version

0 commit comments

Comments
 (0)