diff --git a/cmd/remoteproc-runtime/create.go b/cmd/remoteproc-runtime/create.go index 2b283bf3..1a4ab409 100644 --- a/cmd/remoteproc-runtime/create.go +++ b/cmd/remoteproc-runtime/create.go @@ -20,7 +20,7 @@ var createCmd = &cobra.Command{ if bundlePath == "" { bundlePath = "." } - return runtime.Create(containerID, bundlePath, pidFile) + return runtime.Create(logger, containerID, bundlePath, pidFile) }, } diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index aae14505..9b38cafd 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -101,8 +101,6 @@ Useful for development without access to hardware with Remoteproc support. #### Testing with Docker -⚠️ Docker network must be set to 'Host' (`--network=host`), as the proxy runs in the host's network namespace. - 1. **Build and install the shim and runtime with custom root** ```bash @@ -139,7 +137,6 @@ Useful for development without access to hardware with Remoteproc support. docker run \ --runtime io.containerd.remoteproc.v1 \ --annotation remoteproc.name="test-processor" \ - --network=host \ test-remoteproc-image ``` diff --git a/e2e/limavm/limavm.go b/e2e/limavm/limavm.go index 6fe2d52e..ebaab000 100644 --- a/e2e/limavm/limavm.go +++ b/e2e/limavm/limavm.go @@ -56,17 +56,34 @@ func (vm VM) RunCommand(name string, args ...string) (stdout, stderr string, err return stdout, stderr, nil } +type Runnable interface { + Run(args ...string) (stdout, stderr string, err error) +} + type InstalledBin struct { vm VM pathToBin string } -func (b InstalledBin) String() string { +func (b InstalledBin) Run(args ...string) (stdout, stderr string, err error) { + return b.vm.RunCommand(b.pathToBin, args...) +} + +func (b InstalledBin) Path() string { return b.pathToBin } -func (b InstalledBin) Run(args ...string) (stdout, stderr string, err error) { - return b.vm.RunCommand(b.pathToBin, args...) +type Sudo struct { + vm VM + pathToBin string +} + +func NewSudo(b InstalledBin) Sudo { + return Sudo(b) +} + +func (r Sudo) Run(args ...string) (stdout, stderr string, err error) { + return r.vm.RunCommand("sudo", append([]string{r.pathToBin}, args...)...) } func Require(t *testing.T) { diff --git a/e2e/runtime_test.go b/e2e/runtime_test.go index 3eea0498..42846324 100644 --- a/e2e/runtime_test.go +++ b/e2e/runtime_test.go @@ -141,6 +141,94 @@ func TestRuntime(t *testing.T) { require.FileExists(t, pidFile) assertFileContent(t, pidFile, fmt.Sprintf("%d", pid)) }) + + t.Run("proxy process namespacing", func(t *testing.T) { + installedRuntimeSudo := limavm.NewSudo(installedRuntime) + + t.Run("creates process in requested namespace when root", func(t *testing.T) { + remoteprocName := "lovely-blue-device" + sim := remoteproc.NewSimulator(rootpathPrefix).WithName(remoteprocName) + if err := sim.Start(); err != nil { + t.Fatalf("failed to run simulator: %s", err) + } + defer func() { _ = sim.Stop() }() + + uniqueID := testID(t) + containerName := uniqueID + bundlePath := filepath.Join(dirMountedInVM, uniqueID) + require.NoError(t, generateBundle( + bundlePath, + remoteprocName, + specs.LinuxNamespace{Type: specs.MountNamespace}, + )) + _, stderr, err := installedRuntimeSudo.Run( + "create", + "--bundle", bundlePath, + containerName) + require.NoError(t, err, "stderr: %s", stderr) + t.Cleanup(func() { + _, _, _ = installedRuntimeSudo.Run("delete", containerName) + }) + + pid, err := getContainerPid(installedRuntimeSudo, containerName) + require.NoError(t, err) + + hostMountNS, stderr, err := vm.RunCommand("readlink", "/proc/self/ns/mnt") + require.NoError(t, err, "stderr: %s", stderr) + + proxyMountNS, stderr, err := vm.RunCommand("sudo", "readlink", fmt.Sprintf("/proc/%d/ns/mnt", pid)) + require.NoError(t, err, "stderr: %s", stderr) + assert.NotEqual(t, strings.TrimSpace(hostMountNS), strings.TrimSpace(proxyMountNS)) + + remoteproc.AssertState(t, sim.DeviceDir(), "offline") + + _, stderr, err = installedRuntimeSudo.Run("start", containerName) + require.NoError(t, err, "stderr: %s", stderr) + remoteproc.AssertState(t, sim.DeviceDir(), "running") + }) + + t.Run("creates process in user's namespace when not root", func(t *testing.T) { + remoteprocName := "lovely-blue-device" + sim := remoteproc.NewSimulator(rootpathPrefix).WithName(remoteprocName) + if err := sim.Start(); err != nil { + t.Fatalf("failed to run simulator: %s", err) + } + defer func() { _ = sim.Stop() }() + + uniqueID := testID(t) + containerName := uniqueID + bundlePath := filepath.Join(dirMountedInVM, uniqueID) + require.NoError(t, generateBundle( + bundlePath, + remoteprocName, + specs.LinuxNamespace{Type: specs.MountNamespace}, + )) + _, stderr, err := installedRuntime.Run( + "create", + "--bundle", bundlePath, + containerName) + require.NoError(t, err, "stderr: %s", stderr) + t.Cleanup(func() { + _, _, _ = installedRuntimeSudo.Run("delete", containerName) + }) + + pid, err := getContainerPid(installedRuntimeSudo, containerName) + require.NoError(t, err) + + hostMountNS, stderr, err := vm.RunCommand("readlink", "/proc/self/ns/mnt") + require.NoError(t, err, "stderr: %s", stderr) + + proxyMountNS, stderr, err := vm.RunCommand("sudo", "readlink", fmt.Sprintf("/proc/%d/ns/mnt", pid)) + require.NoError(t, err, "stderr: %s", stderr) + assert.Equal(t, strings.TrimSpace(hostMountNS), strings.TrimSpace(proxyMountNS)) + + remoteproc.AssertState(t, sim.DeviceDir(), "offline") + + _, stderr, err = installedRuntimeSudo.Run("start", containerName) + require.NoError(t, err, "stderr: %s", stderr) + remoteproc.AssertState(t, sim.DeviceDir(), "running") + }) + }) } func testID(t testing.TB) string { @@ -150,14 +238,14 @@ func testID(t testing.TB) string { return name } -func assertContainerStatus(t testing.TB, runtime limavm.InstalledBin, containerName string, wantStatus specs.ContainerState) { +func assertContainerStatus(t testing.TB, runtime limavm.Runnable, containerName string, wantStatus specs.ContainerState) { t.Helper() state, err := getContainerState(runtime, containerName) require.NoError(t, err) assert.Equal(t, wantStatus, state.Status) } -func getContainerPid(runtime limavm.InstalledBin, containerName string) (int, error) { +func getContainerPid(runtime limavm.Runnable, containerName string) (int, error) { state, err := getContainerState(runtime, containerName) if err != nil { return 0, err @@ -165,7 +253,7 @@ func getContainerPid(runtime limavm.InstalledBin, containerName string) (int, er return state.Pid, err } -func getContainerState(runtime limavm.InstalledBin, containerName string) (specs.State, error) { +func getContainerState(runtime limavm.Runnable, containerName string) (specs.State, error) { var state specs.State out, stderr, err := runtime.Run("state", containerName) if err != nil { @@ -178,7 +266,7 @@ func getContainerState(runtime limavm.InstalledBin, containerName string) (specs return state, nil } -func generateBundle(targetDir string, remoteprocName string) error { +func generateBundle(targetDir string, remoteprocName string, namespaces ...specs.LinuxNamespace) error { const bundleRoot = "rootfs" const firmwareName = "hello_world.elf" @@ -201,6 +289,7 @@ func generateBundle(targetDir string, remoteprocName string) error { Annotations: map[string]string{ "remoteproc.name": remoteprocName, }, + Linux: &specs.Linux{Namespaces: namespaces}, } specPath := filepath.Join(targetDir, "config.json") diff --git a/e2e/runtime_via_podman_test.go b/e2e/runtime_via_podman_test.go index 59f78b3b..fe70eff4 100644 --- a/e2e/runtime_via_podman_test.go +++ b/e2e/runtime_via_podman_test.go @@ -41,7 +41,7 @@ func TestPodman(t *testing.T) { stdout, stderr, err := vm.RunCommand( "podman", - fmt.Sprintf("--runtime=%s", installedRuntimeBin), + fmt.Sprintf("--runtime=%s", installedRuntimeBin.Path()), "run", "-d", "--annotation", fmt.Sprintf("remoteproc.name=%s", remoteprocName), imageName) diff --git a/e2e/shim_via_docker_test.go b/e2e/shim_via_docker_test.go index d7d706a4..23f2f562 100644 --- a/e2e/shim_via_docker_test.go +++ b/e2e/shim_via_docker_test.go @@ -45,7 +45,6 @@ func TestDocker(t *testing.T) { containerID, stderr, err := vm.RunCommand( "docker", "run", "-d", - "--network=host", "--runtime", "io.containerd.remoteproc.v1", "--annotation", fmt.Sprintf("remoteproc.name=%s", remoteprocName), imageName) @@ -76,7 +75,6 @@ func TestDocker(t *testing.T) { _, stderr, err := vm.RunCommand( "docker", "run", "-d", - "--network=host", "--runtime", "io.containerd.remoteproc.v1", "--annotation", fmt.Sprintf("remoteproc.name=%s", "other-processor"), imageName) @@ -94,7 +92,6 @@ func TestDocker(t *testing.T) { containerID, stderr, err := vm.RunCommand( "docker", "run", "-d", - "--network=host", "--runtime", "io.containerd.remoteproc.v1", "--annotation", fmt.Sprintf("remoteproc.name=%s", remoteprocName), imageName) diff --git a/internal/proxy/namespaces.go b/internal/proxy/namespaces.go new file mode 100644 index 00000000..e295b596 --- /dev/null +++ b/internal/proxy/namespaces.go @@ -0,0 +1,56 @@ +package proxy + +import ( + "fmt" + "log/slog" + + "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/sys/unix" +) + +var specNamespacesToUnixCloneFlags = map[specs.LinuxNamespaceType]uintptr{ + specs.CgroupNamespace: unix.CLONE_NEWCGROUP, + specs.IPCNamespace: unix.CLONE_NEWIPC, + specs.MountNamespace: unix.CLONE_NEWNS, + specs.NetworkNamespace: unix.CLONE_NEWNET, + specs.PIDNamespace: unix.CLONE_NEWPID, + specs.TimeNamespace: unix.CLONE_NEWTIME, + specs.UserNamespace: unix.CLONE_NEWUSER, + specs.UTSNamespace: unix.CLONE_NEWUTS, +} + +func ParseNamespaceFlags(namespaces []specs.LinuxNamespace) (uintptr, error) { + if namespaces == nil { + return 0, nil + } + + var flags uintptr + for _, ns := range namespaces { + if ns.Path != "" { + continue + } + flag, ok := specNamespacesToUnixCloneFlags[ns.Type] + if !ok { + err := fmt.Errorf("unknown namespace type %q", ns.Type) + return 0, err + } + flags |= flag + } + return flags, nil +} + +/* We mimic runc here - if not root, we ignore namespace isolation due to insufficient permissions + * https://github.com/opencontainers/runc/blob/a75076b4a413f628c4b6aa4c5568b159aa128a56/libcontainer/specconv/example.go */ +func LinuxCloneFlags(logger *slog.Logger, isRoot bool, namespaces []specs.LinuxNamespace) (uintptr, error) { + flags, err := ParseNamespaceFlags(namespaces) + if err != nil { + return 0, err + } + + if !isRoot && flags != 0 { + logger.Warn("running non-root; namespace isolation disabled") + return 0, nil + } + + return flags, nil +} diff --git a/internal/proxy/namespaces_test.go b/internal/proxy/namespaces_test.go new file mode 100644 index 00000000..9124c2fb --- /dev/null +++ b/internal/proxy/namespaces_test.go @@ -0,0 +1,55 @@ +package proxy_test + +import ( + "io" + "log/slog" + "testing" + + proxy "github.com/arm/remoteproc-runtime/internal/proxy" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestLinuxCloneFlags(t *testing.T) { + t.Run("converts known linux namespace flags to unix", func(t *testing.T) { + isRoot := true + namespaces := []specs.LinuxNamespace{ + {Type: specs.CgroupNamespace}, + {Type: specs.UserNamespace}, + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + got, err := proxy.LinuxCloneFlags(logger, isRoot, namespaces) + + require.NoError(t, err) + want := uintptr(unix.CLONE_NEWCGROUP | unix.CLONE_NEWUSER) + require.Equal(t, want, got) + }) + + t.Run("non-root disables cloning", func(t *testing.T) { + isRoot := false + namespaces := []specs.LinuxNamespace{ + {Type: specs.CgroupNamespace}, + {Type: specs.UserNamespace}, + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + got, err := proxy.LinuxCloneFlags(logger, isRoot, namespaces) + + require.NoError(t, err) + require.Equal(t, uintptr(0), got) + }) + + t.Run("errors given unknown namespace", func(t *testing.T) { + isRoot := true + namespaces := []specs.LinuxNamespace{ + {Type: specs.LinuxNamespaceType("weird-name")}, + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + _, err := proxy.LinuxCloneFlags(logger, isRoot, namespaces) + + require.Error(t, err) + }) +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index ee4b0426..c07b72b6 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -2,20 +2,31 @@ package proxy import ( "fmt" + "log/slog" "os" "os/exec" "syscall" + + "github.com/opencontainers/runtime-spec/specs-go" ) -func NewProcess(devicePath string) (int, error) { +func NewProcess(logger *slog.Logger, namespaces []specs.LinuxNamespace, devicePath string) (int, error) { execPath, err := os.Executable() if err != nil { return -1, fmt.Errorf("failed to get executable path: %w", err) } + isRoot := os.Geteuid() == 0 + + namespaceFlags, err := LinuxCloneFlags(logger, isRoot, namespaces) + if err != nil { + return -1, err + } + cmd := exec.Command(execPath, "proxy", "--device-path", devicePath) cmd.SysProcAttr = &syscall.SysProcAttr{ - Setpgid: true, + Setpgid: true, + Cloneflags: namespaceFlags, } if err := cmd.Start(); err != nil { diff --git a/internal/runtime/create.go b/internal/runtime/create.go index 4eaa78e5..421f448e 100644 --- a/internal/runtime/create.go +++ b/internal/runtime/create.go @@ -2,6 +2,7 @@ package runtime import ( "fmt" + "log/slog" "os" "path/filepath" @@ -11,7 +12,7 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" ) -func Create(containerID string, bundlePath string, pidFile string) error { +func Create(logger *slog.Logger, containerID string, bundlePath string, pidFile string) error { spec, err := oci.ReadSpec(bundlePath) if err != nil { return fmt.Errorf("failed to read container specification: %w", err) @@ -36,7 +37,12 @@ func Create(containerID string, bundlePath string, pidFile string) error { return err } - pid, err := proxy.NewProcess(devicePath) + var namespaces []specs.LinuxNamespace + if spec.Linux != nil { + namespaces = spec.Linux.Namespaces + } + + pid, err := proxy.NewProcess(logger, namespaces, devicePath) if err != nil { return fmt.Errorf("failed to start proxy process: %w", err) }