Add sshfs exporter class

Adds a class that allows tests to export a local directory to an
exporter via sshfs.

In order to prevent the exporter from being able to access any file that
the user has permission for on the file system (which would allow it to
read SSH keys for example), the SFTP server is run in an isolated
container by default using podman, although this can be overridden to
use docker or directly execute the SFTP server if the exporters are
trusted.

Signed-off-by: Joshua Watt <[email protected]>

Author: JoshuaWatt

.github/workflows/docker.yml

@@ -27,7 +27,9 @@ jobs:
         docker tag labgrid-client ${{ secrets.DOCKERHUB_PREFIX }}client
         docker tag labgrid-exporter ${{ secrets.DOCKERHUB_PREFIX }}exporter
         docker tag labgrid-coordinator ${{ secrets.DOCKERHUB_PREFIX }}coordinator
+        docker tag labgrid-sftp-server ${{ secrets.DOCKERHUB_PREFIX }}sftp-server
         docker push ${{ secrets.DOCKERHUB_PREFIX }}client
         docker push ${{ secrets.DOCKERHUB_PREFIX }}exporter
         docker push ${{ secrets.DOCKERHUB_PREFIX }}coordinator
+        docker push ${{ secrets.DOCKERHUB_PREFIX }}sftp-server
         docker images

.github/workflows/reusable-unit-tests.yml

@@ -30,8 +30,11 @@ jobs:
           ${{ runner.os }}-pip-
     - name: Install system dependencies
       run: |
-        sudo apt-get install -yq libow-dev openssh-server openssh-client libsnappy-dev ncurses-term graphviz openocd
+        sudo apt-get install -yq libow-dev openssh-server openssh-client libsnappy-dev ncurses-term graphviz openocd sshfs podman
         sudo mkdir -p /var/cache/labgrid/runner && sudo chown runner /var/cache/labgrid/runner
+    - name: Build sftp server
+      run: |
+        podman build --target labgrid-sftp-server -t labgrid/sftp-server:latest -f dockerfiles/Dockerfile .
     - name: Prepare local SSH
       run: |
         # the default of 777 is too open for SSH

dockerfiles/Dockerfile

@@ -121,3 +121,10 @@ COPY --from=ser2net /opt/ser2net /
 VOLUME /opt/conf
 
 CMD ["/entrypoint.sh"]
+
+#
+# SFTP server
+#
+FROM alpine:3.17 as labgrid-sftp-server
+
+RUN apk add openssh-sftp-server

dockerfiles/build.sh

@@ -15,7 +15,7 @@ export DOCKER_BUILDKIT=1
 
 VERSION="$(python -m setuptools_scm)"
 
-for t in client exporter coordinator; do
+for t in client exporter coordinator sftp-server; do
     ${DOCKER} build --build-arg VERSION="$VERSION" \
         --target labgrid-${t} -t labgrid-${t} -f "${SCRIPT_DIR}/Dockerfile" .
 done

labgrid/util/sshfs.py

@@ -0,0 +1,152 @@
+import logging
+import os
+import subprocess
+import random
+import string
+import time
+import shlex
+from contextlib import contextmanager
+
+import attr
+
+from .ssh import sshmanager
+from .timeout import Timeout
+from ..driver.exception import ExecutionError
+from ..resource.common import Resource, NetworkResource
+
+
+DEFAULT_SFTP_SERVER = "podman run --rm -i --mount type=bind,source={path},target={path} labgrid/sftp-server:latest usr/lib/ssh/sftp-server -e {readonly}"
+DEFAULT_RO_OPT = "-R"
+
+
+@attr.s
+class SSHFsExport:
+    """
+    Exports a local directory to a remote device using "reverse" SSH FS
+    """
+
+    local_path = attr.ib(
+        validator=attr.validators.instance_of(str),
+        converter=lambda x: os.path.abspath(str(x)),
+    )
+    resource = attr.ib(
+        validator=attr.validators.instance_of(Resource),
+    )
+    readonly = attr.ib(
+        default=True,
+        validator=attr.validators.instance_of(bool),
+    )
+
+    def __attrs_post_init__(self):
+        if not os.path.isdir(self.local_path):
+            raise FileNotFoundError(f"Local directory {self.local_path} not found")
+        self.logger = logging.getLogger(f"{self}")
+
+    @contextmanager
+    def export(self, remote_path=None):
+        host = self.resource.host
+        conn = sshmanager.open(host)
+
+        # If no remote path was specified, mount on a temporary directory
+        if remote_path is None:
+            tmpname = "".join(random.choices(string.ascii_lowercase, k=10))
+            remote_path = f"/tmp/labgrid-sshfs/{tmpname}/"
+            conn.run_check(f"mkdir -p {remote_path}")
+
+        env = self.resource.target.env
+
+        if env is None:
+            sftp_server_opt = DEFAULT_SFTP_SERVER
+            ro_opt = DEFAULT_RO_OPT
+        else:
+            sftp_server_opt = env.config.get_option("sftp_server", DEFAULT_SFTP_SERVER)
+            ro_opt = env.config.get_option("sftp_server_readonly_opt", DEFAULT_RO_OPT)
+
+        sftp_command = shlex.split(
+            sftp_server_opt.format(
+                path=self.local_path,
+                uid=str(os.getuid()),
+                gid=str(os.getgid()),
+                readonly=ro_opt if self.readonly else "",
+            )
+        )
+
+        sshfs_command = conn.get_prefix() + [
+            "sshfs",
+            "-o",
+            "slave",
+            "-o",
+            "idmap=user",
+            f":{self.local_path}",
+            remote_path,
+        ]
+
+        self.logger.info(
+            "Running %s <-> %s", " ".join(sftp_command), " ".join(sshfs_command)
+        )
+
+        # Reverse sshfs requires that sftp-server running locally and sshfs
+        # running remotely each have their stdout connected to the others
+        # stdin. Connecting sftp-server stdout to sshfs stdin is done using
+        # Popen pipes, but in order to connect sshfs stdout to sftp-stdin, an
+        # external pipe is needed.
+        (rfd, wfd) = os.pipe2(os.O_CLOEXEC)
+        try:
+            with subprocess.Popen(
+                sftp_command,
+                stdout=subprocess.PIPE,
+                stdin=rfd,
+            ) as sftp_server, subprocess.Popen(
+                sshfs_command,
+                stdout=wfd,
+                stdin=sftp_server.stdout,
+            ) as sshfs:
+                # Close all file descriptor open in this process. This way, if
+                # either process exits, the other will get the EPIPE error and
+                # exit also. If this process doesn't close its copy of the
+                # descriptors, that won't happen
+                sftp_server.stdout.close()
+
+                os.close(rfd)
+                rfd = -1
+
+                os.close(wfd)
+                wfd = -1
+
+                # Wait until the mount point appears remotely
+                t = Timeout(30.0)
+
+                while not t.expired:
+                    (_, _, exitcode) = conn.run(f"mountpoint --quiet {remote_path}")
+
+                    if exitcode == 0:
+                        break
+
+                    if sshfs.poll() is not None:
+                        raise ExecutionError(
+                            "sshfs process exited with {sshfs.returncode}"
+                        )
+
+                    if sftp_server.poll() is not None:
+                        raise ExecutionError(
+                            "sftp process exited with {sftp_server.returncode}"
+                        )
+
+                    time.sleep(1)
+
+                if t.expired:
+                    raise TimeoutError("Timeout waiting for SSH fs to mount")
+
+                try:
+                    yield remote_path
+                finally:
+                    sshfs.terminate()
+                    sftp_server.terminate()
+
+        finally:
+            # Cleanup if not done already
+            if rfd >= 0:
+                os.close(rfd)
+
+            if wfd >= 0:
+                os.close(wfd)

tests/test_util.py

@@ -15,6 +15,7 @@
 from labgrid.util.ssh import ForwardError, SSHConnection, sshmanager
 from labgrid.util.proxy import proxymanager
 from labgrid.util.managedfile import ManagedFile
+from labgrid.util.sshfs import SSHFsExport
 from labgrid.driver.exception import ExecutionError
 from labgrid.resource.serialport import NetworkSerialPort
 from labgrid.resource.common import Resource, NetworkResource
@@ -352,6 +353,49 @@ def test_local_managedfile(target, tmpdir):
     assert hash == mf.get_hash()
     assert str(t) == mf.get_remote_path()
 
+@pytest.mark.localsshmanager
+def test_sshfs(target, tmpdir):
+    t = tmpdir.join("test")
+    t_data = """
+Test
+"""
+    t.write(t_data)
+
+    res = NetworkResource(target, "test", "localhost")
+    sshfs = SSHFsExport(tmpdir, res)
+
+    with sshfs.export() as remote_path:
+        remote_test = f"{remote_path}/test"
+        #assert os.path.exists(remote_test)
+        with open(remote_test, "r") as f:
+            remote_data = f.read()
+
+        assert remote_data == t_data
+
+        # Read-only by default
+        with pytest.raises(PermissionError):
+            with open(remote_test, "w") as f:
+                pass
+
+@pytest.mark.localsshmanager
+def test_sshfs_write(target, tmpdir):
+    t = tmpdir.join("test")
+    t.write(
+"""
+Test
+"""
+    )
+
+    res = NetworkResource(target, "test", "localhost")
+    sshfs = SSHFsExport(tmpdir, res, readonly=False)
+
+    with sshfs.export() as remote_path:
+        remote_test = f"{remote_path}test"
+        #assert os.path.exists(remote_test)
+        with open(remote_test, "w") as f:
+            f.write("Hello")
+
+    assert t.read() == "Hello"
 
 def test_find_dict():
     dict_a = {"a": {"a.a": {"a.a.a": "a.a.a_val"}}, "b": "b_val"}