feat: tor managed services support

Author: roshii

src/jmbase/twisted_utils.py

@@ -1,13 +1,17 @@
+import os
 
-from zope.interface import implementer
+import txtorcon
+from twisted.internet import defer, reactor, task
+from twisted.internet.endpoints import (
+    TCP4ClientEndpoint,
+    UNIXClientEndpoint,
+    serverFromString,
+)
 from twisted.internet.error import ReactorNotRunning
-from twisted.internet import reactor, defer
-from twisted.internet.endpoints import (TCP4ClientEndpoint,
-                        UNIXClientEndpoint, serverFromString)
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS
-import txtorcon
+from txtorcon import TorConfig, TorControlProtocol
 from txtorcon.web import tor_agent
-from txtorcon import TorControlProtocol, TorConfig
+from zope.interface import implementer
 
 _custom_stop_reactor_is_set = False
 custom_stop_reactor = None
@@ -174,44 +178,87 @@ def start_tor(self):
         """ This function executes the workflow
         of starting the hidden service and returning its hostname
         """
-        self.info_callback("Attempting to start onion service on port: {} "
-                           "...".format(self.virtual_port))
+        self.info_callback(
+            f"Attempting to start onion service on port: {self.virtual_port} ..."
+        )
+
+        # Check if using Tor-managed mode (via torrc, not control port)
+        if self.hidden_service_dir.startswith("tor-managed:"):
+            self.start_tor_managed_onion()
+            return
+
+        # Ephemeral or txtorcon-managed hidden service (via control port)
+        if str(self.tor_control_host).startswith("unix:"):
+            control_endpoint = UNIXClientEndpoint(reactor, self.tor_control_host[5:])
+        else:
+            control_endpoint = TCP4ClientEndpoint(
+                reactor, self.tor_control_host, self.tor_control_port
+            )
+        d = txtorcon.connect(reactor, control_endpoint)
+
         if self.hidden_service_dir == "":
-            if str(self.tor_control_host).startswith('unix:'):
-                control_endpoint = UNIXClientEndpoint(reactor,
-                                        self.tor_control_host[5:])
-            else:
-                control_endpoint = TCP4ClientEndpoint(reactor,
-                                self.tor_control_host, self.tor_control_port)
-            d = txtorcon.connect(reactor, control_endpoint)
+            # Ephemeral hidden service (no persistence)
             d.addCallback(self.create_onion_ep)
             d.addErrback(self.setup_failed)
-            # TODO: add errbacks to the next two calls in
-            # the chain:
             d.addCallback(self.onion_listen)
             d.addCallback(self.print_host)
         else:
-            ep = "onion:" + str(self.virtual_port) + ":localPort="
-            ep += str(self.serving_port)
-            # endpoints.TCPHiddenServiceEndpoint creates version 2 by
-            # default for backwards compat (err, txtorcon needs to update that ...)
-            ep += ":version=3"
-            ep += ":hiddenServiceDir="+self.hidden_service_dir
-            onion_endpoint = serverFromString(reactor, ep)
-            d = onion_endpoint.listen(self.proto_factory)
+            # txtorcon-managed filesystem hidden service
+            d.addCallback(self.create_filesystem_onion_ep)
+            d.addErrback(self.setup_failed)
             d.addCallback(self.print_host_filesystem)
 
-
     def setup_failed(self, arg):
         # Note that actions based on this failure are deferred to callers:
         self.error_callback("Setup failed: " + str(arg))
 
     def create_onion_ep(self, t):
         self.tor_connection = t
-        portmap_string = config_to_hs_ports(self.virtual_port,
-                                self.serving_host, self.serving_port)
+        portmap_string = config_to_hs_ports(
+            self.virtual_port, self.serving_host, self.serving_port
+        )
         return t.create_onion_service(
-            ports=[portmap_string], private_key=txtorcon.DISCARD)
+            ports=[portmap_string], private_key=txtorcon.DISCARD
+        )
+
+    def create_filesystem_onion_ep(self, t):
+        """Create a persistent hidden service using txtorcon's filesystem support.
+        Requires local Tor control port access.
+        """
+        self.tor_connection = t
+        ep = "onion:" + str(self.virtual_port) + ":localPort="
+        ep += str(self.serving_port)
+        ep += ":version=3"
+        ep += ":hiddenServiceDir=" + self.hidden_service_dir
+        onion_endpoint = serverFromString(reactor, ep)
+        return onion_endpoint.listen(self.proto_factory)
+
+    def start_tor_managed_onion(self):
+        """
+        For Tor-managed hidden services: read hostname, start listening.
+        No control port connection needed.
+        """
+        hs_dir = self.hidden_service_dir.removeprefix("tor-managed:")
+        hostname_file = os.path.join(hs_dir, "hostname")
+
+        def check_and_start():
+            if not os.path.exists(hostname_file):
+                return
+
+            try:
+                with open(hostname_file, "r") as f:
+                    hostname = f.read().strip()
+            except Exception as e:
+                self.error_callback(f"Failed to read {hostname_file}: {e}")
+                poll_loop.stop()
+                return
+
+            poll_loop.stop()
+            self.info_callback(f"Using Tor-managed hidden service: {hostname}")
+            self.onion_hostname_callback(hostname)
+
+        poll_loop = task.LoopingCall(check_and_start)
+        poll_loop.start(0.5)
 
     def onion_listen(self, onion):
         # 'onion' arg is the created EphemeralOnionService object;

src/jmdaemon/onionmc.py

@@ -8,7 +8,7 @@
 from twisted.internet import reactor, task, protocol
 from twisted.protocols import basic
 from twisted.application.internet import ClientService
-from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.internet.endpoints import serverFromString, TCP4ClientEndpoint
 from twisted.internet.address import IPv4Address, IPv6Address
 from txtorcon.socks import (TorSocksEndpoint, HostUnreachableError,
                             SocksError, GeneralServerFailureError)
@@ -697,9 +697,12 @@ def __init__(self,
                 # it'll fire the `setup_error_callback`.
                 self.hs.start_tor()
 
-                # This will serve as our unique identifier, indicating
-                # that we are ready to communicate (in both directions) over Tor.
-                self.onion_hostname = None
+                # For tor-managed services, the hostname is set synchronously by start_tor()
+                # For ephemeral services, we need to wait for the callback
+                if not self.hidden_service_dir.startswith("tor-managed:"):
+                    # This will serve as our unique identifier, indicating
+                    # that we are ready to communicate (in both directions) over Tor.
+                    self.onion_hostname = None
         else:
             # dummy 'hostname' to indicate we can start running immediately:
             self.onion_hostname = NOT_SERVING_ONION_HOSTNAME
@@ -884,7 +887,7 @@ def connect_to_directories(self) -> None:
         if self.genesis_node:
             # we are a directory and we have no directory peers;
             # just start.
-            self.on_welcome(self)
+            self._start_listener()
             return
         # the remaining code is only executed by non-directories:
         for p in self.peers:
@@ -901,6 +904,13 @@ def connect_to_directories(self) -> None:
             self.wait_for_directories)
         self.wait_for_directories_loop.start(2.0)
 
+    def _start_listener(self) -> None:
+        serverstring = f"tcp:{self.onion_serving_port}:interface={self.onion_serving_host}"
+        onion_endpoint = serverFromString(reactor, serverstring)
+        d = onion_endpoint.listen(self.proto_factory)
+        d.addCallback(self.on_welcome)
+        d.addErrback(lambda f: self.setup_error_callback(f"Listen failed: {f}"))
+
     def handshake_as_client(self, peer: OnionPeer) -> None:
         assert peer.status() == PEER_STATUS_CONNECTED
         if self.self_as_peer.directory:
@@ -1461,7 +1471,8 @@ def wait_for_directories(self) -> None:
         # Note that even if the preceding (max) 50 seconds failed to
         # connect all our configured dps, we will keep trying and they
         # can still be used.
-        if not self.on_welcome_sent:
+        # For genesis nodes, on_welcome is called after the listener starts
+        if not self.on_welcome_sent and not self.genesis_node:
             self.on_welcome(self)
             self.on_welcome_sent = True
             self.wait_for_directories_loop.stop()

test/jmbase/test_twisted_utils.py

@@ -0,0 +1,65 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from jmbase.twisted_utils import JMHiddenService
+
+
+def mock_hs(hidden_service_dir: str = "") -> JMHiddenService:
+    return JMHiddenService(
+        Mock(),
+        Mock(),
+        Mock(),
+        Mock(),
+        "127.0.0.1",
+        9051,
+        "127.0.0.1",
+        8080,
+        80,
+        None,
+        hidden_service_dir,
+    )
+
+
+class TestTorManagedHiddenService:
+    @pytest.mark.parametrize(
+        "hidden_service_dir,expect_managed,expect_connect",
+        [
+            ("tor-managed:/path/to/dir", True, False),
+            ("/normal/path", False, True),
+        ],
+    )
+    def test_hidden_service_dir_detection(
+        self, hidden_service_dir, expect_managed, expect_connect
+    ):
+        with (
+            patch.object(JMHiddenService, "start_tor_managed_onion") as mock_managed,
+            patch("jmbase.twisted_utils.txtorcon.connect") as mock_connect,
+        ):
+            hs = mock_hs(hidden_service_dir)
+
+            hs.start_tor()
+
+            if expect_managed:
+                mock_managed.assert_called_once()
+                mock_connect.assert_not_called()
+            else:
+                mock_managed.assert_not_called()
+                mock_connect.assert_called_once()
+
+    def test_ephemeral_service_creation(self):
+        with patch("jmbase.twisted_utils.txtorcon") as mock_txtorcon:
+            mock_t = Mock()
+            mock_t.create_onion_service.return_value = Mock()
+
+            hs = mock_hs()
+            hs.tor_connection = mock_t
+            hs.virtual_port = 80
+            hs.serving_host = "127.0.0.1"
+            hs.serving_port = 8080
+
+            hs.create_onion_ep(mock_t)
+
+            mock_t.create_onion_service.assert_called_once_with(
+                ports=["80 127.0.0.1:8080"], private_key=mock_txtorcon.DISCARD
+            )

test/jmdaemon/test_onionmc.py

@@ -0,0 +1,90 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from jmdaemon.onionmc import OnionMessageChannel
+
+
+@pytest.fixture
+def configdata():
+    return {
+        "btcnet": "mainnet",
+        "tor_control_host": "127.0.0.1",
+        "tor_control_port": 9051,
+        "onion_serving_host": "127.0.0.1",
+        "serving": False,
+        "socks5_host": "127.0.0.1",
+        "socks5_port": 9050,
+        "directory_nodes": "",
+        "passive": False,
+    }
+
+
+class TestOnionMessageChannelListener:
+    def test_start_listener_creates_tcp_endpoint(self, configdata):
+        with (
+            patch("jmdaemon.onionmc.reactor") as mock_reactor,
+            patch("jmdaemon.onionmc.serverFromString") as mock_server_from_string,
+        ):
+            mock_deferred = Mock()
+            mock_endpoint = Mock()
+            mock_endpoint.listen.return_value = mock_deferred
+            mock_server_from_string.return_value = mock_endpoint
+
+            mc = OnionMessageChannel(configdata)
+            mc.onion_serving_host = "127.0.0.1"
+            mc.onion_serving_port = 8080
+            mc.proto_factory = Mock()
+            mc.on_welcome = Mock()
+            mc.setup_error_callback = Mock()
+
+            mc._start_listener()
+
+            expected_serverstring = "tcp:8080:interface=127.0.0.1"
+            mock_server_from_string.assert_called_once_with(
+                mock_reactor, expected_serverstring
+            )
+
+            mock_endpoint.listen.assert_called_once_with(mc.proto_factory)
+
+            mock_deferred.addCallback.assert_called_once_with(mc.on_welcome)
+            mock_deferred.addErrback.assert_called_once()
+
+            errback_callback = mock_deferred.addErrback.call_args[0][0]
+            test_failure = Exception("Test error")
+            errback_callback(test_failure)
+
+            mc.setup_error_callback.assert_called_once_with("Listen failed: Test error")
+
+    @pytest.mark.parametrize(
+        "host,port",
+        [
+            ("192.168.1.1", 9000),
+            ("localhost", 1234),
+            ("0.0.0.0", 80),
+        ],
+    )
+    def test_start_listener_different_ports_and_hosts(self, configdata, host, port):
+        with (
+            patch("jmdaemon.onionmc.reactor") as mock_reactor,
+            patch("jmdaemon.onionmc.serverFromString") as mock_server_from_string,
+        ):
+            mock_endpoint = Mock()
+            mock_server_from_string.return_value = mock_endpoint
+
+            configdata["onion_serving_host"] = host
+            configdata["onion_serving_port"] = port
+
+            mc = OnionMessageChannel(configdata)
+            mc.onion_serving_host = host
+            mc.onion_serving_port = port
+            mc.proto_factory = Mock()
+            mc.on_welcome = Mock()
+            mc.setup_error_callback = Mock()
+
+            mc._start_listener()
+
+            expected_serverstring = f"tcp:{port}:interface={host}"
+            mock_server_from_string.assert_called_once_with(
+                mock_reactor, expected_serverstring
+            )