diff --git a/CHANGES.md b/CHANGES.md
index 1f21696049..b424205e13 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -29,6 +29,7 @@
 
 ## Bug Fixes
 
+* [GH-721](https://github.com/apache/mina-sshd/issues/721) SSH client: schedule session timeout checks on demand only
 * [GH-807](https://github.com/apache/mina-sshd/issues/807) Handle "verified" flag for sk-* keys
 * [GH-809](https://github.com/apache/mina-sshd/pull/809) Fix server-side authentication for FIDO/U2F sk-* keys with flags in `authorized_keys`
 * [GH-827](https://github.com/apache/mina-sshd/issues/827) Don't fail on invalid `known_hosts` lines; log and skip them
diff --git a/sshd-core/src/main/java/org/apache/sshd/common/helpers/AbstractFactoryManager.java b/sshd-core/src/main/java/org/apache/sshd/common/helpers/AbstractFactoryManager.java
index 15bd3edf0b..82f1e05562 100644
--- a/sshd-core/src/main/java/org/apache/sshd/common/helpers/AbstractFactoryManager.java
+++ b/sshd-core/src/main/java/org/apache/sshd/common/helpers/AbstractFactoryManager.java
@@ -55,6 +55,7 @@
 import org.apache.sshd.common.random.Random;
 import org.apache.sshd.common.session.ConnectionService;
 import org.apache.sshd.common.session.ReservedSessionMessagesHandler;
+import org.apache.sshd.common.session.Session;
 import org.apache.sshd.common.session.SessionDisconnectHandler;
 import org.apache.sshd.common.session.SessionListener;
 import org.apache.sshd.common.session.UnknownChannelReferenceHandler;
@@ -472,12 +473,8 @@ public void removePortForwardingEventListener(PortForwardingEventListener listen
     }
 
     protected void setupSessionTimeout(AbstractSessionFactory<?, ?> sessionFactory) {
-        // set up the the session timeout listener and schedule it
         sessionTimeoutListener = createSessionTimeoutListener();
         addSessionListener(sessionTimeoutListener);
-
-        timeoutListenerFuture = getScheduledExecutorService()
-                .scheduleAtFixedRate(sessionTimeoutListener, 1, 1, TimeUnit.SECONDS);
     }
 
     protected void removeSessionTimeout(AbstractSessionFactory<?, ?> sessionFactory) {
@@ -485,11 +482,38 @@ protected void removeSessionTimeout(AbstractSessionFactory<?, ?> sessionFactory)
     }
 
     protected SessionTimeoutListener createSessionTimeoutListener() {
-        return new SessionTimeoutListener();
+        return new SessionTimeoutListener() {
+
+            @Override
+            public void sessionCreated(Session session) {
+                synchronized (this) {
+                    super.sessionCreated(session);
+                    if (!sessions.isEmpty()) {
+                        ensureTimeoutScheduled();
+                    }
+                }
+            }
+
+            @Override
+            public void sessionClosed(Session s) {
+                synchronized (this) {
+                    super.sessionClosed(s);
+                    if (sessions.isEmpty()) {
+                        cancelSessionTimeout();
+                    }
+                }
+            }
+        };
     }
 
-    protected void stopSessionTimeoutListener(AbstractSessionFactory<?, ?> sessionFactory) {
-        // cancel the timeout monitoring task
+    private void ensureTimeoutScheduled() {
+        if (timeoutListenerFuture == null) {
+            timeoutListenerFuture = getScheduledExecutorService().scheduleAtFixedRate(sessionTimeoutListener, 1, 1,
+                    TimeUnit.SECONDS);
+        }
+    }
+
+    private void cancelSessionTimeout() {
         if (timeoutListenerFuture != null) {
             try {
                 timeoutListenerFuture.cancel(true);
@@ -497,6 +521,10 @@ protected void stopSessionTimeoutListener(AbstractSessionFactory<?, ?> sessionFa
                 timeoutListenerFuture = null;
             }
         }
+    }
+
+    protected void stopSessionTimeoutListener(AbstractSessionFactory<?, ?> sessionFactory) {
+        cancelSessionTimeout();
 
         // remove the sessionTimeoutListener completely; should the SSH server/client be restarted, a new one
         // will be created.
diff --git a/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpFileSystemProvider.java b/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpFileSystemProvider.java
index 19830cb5f5..6bb9b1da79 100644
--- a/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpFileSystemProvider.java
+++ b/sshd-sftp/src/main/java/org/apache/sshd/sftp/client/fs/SftpFileSystemProvider.java
@@ -136,7 +136,7 @@ public class SftpFileSystemProvider extends FileSystemProvider {
 
     protected final Logger log;
 
-    private final SshClient clientInstance;
+    private SshClient clientInstance;
     private final SftpClientFactory factory;
     private final SftpVersionSelector versionSelector;
     private final SftpErrorDataHandler errorDataHandler;
@@ -183,11 +183,6 @@ public SftpFileSystemProvider(SshClient client, SftpClientFactory factory,
         this.factory = factory;
         this.versionSelector = selector;
         this.errorDataHandler = errorDataHandler;
-        if (client == null) {
-            // TODO: make this configurable using system properties
-            client = SshClient.setUpDefaultClient();
-            client.start();
-        }
         this.clientInstance = client;
     }
 
@@ -204,10 +199,20 @@ public SftpErrorDataHandler getSftpErrorDataHandler() {
         return errorDataHandler;
     }
 
-    public final SshClient getClientInstance() {
+    public final synchronized SshClient getClientInstance() {
+        if (clientInstance == null) {
+            clientInstance = createClient();
+        }
         return clientInstance;
     }
 
+    private SshClient createClient() {
+        // TODO: make this configurable using system properties
+        SshClient client = SshClient.setUpDefaultClient();
+        client.start();
+        return client;
+    }
+
     public SftpClientFactory getSftpClientFactory() {
         return factory;
     }
@@ -561,10 +566,30 @@ public SftpFileSystem getFileSystem(String id) {
         }
     }
 
+    private SftpFileSystem getOrCreateFileSystem(URI uri) throws IOException {
+        String id = getFileSystemIdentifier(uri);
+        synchronized (fileSystems) {
+            SftpFileSystem fs = fileSystems.get(id);
+            if (fs == null) {
+                fs = newFileSystem(uri, Collections.emptyMap());
+            }
+            return fs;
+        }
+    }
+
     @Override
     public Path getPath(URI uri) {
-        FileSystem fs = getFileSystem(uri);
-        return fs.getPath(uri.getPath());
+        if (!getScheme().equalsIgnoreCase(uri.getScheme())) {
+            throw new IllegalArgumentException("Not a " + getScheme() + " URI: " + uri);
+        }
+        try {
+            FileSystem fs = getOrCreateFileSystem(uri);
+            return fs.getPath(uri.getPath());
+        } catch (IOException e) {
+            FileSystemNotFoundException fe = new FileSystemNotFoundException("No file system for URI " + uri);
+            fe.initCause(e);
+            throw fe;
+        }
     }
 
     @Override