update: sync russh/russh-sftp forks to upstream and unify ssh-key (#207)

Sync bssh-russh to upstream russh 0.61.1 (was 0.60.3): brings the new RustCrypto generation (sha2/sha1 0.11, hmac 0.13, aes 0.9, cbc 0.2, ctr 0.10, digest 0.11, pbkdf2 0.13, ssh-key 0.7.0-rc.10, ssh-encoding 0.3.0-rc.9). The PTY Handle::data() drain fix is re-ported onto the new server/session.rs, and three patches now merged upstream (channel-write-ordering, agent-frame-length-cap, sha1-mac-exclude) are removed so only handle-data-fix.patch remains. Adds [lib] name = "russh" so vendored doctests resolve, a clippy lints allow for the one vendored style lint, and a PTY regression test in tests/pty_handle_data.rs.

Sync bssh-russh-sftp to upstream russh-sftp 2.3.0 (was 2.1.2): the pipelined File I/O helpers (write_all_pipelined / read_to_writer_pipelined) are re-applied on top of 2.3.0, and bssh's SFTP server is adapted to the 2.3.0 Handler::Error change (Into<StatusReply>, which now also carries the error message).

Unify ssh-key to =0.7.0-rc.10 in the bssh crate so the workspace resolves a single ssh-key version instead of 0.6 and 0.7-rc side by side; bump the russh/russh-sftp dependency requirements, enable argon2's std feature (restores rand_core OsRng), and refresh transitive deps via cargo update.

Make both forks' create-patch.sh self-contained (clone upstream, correct patch names, graceful fallback where upstream publishes no tags), generalize sync-upstream.sh to apply every patches/*.patch with upstream-detection, and rewrite both fork READMEs for the new versions.

Author: inureyes

Cargo.lock

@@ -19,14 +19,14 @@ edition = "2024"
 
 [dependencies]
 bytes = "1.11.1"
-tokio = { version = "1.52.1", features = ["full"] }
+tokio = { version = "1.52.3", features = ["full"] }
 # Use our internal russh fork with session loop fixes
 # - Development: uses local path (crates/bssh-russh)
 # - Publishing: uses crates.io version (path ignored)
-russh = { package = "bssh-russh", version = "0.60.3", path = "crates/bssh-russh" }
-# Use our internal russh-sftp fork tracking upstream 2.1.2
+russh = { package = "bssh-russh", version = "0.61.1", path = "crates/bssh-russh" }
+# Use our internal russh-sftp fork tracking upstream 2.3.0
 # (adds pipelined File I/O; serde_bytes perf fix is now upstreamed)
-russh-sftp = { package = "bssh-russh-sftp", version = "2.1.2", path = "crates/bssh-russh-sftp" }
+russh-sftp = { package = "bssh-russh-sftp", version = "2.3.0", path = "crates/bssh-russh-sftp" }
 clap = { version = "4.6.1", features = ["derive", "env"] }
 anyhow = "1.0.102"
 thiserror = "2.0.18"
@@ -62,9 +62,11 @@ shell-words = "1.1.1"
 libc = "0.2"
 ipnetwork = "0.21"
 bcrypt = "0.19"
-argon2 = "0.5"
+argon2 = { version = "0.5", features = ["std"] }
 rand = "0.10"
-ssh-key = { version = "0.6", features = ["std"] }
+# Pinned to match russh's ssh-key (0.61.x uses =0.7.0-rc.10) so the workspace
+# resolves a single ssh-key version instead of 0.6 + 0.7-rc side by side.
+ssh-key = { version = "=0.7.0-rc.10", features = ["std"] }
 async-compression = { version = "0.4", features = ["tokio", "gzip"] }
 serde_json = "1.0"
 opentelemetry = "0.32"

Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "bssh-russh-sftp"
-version = "2.1.2"
+version = "2.3.0"
 authors = ["Jeongkyu Shin <inureyes@gmail.com>"]
-description = "Temporary fork of russh-sftp 2.1.2 adding pipelined SFTP File I/O (write_all_pipelined / read_to_writer_pipelined). Note: the serde_bytes perf fix that originally motivated this fork is now upstreamed in russh-sftp 2.1.2; only the pipelined helpers remain as fork value-add."
+description = "Temporary fork of russh-sftp 2.3.0 adding pipelined SFTP File I/O (write_all_pipelined / read_to_writer_pipelined). These helpers hide per-request RTT for fast bulk transfers and are the only value-add over upstream russh-sftp."
 documentation = "https://docs.rs/bssh-russh-sftp"
 edition = "2021"
 homepage = "https://github.com/lablup/bssh"
@@ -11,23 +11,26 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/lablup/bssh"
 
+# Dependency versions mirror upstream russh-sftp 2.3.0 (AspectUnk/russh-sftp).
+# Update via ./sync-upstream.sh; the only fork addition is the `futures` dep,
+# needed by the forward-ported pipelined helpers in src/client/fs/file.rs.
 [dependencies]
-tokio = { version = "1", default-features = false, features = [
+tokio = { version = "1.52.3", default-features = false, features = [
     "io-util",
     "rt",
     "sync",
     "time",
     "macros",
 ] }
-tokio-util = "0.7.18"
+tokio-util = { version = "0.7.18", default-features = false, features = ["rt"] }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_bytes = "0.11"
+serde_bytes = "0.11.19"
 bitflags = { version = "2.11.1", features = ["serde"] }
 async-trait = { version = "0.1.89", optional = true }
 
 # futures: required by our forward-ported pipelined helpers
 # (write_all_pipelined / read_to_writer_pipelined use FuturesUnordered).
-# Upstream russh-sftp 2.1.2 does not need this dependency.
+# Upstream russh-sftp only needs this as a dev-dependency.
 futures = { version = "0.3.32", default-features = false, features = ["std", "async-await"] }
 
 thiserror = "2.0.18"
@@ -36,5 +39,9 @@ bytes = "1.11.1"
 log = "0.4.29"
 dashmap = "6.1.0"
 
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+wasm-bindgen-futures = "0.4.71"
+gloo-timers = { version = "0.4.0", features = ["futures"] }
+
 [features]
 async-trait = ["dep:async-trait"]

crates/bssh-russh-sftp/Cargo.toml

@@ -1,25 +1,35 @@
 # bssh-russh-sftp
 
-Temporary fork of [russh-sftp](https://crates.io/crates/russh-sftp) with a `serde_bytes` performance fix for SFTP `Write` and `Data` packets.
+**Temporary fork of [russh-sftp](https://crates.io/crates/russh-sftp) (tracking upstream `2.3.0`) adding pipelined SFTP file I/O.**
 
-This crate exists so bssh can ship the packet serialization fix independently while keeping the public crate name usable through Cargo's `package = "bssh-russh-sftp"` dependency alias.
+This crate exists so bssh can ship faster bulk SFTP transfers independently, while keeping the public crate name usable through Cargo's `package = "bssh-russh-sftp"` dependency alias.
 
-## The Problem
+## The Value-Add
 
-`russh-sftp` 2.1.1 derives serde for `Vec<u8>` fields in `SSH_FXP_WRITE` and `SSH_FXP_DATA`. With the crate's custom deserializer, that routes through `deserialize_seq` and reads payload bytes one at a time. Large transfers spend substantial CPU in serde's generic `VecVisitor` path.
+The fork adds two helpers to `client::fs::File` that keep many SFTP requests in flight at once, hiding per-request round-trip latency (mirroring how OpenSSH's `sftp` client keeps ~64 requests outstanding):
 
-## The Fix
+- `File::write_all_pipelined(reader, max_inflight)` — streams a reader to the remote file with up to `max_inflight` concurrent `SSH_FXP_WRITE`s.
+- `File::read_to_writer_pipelined(writer, max_inflight)` — streams the remote file to a writer with up to `max_inflight` concurrent `SSH_FXP_READ`s, reassembling chunks in offset order so the output matches a sequential read.
 
-The fork annotates the binary payload fields with `#[serde(with = "serde_bytes")]` and implements compatible `serialize_bytes` framing in the SFTP serializer. The wire format remains `u32 length + bytes`, but deserialization uses the existing bulk byte-buffer path.
+These are the only additions over upstream. They live in `src/client/fs/file.rs` and are re-applied on each sync from `patches/pipelined-file-io.patch`.
+
+> The `serde_bytes` packet-serialization performance fix that originally motivated this fork was upstreamed in russh-sftp 2.1.2; it is kept for reference under `patches/historical/`.
+
+## Usage
+
+```toml
+[dependencies]
+russh-sftp = { package = "bssh-russh-sftp", version = "2.3.0" }
+```
 
 ## Sync with Upstream
 
 ```bash
 cd crates/bssh-russh-sftp
-./sync-upstream.sh 2.1.1
+./sync-upstream.sh 2.3.0   # omit the version to use upstream's default branch
 ```
 
-Local changes are kept as patch files under `patches/`.
+`sync-upstream.sh` copies upstream `src` verbatim and re-applies every patch directly under `patches/` (anything under `patches/historical/` is excluded). Patches already merged upstream are detected via reverse-apply and skipped.
 
 ## License
 

crates/bssh-russh-sftp/README.md

@@ -1,44 +1,66 @@
 #!/bin/bash
 # create-patch.sh
-# Creates a patch file from the current bssh-russh-sftp changes compared to upstream russh-sftp.
+# Regenerates patches/pipelined-file-io.patch by diffing the current vendored
+# source against a fresh checkout of upstream russh-sftp.
 #
-# Usage: ./create-patch.sh
+# Self-contained: clones upstream into a temp dir (no manually-maintained
+# references/ directory needed), so it always diffs against the exact version.
+#
+# Usage: ./create-patch.sh [version]
+#   version: optional, e.g. "2.3.0" (default: upstream's default branch, since
+#            russh-sftp does not publish git tags)
 
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-BSSH_ROOT="$SCRIPT_DIR/../.."
-UPSTREAM_DIR="$BSSH_ROOT/references/russh-sftp/src"
-CURRENT_DIR="$SCRIPT_DIR/src"
+UPSTREAM_URL="https://github.com/AspectUnk/russh-sftp.git"
+TEMP_DIR="/tmp/russh-sftp-createpatch-$$"
 PATCH_DIR="$SCRIPT_DIR/patches"
-PATCH_FILE="$PATCH_DIR/sftp-serde-bytes-perf.patch"
+PATCH_FILE="$PATCH_DIR/pipelined-file-io.patch"
 
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 NC='\033[0m'
-
 log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
 log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
 
-if [ ! -d "$UPSTREAM_DIR" ]; then
-    echo "Error: Upstream russh-sftp not found at $UPSTREAM_DIR"
-    echo "Please ensure references/russh-sftp exists with the upstream source."
-    exit 1
+cleanup() { [ -d "$TEMP_DIR" ] && rm -rf "$TEMP_DIR"; }
+trap cleanup EXIT
+
+VERSION="${1:-}"
+
+log_info "Cloning upstream russh-sftp..."
+git clone --quiet "$UPSTREAM_URL" "$TEMP_DIR"
+cd "$TEMP_DIR"
+
+if [ -z "$VERSION" ]; then
+    VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "master")
+fi
+if [ "$VERSION" != "master" ]; then
+    # russh-sftp publishes no git tags, so a version string may not be a ref.
+    if ! { git checkout --quiet "v$VERSION" 2>/dev/null || git checkout --quiet "$VERSION" 2>/dev/null; }; then
+        log_warn "No git ref '$VERSION' (russh-sftp publishes no tags); diffing against the default branch."
+        VERSION="master"
+    fi
 fi
+log_info "Diffing against upstream $VERSION ($(git rev-parse --short HEAD))"
 
+UPSTREAM_SRC="$TEMP_DIR/src"
+CURRENT_SRC="$SCRIPT_DIR/src"
 mkdir -p "$PATCH_DIR"
 
-log_info "Creating patch from differences..."
-
-/usr/bin/diff -urN "$UPSTREAM_DIR" "$CURRENT_DIR" \
-    | sed "s|$UPSTREAM_DIR|a/src|g" \
-    | sed "s|$CURRENT_DIR|b/src|g" \
+# The only fork change is the pipelined File I/O in client/fs/file.rs
+# (write_all_pipelined / read_to_writer_pipelined). Emit a -p1 patch.
+diff -u \
+    --label a/src/client/fs/file.rs \
+    --label b/src/client/fs/file.rs \
+    "$UPSTREAM_SRC/client/fs/file.rs" \
+    "$CURRENT_SRC/client/fs/file.rs" \
     > "$PATCH_FILE" || true
 
 if [ -s "$PATCH_FILE" ]; then
     LINES=$(wc -l < "$PATCH_FILE" | tr -d ' ')
     log_info "Patch created: $PATCH_FILE ($LINES lines)"
-
     echo ""
     echo "Patch summary:"
     echo "=============="