feat(volo): introduce shmipc as transport layer protocol (#636)

* feat(volo): introduce shmipc as transport layer protocol

* chore: do not enable shmipc on non-linux environment

---------

Signed-off-by: Yu Li <liyu.yukiteru@bytedance.com>

Author: yukiiiteru

.github/workflows/ci.yaml

@@ -111,7 +111,7 @@ jobs:
           toolchain: ${{matrix.rust}}
       - name: Run tests
         run: |
-          bash scripts/clippy-and-test.sh --no-test
+          bash scripts/clippy-and-test.sh --no-test --no-shmipc
 
   test-windows:
     runs-on: [self-hosted, Windows]
@@ -127,7 +127,7 @@ jobs:
           toolchain: ${{matrix.rust}}
       - name: Run tests
         run: |
-          bash scripts/clippy-and-test.sh --no-test
+          bash scripts/clippy-and-test.sh --no-test --no-shmipc
 
   test-cli:
     runs-on: [self-hosted, Linux, amd64]
@@ -144,6 +144,7 @@ jobs:
       - name: Cli tests
         run: |
           bash scripts/volo-cli-test.sh
+
   coverage:
     runs-on: [self-hosted, Linux, amd64]
     timeout-minutes: 40

Cargo.lock

@@ -34,6 +34,7 @@ metainfo = "0.7.14"
 
 ahash = "0.8"
 anyhow = "1"
+arc-swap = "1"
 async-broadcast = "0.7"
 async-stream = "0.3"
 base64 = "0.22"
@@ -140,6 +141,8 @@ tokio-native-tls = "0.3"
 tungstenite = "0.28"
 tokio-tungstenite = "0.28"
 
+shmipc = "0.1"
+
 [profile.release]
 opt-level = 3
 debug = false

Cargo.toml

@@ -4,6 +4,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+RUN_SHMIPC="yes"
+
 echo_command() {
 	echo "$@"
 
@@ -19,6 +21,13 @@ echo_command() {
 }
 
 run_clippy() {
+	echo_command cargo clippy -p volo -- --deny warnings
+	echo_command cargo clippy -p volo --no-default-features --features rustls-aws-lc-rs -- --deny warnings
+	echo_command cargo clippy -p volo --no-default-features --features rustls-ring -- --deny warnings
+	if [ "${RUN_SHMIPC}" = "yes" ]; then
+		echo_command cargo clippy -p volo --no-default-features --features shmipc -- --deny warnings
+		echo_command cargo clippy -p volo --no-default-features --features tls,shmipc -- --deny warnings
+	fi
 	echo_command cargo clippy -p volo-thrift --no-default-features -- --deny warnings
 	echo_command cargo clippy -p volo-thrift --no-default-features --features multiplex -- --deny warnings
 	echo_command cargo clippy -p volo-thrift --no-default-features --features unsafe-codec -- --deny warnings
@@ -33,9 +42,6 @@ run_clippy() {
 	echo_command cargo clippy -p volo-http --no-default-features --features server,http1,query,form,json,multipart,ws -- --deny warnings
 	echo_command cargo clippy -p volo-http --no-default-features --features server,http2,query,form,json,multipart,ws -- --deny warnings
 	echo_command cargo clippy -p volo-http --no-default-features --features full -- --deny warnings
-	echo_command cargo clippy -p volo -- --deny warnings
-	echo_command cargo clippy -p volo --no-default-features --features rustls-aws-lc-rs -- --deny warnings
-	echo_command cargo clippy -p volo --no-default-features --features rustls-ring -- --deny warnings
 	echo_command cargo clippy -p volo-build -- --deny warnings
 	echo_command cargo clippy -p volo-cli -- --deny warnings
 	echo_command cargo clippy -p volo-macros -- --deny warnings
@@ -69,6 +75,10 @@ main() {
 			RUN_TEST="no"
 			echo "info: unit tests will be ignored"
 			;;
+		--no-shmipc)
+			RUN_SHMIPC="no"
+			echo "info: shmipc will be ignored"
+			;;
 		esac
 	done
 

scripts/clippy-and-test.sh

@@ -234,6 +234,8 @@ fn build_uri(addr: Address, path: &str) -> hyper::Uri {
             .path_and_query(path)
             .build()
             .expect("fail to build unix uri"),
+        #[allow(unreachable_patterns)]
+        _ => unimplemented!("unsupported type of address"),
     }
 }
 

volo-grpc/src/transport/client.rs

@@ -323,6 +323,8 @@ impl From<Address> for Target {
             }
             #[cfg(target_family = "unix")]
             Address::Unix(uds) => Target::Local(uds),
+            #[allow(unreachable_patterns)]
+            _ => unimplemented!("unsupported type of address"),
         }
     }
 }

volo-http/src/client/target.rs

@@ -100,6 +100,8 @@ impl ClientError {
             Address::Ip(addr) => self.addr = Some(addr),
             #[cfg(target_family = "unix")]
             Address::Unix(_) => {}
+            #[allow(unreachable_patterns)]
+            _ => unimplemented!("unsupported type of address"),
         }
         self
     }

volo-http/src/error/client.rs

@@ -217,6 +217,8 @@ impl<S> ClientIpService<S> {
             Some(Address::Ip(socket_addr)) => Some(socket_addr.ip()),
             #[cfg(target_family = "unix")]
             Some(Address::Unix(_)) => None,
+            #[allow(unreachable_patterns)]
+            Some(_) => unimplemented!("unsupported type of address"),
             None => return ClientIp(None),
         };
 

volo-http/src/server/utils/client_ip.rs

@@ -34,7 +34,7 @@ use linkedbytes::LinkedBytes;
 use pilota::thrift::ThriftException;
 use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, Interest};
 use tracing::{trace, warn};
-use volo::{net::ready::AsyncReady, util::buf_reader::BufReader};
+use volo::{net::ext::AsyncExt, util::buf_reader::BufReader};
 
 use self::{framed::MakeFramedCodec, thrift::MakeThriftCodec, ttheader::MakeTTHeaderCodec};
 use super::{Decoder, Encoder, MakeCodec};
@@ -118,7 +118,7 @@ pub struct DefaultEncoder<E, W> {
     linked_bytes: LinkedBytes,
 }
 
-impl<E: ZeroCopyEncoder, W: AsyncWrite + AsyncReady + Unpin + Send + Sync + 'static> Encoder
+impl<E: ZeroCopyEncoder, W: AsyncWrite + AsyncExt + Unpin + Send + Sync + 'static> Encoder
     for DefaultEncoder<E, W>
 {
     #[inline]
@@ -203,7 +203,7 @@ pub struct DefaultDecoder<D, R> {
     reader: BufReader<R>,
 }
 
-impl<D: ZeroCopyDecoder, R: AsyncRead + AsyncReady + Unpin + Send + Sync + 'static> Decoder
+impl<D: ZeroCopyDecoder, R: AsyncRead + AsyncExt + Unpin + Send + Sync + 'static> Decoder
     for DefaultDecoder<D, R>
 {
     #[inline]
@@ -290,8 +290,8 @@ impl Default for DefaultMakeCodec<MakeTTHeaderCodec<MakeFramedCodec<MakeThriftCo
 impl<MkZC, R, W> MakeCodec<R, W> for DefaultMakeCodec<MkZC>
 where
     MkZC: MakeZeroCopyCodec,
-    R: AsyncRead + AsyncReady + Unpin + Send + Sync + 'static,
-    W: AsyncWrite + AsyncReady + Unpin + Send + Sync + 'static,
+    R: AsyncRead + AsyncExt + Unpin + Send + Sync + 'static,
+    W: AsyncWrite + AsyncExt + Unpin + Send + Sync + 'static,
 {
     type Encoder = DefaultEncoder<MkZC::Encoder, W>;
     type Decoder = DefaultDecoder<MkZC::Decoder, R>;

volo-thrift/src/codec/default/mod.rs

@@ -26,6 +26,7 @@ maintenance = { status = "actively-developed" }
 [dependencies]
 motore.workspace = true
 
+arc-swap.workspace = true
 async-broadcast.workspace = true
 dashmap.workspace = true
 faststr.workspace = true
@@ -58,6 +59,7 @@ webpki-roots = { workspace = true, optional = true }
 tokio-rustls = { workspace = true, optional = true }
 native-tls = { workspace = true, optional = true }
 tokio-native-tls = { workspace = true, optional = true }
+shmipc = { workspace = true, optional = true }
 
 [features]
 default = []
@@ -86,3 +88,5 @@ native-tls = [
     "native-tls/alpn",
 ]
 native-tls-vendored = ["native-tls", "tokio-native-tls/vendored"]
+
+shmipc = ["dep:shmipc"]