Skip to content

Commit 4e36cf5

Browse files
xerialclaude
xerial
and
claude
authored
feature: Add HTTP client implementation for Scala Native using libcurl (#4143)
This commit implements HTTP client support for Scala Native by leveraging libcurl bindings. The implementation provides: - CurlBindings: Low-level Scala Native bindings for libcurl including easy interface functions, error codes, and options - CurlChannel: HttpChannel implementation using libcurl - NativeHttpClientBackend: Backend factory for creating HTTP clients - LocalRPCContext: Thread-local RPC context management for Native - Updated Compat.scala with real implementations Requirements: - libcurl 7.56.0 or newer must be installed on the system - For HTTPS: libcurl must be built with SSL support (OpenSSL, etc.) Supported features: - All standard HTTP methods (GET, POST, PUT, DELETE, PATCH, etc.) - Request/response headers and bodies - Connect and read timeouts - Automatic redirect following - Automatic decompression (gzip, deflate) - Both sync and async clients (async wraps sync via Rx) --------- Co-authored-by: Claude <[email protected]>
1 parent c08eb81 commit 4e36cf5

File tree

12 files changed

+863
-75
lines changed

12 files changed

+863
-75
lines changed

‎.github/workflows/release-native.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ jobs:
2020
with:
2121
distribution: 'zulu'
2222
java-version: '17'
23+
- name: Install native dependencies
24+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
2325
- name: Setup GPG
2426
env:
2527
PGP_SECRET: ${{ secrets.PGP_SECRET }}

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,8 @@ jobs:
267267
with:
268268
distribution: 'zulu'
269269
java-version: '21'
270+
- name: Install native dependencies
271+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
270272
- name: Scala Native test
271273
run: JVM_OPTS=-Xmx4g ./sbt "++ 3; projectNative/test"
272274
- name: Publish Test Report
@@ -288,6 +290,8 @@ jobs:
288290
with:
289291
distribution: 'zulu'
290292
java-version: '21'
293+
- name: Install native dependencies
294+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
291295
- name: Scala JVM and Scala.js Test
292296
run: ../sbt "++airspecJVM/test; ++airspecJS/test"
293297
working-directory: ./airspec

‎airframe-http/.js/src/main/scala/wvlet/airframe/http/Compat.scala‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
package wvlet.airframe.http
1515

1616
import org.scalajs.dom.window
17+
import wvlet.airframe.control.ResultClass.Failed
1718
import wvlet.airframe.http.client.{HttpClientBackend, JSHttpClientBackend}
1819

1920
import scala.concurrent.ExecutionContext
@@ -63,4 +64,25 @@ private object Compat extends CompatApi {
6364
override def currentRPCContext: RPCContext = ???
6465
override def attachRPCContext(context: RPCContext): RPCContext = ???
6566
override def detachRPCContext(previous: RPCContext): Unit = ???
67+
68+
/**
69+
* SSL exception classifier for Scala.js. Returns an empty classifier since javax.net.ssl classes are not available
70+
* on JS.
71+
*/
72+
override def sslExceptionClassifier: PartialFunction[Throwable, Failed] = PartialFunction.empty
73+
74+
/**
75+
* Connection exception classifier for Scala.js. Returns an empty classifier since java.net exception classes are not
76+
* available on JS.
77+
*/
78+
override def connectionExceptionClassifier: PartialFunction[Throwable, Failed] = PartialFunction.empty
79+
80+
/**
81+
* Root cause exception classifier for Scala.js. Uses a simpler implementation without java.lang.reflect.
82+
*/
83+
override def rootCauseExceptionClassifier: PartialFunction[Throwable, Failed] = {
84+
case e if e.getCause != null =>
85+
// Trace the true cause
86+
HttpClientException.classifyExecutionFailureScalaJS(e.getCause)
87+
}
6688
}

‎airframe-http/.jvm/src/main/scala/wvlet/airframe/http/Compat.scala‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,18 @@
1212
* limitations under the License.
1313
*/
1414
package wvlet.airframe.http
15+
import wvlet.airframe.control.ResultClass.{Failed, nonRetryableFailure, retryableFailure}
1516
import wvlet.airframe.control.ThreadUtil
1617
import wvlet.airframe.http.client.{HttpClientBackend, JavaHttpClientBackend}
1718
import wvlet.airframe.http.internal.{LocalRPCContext, LogRotationHttpLogger}
1819

19-
import java.net.URLEncoder
20+
import java.io.IOException
21+
import java.lang.reflect.InvocationTargetException
22+
import java.net.*
23+
import java.nio.channels.ClosedChannelException
2024
import java.util.concurrent.atomic.AtomicInteger
21-
import java.util.concurrent.{Executors, ThreadFactory}
25+
import java.util.concurrent.{ExecutionException, Executors, ThreadFactory}
26+
import javax.net.ssl.{SSLException, SSLHandshakeException, SSLKeyException, SSLPeerUnverifiedException}
2227
import scala.concurrent.ExecutionContext
2328

2429
/**
@@ -66,4 +71,63 @@ object Compat extends CompatApi {
6671
override def currentRPCContext: RPCContext = LocalRPCContext.current
6772
override def attachRPCContext(context: RPCContext): RPCContext = LocalRPCContext.attach(context)
6873
override def detachRPCContext(previous: RPCContext): Unit = LocalRPCContext.detach(previous)
74+
75+
/**
76+
* SSL exception classifier for JVM platform. This handles SSL-specific exceptions that are only available on JVM.
77+
*/
78+
override def sslExceptionClassifier: PartialFunction[Throwable, Failed] = { case e: SSLException =>
79+
e match {
80+
// Deterministic SSL exceptions are not retryable
81+
case se: SSLHandshakeException => nonRetryableFailure(e)
82+
case se: SSLKeyException => nonRetryableFailure(e)
83+
case s3: SSLPeerUnverifiedException => nonRetryableFailure(e)
84+
case other =>
85+
// SSLProtocolException and uncategorized SSL exceptions (SSLException) such as unexpected_message may be retryable
86+
retryableFailure(e)
87+
}
88+
}
89+
90+
/**
91+
* Connection exception classifier for JVM platform. This handles java.net exceptions.
92+
*/
93+
override def connectionExceptionClassifier: PartialFunction[Throwable, Failed] = {
94+
// Other types of exception that can happen inside HTTP clients (e.g., Jetty)
95+
case e: java.lang.InterruptedException =>
96+
// Retryable when the http client thread execution is interrupted.
97+
retryableFailure(e)
98+
case e: ProtocolException => retryableFailure(e)
99+
case e: ConnectException => retryableFailure(e)
100+
case e: ClosedChannelException => retryableFailure(e)
101+
case e: SocketTimeoutException => retryableFailure(e)
102+
case e: SocketException =>
103+
e match {
104+
case se: BindException => retryableFailure(e)
105+
case se: ConnectException => retryableFailure(e)
106+
case se: NoRouteToHostException => retryableFailure(e)
107+
case se: PortUnreachableException => retryableFailure(e)
108+
case se if se.getMessage() == "Socket closed" => retryableFailure(e)
109+
case other =>
110+
nonRetryableFailure(e)
111+
}
112+
// HTTP/2 may disconnects the connection with "GOAWAY received" error https://github.com/wvlet/airframe/issues/3421
113+
// See also the code of jdk.internal.net.http.Http2Connection.handleGoAway
114+
case e: IOException if Option(e.getMessage()).exists(_.contains("GOAWAY received")) =>
115+
retryableFailure(e)
116+
// Exceptions from Finagle. Using the string class names so as not to include Finagle dependencies.
117+
case e: Throwable if HttpClientException.isRetryableFinagleException(e) =>
118+
retryableFailure(e)
119+
}
120+
121+
/**
122+
* Root cause exception classifier for JVM platform. This handles java.lang.reflect exceptions.
123+
*/
124+
override def rootCauseExceptionClassifier: PartialFunction[Throwable, Failed] = {
125+
case e: ExecutionException if e.getCause != null =>
126+
HttpClientException.classifyExecutionFailure(e.getCause)
127+
case e: InvocationTargetException =>
128+
HttpClientException.classifyExecutionFailure(e.getTargetException)
129+
case e if e.getCause != null =>
130+
// Trace the true cause
131+
HttpClientException.classifyExecutionFailure(e.getCause)
132+
}
69133
}

‎airframe-http/.native/src/main/scala/wvlet/airframe/http/Compat.scala‎

Lines changed: 113 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,123 @@
1313
*/
1414
package wvlet.airframe.http
1515

16-
import wvlet.airframe.http.client.HttpClientBackend
16+
import wvlet.airframe.control.ResultClass.Failed
17+
import wvlet.airframe.http.client.{HttpClientBackend, NativeHttpClientBackend}
18+
import wvlet.airframe.http.internal.LocalRPCContext
1719

1820
import scala.concurrent.ExecutionContext
1921

2022
/**
21-
* Scala Native specific implementation
23+
* Scala Native specific implementation of HTTP compatibility layer.
24+
*
25+
* This implementation uses libcurl for HTTP client operations and provides platform-specific utilities for URL
26+
* encoding, execution contexts, and RPC context management.
2227
*/
2328
private object Compat extends CompatApi {
24-
override def urlEncode(s: String): String = ???
25-
override def defaultHttpClientBackend: HttpClientBackend = ???
26-
27-
override def defaultExecutionContext: ExecutionContext = ???
28-
override def defaultHttpClientLoggerFactory: HttpLoggerConfig => HttpLogger = ???
29-
override def currentRPCContext: RPCContext = ???
30-
override def attachRPCContext(context: RPCContext): RPCContext = ???
31-
override def detachRPCContext(previous: RPCContext): Unit = ???
29+
30+
/**
31+
* URL encode a string using percent-encoding.
32+
*
33+
* This implementation manually encodes characters following RFC 3986, since Scala Native doesn't have direct access
34+
* to java.net.URLEncoder.
35+
*/
36+
override def urlEncode(s: String): String = {
37+
if (s == null || s.isEmpty) {
38+
s
39+
} else {
40+
val sb = new StringBuilder()
41+
val bytes = s.getBytes("UTF-8")
42+
for (b <- bytes) {
43+
val c = b.toChar
44+
if (isUnreserved(c)) {
45+
sb.append(c)
46+
} else {
47+
// Percent-encode the byte
48+
sb.append('%')
49+
sb.append(toHexChar((b >> 4) & 0x0f))
50+
sb.append(toHexChar(b & 0x0f))
51+
}
52+
}
53+
sb.toString()
54+
}
55+
}
56+
57+
/**
58+
* Check if a character is unreserved per RFC 3986. Unreserved characters don't need encoding.
59+
*/
60+
private def isUnreserved(c: Char): Boolean = {
61+
(c >= 'A' && c <= 'Z') ||
62+
(c >= 'a' && c <= 'z') ||
63+
(c >= '0' && c <= '9') ||
64+
c == '-' || c == '_' || c == '.' || c == '~'
65+
}
66+
67+
private def toHexChar(i: Int): Char = {
68+
if (i < 10) ('0' + i).toChar
69+
else ('A' + (i - 10)).toChar
70+
}
71+
72+
/**
73+
* Scala Native doesn't have a notion of host server address (unlike browsers in Scala.js).
74+
*/
75+
override def hostServerAddress: ServerAddress = ServerAddress.empty
76+
77+
/**
78+
* Return the libcurl-based HTTP client backend for Scala Native.
79+
*/
80+
override def defaultHttpClientBackend: HttpClientBackend = NativeHttpClientBackend
81+
82+
/**
83+
* Return the default execution context for Scala Native.
84+
*
85+
* Uses scala.concurrent.ExecutionContext.global which is available in Scala Native.
86+
*/
87+
override def defaultExecutionContext: ExecutionContext = {
88+
ExecutionContext.global
89+
}
90+
91+
/**
92+
* Return a factory for creating HTTP loggers.
93+
*
94+
* For Scala Native, we use ConsoleHttpLogger which outputs to debug logs.
95+
*/
96+
override def defaultHttpClientLoggerFactory: HttpLoggerConfig => HttpLogger = { (config: HttpLoggerConfig) =>
97+
new HttpLogger.ConsoleHttpLogger(config)
98+
}
99+
100+
/**
101+
* Get the current thread-local RPC context.
102+
*/
103+
override def currentRPCContext: RPCContext = LocalRPCContext.current
104+
105+
/**
106+
* Attach a new RPC context and return the previous context.
107+
*/
108+
override def attachRPCContext(context: RPCContext): RPCContext = LocalRPCContext.attach(context)
109+
110+
/**
111+
* Detach the current RPC context and restore the previous one.
112+
*/
113+
override def detachRPCContext(previous: RPCContext): Unit = LocalRPCContext.detach(previous)
114+
115+
/**
116+
* SSL exception classifier for Scala Native. Returns an empty classifier since javax.net.ssl classes are not
117+
* available on Native.
118+
*/
119+
override def sslExceptionClassifier: PartialFunction[Throwable, Failed] = PartialFunction.empty
120+
121+
/**
122+
* Connection exception classifier for Scala Native. Returns an empty classifier since java.net exception classes
123+
* may not be fully available on Native.
124+
*/
125+
override def connectionExceptionClassifier: PartialFunction[Throwable, Failed] = PartialFunction.empty
126+
127+
/**
128+
* Root cause exception classifier for Scala Native. Uses a simpler implementation without java.lang.reflect.
129+
*/
130+
override def rootCauseExceptionClassifier: PartialFunction[Throwable, Failed] = {
131+
case e if e.getCause != null =>
132+
// Trace the true cause
133+
HttpClientException.classifyExecutionFailure(e.getCause)
134+
}
32135
}

0 commit comments

Comments
 (0)