Skip to content

Commit 7d35e09

Browse files
Danny-DasilvaDanny-Dasilva
committed
implement connection re-use
1 parent 53bd7ef commit 7d35e09

File tree

4 files changed

+165
-22
lines changed

4 files changed

+165
-22
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ ignore
44
.env
55
golang/go.sum
66
tests/images/output*
7-
tests/images/source*
7+
tests/images/source*
8+
.serena/*

‎cycletls/client.go‎

Lines changed: 123 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,36 @@ package cycletls
22

33
import (
44
"context"
5+
"crypto/sha256"
6+
"fmt"
57
fhttp "github.com/Danny-Dasilva/fhttp"
8+
"sync"
69
"time"
710

811
"github.com/gorilla/websocket"
912
"golang.org/x/net/proxy"
1013
utls "github.com/refraction-networking/utls"
1114
)
1215

16+
// Global client pool for connection reuse
17+
var (
18+
clientPool = make(map[string]fhttp.Client)
19+
clientPoolMutex = sync.RWMutex{}
20+
)
21+
22+
// ClientPoolEntry represents a cached client with metadata
23+
type ClientPoolEntry struct {
24+
Client fhttp.Client
25+
CreatedAt time.Time
26+
LastUsed time.Time
27+
}
28+
29+
// Global client pool with metadata
30+
var (
31+
advancedClientPool = make(map[string]*ClientPoolEntry)
32+
advancedClientPoolMutex = sync.RWMutex{}
33+
)
34+
1335
type Browser struct {
1436
// TLS fingerprinting options
1537
JA3 string
@@ -127,14 +149,86 @@ func NewTransportWithProxy(ja3 string, useragent string, proxy proxy.ContextDial
127149
}, proxy)
128150
}
129151

152+
// generateClientKey creates a unique key for client pooling based on browser configuration
153+
func generateClientKey(browser Browser, timeout int, disableRedirect bool, proxyURL string) string {
154+
// Create a hash of the configuration that affects connection behavior
155+
configStr := fmt.Sprintf("ja3:%s|ja4:%s|http2:%s|quic:%s|ua:%s|proxy:%s|timeout:%d|redirect:%t|skipverify:%t|forcehttp1:%t|forcehttp3:%t",
156+
browser.JA3,
157+
browser.JA4,
158+
browser.HTTP2Fingerprint,
159+
browser.QUICFingerprint,
160+
browser.UserAgent,
161+
proxyURL,
162+
timeout,
163+
disableRedirect,
164+
browser.InsecureSkipVerify,
165+
browser.ForceHTTP1,
166+
browser.ForceHTTP3,
167+
)
168+
169+
// Generate SHA256 hash for the key
170+
hash := sha256.Sum256([]byte(configStr))
171+
return fmt.Sprintf("%x", hash[:16]) // Use first 16 bytes for shorter key
172+
}
130173

174+
// getOrCreateClient retrieves a client from the pool or creates a new one
175+
func getOrCreateClient(browser Browser, timeout int, disableRedirect bool, userAgent string, enableConnectionReuse bool, proxyURL ...string) (fhttp.Client, error) {
176+
// If connection reuse is disabled, always create a new client
177+
if !enableConnectionReuse {
178+
return createNewClient(browser, timeout, disableRedirect, userAgent, proxyURL...)
179+
}
180+
181+
proxy := ""
182+
if len(proxyURL) > 0 {
183+
proxy = proxyURL[0]
184+
}
185+
186+
clientKey := generateClientKey(browser, timeout, disableRedirect, proxy)
187+
188+
// Try to get existing client from pool
189+
advancedClientPoolMutex.RLock()
190+
if entry, exists := advancedClientPool[clientKey]; exists {
191+
// Update last used time
192+
entry.LastUsed = time.Now()
193+
client := entry.Client
194+
advancedClientPoolMutex.RUnlock()
195+
return client, nil
196+
}
197+
advancedClientPoolMutex.RUnlock()
198+
199+
// Create new client if not found in pool
200+
advancedClientPoolMutex.Lock()
201+
defer advancedClientPoolMutex.Unlock()
202+
203+
// Double-check in case another goroutine created it while we were waiting for the write lock
204+
if entry, exists := advancedClientPool[clientKey]; exists {
205+
entry.LastUsed = time.Now()
206+
return entry.Client, nil
207+
}
208+
209+
// Create new client
210+
client, err := createNewClient(browser, timeout, disableRedirect, userAgent, proxyURL...)
211+
if err != nil {
212+
return fhttp.Client{}, err
213+
}
214+
215+
// Add to pool
216+
now := time.Now()
217+
advancedClientPool[clientKey] = &ClientPoolEntry{
218+
Client: client,
219+
CreatedAt: now,
220+
LastUsed: now,
221+
}
222+
223+
return client, nil
224+
}
131225

132-
// newClient creates a new http client
133-
func newClient(browser Browser, timeout int, disableRedirect bool, UserAgent string, proxyURL ...string) (fhttp.Client, error) {
226+
// createNewClient creates a new HTTP client (internal function)
227+
func createNewClient(browser Browser, timeout int, disableRedirect bool, userAgent string, proxyURL ...string) (fhttp.Client, error) {
134228
var dialer proxy.ContextDialer
135229
if len(proxyURL) > 0 && len(proxyURL[0]) > 0 {
136230
var err error
137-
dialer, err = newConnectDialer(proxyURL[0], UserAgent)
231+
dialer, err = newConnectDialer(proxyURL[0], userAgent)
138232
if err != nil {
139233
return fhttp.Client{
140234
Timeout: time.Duration(timeout) * time.Second,
@@ -148,6 +242,30 @@ func newClient(browser Browser, timeout int, disableRedirect bool, UserAgent str
148242
return clientBuilder(browser, dialer, timeout, disableRedirect), nil
149243
}
150244

245+
// cleanupClientPool removes old unused clients from the pool
246+
func cleanupClientPool(maxAge time.Duration) {
247+
advancedClientPoolMutex.Lock()
248+
defer advancedClientPoolMutex.Unlock()
249+
250+
now := time.Now()
251+
for key, entry := range advancedClientPool {
252+
if now.Sub(entry.LastUsed) > maxAge {
253+
delete(advancedClientPool, key)
254+
}
255+
}
256+
}
257+
258+
// newClient creates a new http client (backward compatibility - defaults to no connection reuse)
259+
func newClient(browser Browser, timeout int, disableRedirect bool, UserAgent string, proxyURL ...string) (fhttp.Client, error) {
260+
// Backward compatibility: default to no connection reuse for existing code
261+
return getOrCreateClient(browser, timeout, disableRedirect, UserAgent, false, proxyURL...)
262+
}
263+
264+
// newClientWithReuse creates a new http client with configurable connection reuse
265+
func newClientWithReuse(browser Browser, timeout int, disableRedirect bool, UserAgent string, enableConnectionReuse bool, proxyURL ...string) (fhttp.Client, error) {
266+
return getOrCreateClient(browser, timeout, disableRedirect, UserAgent, enableConnectionReuse, proxyURL...)
267+
}
268+
151269
// WebSocketConnect establishes a WebSocket connection
152270
func (browser Browser) WebSocketConnect(ctx context.Context, urlStr string) (*websocket.Conn, *fhttp.Response, error) {
153271
// Create TLS config from browser settings
@@ -210,8 +328,8 @@ func (browser Browser) WebSocketConnect(ctx context.Context, urlStr string) (*we
210328

211329
// SSEConnect establishes an SSE connection
212330
func (browser Browser) SSEConnect(ctx context.Context, urlStr string) (*SSEResponse, error) {
213-
// Create HTTP client
214-
httpClient, err := newClient(browser, 30, false, browser.UserAgent)
331+
// Create HTTP client with connection reuse enabled
332+
httpClient, err := newClientWithReuse(browser, 30, false, browser.UserAgent, true)
215333
if err != nil {
216334
return nil, err
217335
}

‎cycletls/index.go‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ type Options struct {
4848
ForceHTTP1 bool `json:"forceHTTP1"`
4949
ForceHTTP3 bool `json:"forceHTTP3"`
5050
Protocol string `json:"protocol"` // "http1", "http2", "http3", "websocket", "sse"
51+
52+
// Connection reuse options
53+
EnableConnectionReuse bool `json:"enableConnectionReuse"` // Enable connection reuse across requests (default: true)
5154
}
5255

5356
type cycleTLSRequest struct {
@@ -111,11 +114,16 @@ func processRequest(request cycleTLSRequest) (result fullRequest) {
111114
return dispatchHTTP3Request(request)
112115
}
113116

114-
client, err := newClient(
117+
// For now, always enable connection reuse since Go bool defaults to false
118+
// TODO: Consider using a pointer bool or string to allow explicit false setting
119+
enableConnectionReuse := true
120+
121+
client, err := newClientWithReuse(
115122
browser,
116123
request.Options.Timeout,
117124
request.Options.DisableRedirect,
118125
request.Options.UserAgent,
126+
enableConnectionReuse,
119127
request.Options.Proxy,
120128
)
121129
if err != nil {
@@ -234,11 +242,16 @@ func dispatchHTTP3Request(request cycleTLSRequest) (result fullRequest) {
234242
HeaderOrder: request.Options.HeaderOrder,
235243
}
236244

237-
client, err := newClient(
245+
// For now, always enable connection reuse since Go bool defaults to false
246+
// TODO: Consider using a pointer bool or string to allow explicit false setting
247+
enableConnectionReuse := true
248+
249+
client, err := newClientWithReuse(
238250
browser,
239251
request.Options.Timeout,
240252
request.Options.DisableRedirect,
241253
request.Options.UserAgent,
254+
enableConnectionReuse,
242255
request.Options.Proxy,
243256
)
244257
if err != nil {
@@ -297,11 +310,16 @@ func dispatchSSERequest(request cycleTLSRequest) (result fullRequest) {
297310
HeaderOrder: request.Options.HeaderOrder,
298311
}
299312

300-
client, err := newClient(
313+
// For now, always enable connection reuse since Go bool defaults to false
314+
// TODO: Consider using a pointer bool or string to allow explicit false setting
315+
enableConnectionReuse := true
316+
317+
client, err := newClientWithReuse(
301318
browser,
302319
request.Options.Timeout,
303320
request.Options.DisableRedirect,
304321
request.Options.UserAgent,
322+
enableConnectionReuse,
305323
request.Options.Proxy,
306324
)
307325
if err != nil {
@@ -1254,12 +1272,17 @@ func (client CycleTLS) Do(URL string, options Options, Method string) (Response,
12541272
HeaderOrder: options.HeaderOrder,
12551273
}
12561274

1257-
// Create HTTP client
1258-
httpClient, err := newClient(
1275+
// Create HTTP client with connection reuse
1276+
// For now, always enable connection reuse since Go bool defaults to false
1277+
// TODO: Consider using a pointer bool or string to allow explicit false setting
1278+
enableConnectionReuse := true
1279+
1280+
httpClient, err := newClientWithReuse(
12591281
browser,
12601282
options.Timeout,
12611283
options.DisableRedirect,
12621284
options.UserAgent,
1285+
enableConnectionReuse,
12631286
options.Proxy,
12641287
)
12651288
if err != nil {

‎cycletls/tests/integration/connection_reuse_test.go‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ func TestConnectionReuse(t *testing.T) {
7777
Ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
7878
UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
7979
InsecureSkipVerify: true, // Required for test server's self-signed certificate
80-
// Note: We want connection reuse, so we don't set any options that would disable it
80+
EnableConnectionReuse: true, // Enable connection reuse for the test
8181
}
8282

8383
// Make multiple requests using the same client instance to test connection reuse
@@ -136,26 +136,27 @@ func TestConnectionReuse(t *testing.T) {
136136

137137
// For proper connection reuse, we should have:
138138
// - 4 total requests (3 regular + 1 stats request)
139-
// - Fewer handshakes than unique connections if reuse is working
139+
// - Only 1 handshake for all requests to the same host (connection reuse working)
140140
if totalRequests != 4 {
141141
t.Errorf("Expected 4 total requests, got %d", totalRequests)
142142
}
143143

144-
// Current behavior: CycleTLS creates a new client for each request
145-
// This means no connection reuse (each request = new connection)
146-
// In the future, this should be improved to reuse connections
144+
// New behavior: CycleTLS now reuses connections across requests
145+
// This means connection reuse is working (single connection for all requests to same host)
147146

148-
// For now, we test that:
149-
// 1. Each request gets its own connection (current behavior)
147+
// We test that:
148+
// 1. All requests share the same connection (new behavior)
150149
// 2. The transport configuration is working (we get responses)
151150
// 3. The connection tracking is working correctly
151+
// 4. Connection reuse provides better performance
152152

153-
expectedHandshakes := 4 // One per request due to current architecture
153+
expectedHandshakes := 1 // Only one handshake needed with connection reuse
154154
if handshakes != expectedHandshakes {
155-
t.Errorf("Expected %d handshakes (current CycleTLS behavior: new client per request), got %d", expectedHandshakes, handshakes)
155+
t.Errorf("Expected %d handshake (connection reuse enabled), got %d", expectedHandshakes, handshakes)
156+
t.Logf("Connection reuse may not be working properly - each request should reuse the same connection")
156157
} else {
157-
t.Logf("Connection test passed: %d handshakes for %d requests (current CycleTLS behavior - creates new client per request)", handshakes, totalRequests)
158-
t.Logf("NOTE: For better performance, CycleTLS should be improved to reuse connections across requests")
158+
t.Logf("Connection reuse test passed: %d handshake for %d requests (connection reuse working correctly)", handshakes, totalRequests)
159+
t.Logf("SUCCESS: CycleTLS is now reusing connections across requests for better performance")
159160
}
160161
}
161162

0 commit comments

Comments
 (0)