Better failure logging and TLS failure tests

Author: CVanF5

README.md

@@ -157,9 +157,10 @@ For local development and testing without Docker:
 Troubleshooting
 ---------------
 - If EPP endpoints are unreachable or not listening on gRPC, you may see `BAD_GATEWAY` when failure mode allow is off. Toggle `*_failure_mode_allow on` to fail-open during testing.
+- **Enhanced TLS Error Logging**: The module now provides detailed TLS certificate validation error messages (e.g., "invalid peer certificate: UnknownIssuer") instead of generic transport errors. Check error logs for specific TLS issues like unknown issuers or certificate validation failures.
 - Ensure your EPP implementation is configured to return a header mutation for the upstream endpoint. The module will parse response frames and search for `header_mutation` entries.
 - BBR processes JSON directly in the module - ensure request bodies contain valid JSON with a "model" field.
-- Use `error_log` and debug logging to verify module activation. BBR logs body reading and size limit enforcement; EPP logs gRPC errors. Set `error_log` to `debug` to observe processing details.
+- Use `error_log` and debug logging to verify module activation. BBR logs body reading and size limit enforcement; EPP logs gRPC errors with detailed TLS diagnostics. Set `error_log` to `debug` to observe processing details.
 
 Roadmap
 -------
@@ -168,6 +169,8 @@ Roadmap
 - Validate large body handling and back-pressure for BBR; refine chunked reads/writes and resource usage for very large payloads.
 - Connection pooling and caching for improved performance at scale.
 - Enhanced TLS configuration options (client certificates, cipher suites, etc.).
+- ✅ **Completed**: Enhanced TLS error logging with detailed certificate validation messages
+- ✅ **Completed**: Streamlined test infrastructure with consistent vLLM backend usage
 
 License
 -------

src/grpc.rs

@@ -28,6 +28,26 @@ macro_rules! ngx_log_error_http {
     };
 }
 
+/// Extract detailed error information from transport errors
+fn extract_error_details(error: &tonic::transport::Error) -> String {
+    // Try to get the root cause error
+    let mut current_error: &dyn std::error::Error = error;
+    let mut error_chain = Vec::new();
+
+    // Walk the error chain to find the root cause
+    while let Some(source) = current_error.source() {
+        error_chain.push(source.to_string());
+        current_error = source;
+    }
+
+    // If we found a chain, use the most specific error
+    if !error_chain.is_empty() {
+        error_chain.last().unwrap().clone()
+    } else {
+        error.to_string()
+    }
+}
+
 static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
 
 fn get_runtime() -> &'static tokio::runtime::Runtime {
@@ -441,17 +461,18 @@ pub fn epp_headers_blocking(
                 let connect_result = tls_result.connect().await;
 
                 connect_result.map_err(|e| {
+                    let detailed_error = extract_error_details(&e);
                     format!(
-                        "connect error (endpoint: {}, domain: {}): {e}",
-                        endpoint_copy, domain
+                        "TLS connection failed (endpoint: {}, domain: {}): {}",
+                        endpoint_copy, domain, detailed_error
                     )
                 })?
             } else {
                 // PLAINTEXT MODE: No TLS configuration
-                channel_builder
-                    .connect()
-                    .await
-                    .map_err(|e| format!("connect error: {e}"))?
+                channel_builder.connect().await.map_err(|e| {
+                    let detailed_error = extract_error_details(&e);
+                    format!("HTTP connection failed: {}", detailed_error)
+                })?
             };
 
             let mut client = ExternalProcessorClient::new(channel);
@@ -681,17 +702,18 @@ pub fn epp_headers_async<F>(
                     .map_err(|e| format!("tls config error: {e}"))?;
 
                 tls_result.connect().await.map_err(|e| {
+                    let detailed_error = extract_error_details(&e);
                     format!(
-                        "connect error (endpoint: {}, domain: {}): {e}",
-                        endpoint, domain
+                        "TLS connection failed (endpoint: {}, domain: {}): {}",
+                        endpoint, domain, detailed_error
                     )
                 })?
             } else {
                 // No TLS
-                channel_builder
-                    .connect()
-                    .await
-                    .map_err(|e| format!("connect error: {e}"))?
+                channel_builder.connect().await.map_err(|e| {
+                    let detailed_error = extract_error_details(&e);
+                    format!("HTTP connection failed: {}", detailed_error)
+                })?
             };
 
             let mut client = ExternalProcessorClient::new(channel);
@@ -874,17 +896,18 @@ pub async fn epp_headers_blocking_internal(
             .map_err(|e| format!("tls config error: {e}"))?;
 
         tls_result.connect().await.map_err(|e| {
+            let detailed_error = extract_error_details(&e);
             format!(
-                "connect error (endpoint: {}, domain: {}): {e}",
-                endpoint, domain
+                "TLS connection failed (endpoint: {}, domain: {}): {}",
+                endpoint, domain, detailed_error
             )
         })?
     } else {
         // No TLS
-        channel_builder
-            .connect()
-            .await
-            .map_err(|e| format!("connect error: {e}"))?
+        channel_builder.connect().await.map_err(|e| {
+            let detailed_error = extract_error_details(&e);
+            format!("HTTP connection failed: {}", detailed_error)
+        })?
     };
 
     let mut client = ExternalProcessorClient::new(channel);

src/lib.rs

@@ -688,8 +688,8 @@ http_request_handler!(inference_access_handler, |request: &mut http::Request| {
                 // Otherwise continue processing
             }
             core::Status::NGX_ERROR => {
-                // Other BBR error - always return 502 Bad Gateway
-                return http::HTTPStatus::BAD_GATEWAY.into();
+                // Other BBR error - return 500 Internal Server Error
+                return http::HTTPStatus::INTERNAL_SERVER_ERROR.into();
             }
             _ => {
                 // Continue processing
@@ -710,7 +710,7 @@ http_request_handler!(inference_access_handler, |request: &mut http::Request| {
                 unsafe {
                     let msg = b"ngx-inference: EPP module processing failed internally\0";
                     ngx::ffi::ngx_log_error_core(
-                        ngx::ffi::NGX_LOG_INFO as ngx::ffi::ngx_uint_t,
+                        ngx::ffi::NGX_LOG_ERR as ngx::ffi::ngx_uint_t,
                         request.as_mut().connection.as_ref().unwrap().log,
                         0,
                         cstr_ptr(msg.as_ptr()),
@@ -725,10 +725,10 @@ http_request_handler!(inference_access_handler, |request: &mut http::Request| {
                             (*(*r_ptr).connection).log,
                             0,
                             #[allow(clippy::manual_c_str_literals)] // FFI code
-                            cstr_ptr(b"ngx-inference: Module returning HTTP 502 due to EPP processing failure (fail-closed mode)\0".as_ptr()),
+                            cstr_ptr(b"ngx-inference: Module returning HTTP 500 due to EPP processing failure (fail-closed mode)\0".as_ptr()),
                         );
                     }
-                    return http::HTTPStatus::BAD_GATEWAY.into();
+                    return http::HTTPStatus::INTERNAL_SERVER_ERROR.into();
                 }
             }
             _ => {

src/modules/epp.rs

@@ -213,12 +213,7 @@ impl EppProcessor {
                     }
                 }
             }
-            Err(err) => {
-                ngx_log_info_http!(
-                    request,
-                    "ngx-inference: EPP external service error: {}",
-                    err
-                );
+            Err(_err) => {
                 return Err("epp grpc error");
             }
         }

tests/README.md

@@ -9,10 +9,11 @@ This directory contains test scripts and utilities for validating the ngx-infere
 Main test runner that validates BBR (Body-Based Routing) and EPP (External Processing Pipeline) module configurations. This script supports three testing environments:
 
 **Configuration Test Matrix:**
-- **BBR ON + EPP OFF**: Tests model extraction only
-- **BBR OFF + EPP ON**: Tests upstream selection only  
-- **BBR ON + EPP ON**: Tests both modules active
-- **BBR OFF + EPP OFF**: Tests no processing (baseline)
+- **BBR ON + EPP OFF**: Tests model extraction with vLLM backend
+- **BBR OFF + EPP ON**: Tests upstream selection with EPP routing to vLLM
+- **BBR ON + EPP ON**: Tests both modules active with vLLM responses
+- **BBR OFF + EPP OFF**: Tests direct vLLM routing (no processing)
+- **EPP Untrusted TLS**: Tests TLS certificate validation with enhanced error logging
 
 **Execution Modes:**
 - **Local Mode**: Uses locally compiled nginx module with local backend services
@@ -39,7 +40,7 @@ DOCKER_ENVIRONMENT=main ./tests/test-config.sh  # Docker mode
 Main Docker Compose configuration providing the test environment:
 
 - **nginx**: Main web server with ngx-inference module
-- **echo-server**: Node.js service for request inspection and header validation
+- **echo-server**: Node.js service for request inspection and header validation (Docker/local testing only)
 - **mock-extproc**: Mock gRPC External Processing service for EPP module testing
 
 ### Kind Testing Environment (`kind-ngf/`)

tests/configs/bbr_off_epp_off.conf

@@ -4,11 +4,11 @@ server {
     server_name localhost;
 
     # Global default upstream for all inference failures
-    inference_default_upstream "127.0.0.1:8080";
+    inference_default_upstream "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
 
     # vLLM Chat Completions API (both BBR and EPP disabled)
     location /v1/chat/completions {
-        set $backend "echo-server.ngx-inference-test.svc.cluster.local:80";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -17,7 +17,7 @@ server {
 
     # vLLM Models API (both BBR and EPP disabled)
     location /v1/models {
-        set $backend "echo-server.ngx-inference-test.svc.cluster.local:80";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -27,7 +27,7 @@ server {
     # Both BBR and EPP disabled
     location /bbr-test {
         # BBR disabled - just proxy without BBR processing
-        set $backend "127.0.0.1:8080";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -36,7 +36,7 @@ server {
 
     location /epp-test {
         # EPP disabled - just proxy without EPP processing
-        set $backend "127.0.0.1:8080";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -45,7 +45,7 @@ server {
 
     location /responses {
         # Both BBR and EPP disabled
-        set $backend "127.0.0.1:8080";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -58,7 +58,7 @@ server {
     }
 
     location / {
-        set $backend "127.0.0.1:8080";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

tests/configs/bbr_off_epp_on.conf

@@ -4,7 +4,7 @@ server {
     server_name localhost;
 
     # Global default upstream for all inference failures
-    inference_default_upstream "127.0.0.1:8080";
+    inference_default_upstream "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
 
     # vLLM Chat Completions API with EPP
     location /v1/chat/completions {
@@ -39,7 +39,7 @@ server {
     # BBR disabled, EPP enabled
     location /bbr-test {
         # BBR disabled - just proxy without BBR processing
-        set $backend "127.0.0.1:8080";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -81,7 +81,7 @@ server {
     }
 
     location / {
-        set $backend "127.0.0.1:8080";
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

tests/configs/bbr_off_epp_on_untrusted_tls_allow.conf

@@ -0,0 +1,85 @@
+server {
+    listen 8081;
+    listen 80;
+    server_name localhost;
+
+    # Global default upstream for all inference failures - use vLLM server for untrusted TLS test
+    inference_default_upstream "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
+
+    # vLLM Chat Completions API with EPP - Untrusted TLS with failure mode allow
+    location /v1/chat/completions {
+        inference_epp on;
+        inference_epp_endpoint "vllm-llama3-8b-instruct-epp:9002";
+        inference_epp_timeout_ms 5000;
+        inference_epp_failure_mode_allow on;
+        inference_epp_tls on;
+        # Note: No inference_epp_ca_file directive - causes untrusted TLS
+        inference_epp_header_name "x-gateway-destination-endpoint";
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://$inference_upstream;
+    }
+
+    # vLLM Models API with EPP - Untrusted TLS with failure mode allow
+    location /v1/models {
+        inference_epp on;
+        inference_epp_endpoint "vllm-llama3-8b-instruct-epp:9002";
+        inference_epp_timeout_ms 5000;
+        inference_epp_failure_mode_allow on;
+        inference_epp_tls on;
+        # Note: No inference_epp_ca_file directive - causes untrusted TLS
+        inference_epp_header_name "x-gateway-destination-endpoint";
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://$inference_upstream;
+    }
+
+    # EPP test endpoint with untrusted TLS and failure mode allow
+    location /epp-test {
+        inference_epp on;
+        inference_epp_endpoint "vllm-llama3-8b-instruct-epp:9002";
+        inference_epp_timeout_ms 5000;
+        inference_epp_failure_mode_allow on;
+        inference_epp_tls on;
+        # Note: No inference_epp_ca_file directive - causes untrusted TLS
+        inference_epp_header_name "x-gateway-destination-endpoint";
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://$inference_upstream;
+    }
+
+    location /responses {
+        # EPP enabled with untrusted TLS and failure mode allow
+        inference_epp on;
+        inference_epp_endpoint "vllm-llama3-8b-instruct-epp:9002";
+        inference_epp_timeout_ms 5000;
+        inference_epp_failure_mode_allow on;
+        inference_epp_tls on;
+        # Note: No inference_epp_ca_file directive - causes untrusted TLS
+        inference_epp_header_name "x-gateway-destination-endpoint";
+
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://$inference_upstream;
+    }
+
+    location /health {
+        return 200 "OK\n";
+        add_header Content-Type text/plain;
+    }
+
+    location / {
+        set $backend "vllm-llama3-8b-instruct.ngx-inference-test.svc.cluster.local:8000";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_pass http://$backend;
+    }
+}