diff --git a/ci/kokoro/windows/builds/bazel.ps1 b/ci/kokoro/windows/builds/bazel.ps1 index 1bf16b73d0027..52caa3e39b475 100644 --- a/ci/kokoro/windows/builds/bazel.ps1 +++ b/ci/kokoro/windows/builds/bazel.ps1 @@ -40,14 +40,19 @@ $test_flags = $build_flags $test_flags += @("--test_output=errors", "--verbose_failures=true") Write-Host "`n$(Get-Date -Format o) Compiling and running unit tests" -bazelisk $common_flags test $test_flags --test_tag_filters=-integration-test ... +# See #15678 +$exclude_build_targets = @("-//google/cloud/bigtable:internal_query_plan_test", ` + "-//google/cloud/storage/tests:storage_include_test-default", ` + "-//google/cloud/storage/tests:storage_include_test-grpc-metadata", ` + "-//google/cloud/pubsub/samples:all") +bazelisk $common_flags test $test_flags --test_tag_filters=-integration-test ... -- $exclude_build_targets if ($LastExitCode) { Write-Host -ForegroundColor Red "bazel test failed with exit code ${LastExitCode}." Exit ${LastExitCode} } Write-Host "`n$(Get-Date -Format o) Compiling extra programs with bazel $common_flags build $build_flags ..." -bazelisk $common_flags build $build_flags ... +bazelisk $common_flags build $build_flags ... -- $exclude_build_targets if ($LastExitCode) { Write-Host -ForegroundColor Red "bazel build failed with exit code ${LastExitCode}." Exit ${LastExitCode} @@ -57,32 +62,82 @@ if ($LastExitCode) { . ci/kokoro/windows/lib/integration.ps1 function Invoke-REST-Quickstart { - bazelisk $common_flags run $build_flags ` - //google/cloud/storage/quickstart:quickstart -- ` - "${env:GOOGLE_CLOUD_CPP_STORAGE_TEST_BUCKET_NAME}" - if ($LastExitCode) { - Write-Host -ForegroundColor Red "bazel run (storage/quickstart) failed with exit code ${LastExitCode}." - Exit ${LastExitCode} + param($bazel_bin) + try { + $executable = Join-Path $bazel_bin "google/cloud/storage/quickstart/quickstart.exe" + Write-Host "Running REST Quickstart, attempting to run: $executable" + if (-not (Test-Path $executable)) { + Write-Host -ForegroundColor Red "Executable not found at the specified path." + Exit 1 + } + & $executable "${env:GOOGLE_CLOUD_CPP_STORAGE_TEST_BUCKET_NAME}" + if ($LastExitCode) { + Write-Host -ForegroundColor Red "Execution of (storage/quickstart) failed with exit code ${LastExitCode}." + Exit ${LastExitCode} + } + } catch { + Write-Host -ForegroundColor Red "Caught exception while trying to run storage/quickstart: $_" + Exit 1 } } function Invoke-gRPC-Quickstart { - bazelisk $common_flags run $build_flags ` - //google/cloud/pubsub/quickstart:quickstart -- ` - "${env:GOOGLE_CLOUD_PROJECT}" "${env:GOOGLE_CLOUD_CPP_PUBSUB_TEST_QUICKSTART_TOPIC}" - if ($LastExitCode) { - Write-Host -ForegroundColor Red "bazel run (pubsub/quickstart) failed with exit code ${LastExitCode}." - Exit ${LastExitCode} + param($bazel_bin) + try { + $executable = Join-Path $bazel_bin "google/cloud/pubsub/quickstart/quickstart.exe" + Write-Host "Running gRPC Quickstart, attempting to run: $executable" + if (-not (Test-Path $executable)) { + Write-Host -ForegroundColor Red "Executable not found at the specified path." + Exit 1 + } + & $executable "${env:GOOGLE_CLOUD_PROJECT}" "${env:GOOGLE_CLOUD_CPP_PUBSUB_TEST_QUICKSTART_TOPIC}" + if ($LastExitCode) { + Write-Host -ForegroundColor Red "Execution of (pubsub/quickstart) failed with exit code ${LastExitCode}." + Exit ${LastExitCode} + } + } catch { + Write-Host -ForegroundColor Red "Caught exception while trying to run pubsub/quickstart: $_" + Exit 1 } } if (Test-Integration-Enabled) { Write-Host "`n$(Get-Date -Format o) Running minimal quickstart prorams" Install-Roots-Pem - ${env:GRPC_DEFAULT_SSL_ROOTS_FILE_PATH}="${env:KOKORO_GFILE_DIR}/roots.pem" - ${env:GOOGLE_APPLICATION_CREDENTIALS}="${env:KOKORO_GFILE_DIR}/kokoro-run-key.json" - Invoke-REST-Quickstart - Invoke-gRPC-Quickstart + $env:GRPC_DEFAULT_SSL_ROOTS_FILE_PATH = Join-Path $env:KOKORO_GFILE_DIR "roots.pem" + $env:CURL_CA_BUNDLE = Join-Path $env:KOKORO_GFILE_DIR "roots.pem" + $env:GOOGLE_APPLICATION_CREDENTIALS = Join-Path $env:KOKORO_GFILE_DIR "kokoro-run-key.json" + # Troubleshooting output + Write-Host "GOOGLE_APPLICATION_CREDENTIALS=$env:GOOGLE_APPLICATION_CREDENTIALS" + Write-Host "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$env:GRPC_DEFAULT_SSL_ROOTS_FILE_PATH" + Write-Host "CURL_CA_BUNDLE=$env:CURL_CA_BUNDLE" + Get-Content (Join-Path $env:KOKORO_GFILE_DIR "roots.pem") -TotalCount 5 + + bazelisk $common_flags build $build_flags ` + //google/cloud/storage/quickstart:quickstart ` + //google/cloud/pubsub/quickstart:quickstart + + $bazel_bin = (bazelisk $common_flags info $build_flags bazel-bin).Trim() + $bazel_bin = $bazel_bin.Replace('/', '\') + Write-Host "bazel-bin directory: $bazel_bin" + + # DEBUG START: List all quickstart.exe files found in the output directory + Write-Host "DEBUG: Searching for 'quickstart.exe' in $bazel_bin..." + try { + Get-ChildItem -Path $bazel_bin -Filter "quickstart.exe" -Recurse -ErrorAction SilentlyContinue | ForEach-Object { Write-Host "Found: $($_.FullName)" } + } catch { + Write-Host "DEBUG: Failed to list contents of $bazel_bin" + } + # DEBUG END + + # --- NEW DEBUGGING FLAGS --- + # Enable detailed logging in the Google Cloud C++ Client + # "http" logs the headers/payloads (sanitized) and handshake info. + $env:GOOGLE_CLOUD_CPP_ENABLE_TRACING="http" + $env:CURL_VERBOSE=1 + + Invoke-REST-Quickstart $bazel_bin + Invoke-gRPC-Quickstart $bazel_bin } # Shutdown the Bazel server to release any locks diff --git a/ci/kokoro/windows/lib/integration.ps1 b/ci/kokoro/windows/lib/integration.ps1 index c1bd067069277..6a34cd24214b8 100644 --- a/ci/kokoro/windows/lib/integration.ps1 +++ b/ci/kokoro/windows/lib/integration.ps1 @@ -14,29 +14,127 @@ # Helper functions to run the minimal integration tests -$PROJECT_ROOT = (Get-Item -Path ".\" -Verbose).FullName +$PROJECT_ROOT = (Get-Item -Path ".\").FullName $integration_tests_config="${PROJECT_ROOT}/ci/etc/integration-tests-config.ps1" . "${integration_tests_config}" function Test-Integration-Enabled { if ((Test-Path env:KOKORO_GFILE_DIR) -and (Test-Path "${env:KOKORO_GFILE_DIR}/kokoro-run-key.json")) { - return $True + return $True } return $False } +function Debug-Network { + param([string]$targetUrl) + Write-Host -ForegroundColor Cyan "`n--- NETWORK DEBUG START ($targetUrl) ---" + try { + $uri = New-Object System.Uri($targetUrl) + $hostName = $uri.DnsSafeHost + + # 1. DNS Resolution + Write-Host "1. Testing DNS resolution for $hostName..." + $dns = Resolve-DnsName -Name $hostName -ErrorAction SilentlyContinue + if ($dns) { $dns | Format-Table -AutoSize | Out-Host } else { Write-Host -ForegroundColor Red "DNS Resolution FAILED" } + + # 2. Basic TCP Connectivity (checking port 443) + Write-Host "`n2. Testing TCP connectivity to $hostName`:443..." + try { + $tcp = Test-NetConnection -ComputerName $hostName -Port 443 -WarningAction SilentlyContinue + if ($tcp.TcpTestSucceeded) { Write-Host "TCP connection SUCCEEDED" } else { Write-Host -ForegroundColor Red "TCP connection FAILED" } + Write-Host "Detailed Info: $($tcp | Out-String)" + } catch { + Write-Host -ForegroundColor Red "Test-NetConnection failed to run: $_" + } + + # 3. Proxy Detection + Write-Host "`n3. Checking System Proxy for $targetUrl..." + $proxy = [System.Net.WebRequest]::GetSystemWebProxy() + $proxyUri = $proxy.GetProxy($uri) + Write-Host "Effective Proxy: $proxyUri" + Write-Host "Is Bypassed: $($proxy.IsBypassed($uri))" + + } catch { + Write-Host -ForegroundColor Red "An error occurred during network debug: $_" + } + Write-Host -ForegroundColor Cyan "--- NETWORK DEBUG END ---`n" +} + function Install-Roots-Pem { - Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) " ` - "Downloading roots.pem [$_]" + Debug-Network -targetUrl "https://curl.se/ca/cacert.pem" + ForEach($attempt in (1, 2, 3)) { + Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) " ` + "Downloading roots.pem [$attempt]" try { (New-Object System.Net.WebClient).Downloadfile( - 'https://pki.google.com/roots.pem', + 'https://curl.se/ca/cacert.pem', "${env:KOKORO_GFILE_DIR}/roots.pem") + + # --- CHANGE 1: Inspect both Root and Intermediate (CA) stores --- + # Many corporate proxies operate via an Intermediate CA. + $storesToCheck = @("Root", "CA") + + Write-Host "Appending Windows System Certificates to roots.pem..." + + foreach ($storeName in $storesToCheck) { + Write-Host -ForegroundColor Cyan "Processing Store: LocalMachine\$storeName" + $certStore = New-Object System.Security.Cryptography.X509Certificates.X509Store -ArgumentList $storeName, "LocalMachine" + $certStore.Open('ReadOnly') + + $certStore.Certificates | ForEach-Object { + $cert = $_ + # --- CHANGE 2: Log the Subject Name --- + # This lets us verify if the corporate proxy cert is actually present. + Write-Host " Adding: $($cert.Subject)" + + $pem = "`r`n-----BEGIN CERTIFICATE-----`r`n" + + [Convert]::ToBase64String($cert.Export([System.Security.Cryptography.X509Certificates.X509ContentType]::Cert), 'InsertLineBreaks') + + "`r`n-----END CERTIFICATE-----`r`n" + Add-Content -Path "${env:KOKORO_GFILE_DIR}/roots.pem" -Value $pem -Encoding Ascii + } + $certStore.Close() + } + + # --- DEBUG START --- + $pemPath = "${env:KOKORO_GFILE_DIR}/roots.pem" + Write-Host -ForegroundColor Cyan "`nDEBUG: Inspecting roots.pem..." + + $corruption = Select-String -Path $pemPath -Pattern "-----END CERTIFICATE----------BEGIN CERTIFICATE-----" + if ($corruption) { + Write-Host -ForegroundColor Red "FAIL: Found corrupted certificate boundaries (missing newline)!" + } else { + Write-Host -ForegroundColor Green "PASS: No certificate boundary corruption detected." + } + + Write-Host -ForegroundColor Cyan "`nDEBUG: Testing SSL connection to GCS..." + + # Fix: Relax ErrorActionPreference so curl -v stderr doesn't crash the script + $OldEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + + try { + & curl.exe --version + & curl.exe -v https://storage.googleapis.com --cacert $pemPath 2>&1 | Out-Host + if ($LastExitCode -ne 0) { + Write-Host -ForegroundColor Red "Curl exited with error code: $LastExitCode" + } else { + Write-Host -ForegroundColor Green "Curl connection test PASSED." + } + } catch { + Write-Host -ForegroundColor Red "Debug curl command failed unexpectedly: $_" + } finally { + $ErrorActionPreference = $OldEAP + } + # --- DEBUG END --- + return } catch { - Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) download error" + Write-Host -ForegroundColor Yellow "`n$(Get-Date -Format o) download/setup error: $_" + if ($attempt -eq 3) { + Debug-Network -targetUrl "https://storage.googleapis.com" + } } Start-Sleep -Seconds (60 * $attempt) }