|
| 1 | +using System; |
| 2 | +using System.Collections.Generic; |
| 3 | +using System.Globalization; |
| 4 | +using System.Linq; |
| 5 | +using System.Net.Http; |
| 6 | +using System.Security.Cryptography; |
| 7 | +using System.Text; |
| 8 | +using System.Text.Json; |
| 9 | +using System.Threading.Tasks; |
| 10 | +using Amazon.EKS; |
| 11 | +using Amazon.EKS.Model; |
| 12 | +using Testcontainers.Floci; |
| 13 | +using Xunit; |
| 14 | + |
| 15 | +namespace Testcontainers.Floci.Tests; |
| 16 | + |
| 17 | +public sealed class EksServiceTest : IAsyncLifetime |
| 18 | +{ |
| 19 | + private const string ClusterName = "eks-test"; |
| 20 | + private const string Namespace = "floci-test"; |
| 21 | + |
| 22 | + private readonly FlociContainer _floci = new FlociBuilder(TestImages.Floci) |
| 23 | + .WithEks(new EksConfig()) |
| 24 | + .Build(); |
| 25 | + |
| 26 | + public Task InitializeAsync() => _floci.StartAsync(); |
| 27 | + |
| 28 | + public async Task DisposeAsync() |
| 29 | + { |
| 30 | + // Delete the cluster so Floci tears down the sibling k3s container it spawned. |
| 31 | + try |
| 32 | + { |
| 33 | + using var eks = CreateClient(); |
| 34 | + await eks.DeleteClusterAsync(new DeleteClusterRequest { Name = ClusterName }); |
| 35 | + } |
| 36 | + catch (AmazonEKSException) |
| 37 | + { |
| 38 | + // The container is being disposed anyway; nothing actionable here. |
| 39 | + } |
| 40 | + |
| 41 | + await _floci.DisposeAsync(); |
| 42 | + } |
| 43 | + |
| 44 | + private AmazonEKSClient CreateClient() |
| 45 | + { |
| 46 | + return new AmazonEKSClient( |
| 47 | + _floci.AccessKey, |
| 48 | + _floci.SecretKey, |
| 49 | + new AmazonEKSConfig |
| 50 | + { |
| 51 | + ServiceURL = _floci.GetEndpoint(), |
| 52 | + AuthenticationRegion = _floci.Region, |
| 53 | + // CreateCluster can block while Floci pulls the k3s image on first use. |
| 54 | + Timeout = TimeSpan.FromMinutes(5), |
| 55 | + }); |
| 56 | + } |
| 57 | + |
| 58 | + [Fact] |
| 59 | + public async Task CreatesClusterAndDrivesKubernetesApi() |
| 60 | + { |
| 61 | + using var eks = CreateClient(); |
| 62 | + |
| 63 | + // Control plane: create the cluster and wait for it to become ACTIVE. |
| 64 | + await eks.CreateClusterAsync(new CreateClusterRequest |
| 65 | + { |
| 66 | + Name = ClusterName, |
| 67 | + RoleArn = "arn:aws:iam::000000000000:role/eks-role", |
| 68 | + ResourcesVpcConfig = new VpcConfigRequest(), |
| 69 | + }); |
| 70 | + |
| 71 | + var cluster = await WaitForActiveClusterAsync(eks); |
| 72 | + Assert.False(string.IsNullOrEmpty(cluster.Endpoint)); |
| 73 | + |
| 74 | + // Floci returns the API server as https://localhost:<port>; connect via 127.0.0.1 to avoid |
| 75 | + // resolving to IPv6 (::1), which Testcontainers does not publish. |
| 76 | + var apiPort = new Uri(cluster.Endpoint).Port; |
| 77 | + var k8sBase = $"https://127.0.0.1:{apiPort}"; |
| 78 | + var stsHost = new Uri(_floci.GetEndpoint()).Authority; |
| 79 | + |
| 80 | + using var k8s = new HttpClient(new HttpClientHandler |
| 81 | + { |
| 82 | + ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator, |
| 83 | + }); |
| 84 | + |
| 85 | + // Data plane: authenticate with an EKS bearer token (the `aws eks get-token` scheme) and |
| 86 | + // drive the real Kubernetes API — create a namespace + ConfigMap and read it back. This |
| 87 | + // goes beyond control-plane checks and exercises the live k3s API server end to end. |
| 88 | + await WaitForKubernetesReadyAsync(k8s, k8sBase, stsHost); |
| 89 | + |
| 90 | + await PostJsonAsync(k8s, $"{k8sBase}/api/v1/namespaces", stsHost, |
| 91 | + "{\"metadata\":{\"name\":\"" + Namespace + "\"}}"); |
| 92 | + |
| 93 | + await PostJsonAsync(k8s, $"{k8sBase}/api/v1/namespaces/{Namespace}/configmaps", stsHost, |
| 94 | + "{\"metadata\":{\"name\":\"test-config\"},\"data\":{\"greeting\":\"hello-from-floci\"}}"); |
| 95 | + |
| 96 | + using var read = await SendAsync(k8s, HttpMethod.Get, |
| 97 | + $"{k8sBase}/api/v1/namespaces/{Namespace}/configmaps/test-config", stsHost, body: null); |
| 98 | + read.EnsureSuccessStatusCode(); |
| 99 | + |
| 100 | + using var doc = JsonDocument.Parse(await read.Content.ReadAsStringAsync()); |
| 101 | + var greeting = doc.RootElement.GetProperty("data").GetProperty("greeting").GetString(); |
| 102 | + Assert.Equal("hello-from-floci", greeting); |
| 103 | + } |
| 104 | + |
| 105 | + private async Task<Cluster> WaitForActiveClusterAsync(AmazonEKSClient eks) |
| 106 | + { |
| 107 | + for (var attempt = 0; attempt < 60; attempt++) |
| 108 | + { |
| 109 | + var cluster = (await eks.DescribeClusterAsync(new DescribeClusterRequest { Name = ClusterName })).Cluster; |
| 110 | + if (cluster.Status == ClusterStatus.ACTIVE) |
| 111 | + { |
| 112 | + return cluster; |
| 113 | + } |
| 114 | + |
| 115 | + Assert.NotEqual(ClusterStatus.FAILED, cluster.Status); |
| 116 | + await Task.Delay(2000); |
| 117 | + } |
| 118 | + |
| 119 | + throw new Xunit.Sdk.XunitException("EKS cluster did not become ACTIVE within the timeout."); |
| 120 | + } |
| 121 | + |
| 122 | + private async Task WaitForKubernetesReadyAsync(HttpClient k8s, string k8sBase, string stsHost) |
| 123 | + { |
| 124 | + Exception? lastError = null; |
| 125 | + for (var attempt = 0; attempt < 60; attempt++) |
| 126 | + { |
| 127 | + try |
| 128 | + { |
| 129 | + using var resp = await SendAsync(k8s, HttpMethod.Get, $"{k8sBase}/api/v1/namespaces", stsHost, body: null); |
| 130 | + if (resp.IsSuccessStatusCode) |
| 131 | + { |
| 132 | + return; |
| 133 | + } |
| 134 | + |
| 135 | + lastError = new Exception($"namespaces list returned {(int)resp.StatusCode}"); |
| 136 | + } |
| 137 | + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) |
| 138 | + { |
| 139 | + lastError = ex; |
| 140 | + } |
| 141 | + |
| 142 | + await Task.Delay(2000); |
| 143 | + } |
| 144 | + |
| 145 | + throw new Xunit.Sdk.XunitException( |
| 146 | + $"Kubernetes API did not become usable within the timeout. Last error: {lastError?.Message}"); |
| 147 | + } |
| 148 | + |
| 149 | + private async Task PostJsonAsync(HttpClient k8s, string url, string stsHost, string json) |
| 150 | + { |
| 151 | + using var resp = await SendAsync(k8s, HttpMethod.Post, url, stsHost, json); |
| 152 | + resp.EnsureSuccessStatusCode(); |
| 153 | + } |
| 154 | + |
| 155 | + private async Task<HttpResponseMessage> SendAsync( |
| 156 | + HttpClient k8s, HttpMethod method, string url, string stsHost, string? body) |
| 157 | + { |
| 158 | + var request = new HttpRequestMessage(method, url); |
| 159 | + request.Headers.TryAddWithoutValidation("Authorization", "Bearer " + GenerateEksToken(stsHost)); |
| 160 | + if (body != null) |
| 161 | + { |
| 162 | + request.Content = new StringContent(body, Encoding.UTF8, "application/json"); |
| 163 | + } |
| 164 | + |
| 165 | + return await k8s.SendAsync(request); |
| 166 | + } |
| 167 | + |
| 168 | + // Builds an EKS bearer token: a SigV4 query-presigned STS GetCallerIdentity URL (carrying the |
| 169 | + // x-k8s-aws-id header) base64url-wrapped as "k8s-aws-v1.<url>" — exactly what `aws eks |
| 170 | + // get-token` produces. Floci's IAM-auth webhook validates it and maps it to cluster-admin. |
| 171 | + private string GenerateEksToken(string stsHost) |
| 172 | + { |
| 173 | + const string service = "sts"; |
| 174 | + var region = _floci.Region; |
| 175 | + var now = DateTime.UtcNow; |
| 176 | + var amzDate = now.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture); |
| 177 | + var dateStamp = now.ToString("yyyyMMdd", CultureInfo.InvariantCulture); |
| 178 | + |
| 179 | + var query = new SortedDictionary<string, string>(StringComparer.Ordinal) |
| 180 | + { |
| 181 | + ["Action"] = "GetCallerIdentity", |
| 182 | + ["Version"] = "2011-06-15", |
| 183 | + ["X-Amz-Algorithm"] = "AWS4-HMAC-SHA256", |
| 184 | + ["X-Amz-Credential"] = $"{_floci.AccessKey}/{dateStamp}/{region}/{service}/aws4_request", |
| 185 | + ["X-Amz-Date"] = amzDate, |
| 186 | + ["X-Amz-Expires"] = "900", |
| 187 | + ["X-Amz-SignedHeaders"] = "host;x-k8s-aws-id", |
| 188 | + }; |
| 189 | + var canonicalQuery = string.Join("&", query.Select(kv => $"{UriEncode(kv.Key)}={UriEncode(kv.Value)}")); |
| 190 | + |
| 191 | + var canonicalRequest = |
| 192 | + $"GET\n/\n{canonicalQuery}\nhost:{stsHost}\nx-k8s-aws-id:{ClusterName}\n\nhost;x-k8s-aws-id\n{Hex(Sha256(string.Empty))}"; |
| 193 | + var scope = $"{dateStamp}/{region}/{service}/aws4_request"; |
| 194 | + var stringToSign = $"AWS4-HMAC-SHA256\n{amzDate}\n{scope}\n{Hex(Sha256(canonicalRequest))}"; |
| 195 | + |
| 196 | + var signingKey = HmacSha256( |
| 197 | + HmacSha256( |
| 198 | + HmacSha256( |
| 199 | + HmacSha256(Encoding.UTF8.GetBytes("AWS4" + _floci.SecretKey), dateStamp), |
| 200 | + region), |
| 201 | + service), |
| 202 | + "aws4_request"); |
| 203 | + var signature = Hex(HmacSha256(signingKey, stringToSign)); |
| 204 | + |
| 205 | + var presignedUrl = $"http://{stsHost}/?{canonicalQuery}&X-Amz-Signature={signature}"; |
| 206 | + return "k8s-aws-v1." + Base64UrlNoPad(Encoding.UTF8.GetBytes(presignedUrl)); |
| 207 | + } |
| 208 | + |
| 209 | + private static string UriEncode(string value) |
| 210 | + { |
| 211 | + var sb = new StringBuilder(); |
| 212 | + foreach (var b in Encoding.UTF8.GetBytes(value)) |
| 213 | + { |
| 214 | + var c = (char)b; |
| 215 | + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') |
| 216 | + || c is '-' or '_' or '.' or '~') |
| 217 | + { |
| 218 | + sb.Append(c); |
| 219 | + } |
| 220 | + else |
| 221 | + { |
| 222 | + sb.Append('%').Append(b.ToString("X2", CultureInfo.InvariantCulture)); |
| 223 | + } |
| 224 | + } |
| 225 | + |
| 226 | + return sb.ToString(); |
| 227 | + } |
| 228 | + |
| 229 | + private static string Base64UrlNoPad(byte[] bytes) => |
| 230 | + Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); |
| 231 | + |
| 232 | + private static string Hex(byte[] bytes) => Convert.ToHexString(bytes).ToLowerInvariant(); |
| 233 | + |
| 234 | + private static byte[] Sha256(string value) => SHA256.HashData(Encoding.UTF8.GetBytes(value)); |
| 235 | + |
| 236 | + private static byte[] HmacSha256(byte[] key, string data) |
| 237 | + { |
| 238 | + using var hmac = new HMACSHA256(key); |
| 239 | + return hmac.ComputeHash(Encoding.UTF8.GetBytes(data)); |
| 240 | + } |
| 241 | +} |
0 commit comments