Add integration test infrastructure (#419)

hargut · shaneutt · commit 41f31b7e42a9 · 2025-08-08T14:43:26.000-04:00
Signed-off-by: Harald Gutmann &lt;harald@gutmann.one&gt;
Signed-off-by: Shane Utt &lt;shaneutt@linux.com&gt;
diff --git a/Cargo.lock b/Cargo.lock
@@ -1344,6 +1344,15 @@ version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 
+[[package]]
+name = "matchers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+dependencies = [
+ "regex-automata 0.1.10",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.5"
@@ -1765,8 +1774,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata",
- "regex-syntax",
+ "regex-automata 0.4.9",
+ "regex-syntax 0.8.5",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+dependencies = [
+ "regex-syntax 0.6.29",
 ]
 
 [[package]]
@@ -1777,7 +1795,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -1786,6 +1804,12 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
 
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
 [[package]]
 name = "regex-syntax"
 version = "0.8.5"
@@ -2176,6 +2200,18 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "tests-integration"
+version = "0.3.0"
+dependencies = [
+ "k8s-openapi",
+ "kube",
+ "thiserror 2.0.12",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
@@ -2453,8 +2489,12 @@ version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
 dependencies = [
+ "matchers",
+ "once_cell",
+ "regex",
  "sharded-slab",
  "thread_local",
+ "tracing",
  "tracing-core",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,6 +5,7 @@ members = [
 	"dataplane/api-server",
 	"dataplane/common",
 	"dataplane/loader",
+	"tests-integration",
 	"tools/udp-test-server",
 	"xtask",
 ]
diff --git a/build/Containerfile.controlplane b/build/Containerfile.controlplane
@@ -18,6 +18,8 @@ COPY controlplane/ controlplane/
 
 COPY dataplane/ dataplane/
 
+COPY tests-integration/ tests-integration/
+
 COPY tools/ tools/
 
 COPY xtask/ xtask/
diff --git a/build/Containerfile.dataplane b/build/Containerfile.dataplane
@@ -40,6 +40,8 @@ COPY controlplane/ controlplane/
 
 COPY dataplane/ dataplane/
 
+COPY tests-integration/ tests-integration/
+
 COPY tools/ tools/
 
 COPY xtask/ xtask/
diff --git a/build/Containerfile.udp_test_server b/build/Containerfile.udp_test_server
@@ -8,8 +8,6 @@ RUN apk add --no-cache musl-dev clang lld
 
 WORKDIR /workspace
 
-WORKDIR /workspace
-
 ARG PROJECT_DIR=/workspace
 
 ARG BUILD_DIR=$PROJECT_DIR/build
@@ -20,6 +18,8 @@ COPY controlplane/ controlplane/
 
 COPY dataplane/ dataplane/
 
+COPY tests-integration/ tests-integration/
+
 COPY tools/ tools/
 
 COPY xtask/ xtask/
diff --git a/config/dataplane/dataplane.yaml b/config/dataplane/dataplane.yaml
@@ -24,6 +24,8 @@ spec:
         securityContext:
           privileged: true
         args: ["-i", "eth0"]
+        ports:
+          - containerPort: 9874
         env:
         - name: RUST_LOG
           value: debug
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -44,6 +44,4 @@ kind: Kustomization
 resources:
 - ../rbac
 - ../manager
-- ../dataplane
-patches:
-- path: manager_auth_proxy_patch.yaml
+- ../dataplane
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -35,7 +35,7 @@ spec:
         #   type: RuntimeDefault
       containers:
       - command:
-        - /workspace/manager
+        - /controller
         args:
         - --leader-elect
         image: ghcr.io/kubernetes-sigs/blixt-controlplane:latest
@@ -47,14 +47,12 @@ spec:
             drop:
               - "ALL"
         livenessProbe:
-          httpGet:
-            path: /healthz
+          grpc:
             port: 8081
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
-          httpGet:
-            path: /readyz
+          grpc:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
diff --git a/config/tests/metallb/ipaddresspool.yaml b/config/tests/metallb/ipaddresspool.yaml
@@ -0,0 +1,9 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: blixt-system-pool
+spec:
+  addresses:
+    - 10.89.0.100-10.89.0.250
+  autoAssign: true
+  avoidBuggyIPs: false
diff --git a/config/tests/metallb/kustomization.yaml b/config/tests/metallb/kustomization.yaml
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: metallb-system
+resources:
+  - ipaddresspool.yaml
+  - l2advertisement.yaml
diff --git a/config/tests/metallb/l2advertisement.yaml b/config/tests/metallb/l2advertisement.yaml
@@ -0,0 +1,4 @@
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: blixt-system-pool
diff --git a/controlplane/src/main.rs b/controlplane/src/main.rs
@@ -13,11 +13,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-
 use controlplane::*;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
 
 use kube::Client;
+use tokio::task::JoinHandle;
 use tokio::try_join;
+use tonic::transport::Server;
 use tracing::*;
 
 #[tokio::main]
@@ -46,8 +48,35 @@ pub async fn run() {
     if let Err(error) = try_join!(
         gateway_controller(ctx.clone()),
         gatewayclass_controller(ctx),
+        setup_health_checks(IpAddr::from(Ipv4Addr::new(0, 0, 0, 0)), 8080)
     ) {
         error!("failed to start controllers: {error:?}");
         std::process::exit(1);
     }
 }
+
+// TODO: integrate with DataplaneClientManager connection status
+// only get healthy once the dataplane pod connections are established
+async fn setup_health_checks(addr: IpAddr, port: u16) -> Result<JoinHandle<()>> {
+    let healthchecks = tokio::spawn(async move {
+        let (_, health_service) = tonic_health::server::health_reporter();
+        let server_builder = Server::builder();
+
+        // by convention we add 1 to the API listen port and use that
+        // for the health check port.
+        let port = port + 1;
+
+        let addr = match addr {
+            IpAddr::V4(v4) => SocketAddr::V4(SocketAddrV4::new(v4, port)),
+            IpAddr::V6(v6) => SocketAddr::V6(SocketAddrV6::new(v6, port, 0, 0)),
+        };
+
+        let server = server_builder.serve(addr, health_service);
+
+        debug!("gRPC Health Checking service listens on {addr}");
+        server
+            .await
+            .expect("Failed to serve gRPC Health Checking service");
+    });
+    Ok(healthchecks)
+}
diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "tests-integration"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+version.workspace = true
+publish = false
+
+[dependencies]
+k8s-openapi= { workspace = true, features = ["v1_32"] }
+kube = { workspace = true, features = ["client", "rustls-tls", "ring"] }
+thiserror.workspace = true
+tokio = { workspace = true, features = ["rt", "macros", "process"] }
+tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
+tracing.workspace = true
diff --git a/tests-integration/src/infrastructure/kind_cluster.rs b/tests-integration/src/infrastructure/kind_cluster.rs
diff --git a/tests-integration/src/infrastructure/kustomize.rs b/tests-integration/src/infrastructure/kustomize.rs
diff --git a/tests-integration/src/infrastructure/mod.rs b/tests-integration/src/infrastructure/mod.rs
diff --git a/tests-integration/src/lib.rs b/tests-integration/src/lib.rs
diff --git a/tests-integration/tests/clusters.rs b/tests-integration/tests/clusters.rs

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@ members = [`
`5`	`5`	`"dataplane/api-server",`
`6`	`6`	`"dataplane/common",`
`7`	`7`	`"dataplane/loader",`
	`8`	`+ "tests-integration",`
`8`	`9`	`"tools/udp-test-server",`
`9`	`10`	`"xtask",`
`10`	`11`	`]`