fix(helm): add kubeapi listener

.gitignore

@@ -20,3 +20,6 @@ bin/
 
 # Ignore .env files.
 .env*
+
+# Ignore kubeconfig files.
+*kubeconfig*

Pulumi.foundation.yaml

@@ -0,0 +1,4 @@
+config:
+  pulumi:disable-default-providers:
+    # TODO: Disable all default providers once we have properly injected the cloudflare provider.
+    - 'kubernetes'

charts/edge/templates/kubeapi.yaml

@@ -0,0 +1,34 @@
+{{- range $service := .Values.services.kubeapi }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "edge.fullname" $ }}-kubeapi-{{ $service.name }}
+  labels:
+    {{- include "edge.labels" $ | nindent 4 }}
+spec:
+  type: ExternalName
+  externalName: {{ $service.backend.host }}
+  ports:
+    - name: https
+      protocol: TCP
+      port: 6443
+      targetPort: {{ default 6443 $service.backend.port }}
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRouteTCP
+metadata:
+  name: {{ include "edge.fullname" $ }}-kubeapi-{{ $service.name }}
+  labels:
+    {{- include "edge.labels" $ | nindent 4 }}
+spec:
+  entryPoints:
+    - kubeapi
+  routes:
+    - match: HostSNI(`{{ $service.host }}`)
+      services:
+        - name: {{ include "edge.fullname" $ }}-kubeapi-{{ $service.name }}
+          port: 6443
+  tls:
+    passthrough: true
+{{- end }}

charts/edge/values.yaml

@@ -34,8 +34,8 @@ services:
       backend:
         host: 10.3.11.103.nip.io
 
+  kubeapi:
     - name: zebra-k8s
       host: k8s.zebra.nicklasfrahm.dev
       backend:
         host: zebra.srv.nicklasfrahm.dev
-        port: 6443

deploy/argocd/root/templates/ingress/traefik.yaml

@@ -83,6 +83,13 @@ spec:
                 protocol: TCP
                 tls:
                   enabled: false
+              kubeapi:
+                port: 9443
+                expose: true
+                exposedPort: 6443
+                protocol: TCP
+                tls:
+                  enabled: false
 
             providers:
               kubernetesIngress:

deploy/k3se/zebra.yaml

@@ -11,8 +11,6 @@ cluster:
     # is used to determine the server URL of the cluster.
     tls-san:
       - k8s.zebra.nicklasfrahm.dev
-    advertise-port: 443
-    https-listen-port: 6443
     disable:
       - traefik
     flannel-iface: enp3s0

foundation.go

@@ -3,12 +3,16 @@ package main
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/go-playground/validator/v10"
+	corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
+	metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	"gopkg.in/yaml.v3"
 
 	"github.com/nicklasfrahm/infrastructure/pkg/pulumi/dns"
+	"github.com/nicklasfrahm/infrastructure/pkg/pulumi/kubernetes"
 )
 
 const (
@@ -23,36 +27,88 @@ const (
 // NewFoundationStack deploys DNS, Kubernetes at the network edge
 // and underlay networking.
 func NewFoundationStack(ctx *pulumi.Context) error {
-	if err := configureDNS(ctx); err != nil {
+	zoneResources, err := configureDNS(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := configureClusters(ctx, pulumi.DependsOn(zoneResources)); err != nil {
 		return err
 	}
 
 	return nil
 }
 
 // configureDNS loads the DNS specification and configures DNS.
-func configureDNS(ctx *pulumi.Context) error {
+func configureDNS(ctx *pulumi.Context, opts ...pulumi.ResourceOption) ([]pulumi.Resource, error) {
 	dnsSpecBytes, err := os.ReadFile(dnsSpecPath)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	var dnsSpec dns.Spec
 	if err := yaml.Unmarshal(dnsSpecBytes, &dnsSpec); err != nil {
-		return err
+		return nil, err
 	}
 
 	if err := validator.New().Struct(dnsSpec); err != nil {
-		return err
+		return nil, err
 	}
 
+	resources := make([]pulumi.Resource, len(dnsSpec.Zones))
 	for i := 0; i < len(dnsSpec.Zones); i++ {
 		zone := &dnsSpec.Zones[i]
 
-		_, err := dns.NewZone(ctx, fmt.Sprintf("%s-c.zone-%s", StackFoundation, zone.Name), zone)
+		resource, err := dns.NewZone(ctx, fmt.Sprintf("%s-c.zone-%s", StackFoundation, zone.Name), zone, opts...)
+		if err != nil {
+			return nil, err
+		}
+
+		resources[i] = resource
+	}
+
+	return resources, nil
+}
+
+// configureEdgeClusters creates k3s clusters at the network edge.
+func configureClusters(ctx *pulumi.Context, opts ...pulumi.ResourceOption) error {
+	entries, err := os.ReadDir(kubernetes.K3seSpecDir)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		clusterName := strings.TrimSuffix(entry.Name(), ".yaml")
+		cluster, err := kubernetes.NewK3se(ctx, fmt.Sprintf("%s-c.k3se-%s", StackFoundation, clusterName), &kubernetes.K3seArgs{
+			Name: clusterName,
+		}, opts...)
 		if err != nil {
 			return err
 		}
+
+		if clusterName == "charlie" {
+			_, err := corev1.NewNamespace(ctx, fmt.Sprintf("%s-%s-r.namespace-test", StackFoundation, clusterName), &corev1.NamespaceArgs{
+				Metadata: &metav1.ObjectMetaArgs{
+					Name: pulumi.String("pulumi-test"),
+				},
+			}, pulumi.Parent(cluster), pulumi.Provider(cluster.Provider))
+			if err != nil {
+				return err
+			}
+		}
+
+		// TODO: Remove this.
+		cluster.Server.ApplyT(func(server string) string {
+			if server != "" {
+				pulumi.Printf("%s: %s\n", clusterName, server)
+			}
+
+			return server
+		})
 	}
 
 	return nil

go.mod

@@ -6,15 +6,18 @@ toolchain go1.21.3
 
 require (
 	github.com/cloudflare/cloudflare-go v0.79.0
-	github.com/go-playground/validator/v10 v10.15.5
+	github.com/go-playground/validator/v10 v10.16.0
 	github.com/gofiber/fiber/v2 v2.50.0
 	github.com/joho/godotenv v1.5.1
-	github.com/pulumi/pulumi-cloudflare/sdk/v5 v5.13.0
-	github.com/pulumi/pulumi/sdk/v3 v3.89.0
+	github.com/pulumi/pulumi-cloudflare/sdk/v5 v5.14.0
+	github.com/pulumi/pulumi-command/sdk v0.9.2
+	github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.5.4
+	github.com/pulumi/pulumi/sdk/v3 v3.93.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apimachinery v0.28.3
+	k8s.io/client-go v0.28.3
 )
 
 require (
@@ -33,7 +36,7 @@ require (
 	github.com/charmbracelet/bubbletea v0.24.2 // indirect
 	github.com/charmbracelet/lipgloss v0.9.1 // indirect
 	github.com/cheggaaa/pb v1.0.29 // indirect
-	github.com/cloudflare/circl v1.3.5 // indirect
+	github.com/cloudflare/circl v1.3.6 // indirect
 	github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -43,8 +46,8 @@ require (
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
-	github.com/go-git/go-git/v5 v5.9.0 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-git/go-git/v5 v5.10.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
@@ -61,6 +64,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/hashicorp/hcl/v2 v2.19.1 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -86,7 +90,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pkg/term v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/pulumi/esc v0.5.6 // indirect
+	github.com/pulumi/esc v0.6.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
@@ -104,26 +108,29 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/zclconf/go-cty v1.14.1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.4.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/oauth2 v0.14.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.4.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
-	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	lukechampine.com/frand v1.4.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20211028080628-e2786a622600 // indirect
 )