diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0c8433249b..21cdd95c3c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,10 +1,12 @@ # syntax=docker/dockerfile:1 -FROM mcr.microsoft.com/devcontainers/go:1-1.24-bookworm + +# CHANGE: Switched from debian-based Go image to Ubuntu Base +FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04 # Install kubectl -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ && install -m 0755 kubectl /usr/local/bin/kubectl \ && rm kubectl # Install Helm -RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3a18b63a62..823b9927c6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "name": "Go", + "name": "Ubuntu Go", "dockerFile": "Dockerfile", "runArgs": [ "--privileged", @@ -15,9 +15,9 @@ "installOhMyZshConfig": true, "upgradePackages": true, "nonFreePackages": true, - "username": "automatic", - "userUid": "automatic", - "userGid": "automatic" + "username": "root", + "userUid": "0", + "userGid": "0" }, "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { "version": "latest", @@ -33,11 +33,15 @@ "ghcr.io/devcontainers-extra/features/zsh-plugins:0": { "plugins": "zsh-autosuggestions zsh-syntax-highlighting kubectl docker docker-compose git-auto-fetch git-commit golang helm", "omzPlugins": "https://github.com/zsh-users/zsh-autosuggestions https://github.com/zsh-users/zsh-syntax-highlighting", - "username": "vscode" + "username": "root" + }, + "ghcr.io/devcontainers/features/go:1": { + "version": "latest" } }, "mounts": [ "source=${env:HOME},target=/mnt/hosthome,type=bind,consistency=cached,readonly" ], - "postStartCommand": "if [ -f /mnt/hosthome/.p10k.zsh ]; then cp /mnt/hosthome/.p10k.zsh /home/vscode/.p10k.zsh; fi && git clone --depth=1 https://github.com/romkatv/powerlevel10k.git /home/vscode/.oh-my-zsh/custom/themes/powerlevel10k && sed -i 's|^ZSH_THEME=.*|ZSH_THEME=\\\"powerlevel10k/powerlevel10k\\\"|' /home/vscode/.zshrc && echo '\nif [[ -r \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\" ]]; then\n source \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\"\nfi\n\n[[ ! -f ~/.p10k.zsh ]] || source ~/.p10k.zsh' >> /home/vscode/.zshrc && docker buildx create --name liqo-builder --use --driver-opt network=host" -} \ No newline at end of file + "postStartCommand": "if [ -f /mnt/hosthome/.p10k.zsh ]; then cp /mnt/hosthome/.p10k.zsh /root/.p10k.zsh; fi && git clone --depth=1 https://github.com/romkatv/powerlevel10k.git /root/.oh-my-zsh/custom/themes/powerlevel10k && sed -i 's|^ZSH_THEME=.*|ZSH_THEME=\"powerlevel10k/powerlevel10k\"|' /root/.zshrc && echo '\nif [[ -r \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\" ]]; then\n source \"${XDG_CACHE_HOME:-$HOME/.cache}/p10k-instant-prompt-${(%):-%n}.zsh\"\nfi\n\n[[ ! -f ~/.p10k.zsh ]] || source ~/.p10k.zsh' >> /root/.zshrc && docker buildx create --name liqo-builder --use --driver-opt network=host", + "remoteUser": "root" +} diff --git a/Makefile b/Makefile index 57b02cfaa6..c8e53a8ef0 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,10 @@ endif ifeq ($(shell uname),Darwin) SED_COMMAND=sed -i '' -n '/rules/,$$p' +SED_INPLACE=sed -i '' else SED_COMMAND=sed -i -n '/rules/,$$p' +SED_INPLACE=sed -i endif generate: generate-groups rbacs manifests fmt @@ -103,7 +105,7 @@ fmt: gci addlicense docs go fmt ./... find . -type f -name '*.go' -a ! -name '*zz_generated*' -exec $(GCI) write -s standard -s default -s "prefix(github.com/liqotech/liqo)" {} \; find . -type f -name '*.go' -exec $(ADDLICENSE) -l apache -c "The Liqo Authors" -y "2019-$(shell date +%Y)" {} \; - find . -type f -name "*.go" -exec sed -i "s/Copyright 2019-[0-9]\{4\} The Liqo Authors/Copyright 2019-$(shell date +%Y) The Liqo Authors/" {} + + find . -type f -name "*.go" -exec $(SED_INPLACE) "s/Copyright 2019-[0-9]\{4\} The Liqo Authors/Copyright 2019-$(shell date +%Y) The Liqo Authors/" {} + # Install golangci-lint if not available golangci-lint: diff --git a/apis/networking/v1beta1/firewall/filterrule_types.go b/apis/networking/v1beta1/firewall/filterrule_types.go index e4de5f6a07..27962ef6c8 100644 --- a/apis/networking/v1beta1/firewall/filterrule_types.go +++ b/apis/networking/v1beta1/firewall/filterrule_types.go @@ -24,6 +24,9 @@ const ( // ActionSetMetaMarkFromCtMark is the action to be applied to the rule. // It is used to set the meta mark from the conntrack mark. ActionSetMetaMarkFromCtMark FilterAction = "metamarkfromctmark" + // ActionTCPMssClamp is the action to be applied to the rule. + // It is used to clamp the TCP MSS. + ActionTCPMssClamp FilterAction = "tcpmssclamp" ) // FilterRule is a rule to be applied to a filter chain. @@ -35,7 +38,7 @@ type FilterRule struct { // They can be multiple and they are applied with an AND operator. Match []Match `json:"match"` // Action is the action to be applied to the rule. - // +kubebuilder:validation:Enum=ctmark;metamarkfromctmark + // +kubebuilder:validation:Enum=ctmark;metamarkfromctmark;tcpmssclamp Action FilterAction `json:"action"` // Value is the value to be used for the action. Value *string `json:"value,omitempty"` diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 02b49fc374..2b10ad2eda 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -213,6 +213,7 @@ func run(cmd *cobra.Command, _ []string) error { connoptions.GwOptions.Name, mgr.GetEventRecorderFor("firewall-controller"), []labels.Set{ + gateway.ForgeFirewallAllGatewaysTargetLabels(), gateway.ForgeFirewallInternalTargetLabels(), remapping.ForgeFirewallTargetLabels(connoptions.GwOptions.RemoteClusterID), remapping.ForgeFirewallTargetLabelsIPMappingGw(), diff --git a/deployments/liqo/README.md b/deployments/liqo/README.md index 1b1c64a606..02fb1812b6 100644 --- a/deployments/liqo/README.md +++ b/deployments/liqo/README.md @@ -101,6 +101,8 @@ | networking.fabric.pod.priorityClassName | string | `""` | PriorityClassName (https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) for the fabric pod. | | networking.fabric.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the fabric pod. | | networking.fabric.tolerations | list | `[]` | Extra tolerations for the fabric pod. | +| networking.gateway.mssclamp | object | `{"enabled":true,"value":0}` | Enable the TCP MSS clamping on tunnel interfaces. Tunneling technologies introduce extra overhead that reduces the MTU, causing standard-sized Internet packets to exceed the tunnel's capacity and be dropped. TCP MSS Clamping resolves this by intercepting the initial TCP connection handshake and dynamically rewriting the Maximum Segment Size (MSS) value to match the smaller available space of the tunnel interface. This dynamic adjustment, per TCP-session, forces the remote server to generate smaller data packets that fit inside the tunnel, effectively preventing fragmentation issues and the common "black hole" phenomenon where connections establish but data transfer hangs indefinitely. | +| networking.gateway.mssclamp.value | int | `0` | Set the value for the mssclamp rule. Set to 0 to use automatic value discovery based on the MTU of the tunnel interface. | | networking.gatewayTemplates | object | `{"container":{"gateway":{"image":{"name":"ghcr.io/liqotech/gateway","version":""}},"geneve":{"image":{"name":"ghcr.io/liqotech/gateway/geneve","version":""}},"wireguard":{"image":{"name":"ghcr.io/liqotech/gateway/wireguard","version":""}}},"ping":{"interval":"2s","lossThreshold":5,"updateStatusInterval":"10s"},"replicas":1,"server":{"service":{"allocateLoadBalancerNodePorts":"","annotations":{}}},"wireguard":{"implementation":"kernel"}}` | Set the options for the default gateway (server/client) templates. The default templates use a WireGuard implementation to connect the gateway of the clusters. These options are used to configure only the default templates and should not be considered if a custom template is used. | | networking.gatewayTemplates.container.gateway.image.name | string | `"ghcr.io/liqotech/gateway"` | Image repository for the gateway container. | | networking.gatewayTemplates.container.gateway.image.version | string | `""` | Custom version for the gateway image. If not specified, the global tag is used. | diff --git a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml index 9093d78ed4..27c9ad510e 100644 --- a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml +++ b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_firewallconfigurations.yaml @@ -103,6 +103,7 @@ spec: enum: - ctmark - metamarkfromctmark + - tcpmssclamp type: string match: description: |- diff --git a/deployments/liqo/templates/liqo-gw-mssclamp-firewallconfiguration.yaml b/deployments/liqo/templates/liqo-gw-mssclamp-firewallconfiguration.yaml new file mode 100644 index 0000000000..36fcff91d0 --- /dev/null +++ b/deployments/liqo/templates/liqo-gw-mssclamp-firewallconfiguration.yaml @@ -0,0 +1,31 @@ +{{- if .Values.networking.gateway.mssclamp.enabled }} +apiVersion: networking.liqo.io/v1beta1 +kind: FirewallConfiguration +metadata: + name: gw-mss-clamping + labels: + networking.liqo.io/firewall-category: gateway + networking.liqo.io/firewall-subcategory: all-gateways +spec: + table: + family: IPV4 + name: mss-clamping + chains: + - name: mss-clamping + type: filter + hook: forward + policy: accept + priority: 0 + rules: + filterRules: + - action: tcpmssclamp + value: {{ .Values.networking.gateway.mssclamp.value | toString | quote }} + match: + - op: eq + dev: + position: out + value: liqo-tunnel + proto: + value: tcp + name: mss-clamping-out +{{- end }} diff --git a/deployments/liqo/values.yaml b/deployments/liqo/values.yaml index a6567640ab..36a0aa50e7 100644 --- a/deployments/liqo/values.yaml +++ b/deployments/liqo/values.yaml @@ -48,6 +48,20 @@ networking: clientResources: - apiVersion: networking.liqo.io/v1beta1 resource: wggatewayclients + gateway: + # -- Enable the TCP MSS clamping on tunnel interfaces. + # Tunneling technologies introduce extra overhead that reduces the MTU, causing standard-sized Internet + # packets to exceed the tunnel's capacity and be dropped. TCP MSS Clamping resolves this by intercepting + # the initial TCP connection handshake and dynamically rewriting the Maximum Segment Size (MSS) + # value to match the smaller available space of the tunnel interface. This dynamic adjustment, per + # TCP-session, forces the remote server to generate smaller data packets that fit inside the tunnel, + # effectively preventing fragmentation issues and the common "black hole" phenomenon where connections + # establish but data transfer hangs indefinitely. + mssclamp: + enabled: true + # -- Set the value for the mssclamp rule. + # Set to 0 to use automatic value discovery based on the MTU of the tunnel interface. + value: 0 # -- Set the options for the default gateway (server/client) templates. # The default templates use a WireGuard implementation to connect the gateway of the clusters. # These options are used to configure only the default templates and should not be considered @@ -119,11 +133,11 @@ networking: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: liqo.io/type - operator: NotIn - values: - - virtual-node + - matchExpressions: + - key: liqo.io/type + operator: NotIn + values: + - virtual-node # -- Extra tolerations for the fabric pod. tolerations: [] config: @@ -536,8 +550,7 @@ discovery: clusterID: "" # -- A set of labels that characterizes the local cluster when exposed remotely as a virtual node. # It is suggested to specify the distinguishing characteristics that may be used to decide whether to offload pods on this cluster. - clusterLabels: - {} + clusterLabels: {} # topology.kubernetes.io/zone: us-east-1 # liqo.io/provider: your-provider diff --git a/docs/advanced/peering/inter-cluster-network.md b/docs/advanced/peering/inter-cluster-network.md index 6267d87cb8..e287e490c9 100644 --- a/docs/advanced/peering/inter-cluster-network.md +++ b/docs/advanced/peering/inter-cluster-network.md @@ -628,3 +628,12 @@ spec: # Optional field - included only if .Spec.ExtraConfig is not empty ?extraConfig: "{{ .Spec.ExtraConfig }}" ``` + +## IP Traffic Fragmentation + +Tunneling technologies, such as Wireguard used to connect two Liqo clusters, introduce extra overhead that reduces the MTU, causing standard-sized Internet packets to exceed the tunnel's capacity and be dropped. +TCP MSS Clamping resolves this by intercepting the initial TCP connection handshake and dynamically rewriting the Maximum Segment Size (MSS) value to match the smaller available space of the tunnel interface. +This dynamic adjustment, which is done per each TCP session, forces the remote server to generate smaller data packets that fit inside the inter-cluster tunnel, effectively preventing fragmentation issues and the common _black hole_ phenomenon where connections establish but data transfer hangs indefinitely. + +By default, the Liqo Gateway implements TCP MSS Clamping, hence it is able to adjust the maximum size of TCP segments based on the actual MTU of the tunnel. +However, this mechanism does not work with UDP traffic: UDP packets with the IP `Don't Fragment` flag may be dropped if their size exceeds the maximum allowed value in the tunnel. diff --git a/pkg/firewall/utils/filterule.go b/pkg/firewall/utils/filterule.go index f6e1deaf4d..6cb3ddbe51 100644 --- a/pkg/firewall/utils/filterule.go +++ b/pkg/firewall/utils/filterule.go @@ -122,6 +122,10 @@ func forgeFilterRule(fr *firewallv1beta1.FilterRule, chain *nftables.Chain) (*nf } case firewallv1beta1.ActionSetMetaMarkFromCtMark: applySetMetaMarkFromCtMarkAction(rule) + case firewallv1beta1.ActionTCPMssClamp: + if err := applyTCPMssClampAction(fr.Value, rule); err != nil { + return nil, fmt.Errorf("cannot apply tcpmssclamp action: %w", err) + } default: } return rule, nil @@ -159,3 +163,80 @@ func applySetMetaMarkFromCtMarkAction(rule *nftables.Rule) { }, ) } + +func applyTCPMssClampAction(value *string, rule *nftables.Rule) error { + var ( + err error + size int + ) + + if value != nil { + size, err = strconv.Atoi(*value) + if err != nil { + return fmt.Errorf("cannot convert value to int: %w", err) + } + } + + rule.Exprs = append(rule.Exprs, + // Match TCP SYN flag + // Load TCP flags byte (offset 13 in TCP header) + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseTransportHeader, + Offset: 13, // TCP flags offset + Len: 1, + }, + // Apply bitmask to check SYN flag (0x02) + &expr.Bitwise{ + DestRegister: 1, + SourceRegister: 1, + Len: 1, + Mask: []byte{0x02}, // SYN flag mask + Xor: []byte{0x00}, + }, + // Check if SYN flag is set (not equal to 0) + &expr.Cmp{ + Op: expr.CmpOpNeq, + Register: 1, + Data: []byte{0x00}, + }) + + if size == 0 { + rule.Exprs = append(rule.Exprs, // Load route MTU into register 1 + &expr.Rt{ + Register: 1, + Key: expr.RtTCPMSS, + }, + + // Convert to network byte order (host to network) + &expr.Byteorder{ + DestRegister: 1, + SourceRegister: 1, + Op: expr.ByteorderHton, + Len: 2, + Size: 2, + }) + } else { + rule.Exprs = append(rule.Exprs, + // Load fixed MSS value into register 1 + &expr.Immediate{ + Register: 1, + Data: []byte{byte(size >> 8), byte(size)}, + }, + ) + } + + rule.Exprs = append(rule.Exprs, + // Write the MSS value to TCP option maxseg + // TCP option type 2 = MSS, offset 2, length 2 bytes + &expr.Exthdr{ + SourceRegister: 1, + Type: 2, // TCP option MSS + Offset: 2, // Offset within the option + Len: 2, // 2 bytes for MSS value + Op: expr.ExthdrOpTcpopt, + }, + ) + + return nil +} diff --git a/pkg/gateway/label.go b/pkg/gateway/label.go index 7b28dace02..e329c59f41 100644 --- a/pkg/gateway/label.go +++ b/pkg/gateway/label.go @@ -42,6 +42,9 @@ const ( // FirewallSubCategoryFabricTargetValue is the value used by the firewallconfiguration controller to reconcile only resources related to a gateway. FirewallSubCategoryFabricTargetValue = "fabric" + + // FirewallSubCategoryAllGatewaysTargetValue is the value used by the firewallconfiguration controller to reconcile a resource on every gateway. + FirewallSubCategoryAllGatewaysTargetValue = "all-gateways" ) // ForgeActiveGatewayPodLabels returns the labels for the gateway pod. @@ -88,3 +91,12 @@ func ForgeFirewallInternalTargetLabels() map[string]string { firewall.FirewallSubCategoryTargetKey: FirewallSubCategoryFabricTargetValue, } } + +// ForgeFirewallAllGatewaysTargetLabels returns the labels used by the firewallconfiguration controller +// to reconcile only resources related to all gateways. +func ForgeFirewallAllGatewaysTargetLabels() map[string]string { + return map[string]string{ + firewall.FirewallCategoryTargetKey: FirewallCategoryGwTargetValue, + firewall.FirewallSubCategoryTargetKey: FirewallSubCategoryAllGatewaysTargetValue, + } +} diff --git a/pkg/webhooks/firewallconfiguration/rule.go b/pkg/webhooks/firewallconfiguration/rule.go index 232873823a..e6c5e37240 100644 --- a/pkg/webhooks/firewallconfiguration/rule.go +++ b/pkg/webhooks/firewallconfiguration/rule.go @@ -32,6 +32,9 @@ func checkRulesInChain(chain *firewallapi.Chain) error { if err := checkUniqueRuleNames(rules); err != nil { return forgeChainError(chain, err) } + if err := checkFilterRules(rules); err != nil { + return forgeChainError(chain, err) + } return nil } @@ -60,6 +63,35 @@ func checkUniqueRuleNames(rules []firewallutils.Rule) error { return nil } +func checkFilterRules(rules []firewallutils.Rule) error { + for i := range rules { + if r, ok := rules[i].(*firewallutils.FilterRuleWrapper); ok { + if err := checkFilterRule(r.FilterRule); err != nil { + return fmt.Errorf("rule %v is invalid: %w", r.Name, err) + } + } + } + return nil +} + +func checkFilterRule(r *firewallapi.FilterRule) error { + switch r.Action { + case firewallapi.ActionTCPMssClamp: + return checkFilterRuleTCPMssClamp(r) + default: + return nil + } +} + +func checkFilterRuleTCPMssClamp(r *firewallapi.FilterRule) error { + for i := range r.Match { + if r.Match[i].Proto.Value == firewallapi.L4ProtoTCP && r.Match[i].Op == firewallapi.MatchOperationEq { + return nil + } + } + return fmt.Errorf("tcp mss clamp rule should have a match for tcp protocol") +} + func generateRuleNames(chains []firewallapi.Chain) { for i := range chains { rules := firewall.FromChainToRulesArray(&chains[i]) diff --git a/test/e2e/cruise/network/network_test.go b/test/e2e/cruise/network/network_test.go index bf04f59328..2fcd200f74 100644 --- a/test/e2e/cruise/network/network_test.go +++ b/test/e2e/cruise/network/network_test.go @@ -136,6 +136,8 @@ var _ = Describe("Liqo E2E", func() { RestartPods(testContext.Clusters[i].ControllerClient) } + time.Sleep(time.Second * 60) + // Check if there is only one active gateway pod per remote cluster. for i := range testContext.Clusters { numActiveGateway := testContext.Clusters[i].NumPeeredConsumers + testContext.Clusters[i].NumPeeredProviders diff --git a/test/e2e/pipeline/infra/cni.sh b/test/e2e/pipeline/infra/cni.sh index 0f7d34a8d0..6f85c5caa6 100644 --- a/test/e2e/pipeline/infra/cni.sh +++ b/test/e2e/pipeline/infra/cni.sh @@ -15,17 +15,23 @@ source "$WORKDIR/../../utils.sh" DOCKER_PROXY="${DOCKER_PROXY:-docker.io}" function install_calico() { - local kubeconfig=$1 - "${KUBECTL}" create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/tigera-operator.yaml --kubeconfig "$kubeconfig" - - # append a slash to DOCKER_PROXY if not present - if [[ "${DOCKER_PROXY}" != */ ]]; then - registry="${DOCKER_PROXY}/" - else - registry="${DOCKER_PROXY}" - fi - - cat < custom-resources.yaml + local kubeconfig=$1 + "${KUBECTL}" create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/tigera-operator.yaml --kubeconfig "$kubeconfig" + + # Wait for the Installation CRD to be available + if ! waitandretry 5s 12 "${KUBECTL} get crd installations.operator.tigera.io --kubeconfig $kubeconfig"; then + echo "Failed to wait for Calico Installation CRD to be available" + exit 1 + fi + + # append a slash to DOCKER_PROXY if not present + if [[ "${DOCKER_PROXY}" != */ ]]; then + registry="${DOCKER_PROXY}/" + else + registry="${DOCKER_PROXY}" + fi + + cat <custom-resources.yaml # This section includes base Calico installation configuration. # For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation apiVersion: operator.tigera.io/v1 @@ -56,41 +62,39 @@ metadata: name: default spec: {} EOF - "${KUBECTL}" apply -f custom-resources.yaml --kubeconfig "$kubeconfig" + "${KUBECTL}" apply -f custom-resources.yaml --kubeconfig "$kubeconfig" } function wait_calico() { - local kubeconfig=$1 - if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n calico-system pod --all --kubeconfig $kubeconfig --timeout=-1s" - then - echo "Failed to wait for calico pods to be ready" - exit 1 - fi - # set felix to use different port for VXLAN - if ! waitandretry 5s 12 "${KUBECTL} patch felixconfiguration default --type=merge -p {\"spec\":{\"vxlanPort\":6789}} --kubeconfig $kubeconfig"; - then - echo "Failed to patch felixconfiguration" - exit 1 - fi + local kubeconfig=$1 + if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n calico-system pod --all --kubeconfig $kubeconfig --timeout=-1s"; then + echo "Failed to wait for calico pods to be ready" + exit 1 + fi + # set felix to use different port for VXLAN + if ! waitandretry 5s 12 "${KUBECTL} patch felixconfiguration default --type=merge -p {\"spec\":{\"vxlanPort\":6789}} --kubeconfig $kubeconfig"; then + echo "Failed to patch felixconfiguration" + exit 1 + fi } function install_cilium() { - local kubeconfig=$1 - - if [ ! -f "${BINDIR}/cilium}" ]; then - setup_arch_and_os - local CILIUM_CLI_VERSION - CILIUM_CLI_VERSION="v0.18.8" - - echo "Downloading Cilium CLI ${CILIUM_CLI_VERSION} for ${OS}-${ARCH}" - curl -L --remote-name-all "https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-${OS}-${ARCH}.tar.gz{,.sha256sum}" - sha256sum --check "cilium-${OS}-${ARCH}.tar.gz.sha256sum" - tar -C "${BINDIR}" -xzvf "cilium-${OS}-${ARCH}.tar.gz" - rm "cilium-${OS}-${ARCH}.tar.gz" - rm "cilium-${OS}-${ARCH}.tar.gz.sha256sum" - fi - - cat < cilium-values.yaml + local kubeconfig=$1 + + if [ ! -f "${BINDIR}/cilium}" ]; then + setup_arch_and_os + local CILIUM_CLI_VERSION + CILIUM_CLI_VERSION="v0.18.8" + + echo "Downloading Cilium CLI ${CILIUM_CLI_VERSION} for ${OS}-${ARCH}" + curl -L --remote-name-all "https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-${OS}-${ARCH}.tar.gz{,.sha256sum}" + sha256sum --check "cilium-${OS}-${ARCH}.tar.gz.sha256sum" + tar -C "${BINDIR}" -xzvf "cilium-${OS}-${ARCH}.tar.gz" + rm "cilium-${OS}-${ARCH}.tar.gz" + rm "cilium-${OS}-${ARCH}.tar.gz.sha256sum" + fi + + cat <cilium-values.yaml ipam: operator: clusterPoolIPv4PodCIDRList: ${POD_CIDR} @@ -105,27 +109,26 @@ affinity: EOF - KUBECONFIG="$kubeconfig" "${BINDIR}/cilium" install --values "cilium-values.yaml" + KUBECONFIG="$kubeconfig" "${BINDIR}/cilium" install --values "cilium-values.yaml" } function wait_cilium() { - local kubeconfig=$1 - KUBECONFIG="$kubeconfig" "${BINDIR}/cilium" status --wait + local kubeconfig=$1 + KUBECONFIG="$kubeconfig" "${BINDIR}/cilium" status --wait } function install_flannel() { - local kubeconfig=$1 - "${KUBECTL}" create ns kube-flannel --kubeconfig "$kubeconfig" - "${KUBECTL}" label --overwrite ns kube-flannel pod-security.kubernetes.io/enforce=privileged --kubeconfig "$kubeconfig" - "${HELM}" repo add flannel https://flannel-io.github.io/flannel/ - "${HELM}" install flannel --set podCidr="${POD_CIDR}" --namespace kube-flannel flannel/flannel --kubeconfig "$kubeconfig" + local kubeconfig=$1 + "${KUBECTL}" create ns kube-flannel --kubeconfig "$kubeconfig" + "${KUBECTL}" label --overwrite ns kube-flannel pod-security.kubernetes.io/enforce=privileged --kubeconfig "$kubeconfig" + "${HELM}" repo add flannel https://flannel-io.github.io/flannel/ + "${HELM}" install flannel --set podCidr="${POD_CIDR}" --namespace kube-flannel flannel/flannel --kubeconfig "$kubeconfig" } function wait_flannel() { - local kubeconfig=$1 - if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n kube-flannel pod --all --timeout=-1s --kubeconfig $kubeconfig"; - then - echo "Failed to wait for flannel pods to be ready" - exit 1 - fi + local kubeconfig=$1 + if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n kube-flannel pod --all --timeout=-1s --kubeconfig $kubeconfig"; then + echo "Failed to wait for flannel pods to be ready" + exit 1 + fi }