diff --git a/.github/actions/kind-create/action.yaml b/.github/actions/kind-create/action.yaml index 52e0ddeba..00581be78 100644 --- a/.github/actions/kind-create/action.yaml +++ b/.github/actions/kind-create/action.yaml @@ -45,3 +45,12 @@ runs: chmod 600 $kubeconfig_path echo "kubeconfig=$(echo $kubeconfig_path)" >> $GITHUB_OUTPUT shell: bash + + - name: Install cloud-provider-kind + id: cloud-provider-kind + run: | + echo "Install cloud-provider-kind" + go install sigs.k8s.io/cloud-provider-kind@latest + kubectl label node e2e-kind-control-plane node.kubernetes.io/exclude-from-external-load-balancers- + ~/go/bin/cloud-provider-kind & + shell: bash diff --git a/.github/workflows/build-push-kafka-docker.yml b/.github/workflows/build-push-kafka-docker.yml index 54efb3932..f9a99f8f2 100644 --- a/.github/workflows/build-push-kafka-docker.yml +++ b/.github/workflows/build-push-kafka-docker.yml @@ -24,8 +24,10 @@ jobs: echo ::set-output name=version::${VERSION} echo ::set-output name=tags::${TAGS} echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub if: startsWith(github.ref, 'refs/tags/') uses: docker/login-action@v1 @@ -33,9 +35,10 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: docker/kafka + platforms: linux/amd64,linux/arm64 push: ${{ startsWith(github.ref, 'refs/tags/') }} tags: ${{ steps.prep.outputs.tags }} labels: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 99c6b7266..914eaacc3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 3d9b0602f..f0eb0e71f 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -18,6 +18,12 @@ jobs: uses: actions/setup-go@v4 with: go-version: 1.21 + + # Enable Tmate Session if you'd like to Debut the E2E Kind Cluster + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # detached: true - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index f2d49110c..dc3ffa611 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -33,7 +33,7 @@ jobs: - name: Add Helm repositories run: | - helm repo add banzaicloud-stable "https://kubernetes-charts.banzaicloud.com" + # helm repo add banzaicloud-stable "https://kubernetes-charts.banzaicloud.com" helm repo add incubator "https://charts.helm.sh/incubator" helm repo add stable "https://charts.helm.sh/stable" diff --git a/Makefile b/Makefile index d202e3195..faf45722b 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,20 @@ docker-build: ## Build the operator docker image. docker-push: ## Push the operator docker image. docker push ${IMG} +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/arm64,linux/amd64 +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + - docker buildx create --name koperator-builder + docker buildx use koperator-builder + docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile . + - docker buildx rm koperator-builder + bin/controller-gen: bin/controller-gen-$(CONTROLLER_GEN_VERSION) ## Symlink controller-gen- into versionless controller-gen. @ln -sf controller-gen-$(CONTROLLER_GEN_VERSION) bin/controller-gen diff --git a/api/go.mod b/api/go.mod index 8d42183b5..827852e86 100644 --- a/api/go.mod +++ b/api/go.mod @@ -7,7 +7,7 @@ require ( emperror.dev/errors v0.8.1 github.com/banzaicloud/istio-client-go v0.0.17 github.com/cert-manager/cert-manager v1.13.2 - github.com/stretchr/testify v1.8.4 + // github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa gotest.tools v2.2.0+incompatible k8s.io/api v0.28.4 @@ -16,7 +16,7 @@ require ( ) require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + // github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/go-cmp v0.5.9 // indirect @@ -25,18 +25,26 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + // github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + // gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) +require github.com/stretchr/testify v1.8.4 + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + // remove once https://github.com/cert-manager/cert-manager/issues/5953 is fixed replace github.com/Venafi/vcert/v4 => github.com/jetstack/vcert/v4 v4.9.6-0.20230127103832-3aa3dfd6613d diff --git a/api/v1beta1/kafkacluster_types.go b/api/v1beta1/kafkacluster_types.go index 7cf05da87..f710cd0b6 100644 --- a/api/v1beta1/kafkacluster_types.go +++ b/api/v1beta1/kafkacluster_types.go @@ -178,7 +178,7 @@ type KafkaClusterSpec struct { RollingUpgradeConfig RollingUpgradeConfig `json:"rollingUpgradeConfig"` // Selector for broker pods that need to be recycled/reconciled TaintedBrokersSelector *metav1.LabelSelector `json:"taintedBrokersSelector,omitempty"` - // +kubebuilder:validation:Enum=envoy;istioingress + // +kubebuilder:validation:Enum=envoy;contour;istioingress // IngressController specifies the type of the ingress controller to be used for external listeners. The `istioingress` ingress controller type requires the `spec.istioControlPlane` field to be populated as well. IngressController string `json:"ingressController,omitempty"` // IstioControlPlane is a reference to the IstioControlPlane resource for envoy configuration. It must be specified if istio ingress is used. @@ -190,13 +190,14 @@ type KafkaClusterSpec struct { // when false, they will be kept so the Kafka cluster remains available for those Kafka clients which are still using the previous ingress setting. // +kubebuilder:default=false // +optional - RemoveUnusedIngressResources bool `json:"removeUnusedIngressResources,omitempty"` - PropagateLabels bool `json:"propagateLabels,omitempty"` - CruiseControlConfig CruiseControlConfig `json:"cruiseControlConfig"` - EnvoyConfig EnvoyConfig `json:"envoyConfig,omitempty"` - MonitoringConfig MonitoringConfig `json:"monitoringConfig,omitempty"` - AlertManagerConfig *AlertManagerConfig `json:"alertManagerConfig,omitempty"` - IstioIngressConfig IstioIngressConfig `json:"istioIngressConfig,omitempty"` + RemoveUnusedIngressResources bool `json:"removeUnusedIngressResources,omitempty"` + PropagateLabels bool `json:"propagateLabels,omitempty"` + CruiseControlConfig CruiseControlConfig `json:"cruiseControlConfig"` + EnvoyConfig EnvoyConfig `json:"envoyConfig,omitempty"` + ContourIngressConfig ContourIngressConfig `json:"contourIngressConfig,omitempty"` + MonitoringConfig MonitoringConfig `json:"monitoringConfig,omitempty"` + AlertManagerConfig *AlertManagerConfig `json:"alertManagerConfig,omitempty"` + IstioIngressConfig IstioIngressConfig `json:"istioIngressConfig,omitempty"` // Envs defines environment variables for Kafka broker Pods. // Adding the "+" prefix to the name prepends the value to that environment variable instead of overwriting it. // Add the "+" suffix to append. @@ -239,15 +240,17 @@ type RollingUpgradeConfig struct { // alerts with 'rollingupgrade' FailureThreshold int `json:"failureThreshold"` - // ConcurrentBrokerRestartsAllowed controls how many brokers can be restarted in parallel during a rolling upgrade. If + // ConcurrentBrokerRestartCountPerRack controls how many brokers can be restarted in parallel during a rolling upgrade. If // it is set to a value greater than 1, the operator will restart up to that amount of brokers in parallel, if the // brokers are within the same rack (as specified by "broker.rack" in broker read-only configs). Since using Kafka broker // racks spreads out the replicas, we know that restarting multiple brokers in the same rack will not cause more than // 1/Nth of the replicas of a topic-partition to be unavailable at the same time, where N is the number of racks used. // This is a safe way to speed up the rolling upgrade. Note that for the rack distribution explained above, Cruise Control - // requires `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal` to be configured. + // requires `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal` to be configured. Default value is 1. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=1 // +optional - ConcurrentBrokerRestartsAllowed int `json:"concurrentBrokerRestartsAllowed,omitempty"` + ConcurrentBrokerRestartCountPerRack int `json:"concurrentBrokerRestartCountPerRack,omitempty"` } // DisruptionBudget defines the configuration for PodDisruptionBudget where the workload is managed by the kafka-operator @@ -622,6 +625,10 @@ func (c IngressServiceSettings) GetServiceType() corev1.ServiceType { return c.ServiceType } +func (c ContourIngressConfig) GetBrokerFqdn(brokerId int32) string { + return strings.Replace(c.BrokerFQDNTemplate, "%id", strconv.Itoa(int(brokerId)), 1) +} + // Replace %id in brokerHostnameTemplate with actual broker id func (c EnvoyConfig) GetBrokerHostname(brokerId int32) string { return strings.Replace(c.BrokerHostnameTemplate, "%id", strconv.Itoa(int(brokerId)), 1) @@ -702,7 +709,7 @@ type ExternalListenerConfig struct { // IngressControllerTargetPort defines the container port that the ingress controller uses for handling external traffic. // If not defined, 29092 will be used as the default IngressControllerTargetPort value. IngressControllerTargetPort *int32 `json:"ingressControllerTargetPort,omitempty"` - // +kubebuilder:validation:Enum=LoadBalancer;NodePort + // +kubebuilder:validation:Enum=LoadBalancer;NodePort;ClusterIP // accessMethod defines the method which the external listener is exposed through. // Two types are supported LoadBalancer and NodePort. // The recommended and default is the LoadBalancer. @@ -725,8 +732,16 @@ type Config struct { type IngressConfig struct { IngressServiceSettings `json:",inline"` - IstioIngressConfig *IstioIngressConfig `json:"istioIngressConfig,omitempty"` - EnvoyConfig *EnvoyConfig `json:"envoyConfig,omitempty"` + IstioIngressConfig *IstioIngressConfig `json:"istioIngressConfig,omitempty"` + EnvoyConfig *EnvoyConfig `json:"envoyConfig,omitempty"` + ContourIngressConfig *ContourIngressConfig `json:"contourIngressConfig,omitempty"` +} + +type ContourIngressConfig struct { + // TLS secret used for Contour IngressRoute resource + TLSSecretName string `json:"tlsSecretName"` + // Broker hostname template for Contour IngressRoute resource to generate broker hostnames. + BrokerFQDNTemplate string `json:"brokerFQDNTemplate"` } // InternalListenerConfig defines the internal listener config for Kafka @@ -764,6 +779,9 @@ type CommonListenerSpec struct { // At least one of the listeners should have this flag enabled // +optional UsedForInnerBrokerCommunication bool `json:"usedForInnerBrokerCommunication"` + // UsedForKafkaAdminCommunication allows for a different port to be returned when the koperator is checking for the port to use to check if kafka is operating. + // +optional + UsedForKafkaAdminCommunication bool `json:"usedForKafkaAdminCommunication,omitempty"` } func (c *CommonListenerSpec) GetServerSSLCertSecretName() string { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 41e4b6d49..c617af827 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -263,6 +263,21 @@ func (in *Config) DeepCopy() *Config { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContourIngressConfig) DeepCopyInto(out *ContourIngressConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContourIngressConfig. +func (in *ContourIngressConfig) DeepCopy() *ContourIngressConfig { + if in == nil { + return nil + } + out := new(ContourIngressConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CruiseControlConfig) DeepCopyInto(out *CruiseControlConfig) { *out = *in @@ -617,6 +632,11 @@ func (in *IngressConfig) DeepCopyInto(out *IngressConfig) { *out = new(EnvoyConfig) (*in).DeepCopyInto(*out) } + if in.ContourIngressConfig != nil { + in, out := &in.ContourIngressConfig, &out.ContourIngressConfig + *out = new(ContourIngressConfig) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressConfig. @@ -861,6 +881,7 @@ func (in *KafkaClusterSpec) DeepCopyInto(out *KafkaClusterSpec) { } in.CruiseControlConfig.DeepCopyInto(&out.CruiseControlConfig) in.EnvoyConfig.DeepCopyInto(&out.EnvoyConfig) + out.ContourIngressConfig = in.ContourIngressConfig out.MonitoringConfig = in.MonitoringConfig if in.AlertManagerConfig != nil { in, out := &in.AlertManagerConfig, &out.AlertManagerConfig diff --git a/charts/kafka-operator/crds/kafkaclusters.yaml b/charts/kafka-operator/crds/kafkaclusters.yaml index d1bc3bcd4..d6376c498 100644 --- a/charts/kafka-operator/crds/kafkaclusters.yaml +++ b/charts/kafka-operator/crds/kafkaclusters.yaml @@ -12919,6 +12919,19 @@ spec: type: string clusterWideConfig: type: string + contourIngressConfig: + properties: + brokerFQDNTemplate: + description: Broker hostname template for Contour IngressRoute + resource to generate broker hostnames. + type: string + tlsSecretName: + description: TLS secret used for Contour IngressRoute resource + type: string + required: + - brokerFQDNTemplate + - tlsSecretName + type: object cruiseControlConfig: description: CruiseControlConfig defines the config for Cruise Control properties: @@ -18837,6 +18850,7 @@ spec: as well. enum: - envoy + - contour - istioingress type: string istioControlPlane: @@ -19197,6 +19211,7 @@ spec: enum: - LoadBalancer - NodePort + - ClusterIP type: string anyCastPort: description: configuring AnyCastPort allows kafka cluster @@ -19216,6 +19231,21 @@ spec: ingressConfig: additionalProperties: properties: + contourIngressConfig: + properties: + brokerFQDNTemplate: + description: Broker hostname template for + Contour IngressRoute resource to generate + broker hostnames. + type: string + tlsSecretName: + description: TLS secret used for Contour IngressRoute + resource + type: string + required: + - brokerFQDNTemplate + - tlsSecretName + type: object envoyConfig: description: EnvoyConfig defines the config for Envoy @@ -21673,6 +21703,11 @@ spec: description: At least one of the listeners should have this flag enabled type: boolean + usedForKafkaAdminCommunication: + description: UsedForKafkaAdminCommunication allows for a + different port to be returned when the koperator is checking + for the port to use to check if kafka is operating. + type: boolean required: - containerPort - externalStartingPort @@ -21749,6 +21784,11 @@ spec: description: At least one of the listeners should have this flag enabled type: boolean + usedForKafkaAdminCommunication: + description: UsedForKafkaAdminCommunication allows for a + different port to be returned when the koperator is checking + for the port to use to check if kafka is operating. + type: boolean required: - containerPort - name @@ -21841,9 +21881,10 @@ spec: description: RollingUpgradeConfig defines the desired config of the RollingUpgrade properties: - concurrentBrokerRestartsAllowed: - description: ConcurrentBrokerRestartsAllowed controls how many - brokers can be restarted in parallel during a rolling upgrade. + concurrentBrokerRestartCountPerRack: + default: 1 + description: ConcurrentBrokerRestartCountPerRack controls how + many brokers can be restarted in parallel during a rolling upgrade. If it is set to a value greater than 1, the operator will restart up to that amount of brokers in parallel, if the brokers are within the same rack (as specified by "broker.rack" in broker @@ -21854,7 +21895,8 @@ spec: N is the number of racks used. This is a safe way to speed up the rolling upgrade. Note that for the rack distribution explained above, Cruise Control requires `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal` - to be configured. + to be configured. Default value is 1. + minimum: 1 type: integer failureThreshold: description: FailureThreshold controls how many failures the cluster diff --git a/charts/kafka-operator/templates/operator-rbac.yaml b/charts/kafka-operator/templates/operator-rbac.yaml index 29557e617..5c47bcc80 100644 --- a/charts/kafka-operator/templates/operator-rbac.yaml +++ b/charts/kafka-operator/templates/operator-rbac.yaml @@ -44,6 +44,12 @@ rules: - '*' verbs: - '*' +- apiGroups: + - projectcontour.io + resources: + - '*' + verbs: + - '*' - apiGroups: - "" resources: diff --git a/charts/kafka-operator/values.yaml b/charts/kafka-operator/values.yaml index a01c18585..fdb940e6e 100644 --- a/charts/kafka-operator/values.yaml +++ b/charts/kafka-operator/values.yaml @@ -23,7 +23,7 @@ operator: # When this field is not empty and Cert-manager is used, # the Cert-manager's Custom Resource Namespace must be included in the comma separated list. # When it is empty, all namespaces will be watched. - namespaces: "" + namespaces: "kafka, cert-manager" verboseLogging: false developmentLogging: false resources: diff --git a/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml b/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml index d1bc3bcd4..d6376c498 100644 --- a/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml +++ b/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml @@ -12919,6 +12919,19 @@ spec: type: string clusterWideConfig: type: string + contourIngressConfig: + properties: + brokerFQDNTemplate: + description: Broker hostname template for Contour IngressRoute + resource to generate broker hostnames. + type: string + tlsSecretName: + description: TLS secret used for Contour IngressRoute resource + type: string + required: + - brokerFQDNTemplate + - tlsSecretName + type: object cruiseControlConfig: description: CruiseControlConfig defines the config for Cruise Control properties: @@ -18837,6 +18850,7 @@ spec: as well. enum: - envoy + - contour - istioingress type: string istioControlPlane: @@ -19197,6 +19211,7 @@ spec: enum: - LoadBalancer - NodePort + - ClusterIP type: string anyCastPort: description: configuring AnyCastPort allows kafka cluster @@ -19216,6 +19231,21 @@ spec: ingressConfig: additionalProperties: properties: + contourIngressConfig: + properties: + brokerFQDNTemplate: + description: Broker hostname template for + Contour IngressRoute resource to generate + broker hostnames. + type: string + tlsSecretName: + description: TLS secret used for Contour IngressRoute + resource + type: string + required: + - brokerFQDNTemplate + - tlsSecretName + type: object envoyConfig: description: EnvoyConfig defines the config for Envoy @@ -21673,6 +21703,11 @@ spec: description: At least one of the listeners should have this flag enabled type: boolean + usedForKafkaAdminCommunication: + description: UsedForKafkaAdminCommunication allows for a + different port to be returned when the koperator is checking + for the port to use to check if kafka is operating. + type: boolean required: - containerPort - externalStartingPort @@ -21749,6 +21784,11 @@ spec: description: At least one of the listeners should have this flag enabled type: boolean + usedForKafkaAdminCommunication: + description: UsedForKafkaAdminCommunication allows for a + different port to be returned when the koperator is checking + for the port to use to check if kafka is operating. + type: boolean required: - containerPort - name @@ -21841,9 +21881,10 @@ spec: description: RollingUpgradeConfig defines the desired config of the RollingUpgrade properties: - concurrentBrokerRestartsAllowed: - description: ConcurrentBrokerRestartsAllowed controls how many - brokers can be restarted in parallel during a rolling upgrade. + concurrentBrokerRestartCountPerRack: + default: 1 + description: ConcurrentBrokerRestartCountPerRack controls how + many brokers can be restarted in parallel during a rolling upgrade. If it is set to a value greater than 1, the operator will restart up to that amount of brokers in parallel, if the brokers are within the same rack (as specified by "broker.rack" in broker @@ -21854,7 +21895,8 @@ spec: N is the number of racks used. This is a safe way to speed up the rolling upgrade. Note that for the rack distribution explained above, Cruise Control requires `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal` - to be configured. + to be configured. Default value is 1. + minimum: 1 type: integer failureThreshold: description: FailureThreshold controls how many failures the cluster diff --git a/config/samples/banzaicloud_v1beta1_kafkacluster.yaml b/config/samples/banzaicloud_v1beta1_kafkacluster.yaml index 9bf828e10..85719d1c6 100644 --- a/config/samples/banzaicloud_v1beta1_kafkacluster.yaml +++ b/config/samples/banzaicloud_v1beta1_kafkacluster.yaml @@ -59,13 +59,13 @@ spec: # alerts with 'rollingupgrade' # failureThreshold: 1 - # concurrentBrokerRestartsAllowed controls how many brokers can be restarted in parallel during a rolling upgrade. If + # concurrentBrokerRestartCountPerRack controls how many brokers can be restarted in parallel during a rolling upgrade. If # it is set to a value greater than 1, the operator will restart up to that amount of brokers in parallel, if the # brokers are within the same AZ (as specified by "broker.rack" in broker read-only configs). Since using Kafka broker # racks spreads out the replicas, we know that restarting multiple brokers in the same rack will not cause more than # 1/Nth of the replicas of a topic-partition to be unavailable at the same time, where N is the number of racks used. # This is a safe way to speed up the rolling upgrade. - # concurrentBrokerRestartsAllowed: 1 + # concurrentBrokerRestartCountPerRack: 1 # brokerConfigGroups specifies multiple broker configs with unique name brokerConfigGroups: diff --git a/config/samples/simplekafkacluster_ssl.yaml b/config/samples/simplekafkacluster_ssl.yaml index 59741fac4..61912ea9c 100644 --- a/config/samples/simplekafkacluster_ssl.yaml +++ b/config/samples/simplekafkacluster_ssl.yaml @@ -50,16 +50,16 @@ spec: containerPort: 29092 usedForInnerBrokerCommunication: true # sslClientAuth defaults to be "required" for two-way SSL authentication, possible values are: "required", "requested", and "none" - # sslClientAuth: "requested" + sslClientAuth: "requested" - type: "ssl" name: "controller" containerPort: 29093 usedForInnerBrokerCommunication: false usedForControllerCommunication: true # sslClientAuth defaults to be "required" for two-way SSL authentication, possible values are: "required", "requested", and "none" - # sslClientAuth: "requested" + sslClientAuth: "requested" sslSecrets: - tlsSecretName: "kafka-ca-certs" + tlsSecretName: "kafka-ca-certificate" create: true cruiseControlConfig: # podSecurityContext: diff --git a/config/samples/simplekafkacluster_with_contour.yaml b/config/samples/simplekafkacluster_with_contour.yaml new file mode 100644 index 000000000..130c9537e --- /dev/null +++ b/config/samples/simplekafkacluster_with_contour.yaml @@ -0,0 +1,294 @@ +apiVersion: kafka.banzaicloud.io/v1beta1 +kind: KafkaCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: kafka +spec: + monitoringConfig: + jmxImage: "ghcr.io/banzaicloud/jmx-javaagent:0.16.1" + headlessServiceEnabled: true + zkAddresses: + - "zookeeper-server-client.zookeeper:2181" + propagateLabels: false + oneBrokerPerNode: false + clusterImage: "ghcr.io/banzaicloud/kafka:2.13-3.4.1" + ingressController: "contour" + readOnlyConfig: | + auto.create.topics.enable=false + cruise.control.metrics.topic.auto.create=true + cruise.control.metrics.topic.num.partitions=1 + cruise.control.metrics.topic.replication.factor=2 + brokerConfigGroups: + default: + # podSecurityContext: + # runAsNonRoot: false + # securityContext: + # privileged: true + storageConfigs: + - mountPath: "/kafka-logs" + pvcSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + brokerAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9020" + brokerIngressMapping: + - "contour" + # brokerLabels: + # kafka_broker_group: "default_group" + brokers: + - id: 0 + brokerConfigGroup: "default" + # brokerConfig: + # envs: + # - name: +CLASSPATH + # value: "/opt/kafka/libs/dev/*:" + # - name: CLASSPATH+ + # value: ":/opt/kafka/libs/extra-jars/*" + - id: 1 + brokerConfigGroup: "default" + - id: 2 + brokerConfigGroup: "default" + rollingUpgradeConfig: + failureThreshold: 1 + listenersConfig: + internalListeners: + - type: "plaintext" + name: "internal" + containerPort: 29092 + usedForInnerBrokerCommunication: true + - type: "plaintext" + name: "controller" + containerPort: 29093 + usedForInnerBrokerCommunication: false + usedForControllerCommunication: true + externalListeners: + - accessMethod: ClusterIP + anyCastPort: 8443 + containerPort: 29095 + externalStartingPort: -1 + name: contour + type: plaintext + usedForInnerBrokerCommunication: false + serviceAnnotations: + kubernetes.io/ingress.class: contour + config: + defaultIngressConfig: "" + ingressConfig: + contour: + hostnameOverride: kafka.cluster.local + contourIngressConfig: + tlsSecretName: heptio-contour/cluster-ssl + brokerFQDNTemplate: kafka-%id.cluster.local + cruiseControlConfig: + # podSecurityContext: + # runAsNonRoot: false + # securityContext: + # privileged: true + cruiseControlTaskSpec: + RetryDurationMinutes: 5 + topicConfig: + partitions: 12 + replicationFactor: 3 +# resourceRequirements: +# requests: +# cpu: 500m +# memory: 1Gi +# limits: +# cpu: 500m +# memory: 1Gi +# image: "ghcr.io/banzaicloud/cruise-control:2.5.86" + config: | + # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. + # + # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. + # Configuration for the metadata client. + # ======================================= + # The maximum interval in milliseconds between two metadata refreshes. + #metadata.max.age.ms=300000 + # Client id for the Cruise Control. It is used for the metadata client. + #client.id=kafka-cruise-control + # The size of TCP send buffer bytes for the metadata client. + #send.buffer.bytes=131072 + # The size of TCP receive buffer size for the metadata client. + #receive.buffer.bytes=131072 + # The time to wait before disconnect an idle TCP connection. + #connections.max.idle.ms=540000 + # The time to wait before reconnect to a given host. + #reconnect.backoff.ms=50 + # The time to wait for a response from a host after sending a request. + #request.timeout.ms=30000 + # Configurations for the load monitor + # ======================================= + # The number of metric fetcher thread to fetch metrics for the Kafka cluster + num.metric.fetchers=1 + # The metric sampler class + metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + # Configurations for CruiseControlMetricsReporterSampler + metric.reporter.topic.pattern=__CruiseControlMetrics + # The sample store class name + sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + # The config for the Kafka sample store to save the partition metric samples + partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + # The config for the Kafka sample store to save the model training samples + broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + # The replication factor of Kafka metric sample store topic + sample.store.topic.replication.factor=2 + # The config for the number of Kafka sample store consumer threads + num.sample.loading.threads=8 + # The partition assignor class for the metric samplers + metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + # The metric sampling interval in milliseconds + metric.sampling.interval.ms=120000 + metric.anomaly.detection.interval.ms=180000 + # The partition metrics window size in milliseconds + partition.metrics.window.ms=300000 + # The number of partition metric windows to keep in memory + num.partition.metrics.windows=1 + # The minimum partition metric samples required for a partition in each window + min.samples.per.partition.metrics.window=1 + # The broker metrics window size in milliseconds + broker.metrics.window.ms=300000 + # The number of broker metric windows to keep in memory + num.broker.metrics.windows=20 + # The minimum broker metric samples required for a partition in each window + min.samples.per.broker.metrics.window=1 + # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) + capacity.config.file=config/capacity.json + #capacity.config.file=config/capacityJBOD.json + # Configurations for the analyzer + # ======================================= + # The list of goals to optimize the Kafka cluster for with pre-computed proposals + default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + # The list of supported goals + goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + # The list of supported hard goals + hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The minimum percentage of well monitored partitions out of all the partitions + min.monitored.partition.percentage=0.95 + # The balance threshold for CPU + cpu.balance.threshold=1.1 + # The balance threshold for disk + disk.balance.threshold=1.1 + # The balance threshold for network inbound utilization + network.inbound.balance.threshold=1.1 + # The balance threshold for network outbound utilization + network.outbound.balance.threshold=1.1 + # The balance threshold for the replica count + replica.count.balance.threshold=1.1 + # The capacity threshold for CPU in percentage + cpu.capacity.threshold=0.8 + # The capacity threshold for disk in percentage + disk.capacity.threshold=0.8 + # The capacity threshold for network inbound utilization in percentage + network.inbound.capacity.threshold=0.8 + # The capacity threshold for network outbound utilization in percentage + network.outbound.capacity.threshold=0.8 + # The threshold to define the cluster to be in a low CPU utilization state + cpu.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + disk.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low network inbound utilization state + network.inbound.low.utilization.threshold=0.0 + # The threshold to define the cluster to be in a low disk utilization state + network.outbound.low.utilization.threshold=0.0 + # The metric anomaly percentile upper threshold + metric.anomaly.percentile.upper.threshold=90.0 + # The metric anomaly percentile lower threshold + metric.anomaly.percentile.lower.threshold=10.0 + # How often should the cached proposal be expired and recalculated if necessary + proposal.expiration.ms=60000 + # The maximum number of replicas that can reside on a broker at any given time. + max.replicas.per.broker=10000 + # The number of threads to use for proposal candidate precomputing. + num.proposal.precompute.threads=1 + # the topics that should be excluded from the partition movement. + #topics.excluded.from.partition.movement + # Configurations for the executor + # ======================================= + # The max number of partitions to move in/out on a given broker at a given time. + num.concurrent.partition.movements.per.broker=10 + # The interval between two execution progress checks. + execution.progress.check.interval.ms=10000 + # Configurations for anomaly detector + # ======================================= + # The goal violation notifier class + anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + # The metric anomaly finder class + metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + # The anomaly detection interval + anomaly.detection.interval.ms=10000 + # The goal violation to detect. + anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + # The interested metrics for metric anomaly analyzer. + metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN + ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. + #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + # The zk path to store failed broker information. + failed.brokers.zk.path=/CruiseControlBrokerList + # Topic config provider class + topic.config.provider.class=com.linkedin.kafka.cruisecontrol.config.KafkaTopicConfigProvider + # The cluster configurations for the KafkaTopicConfigProvider + cluster.configs.file=config/clusterConfigs.json + # The maximum time in milliseconds to store the response and access details of a completed user task. + completed.user.task.retention.time.ms=21600000 + # The maximum time in milliseconds to retain the demotion history of brokers. + demotion.history.retention.time.ms=86400000 + # The maximum number of completed user tasks for which the response and access details will be cached. + max.cached.completed.user.tasks=500 + # The maximum number of user tasks for concurrently running in async endpoints across all users. + max.active.user.tasks=25 + # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled + self.healing.enabled=true + # Enable self healing for broker failure detector + #self.healing.broker.failure.enabled=true + # Enable self healing for goal violation detector + #self.healing.goal.violation.enabled=true + # Enable self healing for metric anomaly detector + #self.healing.metric.anomaly.enabled=true + # configurations for the webserver + # ================================ + # HTTP listen port + webserver.http.port=9090 + # HTTP listen address + webserver.http.address=0.0.0.0 + # Whether CORS support is enabled for API or not + webserver.http.cors.enabled=false + # Value for Access-Control-Allow-Origin + webserver.http.cors.origin=http://localhost:8080/ + # Value for Access-Control-Request-Method + webserver.http.cors.allowmethods=OPTIONS,GET,POST + # Headers that should be exposed to the Browser (Webapp) + # This is a special header that is used by the + # User Tasks subsystem and should be explicitly + # Enabled when CORS mode is used as part of the + # Admin Interface + webserver.http.cors.exposeheaders=User-Task-ID + # REST API default prefix + # (dont forget the ending *) + webserver.api.urlprefix=/kafkacruisecontrol/* + # Location where the Cruise Control frontend is deployed + webserver.ui.diskpath=./cruise-control-ui/dist/ + # URL path prefix for UI + # (dont forget the ending *) + webserver.ui.urlprefix=/* + # Time After which request is converted to Async + webserver.request.maxBlockTimeMs=10000 + # Default Session Expiry Period + webserver.session.maxExpiryTimeMs=60000 + # Session cookie path + webserver.session.path=/ + # Server Access Logs + webserver.accesslog.enabled=true + # Location of HTTP Request Logs + webserver.accesslog.path=access.log + # HTTP Request Log retention days + webserver.accesslog.retention.days=14 + clusterConfig: | + { + "min.insync.replicas": 3 + } diff --git a/config/test/crd/projectcontour/crds.yaml b/config/test/crd/projectcontour/crds.yaml new file mode 100644 index 000000000..c240baeaf --- /dev/null +++ b/config/test/crd/projectcontour/crds.yaml @@ -0,0 +1,8562 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: contourconfigurations.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: ContourConfiguration + listKind: ContourConfigurationList + plural: contourconfigurations + shortNames: + - contourconfig + singular: contourconfiguration + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ContourConfiguration is the schema for a Contour instance. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + ContourConfigurationSpec represents a configuration of a Contour controller. + It contains most of all the options that can be customized, the + other remaining options being command line flags. + properties: + debug: + description: |- + Debug contains parameters to enable debug logging + and debug interfaces inside Contour. + properties: + address: + description: |- + Defines the Contour debug address interface. + Contour's default is "127.0.0.1". + type: string + port: + description: |- + Defines the Contour debug address port. + Contour's default is 6060. + type: integer + type: object + enableExternalNameService: + description: |- + EnableExternalNameService allows processing of ExternalNameServices + Contour's default is false for security reasons. + type: boolean + envoy: + description: |- + Envoy contains parameters for Envoy as well + as how to optionally configure a managed Envoy fleet. + properties: + clientCertificate: + description: |- + ClientCertificate defines the namespace/name of the Kubernetes + secret containing the client certificate and private key + to be used when establishing TLS connection to upstream + cluster. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + cluster: + description: |- + Cluster holds various configurable Envoy cluster values that can + be set in the config file. + properties: + circuitBreakers: + description: |- + GlobalCircuitBreakerDefaults specifies default circuit breaker budget across all services. + If defined, this will be used as the default for all services. + properties: + maxConnections: + description: The maximum number of connections that a + single Envoy instance allows to the Kubernetes Service; + defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests that + a single Envoy instance allows to the Kubernetes Service; + defaults to 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single Envoy + instance allows to the Kubernetes Service; defaults + to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries a + single Envoy instance allows to the Kubernetes Service; + defaults to 3. + format: int32 + type: integer + type: object + dnsLookupFamily: + description: |- + DNSLookupFamily defines how external names are looked up + When configured as V4, the DNS resolver will only perform a lookup + for addresses in the IPv4 family. If V6 is configured, the DNS resolver + will only perform a lookup for addresses in the IPv6 family. + If AUTO is configured, the DNS resolver will first perform a lookup + for addresses in the IPv6 family and fallback to a lookup for addresses + in the IPv4 family. If ALL is specified, the DNS resolver will perform a lookup for + both IPv4 and IPv6 families, and return all resolved addresses. + When this is used, Happy Eyeballs will be enabled for upstream connections. + Refer to Happy Eyeballs Support for more information. + Note: This only applies to externalName clusters. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html#envoy-v3-api-enum-config-cluster-v3-cluster-dnslookupfamily + for more information. + Values: `auto` (default), `v4`, `v6`, `all`. + Other values will produce an error. + type: string + maxRequestsPerConnection: + description: |- + Defines the maximum requests for upstream connections. If not specified, there is no limit. + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions + for more information. + format: int32 + minimum: 1 + type: integer + per-connection-buffer-limit-bytes: + description: |- + Defines the soft limit on size of the cluster’s new connection read and write buffers in bytes. + If unspecified, an implementation defined default is applied (1MiB). + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-field-config-cluster-v3-cluster-per-connection-buffer-limit-bytes + for more information. + format: int32 + minimum: 1 + type: integer + upstreamTLS: + description: UpstreamTLS contains the TLS policy parameters + for upstream connections + properties: + cipherSuites: + description: |- + CipherSuites defines the TLS ciphers to be supported by Envoy TLS + listeners when negotiating TLS 1.2. Ciphers are validated against the + set that Envoy supports by default. This parameter should only be used + by advanced users. Note that these will be ignored when TLS 1.3 is in + use. + This field is optional; when it is undefined, a Contour-managed ciphersuite list + will be used, which may be updated to keep it secure. + Contour's default list is: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + Ciphers provided are validated against the following list: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES128-GCM-SHA256" + - "ECDHE-RSA-AES128-GCM-SHA256" + - "ECDHE-ECDSA-AES128-SHA" + - "ECDHE-RSA-AES128-SHA" + - "AES128-GCM-SHA256" + - "AES128-SHA" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + - "ECDHE-ECDSA-AES256-SHA" + - "ECDHE-RSA-AES256-SHA" + - "AES256-GCM-SHA384" + - "AES256-SHA" + Contour recommends leaving this undefined unless you are sure you must. + See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#extensions-transport-sockets-tls-v3-tlsparameters + Note: This list is a superset of what is valid for stock Envoy builds and those using BoringSSL FIPS. + items: + type: string + type: array + maximumProtocolVersion: + description: |- + MaximumProtocolVersion is the maximum TLS version this vhost should + negotiate. + Values: `1.2`, `1.3`(default). + Other values will produce an error. + type: string + minimumProtocolVersion: + description: |- + MinimumProtocolVersion is the minimum TLS version this vhost should + negotiate. + Values: `1.2` (default), `1.3`. + Other values will produce an error. + type: string + type: object + type: object + defaultHTTPVersions: + description: |- + DefaultHTTPVersions defines the default set of HTTPS + versions the proxy should accept. HTTP versions are + strings of the form "HTTP/xx". Supported versions are + "HTTP/1.1" and "HTTP/2". + Values: `HTTP/1.1`, `HTTP/2` (default: both). + Other values will produce an error. + items: + description: HTTPVersionType is the name of a supported HTTP + version. + type: string + type: array + health: + description: |- + Health defines the endpoint Envoy uses to serve health checks. + Contour's default is { address: "0.0.0.0", port: 8002 }. + properties: + address: + description: Defines the health address interface. + minLength: 1 + type: string + port: + description: Defines the health port. + type: integer + type: object + http: + description: |- + Defines the HTTP Listener for Envoy. + Contour's default is { address: "0.0.0.0", port: 8080, accessLog: "/dev/stdout" }. + properties: + accessLog: + description: AccessLog defines where Envoy logs are outputted + for this listener. + type: string + address: + description: Defines an Envoy Listener Address. + minLength: 1 + type: string + port: + description: Defines an Envoy listener Port. + type: integer + type: object + https: + description: |- + Defines the HTTPS Listener for Envoy. + Contour's default is { address: "0.0.0.0", port: 8443, accessLog: "/dev/stdout" }. + properties: + accessLog: + description: AccessLog defines where Envoy logs are outputted + for this listener. + type: string + address: + description: Defines an Envoy Listener Address. + minLength: 1 + type: string + port: + description: Defines an Envoy listener Port. + type: integer + type: object + listener: + description: Listener hold various configurable Envoy listener + values. + properties: + connectionBalancer: + description: |- + ConnectionBalancer. If the value is exact, the listener will use the exact connection balancer + See https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/listener.proto#envoy-api-msg-listener-connectionbalanceconfig + for more information. + Values: (empty string): use the default ConnectionBalancer, `exact`: use the Exact ConnectionBalancer. + Other values will produce an error. + type: string + disableAllowChunkedLength: + description: |- + DisableAllowChunkedLength disables the RFC-compliant Envoy behavior to + strip the "Content-Length" header if "Transfer-Encoding: chunked" is + also set. This is an emergency off-switch to revert back to Envoy's + default behavior in case of failures. Please file an issue if failures + are encountered. + See: https://github.com/projectcontour/contour/issues/3221 + Contour's default is false. + type: boolean + disableMergeSlashes: + description: |- + DisableMergeSlashes disables Envoy's non-standard merge_slashes path transformation option + which strips duplicate slashes from request URL paths. + Contour's default is false. + type: boolean + httpMaxConcurrentStreams: + description: |- + Defines the value for SETTINGS_MAX_CONCURRENT_STREAMS Envoy will advertise in the + SETTINGS frame in HTTP/2 connections and the limit for concurrent streams allowed + for a peer on a single HTTP/2 connection. It is recommended to not set this lower + than 100 but this field can be used to bound resource usage by HTTP/2 connections + and mitigate attacks like CVE-2023-44487. The default value when this is not set is + unlimited. + format: int32 + minimum: 1 + type: integer + maxConnectionsPerListener: + description: |- + Defines the limit on number of active connections to a listener. The limit is applied + per listener. The default value when this is not set is unlimited. + format: int32 + minimum: 1 + type: integer + maxRequestsPerConnection: + description: |- + Defines the maximum requests for downstream connections. If not specified, there is no limit. + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions + for more information. + format: int32 + minimum: 1 + type: integer + maxRequestsPerIOCycle: + description: |- + Defines the limit on number of HTTP requests that Envoy will process from a single + connection in a single I/O cycle. Requests over this limit are processed in subsequent + I/O cycles. Can be used as a mitigation for CVE-2023-44487 when abusive traffic is + detected. Configures the http.max_requests_per_io_cycle Envoy runtime setting. The default + value when this is not set is no limit. + format: int32 + minimum: 1 + type: integer + per-connection-buffer-limit-bytes: + description: |- + Defines the soft limit on size of the listener’s new connection read and write buffers in bytes. + If unspecified, an implementation defined default is applied (1MiB). + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener.proto#envoy-v3-api-field-config-listener-v3-listener-per-connection-buffer-limit-bytes + for more information. + format: int32 + minimum: 1 + type: integer + serverHeaderTransformation: + description: |- + Defines the action to be applied to the Server header on the response path. + When configured as overwrite, overwrites any Server header with "envoy". + When configured as append_if_absent, if a Server header is present, pass it through, otherwise set it to "envoy". + When configured as pass_through, pass through the value of the Server header, and do not append a header if none is present. + Values: `overwrite` (default), `append_if_absent`, `pass_through` + Other values will produce an error. + Contour's default is overwrite. + type: string + socketOptions: + description: |- + SocketOptions defines configurable socket options for the listeners. + Single set of options are applied to all listeners. + properties: + tos: + description: |- + Defines the value for IPv4 TOS field (including 6 bit DSCP field) for IP packets originating from Envoy listeners. + Single value is applied to all listeners. + If listeners are bound to IPv6-only addresses, setting this option will cause an error. + format: int32 + maximum: 255 + minimum: 0 + type: integer + trafficClass: + description: |- + Defines the value for IPv6 Traffic Class field (including 6 bit DSCP field) for IP packets originating from the Envoy listeners. + Single value is applied to all listeners. + If listeners are bound to IPv4-only addresses, setting this option will cause an error. + format: int32 + maximum: 255 + minimum: 0 + type: integer + type: object + tls: + description: TLS holds various configurable Envoy TLS listener + values. + properties: + cipherSuites: + description: |- + CipherSuites defines the TLS ciphers to be supported by Envoy TLS + listeners when negotiating TLS 1.2. Ciphers are validated against the + set that Envoy supports by default. This parameter should only be used + by advanced users. Note that these will be ignored when TLS 1.3 is in + use. + This field is optional; when it is undefined, a Contour-managed ciphersuite list + will be used, which may be updated to keep it secure. + Contour's default list is: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + Ciphers provided are validated against the following list: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES128-GCM-SHA256" + - "ECDHE-RSA-AES128-GCM-SHA256" + - "ECDHE-ECDSA-AES128-SHA" + - "ECDHE-RSA-AES128-SHA" + - "AES128-GCM-SHA256" + - "AES128-SHA" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + - "ECDHE-ECDSA-AES256-SHA" + - "ECDHE-RSA-AES256-SHA" + - "AES256-GCM-SHA384" + - "AES256-SHA" + Contour recommends leaving this undefined unless you are sure you must. + See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#extensions-transport-sockets-tls-v3-tlsparameters + Note: This list is a superset of what is valid for stock Envoy builds and those using BoringSSL FIPS. + items: + type: string + type: array + maximumProtocolVersion: + description: |- + MaximumProtocolVersion is the maximum TLS version this vhost should + negotiate. + Values: `1.2`, `1.3`(default). + Other values will produce an error. + type: string + minimumProtocolVersion: + description: |- + MinimumProtocolVersion is the minimum TLS version this vhost should + negotiate. + Values: `1.2` (default), `1.3`. + Other values will produce an error. + type: string + type: object + useProxyProtocol: + description: |- + Use PROXY protocol for all listeners. + Contour's default is false. + type: boolean + type: object + logging: + description: Logging defines how Envoy's logs can be configured. + properties: + accessLogFormat: + description: |- + AccessLogFormat sets the global access log format. + Values: `envoy` (default), `json`. + Other values will produce an error. + type: string + accessLogFormatString: + description: |- + AccessLogFormatString sets the access log format when format is set to `envoy`. + When empty, Envoy's default format is used. + type: string + accessLogJSONFields: + description: |- + AccessLogJSONFields sets the fields that JSON logging will + output when AccessLogFormat is json. + items: + type: string + type: array + accessLogLevel: + description: |- + AccessLogLevel sets the verbosity level of the access log. + Values: `info` (default, all requests are logged), `error` (all non-success requests, i.e. 300+ response code, are logged), `critical` (all 5xx requests are logged) and `disabled`. + Other values will produce an error. + type: string + type: object + metrics: + description: |- + Metrics defines the endpoint Envoy uses to serve metrics. + Contour's default is { address: "0.0.0.0", port: 8002 }. + properties: + address: + description: Defines the metrics address interface. + maxLength: 253 + minLength: 1 + type: string + port: + description: Defines the metrics port. + type: integer + tls: + description: |- + TLS holds TLS file config details. + Metrics and health endpoints cannot have same port number when metrics is served over HTTPS. + properties: + caFile: + description: CA filename. + type: string + certFile: + description: Client certificate filename. + type: string + keyFile: + description: Client key filename. + type: string + type: object + type: object + network: + description: Network holds various configurable Envoy network + values. + properties: + adminPort: + description: |- + Configure the port used to access the Envoy Admin interface. + If configured to port "0" then the admin interface is disabled. + Contour's default is 9001. + type: integer + numTrustedHops: + description: |- + XffNumTrustedHops defines the number of additional ingress proxy hops from the + right side of the x-forwarded-for HTTP header to trust when determining the origin + client’s IP address. + See https://www.envoyproxy.io/docs/envoy/v1.17.0/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto?highlight=xff_num_trusted_hops + for more information. + Contour's default is 0. + format: int32 + type: integer + type: object + service: + description: |- + Service holds Envoy service parameters for setting Ingress status. + Contour's default is { namespace: "projectcontour", name: "envoy" }. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + timeouts: + description: |- + Timeouts holds various configurable timeouts that can + be set in the config file. + properties: + connectTimeout: + description: |- + ConnectTimeout defines how long the proxy should wait when establishing connection to upstream service. + If not set, a default value of 2 seconds will be used. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-field-config-cluster-v3-cluster-connect-timeout + for more information. + type: string + connectionIdleTimeout: + description: |- + ConnectionIdleTimeout defines how long the proxy should wait while there are + no active requests (for HTTP/1.1) or streams (for HTTP/2) before terminating + an HTTP connection. Set to "infinity" to disable the timeout entirely. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-httpprotocoloptions-idle-timeout + for more information. + type: string + connectionShutdownGracePeriod: + description: |- + ConnectionShutdownGracePeriod defines how long the proxy will wait between sending an + initial GOAWAY frame and a second, final GOAWAY frame when terminating an HTTP/2 connection. + During this grace period, the proxy will continue to respond to new streams. After the final + GOAWAY frame has been sent, the proxy will refuse new streams. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-drain-timeout + for more information. + type: string + delayedCloseTimeout: + description: |- + DelayedCloseTimeout defines how long envoy will wait, once connection + close processing has been initiated, for the downstream peer to close + the connection before Envoy closes the socket associated with the connection. + Setting this timeout to 'infinity' will disable it, equivalent to setting it to '0' + in Envoy. Leaving it unset will result in the Envoy default value being used. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-delayed-close-timeout + for more information. + type: string + maxConnectionDuration: + description: |- + MaxConnectionDuration defines the maximum period of time after an HTTP connection + has been established from the client to the proxy before it is closed by the proxy, + regardless of whether there has been activity or not. Omit or set to "infinity" for + no max duration. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-httpprotocoloptions-max-connection-duration + for more information. + type: string + requestTimeout: + description: |- + RequestTimeout sets the client request timeout globally for Contour. Note that + this is a timeout for the entire request, not an idle timeout. Omit or set to + "infinity" to disable the timeout entirely. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-request-timeout + for more information. + type: string + streamIdleTimeout: + description: |- + StreamIdleTimeout defines how long the proxy should wait while there is no + request activity (for HTTP/1.1) or stream activity (for HTTP/2) before + terminating the HTTP request or stream. Set to "infinity" to disable the + timeout entirely. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-stream-idle-timeout + for more information. + type: string + type: object + type: object + featureFlags: + description: |- + FeatureFlags defines toggle to enable new contour features. + Available toggles are: + useEndpointSlices - Configures contour to fetch endpoint data + from k8s endpoint slices. defaults to true, + If false then reads endpoint data from the k8s endpoints. + items: + type: string + type: array + gateway: + description: |- + Gateway contains parameters for the gateway-api Gateway that Contour + is configured to serve traffic. + properties: + gatewayRef: + description: |- + GatewayRef defines the specific Gateway that this Contour + instance corresponds to. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - gatewayRef + type: object + globalExtAuth: + description: |- + GlobalExternalAuthorization allows envoys external authorization filter + to be enabled for all virtual hosts. + properties: + authPolicy: + description: |- + AuthPolicy sets a default authorization policy for client requests. + This policy will be used unless overridden by individual routes. + properties: + context: + additionalProperties: + type: string + description: |- + Context is a set of key/value pairs that are sent to the + authentication server in the check request. If a context + is provided at an enclosing scope, the entries are merged + such that the inner scope overrides matching keys from the + outer scope. + type: object + disabled: + description: |- + When true, this field disables client request authentication + for the scope of the policy. + type: boolean + type: object + extensionRef: + description: ExtensionServiceRef specifies the extension resource + that will authorize client requests. + properties: + apiVersion: + description: |- + API version of the referent. + If this field is not specified, the default "projectcontour.io/v1alpha1" will be used + minLength: 1 + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referent. + If this field is not specifies, the namespace of the resource that targets the referent will be used. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + minLength: 1 + type: string + type: object + failOpen: + description: |- + If FailOpen is true, the client request is forwarded to the upstream service + even if the authorization server fails to respond. This field should not be + set in most cases. It is intended for use only while migrating applications + from internal authorization to Contour external authorization. + type: boolean + responseTimeout: + description: |- + ResponseTimeout configures maximum time to wait for a check response from the authorization server. + Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). + Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + The string "infinity" is also a valid input and specifies no timeout. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + withRequestBody: + description: WithRequestBody specifies configuration for sending + the client request's body to authorization server. + properties: + allowPartialMessage: + description: If AllowPartialMessage is true, then Envoy will + buffer the body until MaxRequestBytes are reached. + type: boolean + maxRequestBytes: + default: 1024 + description: MaxRequestBytes sets the maximum size of message + body ExtAuthz filter will hold in-memory. + format: int32 + minimum: 1 + type: integer + packAsBytes: + description: If PackAsBytes is true, the body sent to Authorization + Server is in raw bytes. + type: boolean + type: object + type: object + health: + description: |- + Health defines the endpoints Contour uses to serve health checks. + Contour's default is { address: "0.0.0.0", port: 8000 }. + properties: + address: + description: Defines the health address interface. + minLength: 1 + type: string + port: + description: Defines the health port. + type: integer + type: object + httpproxy: + description: HTTPProxy defines parameters on HTTPProxy. + properties: + disablePermitInsecure: + description: |- + DisablePermitInsecure disables the use of the + permitInsecure field in HTTPProxy. + Contour's default is false. + type: boolean + fallbackCertificate: + description: |- + FallbackCertificate defines the namespace/name of the Kubernetes secret to + use as fallback when a non-SNI request is received. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + rootNamespaces: + description: Restrict Contour to searching these namespaces for + root ingress routes. + items: + type: string + type: array + type: object + ingress: + description: Ingress contains parameters for ingress options. + properties: + classNames: + description: Ingress Class Names Contour should use. + items: + type: string + type: array + statusAddress: + description: Address to set in Ingress object status. + type: string + type: object + metrics: + description: |- + Metrics defines the endpoint Contour uses to serve metrics. + Contour's default is { address: "0.0.0.0", port: 8000 }. + properties: + address: + description: Defines the metrics address interface. + maxLength: 253 + minLength: 1 + type: string + port: + description: Defines the metrics port. + type: integer + tls: + description: |- + TLS holds TLS file config details. + Metrics and health endpoints cannot have same port number when metrics is served over HTTPS. + properties: + caFile: + description: CA filename. + type: string + certFile: + description: Client certificate filename. + type: string + keyFile: + description: Client key filename. + type: string + type: object + type: object + policy: + description: Policy specifies default policy applied if not overridden + by the user + properties: + applyToIngress: + description: |- + ApplyToIngress determines if the Policies will apply to ingress objects + Contour's default is false. + type: boolean + requestHeaders: + description: RequestHeadersPolicy defines the request headers + set/removed on all routes + properties: + remove: + items: + type: string + type: array + set: + additionalProperties: + type: string + type: object + type: object + responseHeaders: + description: ResponseHeadersPolicy defines the response headers + set/removed on all routes + properties: + remove: + items: + type: string + type: array + set: + additionalProperties: + type: string + type: object + type: object + type: object + rateLimitService: + description: |- + RateLimitService optionally holds properties of the Rate Limit Service + to be used for global rate limiting. + properties: + defaultGlobalRateLimitPolicy: + description: |- + DefaultGlobalRateLimitPolicy allows setting a default global rate limit policy for every HTTPProxy. + HTTPProxy can overwrite this configuration. + properties: + descriptors: + description: |- + Descriptors defines the list of descriptors that will + be generated and sent to the rate limit service. Each + descriptor contains 1+ key-value pair entries. + items: + description: RateLimitDescriptor defines a list of key-value + pair generators. + properties: + entries: + description: Entries is the list of key-value pair generators. + items: + description: |- + RateLimitDescriptorEntry is a key-value pair generator. Exactly + one field on this struct must be non-nil. + properties: + genericKey: + description: GenericKey defines a descriptor entry + with a static key and value. + properties: + key: + description: |- + Key defines the key of the descriptor entry. If not set, the + key is set to "generic_key". + type: string + value: + description: Value defines the value of the + descriptor entry. + minLength: 1 + type: string + type: object + remoteAddress: + description: |- + RemoteAddress defines a descriptor entry with a key of "remote_address" + and a value equal to the client's IP address (from x-forwarded-for). + type: object + requestHeader: + description: |- + RequestHeader defines a descriptor entry that's populated only if + a given header is present on the request. The descriptor key is static, + and the descriptor value is equal to the value of the header. + properties: + descriptorKey: + description: DescriptorKey defines the key + to use on the descriptor entry. + minLength: 1 + type: string + headerName: + description: HeaderName defines the name of + the header to look for on the request. + minLength: 1 + type: string + type: object + requestHeaderValueMatch: + description: |- + RequestHeaderValueMatch defines a descriptor entry that's populated + if the request's headers match a set of 1+ match criteria. The + descriptor key is "header_match", and the descriptor value is static. + properties: + expectMatch: + default: true + description: |- + ExpectMatch defines whether the request must positively match the match + criteria in order to generate a descriptor entry (i.e. true), or not + match the match criteria in order to generate a descriptor entry (i.e. false). + The default is true. + type: boolean + headers: + description: |- + Headers is a list of 1+ match criteria to apply against the request + to determine whether to populate the descriptor entry or not. + items: + description: |- + HeaderMatchCondition specifies how to conditionally match against HTTP + headers. The Name field is required, only one of Present, NotPresent, + Contains, NotContains, Exact, NotExact and Regex can be set. + For negative matching rules only (e.g. NotContains or NotExact) you can set + TreatMissingAsEmpty. + IgnoreCase has no effect for Regex. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the header value. + type: string + exact: + description: Exact specifies a string + that the header value must be equal + to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the header to match against. Name is required. + Header names are case insensitive. + type: string + notcontains: + description: |- + NotContains specifies a substring that must not be present + in the header value. + type: string + notexact: + description: |- + NoExact specifies a string that the header value must not be + equal to. The condition is true if the header has any other value. + type: string + notpresent: + description: |- + NotPresent specifies that condition is true when the named header + is not present. Note that setting NotPresent to false does not + make the condition true if the named header is present. + type: boolean + present: + description: |- + Present specifies that condition is true when the named header + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named header + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the header + value. + type: string + treatMissingAsEmpty: + description: |- + TreatMissingAsEmpty specifies if the header match rule specified header + does not exist, this header value will be treated as empty. Defaults to false. + Unlike the underlying Envoy implementation this is **only** supported for + negative matches (e.g. NotContains, NotExact). + type: boolean + required: + - name + type: object + minItems: 1 + type: array + value: + description: Value defines the value of the + descriptor entry. + minLength: 1 + type: string + type: object + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + disabled: + description: |- + Disabled configures the HTTPProxy to not use + the default global rate limit policy defined by the Contour configuration. + type: boolean + type: object + domain: + description: Domain is passed to the Rate Limit Service. + type: string + enableResourceExhaustedCode: + description: |- + EnableResourceExhaustedCode enables translating error code 429 to + grpc code RESOURCE_EXHAUSTED. When disabled it's translated to UNAVAILABLE + type: boolean + enableXRateLimitHeaders: + description: |- + EnableXRateLimitHeaders defines whether to include the X-RateLimit + headers X-RateLimit-Limit, X-RateLimit-Remaining, and X-RateLimit-Reset + (as defined by the IETF Internet-Draft linked below), on responses + to clients when the Rate Limit Service is consulted for a request. + ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html + type: boolean + extensionService: + description: ExtensionService identifies the extension service + defining the RLS. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + failOpen: + description: |- + FailOpen defines whether to allow requests to proceed when the + Rate Limit Service fails to respond with a valid rate limit + decision within the timeout defined on the extension service. + type: boolean + required: + - extensionService + type: object + tracing: + description: Tracing defines properties for exporting trace data to + OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with unique + tag name. + items: + description: |- + CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: |- + Literal is a static custom tag value. + Precisely one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: |- + RequestHeaderName indicates which request header + the label value is obtained from. + Precisely one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: |- + IncludePodDetail defines a flag. + If it is true, contour will add the pod name and namespace to the span of the trace. + the default is true. + Note: The Envoy pods MUST have the HOSTNAME and CONTOUR_NAMESPACE environment variables set for this to work properly. + type: boolean + maxPathTagLength: + description: |- + MaxPathTagLength defines maximum length of the request path + to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: |- + OverallSampling defines the sampling rate of trace data. + contour's default is 100. + type: string + serviceName: + description: |- + ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object + xdsServer: + description: XDSServer contains parameters for the xDS server. + properties: + address: + description: |- + Defines the xDS gRPC API address which Contour will serve. + Contour's default is "0.0.0.0". + minLength: 1 + type: string + port: + description: |- + Defines the xDS gRPC API port which Contour will serve. + Contour's default is 8001. + type: integer + tls: + description: |- + TLS holds TLS file config details. + Contour's default is { caFile: "/certs/ca.crt", certFile: "/certs/tls.cert", keyFile: "/certs/tls.key", insecure: false }. + properties: + caFile: + description: CA filename. + type: string + certFile: + description: Client certificate filename. + type: string + insecure: + description: Allow serving the xDS gRPC API without TLS. + type: boolean + keyFile: + description: Client key filename. + type: string + type: object + type: + description: |- + Defines the XDSServer to use for `contour serve`. + Values: `envoy` (default), `contour (deprecated)`. + Other values will produce an error. + type: string + type: object + type: object + status: + description: ContourConfigurationStatus defines the observed state of + a ContourConfiguration resource. + properties: + conditions: + description: |- + Conditions contains the current status of the Contour resource. + Contour will update a single condition, `Valid`, that is in normal-true polarity. + Contour will not modify any other Conditions set in this block, + in case some other controller wants to add a Condition. + items: + description: |- + DetailedCondition is an extension of the normal Kubernetes conditions, with two extra + fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) + of the condition. + `errors` holds information about sub-conditions which are fatal to that condition and render its state False. + `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. + Remember that Conditions have a type, a status, and a reason. + The type is the type of the condition, the most important one in this CRD set is `Valid`. + `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. + In more detail, `status: true` means that the object is has been ingested into Contour with no errors. + `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` + slice in this case. + `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. + The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` + slice if `status` is `false`. + For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. + When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. + When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. + In either case, there may be entries in the `warnings` slice. + Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason + (if there is one and only one entry in total across both the `errors` and `warnings` slices), or + `MultipleReasons` if there is more than one entry. + properties: + errors: + description: |- + Errors contains a slice of relevant error subconditions for this object. + Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. + An empty slice here indicates no errors. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: |- + Warnings contains a slice of relevant warning subconditions for this object. + Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. + An empty slice here indicates no warnings. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: contourdeployments.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: ContourDeployment + listKind: ContourDeploymentList + plural: contourdeployments + shortNames: + - contourdeploy + singular: contourdeployment + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ContourDeployment is the schema for a Contour Deployment. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + ContourDeploymentSpec specifies options for how a Contour + instance should be provisioned. + properties: + contour: + description: |- + Contour specifies deployment-time settings for the Contour + part of the installation, i.e. the xDS server/control plane + and associated resources, including things like replica count + for the Deployment, and node placement constraints for the pods. + properties: + deployment: + description: Deployment describes the settings for running contour + as a `Deployment`. + properties: + replicas: + description: Replicas is the desired number of replicas. + format: int32 + minimum: 0 + type: integer + strategy: + description: Strategy describes the deployment strategy to + use to replace existing pods with new pods. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + type: object + disabledFeatures: + description: |- + DisabledFeatures defines an array of resources that will be ignored by + contour reconciler. + items: + enum: + - grpcroutes + - tlsroutes + - extensionservices + - backendtlspolicies + type: string + maxItems: 42 + minItems: 1 + type: array + kubernetesLogLevel: + description: |- + KubernetesLogLevel Enable Kubernetes client debug logging with log level. If unset, + defaults to 0. + maximum: 9 + minimum: 0 + type: integer + logLevel: + description: |- + LogLevel sets the log level for Contour + Allowed values are "info", "debug". + type: string + nodePlacement: + description: NodePlacement describes node scheduling configuration + of Contour pods. + properties: + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is the simplest recommended form of node selection constraint + and specifies a map of key-value pairs. For the pod to be eligible + to run on a node, the node must have each of the indicated key-value pairs + as labels (it can have additional labels as well). + If unset, the pod(s) will be scheduled to any available node. + type: object + tolerations: + description: |- + Tolerations work with taints to ensure that pods are not scheduled + onto inappropriate nodes. One or more taints are applied to a node; this + marks that the node should not accept any pods that do not tolerate the + taints. + The default is an empty list. + See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + for additional details. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + podAnnotations: + additionalProperties: + type: string + description: |- + PodAnnotations defines annotations to add to the Contour pods. + the annotations for Prometheus will be appended or overwritten with predefined value. + type: object + replicas: + description: |- + Deprecated: Use `DeploymentSettings.Replicas` instead. + Replicas is the desired number of Contour replicas. If if unset, + defaults to 2. + if both `DeploymentSettings.Replicas` and this one is set, use `DeploymentSettings.Replicas`. + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Compute Resources required by contour container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + watchNamespaces: + description: |- + WatchNamespaces is an array of namespaces. Setting it will instruct the contour instance + to only watch this subset of namespaces. + items: + description: |- + Namespace refers to a Kubernetes namespace. It must be a RFC 1123 label. + This validation is based off of the corresponding Kubernetes validation: + https://github.com/kubernetes/apimachinery/blob/02cfb53916346d085a6c6c7c66f882e3c6b0eca6/pkg/util/validation/validation.go#L187 + This is used for Namespace name validation here: + https://github.com/kubernetes/apimachinery/blob/02cfb53916346d085a6c6c7c66f882e3c6b0eca6/pkg/api/validation/generic.go#L63 + Valid values include: + * "example" + Invalid values include: + * "example.com" - "." is an invalid character + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + maxItems: 42 + minItems: 1 + type: array + type: object + envoy: + description: |- + Envoy specifies deployment-time settings for the Envoy + part of the installation, i.e. the xDS client/data plane + and associated resources, including things like the workload + type to use (DaemonSet or Deployment), node placement constraints + for the pods, and various options for the Envoy service. + properties: + baseID: + description: |- + The base ID to use when allocating shared memory regions. + if Envoy needs to be run multiple times on the same machine, each running Envoy will need a unique base ID + so that the shared memory regions do not conflict. + defaults to 0. + format: int32 + minimum: 0 + type: integer + daemonSet: + description: |- + DaemonSet describes the settings for running envoy as a `DaemonSet`. + if `WorkloadType` is `Deployment`,it's must be nil + properties: + updateStrategy: + description: Strategy describes the deployment strategy to + use to replace existing DaemonSet pods with new pods. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if type = "RollingUpdate". + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. Same as Deployment `strategy.rollingUpdate`. + See https://github.com/kubernetes/kubernetes/issues/35345 + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of nodes with an existing available DaemonSet pod that + can have an updated DaemonSet pod during during an update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up to a minimum of 1. + Default value is 0. + Example: when this is set to 30%, at most 30% of the total number of nodes + that should be running the daemon pod (i.e. status.desiredNumberScheduled) + can have their a new pod created before the old pod is marked as deleted. + The update starts by launching new pods on 30% of nodes. Once an updated + pod is available (Ready for at least minReadySeconds) the old DaemonSet pod + on that node is marked deleted. If the old pod becomes unavailable for any + reason (Ready transitions to false, is evicted, or is drained) an updated + pod is immediatedly created on that node without considering surge limits. + Allowing surge implies the possibility that the resources consumed by the + daemonset on any given node can double if the readiness check fails, and + so resource intensive daemonsets should take into account that they may + cause evictions during disruption. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of DaemonSet pods that can be unavailable during the + update. Value can be an absolute number (ex: 5) or a percentage of total + number of DaemonSet pods at the start of the update (ex: 10%). Absolute + number is calculated from percentage by rounding up. + This cannot be 0 if MaxSurge is 0 + Default value is 1. + Example: when this is set to 30%, at most 30% of the total number of nodes + that should be running the daemon pod (i.e. status.desiredNumberScheduled) + can have their pods stopped for an update at any given time. The update + starts by stopping at most 30% of those DaemonSet pods and then brings + up new DaemonSet pods in their place. Once the new pods are available, + it then proceeds onto other DaemonSet pods, thus ensuring that at least + 70% of original number of DaemonSet pods are available at all times during + the update. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of daemon set update. Can be "RollingUpdate" + or "OnDelete". Default is RollingUpdate. + type: string + type: object + type: object + deployment: + description: |- + Deployment describes the settings for running envoy as a `Deployment`. + if `WorkloadType` is `DaemonSet`,it's must be nil + properties: + replicas: + description: Replicas is the desired number of replicas. + format: int32 + minimum: 0 + type: integer + strategy: + description: Strategy describes the deployment strategy to + use to replace existing pods with new pods. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + type: object + extraVolumeMounts: + description: ExtraVolumeMounts holds the extra volume mounts to + add (normally used with extraVolumes). + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + If ReadOnly is false, this field has no meaning and must be unspecified. + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + extraVolumes: + description: ExtraVolumes holds the extra volumes to add. + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: + None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the + blob storage + type: string + fsType: + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single + blob disk per storage account Managed: azure managed + data disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that + contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted + root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must not + be absolute or contain the ''..'' path. Must + be utf-8 encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use + for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds + extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. + This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- + TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not + mount host directories as read/write. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support + iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI + target and initiator authentication + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + Alpha, gated by the ClusterTrustBundleProjection feature gate. + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume + root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the + downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool + associated with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy + Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + logLevel: + description: |- + LogLevel sets the log level for Envoy. + Allowed values are "trace", "debug", "info", "warn", "error", "critical", "off". + type: string + networkPublishing: + description: NetworkPublishing defines how to expose Envoy to + a network. + properties: + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, + and LoadBalancer IPs). + If unset, defaults to "Local". + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). + type: string + serviceAnnotations: + additionalProperties: + type: string + description: |- + ServiceAnnotations is the annotations to add to + the provisioned Envoy service. + type: object + type: + description: |- + NetworkPublishingType is the type of publishing strategy to use. Valid values are: + * LoadBalancerService + In this configuration, network endpoints for Envoy use container networking. + A Kubernetes LoadBalancer Service is created to publish Envoy network + endpoints. + See: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer + * NodePortService + Publishes Envoy network endpoints using a Kubernetes NodePort Service. + In this configuration, Envoy network endpoints use container networking. A Kubernetes + NodePort Service is created to publish the network endpoints. + See: https://kubernetes.io/docs/concepts/services-networking/service/#nodeport + NOTE: + When provisioning an Envoy `NodePortService`, use Gateway Listeners' port numbers to populate + the Service's node port values, there's no way to auto-allocate them. + See: https://github.com/projectcontour/contour/issues/4499 + * ClusterIPService + Publishes Envoy network endpoints using a Kubernetes ClusterIP Service. + In this configuration, Envoy network endpoints use container networking. A Kubernetes + ClusterIP Service is created to publish the network endpoints. + See: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + If unset, defaults to LoadBalancerService. + type: string + type: object + nodePlacement: + description: NodePlacement describes node scheduling configuration + of Envoy pods. + properties: + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is the simplest recommended form of node selection constraint + and specifies a map of key-value pairs. For the pod to be eligible + to run on a node, the node must have each of the indicated key-value pairs + as labels (it can have additional labels as well). + If unset, the pod(s) will be scheduled to any available node. + type: object + tolerations: + description: |- + Tolerations work with taints to ensure that pods are not scheduled + onto inappropriate nodes. One or more taints are applied to a node; this + marks that the node should not accept any pods that do not tolerate the + taints. + The default is an empty list. + See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + for additional details. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + overloadMaxHeapSize: + description: |- + OverloadMaxHeapSize defines the maximum heap memory of the envoy controlled by the overload manager. + When the value is greater than 0, the overload manager is enabled, + and when envoy reaches 95% of the maximum heap size, it performs a shrink heap operation, + When it reaches 98% of the maximum heap size, Envoy Will stop accepting requests. + More info: https://projectcontour.io/docs/main/config/overload-manager/ + format: int64 + type: integer + podAnnotations: + additionalProperties: + type: string + description: |- + PodAnnotations defines annotations to add to the Envoy pods. + the annotations for Prometheus will be appended or overwritten with predefined value. + type: object + replicas: + description: |- + Deprecated: Use `DeploymentSettings.Replicas` instead. + Replicas is the desired number of Envoy replicas. If WorkloadType + is not "Deployment", this field is ignored. Otherwise, if unset, + defaults to 2. + if both `DeploymentSettings.Replicas` and this one is set, use `DeploymentSettings.Replicas`. + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Compute Resources required by envoy container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + workloadType: + description: |- + WorkloadType is the type of workload to install Envoy + as. Choices are DaemonSet and Deployment. If unset, defaults + to DaemonSet. + type: string + type: object + resourceLabels: + additionalProperties: + type: string + description: |- + ResourceLabels is a set of labels to add to the provisioned Contour resources. + Deprecated: use Gateway.Spec.Infrastructure.Labels instead. This field will be + removed in a future release. + type: object + runtimeSettings: + description: |- + RuntimeSettings is a ContourConfiguration spec to be used when + provisioning a Contour instance that will influence aspects of + the Contour instance's runtime behavior. + properties: + debug: + description: |- + Debug contains parameters to enable debug logging + and debug interfaces inside Contour. + properties: + address: + description: |- + Defines the Contour debug address interface. + Contour's default is "127.0.0.1". + type: string + port: + description: |- + Defines the Contour debug address port. + Contour's default is 6060. + type: integer + type: object + enableExternalNameService: + description: |- + EnableExternalNameService allows processing of ExternalNameServices + Contour's default is false for security reasons. + type: boolean + envoy: + description: |- + Envoy contains parameters for Envoy as well + as how to optionally configure a managed Envoy fleet. + properties: + clientCertificate: + description: |- + ClientCertificate defines the namespace/name of the Kubernetes + secret containing the client certificate and private key + to be used when establishing TLS connection to upstream + cluster. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + cluster: + description: |- + Cluster holds various configurable Envoy cluster values that can + be set in the config file. + properties: + circuitBreakers: + description: |- + GlobalCircuitBreakerDefaults specifies default circuit breaker budget across all services. + If defined, this will be used as the default for all services. + properties: + maxConnections: + description: The maximum number of connections that + a single Envoy instance allows to the Kubernetes + Service; defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests + that a single Envoy instance allows to the Kubernetes + Service; defaults to 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single + Envoy instance allows to the Kubernetes Service; + defaults to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries + a single Envoy instance allows to the Kubernetes + Service; defaults to 3. + format: int32 + type: integer + type: object + dnsLookupFamily: + description: |- + DNSLookupFamily defines how external names are looked up + When configured as V4, the DNS resolver will only perform a lookup + for addresses in the IPv4 family. If V6 is configured, the DNS resolver + will only perform a lookup for addresses in the IPv6 family. + If AUTO is configured, the DNS resolver will first perform a lookup + for addresses in the IPv6 family and fallback to a lookup for addresses + in the IPv4 family. If ALL is specified, the DNS resolver will perform a lookup for + both IPv4 and IPv6 families, and return all resolved addresses. + When this is used, Happy Eyeballs will be enabled for upstream connections. + Refer to Happy Eyeballs Support for more information. + Note: This only applies to externalName clusters. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html#envoy-v3-api-enum-config-cluster-v3-cluster-dnslookupfamily + for more information. + Values: `auto` (default), `v4`, `v6`, `all`. + Other values will produce an error. + type: string + maxRequestsPerConnection: + description: |- + Defines the maximum requests for upstream connections. If not specified, there is no limit. + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions + for more information. + format: int32 + minimum: 1 + type: integer + per-connection-buffer-limit-bytes: + description: |- + Defines the soft limit on size of the cluster’s new connection read and write buffers in bytes. + If unspecified, an implementation defined default is applied (1MiB). + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-field-config-cluster-v3-cluster-per-connection-buffer-limit-bytes + for more information. + format: int32 + minimum: 1 + type: integer + upstreamTLS: + description: UpstreamTLS contains the TLS policy parameters + for upstream connections + properties: + cipherSuites: + description: |- + CipherSuites defines the TLS ciphers to be supported by Envoy TLS + listeners when negotiating TLS 1.2. Ciphers are validated against the + set that Envoy supports by default. This parameter should only be used + by advanced users. Note that these will be ignored when TLS 1.3 is in + use. + This field is optional; when it is undefined, a Contour-managed ciphersuite list + will be used, which may be updated to keep it secure. + Contour's default list is: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + Ciphers provided are validated against the following list: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES128-GCM-SHA256" + - "ECDHE-RSA-AES128-GCM-SHA256" + - "ECDHE-ECDSA-AES128-SHA" + - "ECDHE-RSA-AES128-SHA" + - "AES128-GCM-SHA256" + - "AES128-SHA" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + - "ECDHE-ECDSA-AES256-SHA" + - "ECDHE-RSA-AES256-SHA" + - "AES256-GCM-SHA384" + - "AES256-SHA" + Contour recommends leaving this undefined unless you are sure you must. + See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#extensions-transport-sockets-tls-v3-tlsparameters + Note: This list is a superset of what is valid for stock Envoy builds and those using BoringSSL FIPS. + items: + type: string + type: array + maximumProtocolVersion: + description: |- + MaximumProtocolVersion is the maximum TLS version this vhost should + negotiate. + Values: `1.2`, `1.3`(default). + Other values will produce an error. + type: string + minimumProtocolVersion: + description: |- + MinimumProtocolVersion is the minimum TLS version this vhost should + negotiate. + Values: `1.2` (default), `1.3`. + Other values will produce an error. + type: string + type: object + type: object + defaultHTTPVersions: + description: |- + DefaultHTTPVersions defines the default set of HTTPS + versions the proxy should accept. HTTP versions are + strings of the form "HTTP/xx". Supported versions are + "HTTP/1.1" and "HTTP/2". + Values: `HTTP/1.1`, `HTTP/2` (default: both). + Other values will produce an error. + items: + description: HTTPVersionType is the name of a supported + HTTP version. + type: string + type: array + health: + description: |- + Health defines the endpoint Envoy uses to serve health checks. + Contour's default is { address: "0.0.0.0", port: 8002 }. + properties: + address: + description: Defines the health address interface. + minLength: 1 + type: string + port: + description: Defines the health port. + type: integer + type: object + http: + description: |- + Defines the HTTP Listener for Envoy. + Contour's default is { address: "0.0.0.0", port: 8080, accessLog: "/dev/stdout" }. + properties: + accessLog: + description: AccessLog defines where Envoy logs are outputted + for this listener. + type: string + address: + description: Defines an Envoy Listener Address. + minLength: 1 + type: string + port: + description: Defines an Envoy listener Port. + type: integer + type: object + https: + description: |- + Defines the HTTPS Listener for Envoy. + Contour's default is { address: "0.0.0.0", port: 8443, accessLog: "/dev/stdout" }. + properties: + accessLog: + description: AccessLog defines where Envoy logs are outputted + for this listener. + type: string + address: + description: Defines an Envoy Listener Address. + minLength: 1 + type: string + port: + description: Defines an Envoy listener Port. + type: integer + type: object + listener: + description: Listener hold various configurable Envoy listener + values. + properties: + connectionBalancer: + description: |- + ConnectionBalancer. If the value is exact, the listener will use the exact connection balancer + See https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/listener.proto#envoy-api-msg-listener-connectionbalanceconfig + for more information. + Values: (empty string): use the default ConnectionBalancer, `exact`: use the Exact ConnectionBalancer. + Other values will produce an error. + type: string + disableAllowChunkedLength: + description: |- + DisableAllowChunkedLength disables the RFC-compliant Envoy behavior to + strip the "Content-Length" header if "Transfer-Encoding: chunked" is + also set. This is an emergency off-switch to revert back to Envoy's + default behavior in case of failures. Please file an issue if failures + are encountered. + See: https://github.com/projectcontour/contour/issues/3221 + Contour's default is false. + type: boolean + disableMergeSlashes: + description: |- + DisableMergeSlashes disables Envoy's non-standard merge_slashes path transformation option + which strips duplicate slashes from request URL paths. + Contour's default is false. + type: boolean + httpMaxConcurrentStreams: + description: |- + Defines the value for SETTINGS_MAX_CONCURRENT_STREAMS Envoy will advertise in the + SETTINGS frame in HTTP/2 connections and the limit for concurrent streams allowed + for a peer on a single HTTP/2 connection. It is recommended to not set this lower + than 100 but this field can be used to bound resource usage by HTTP/2 connections + and mitigate attacks like CVE-2023-44487. The default value when this is not set is + unlimited. + format: int32 + minimum: 1 + type: integer + maxConnectionsPerListener: + description: |- + Defines the limit on number of active connections to a listener. The limit is applied + per listener. The default value when this is not set is unlimited. + format: int32 + minimum: 1 + type: integer + maxRequestsPerConnection: + description: |- + Defines the maximum requests for downstream connections. If not specified, there is no limit. + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-msg-config-core-v3-httpprotocoloptions + for more information. + format: int32 + minimum: 1 + type: integer + maxRequestsPerIOCycle: + description: |- + Defines the limit on number of HTTP requests that Envoy will process from a single + connection in a single I/O cycle. Requests over this limit are processed in subsequent + I/O cycles. Can be used as a mitigation for CVE-2023-44487 when abusive traffic is + detected. Configures the http.max_requests_per_io_cycle Envoy runtime setting. The default + value when this is not set is no limit. + format: int32 + minimum: 1 + type: integer + per-connection-buffer-limit-bytes: + description: |- + Defines the soft limit on size of the listener’s new connection read and write buffers in bytes. + If unspecified, an implementation defined default is applied (1MiB). + see https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener.proto#envoy-v3-api-field-config-listener-v3-listener-per-connection-buffer-limit-bytes + for more information. + format: int32 + minimum: 1 + type: integer + serverHeaderTransformation: + description: |- + Defines the action to be applied to the Server header on the response path. + When configured as overwrite, overwrites any Server header with "envoy". + When configured as append_if_absent, if a Server header is present, pass it through, otherwise set it to "envoy". + When configured as pass_through, pass through the value of the Server header, and do not append a header if none is present. + Values: `overwrite` (default), `append_if_absent`, `pass_through` + Other values will produce an error. + Contour's default is overwrite. + type: string + socketOptions: + description: |- + SocketOptions defines configurable socket options for the listeners. + Single set of options are applied to all listeners. + properties: + tos: + description: |- + Defines the value for IPv4 TOS field (including 6 bit DSCP field) for IP packets originating from Envoy listeners. + Single value is applied to all listeners. + If listeners are bound to IPv6-only addresses, setting this option will cause an error. + format: int32 + maximum: 255 + minimum: 0 + type: integer + trafficClass: + description: |- + Defines the value for IPv6 Traffic Class field (including 6 bit DSCP field) for IP packets originating from the Envoy listeners. + Single value is applied to all listeners. + If listeners are bound to IPv4-only addresses, setting this option will cause an error. + format: int32 + maximum: 255 + minimum: 0 + type: integer + type: object + tls: + description: TLS holds various configurable Envoy TLS + listener values. + properties: + cipherSuites: + description: |- + CipherSuites defines the TLS ciphers to be supported by Envoy TLS + listeners when negotiating TLS 1.2. Ciphers are validated against the + set that Envoy supports by default. This parameter should only be used + by advanced users. Note that these will be ignored when TLS 1.3 is in + use. + This field is optional; when it is undefined, a Contour-managed ciphersuite list + will be used, which may be updated to keep it secure. + Contour's default list is: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + Ciphers provided are validated against the following list: + - "[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]" + - "[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]" + - "ECDHE-ECDSA-AES128-GCM-SHA256" + - "ECDHE-RSA-AES128-GCM-SHA256" + - "ECDHE-ECDSA-AES128-SHA" + - "ECDHE-RSA-AES128-SHA" + - "AES128-GCM-SHA256" + - "AES128-SHA" + - "ECDHE-ECDSA-AES256-GCM-SHA384" + - "ECDHE-RSA-AES256-GCM-SHA384" + - "ECDHE-ECDSA-AES256-SHA" + - "ECDHE-RSA-AES256-SHA" + - "AES256-GCM-SHA384" + - "AES256-SHA" + Contour recommends leaving this undefined unless you are sure you must. + See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/transport_sockets/tls/v3/common.proto#extensions-transport-sockets-tls-v3-tlsparameters + Note: This list is a superset of what is valid for stock Envoy builds and those using BoringSSL FIPS. + items: + type: string + type: array + maximumProtocolVersion: + description: |- + MaximumProtocolVersion is the maximum TLS version this vhost should + negotiate. + Values: `1.2`, `1.3`(default). + Other values will produce an error. + type: string + minimumProtocolVersion: + description: |- + MinimumProtocolVersion is the minimum TLS version this vhost should + negotiate. + Values: `1.2` (default), `1.3`. + Other values will produce an error. + type: string + type: object + useProxyProtocol: + description: |- + Use PROXY protocol for all listeners. + Contour's default is false. + type: boolean + type: object + logging: + description: Logging defines how Envoy's logs can be configured. + properties: + accessLogFormat: + description: |- + AccessLogFormat sets the global access log format. + Values: `envoy` (default), `json`. + Other values will produce an error. + type: string + accessLogFormatString: + description: |- + AccessLogFormatString sets the access log format when format is set to `envoy`. + When empty, Envoy's default format is used. + type: string + accessLogJSONFields: + description: |- + AccessLogJSONFields sets the fields that JSON logging will + output when AccessLogFormat is json. + items: + type: string + type: array + accessLogLevel: + description: |- + AccessLogLevel sets the verbosity level of the access log. + Values: `info` (default, all requests are logged), `error` (all non-success requests, i.e. 300+ response code, are logged), `critical` (all 5xx requests are logged) and `disabled`. + Other values will produce an error. + type: string + type: object + metrics: + description: |- + Metrics defines the endpoint Envoy uses to serve metrics. + Contour's default is { address: "0.0.0.0", port: 8002 }. + properties: + address: + description: Defines the metrics address interface. + maxLength: 253 + minLength: 1 + type: string + port: + description: Defines the metrics port. + type: integer + tls: + description: |- + TLS holds TLS file config details. + Metrics and health endpoints cannot have same port number when metrics is served over HTTPS. + properties: + caFile: + description: CA filename. + type: string + certFile: + description: Client certificate filename. + type: string + keyFile: + description: Client key filename. + type: string + type: object + type: object + network: + description: Network holds various configurable Envoy network + values. + properties: + adminPort: + description: |- + Configure the port used to access the Envoy Admin interface. + If configured to port "0" then the admin interface is disabled. + Contour's default is 9001. + type: integer + numTrustedHops: + description: |- + XffNumTrustedHops defines the number of additional ingress proxy hops from the + right side of the x-forwarded-for HTTP header to trust when determining the origin + client’s IP address. + See https://www.envoyproxy.io/docs/envoy/v1.17.0/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto?highlight=xff_num_trusted_hops + for more information. + Contour's default is 0. + format: int32 + type: integer + type: object + service: + description: |- + Service holds Envoy service parameters for setting Ingress status. + Contour's default is { namespace: "projectcontour", name: "envoy" }. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + timeouts: + description: |- + Timeouts holds various configurable timeouts that can + be set in the config file. + properties: + connectTimeout: + description: |- + ConnectTimeout defines how long the proxy should wait when establishing connection to upstream service. + If not set, a default value of 2 seconds will be used. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#envoy-v3-api-field-config-cluster-v3-cluster-connect-timeout + for more information. + type: string + connectionIdleTimeout: + description: |- + ConnectionIdleTimeout defines how long the proxy should wait while there are + no active requests (for HTTP/1.1) or streams (for HTTP/2) before terminating + an HTTP connection. Set to "infinity" to disable the timeout entirely. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-httpprotocoloptions-idle-timeout + for more information. + type: string + connectionShutdownGracePeriod: + description: |- + ConnectionShutdownGracePeriod defines how long the proxy will wait between sending an + initial GOAWAY frame and a second, final GOAWAY frame when terminating an HTTP/2 connection. + During this grace period, the proxy will continue to respond to new streams. After the final + GOAWAY frame has been sent, the proxy will refuse new streams. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-drain-timeout + for more information. + type: string + delayedCloseTimeout: + description: |- + DelayedCloseTimeout defines how long envoy will wait, once connection + close processing has been initiated, for the downstream peer to close + the connection before Envoy closes the socket associated with the connection. + Setting this timeout to 'infinity' will disable it, equivalent to setting it to '0' + in Envoy. Leaving it unset will result in the Envoy default value being used. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-delayed-close-timeout + for more information. + type: string + maxConnectionDuration: + description: |- + MaxConnectionDuration defines the maximum period of time after an HTTP connection + has been established from the client to the proxy before it is closed by the proxy, + regardless of whether there has been activity or not. Omit or set to "infinity" for + no max duration. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/protocol.proto#envoy-v3-api-field-config-core-v3-httpprotocoloptions-max-connection-duration + for more information. + type: string + requestTimeout: + description: |- + RequestTimeout sets the client request timeout globally for Contour. Note that + this is a timeout for the entire request, not an idle timeout. Omit or set to + "infinity" to disable the timeout entirely. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-request-timeout + for more information. + type: string + streamIdleTimeout: + description: |- + StreamIdleTimeout defines how long the proxy should wait while there is no + request activity (for HTTP/1.1) or stream activity (for HTTP/2) before + terminating the HTTP request or stream. Set to "infinity" to disable the + timeout entirely. + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-stream-idle-timeout + for more information. + type: string + type: object + type: object + featureFlags: + description: |- + FeatureFlags defines toggle to enable new contour features. + Available toggles are: + useEndpointSlices - Configures contour to fetch endpoint data + from k8s endpoint slices. defaults to true, + If false then reads endpoint data from the k8s endpoints. + items: + type: string + type: array + gateway: + description: |- + Gateway contains parameters for the gateway-api Gateway that Contour + is configured to serve traffic. + properties: + gatewayRef: + description: |- + GatewayRef defines the specific Gateway that this Contour + instance corresponds to. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - gatewayRef + type: object + globalExtAuth: + description: |- + GlobalExternalAuthorization allows envoys external authorization filter + to be enabled for all virtual hosts. + properties: + authPolicy: + description: |- + AuthPolicy sets a default authorization policy for client requests. + This policy will be used unless overridden by individual routes. + properties: + context: + additionalProperties: + type: string + description: |- + Context is a set of key/value pairs that are sent to the + authentication server in the check request. If a context + is provided at an enclosing scope, the entries are merged + such that the inner scope overrides matching keys from the + outer scope. + type: object + disabled: + description: |- + When true, this field disables client request authentication + for the scope of the policy. + type: boolean + type: object + extensionRef: + description: ExtensionServiceRef specifies the extension resource + that will authorize client requests. + properties: + apiVersion: + description: |- + API version of the referent. + If this field is not specified, the default "projectcontour.io/v1alpha1" will be used + minLength: 1 + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referent. + If this field is not specifies, the namespace of the resource that targets the referent will be used. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + minLength: 1 + type: string + type: object + failOpen: + description: |- + If FailOpen is true, the client request is forwarded to the upstream service + even if the authorization server fails to respond. This field should not be + set in most cases. It is intended for use only while migrating applications + from internal authorization to Contour external authorization. + type: boolean + responseTimeout: + description: |- + ResponseTimeout configures maximum time to wait for a check response from the authorization server. + Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). + Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + The string "infinity" is also a valid input and specifies no timeout. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + withRequestBody: + description: WithRequestBody specifies configuration for sending + the client request's body to authorization server. + properties: + allowPartialMessage: + description: If AllowPartialMessage is true, then Envoy + will buffer the body until MaxRequestBytes are reached. + type: boolean + maxRequestBytes: + default: 1024 + description: MaxRequestBytes sets the maximum size of + message body ExtAuthz filter will hold in-memory. + format: int32 + minimum: 1 + type: integer + packAsBytes: + description: If PackAsBytes is true, the body sent to + Authorization Server is in raw bytes. + type: boolean + type: object + type: object + health: + description: |- + Health defines the endpoints Contour uses to serve health checks. + Contour's default is { address: "0.0.0.0", port: 8000 }. + properties: + address: + description: Defines the health address interface. + minLength: 1 + type: string + port: + description: Defines the health port. + type: integer + type: object + httpproxy: + description: HTTPProxy defines parameters on HTTPProxy. + properties: + disablePermitInsecure: + description: |- + DisablePermitInsecure disables the use of the + permitInsecure field in HTTPProxy. + Contour's default is false. + type: boolean + fallbackCertificate: + description: |- + FallbackCertificate defines the namespace/name of the Kubernetes secret to + use as fallback when a non-SNI request is received. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + rootNamespaces: + description: Restrict Contour to searching these namespaces + for root ingress routes. + items: + type: string + type: array + type: object + ingress: + description: Ingress contains parameters for ingress options. + properties: + classNames: + description: Ingress Class Names Contour should use. + items: + type: string + type: array + statusAddress: + description: Address to set in Ingress object status. + type: string + type: object + metrics: + description: |- + Metrics defines the endpoint Contour uses to serve metrics. + Contour's default is { address: "0.0.0.0", port: 8000 }. + properties: + address: + description: Defines the metrics address interface. + maxLength: 253 + minLength: 1 + type: string + port: + description: Defines the metrics port. + type: integer + tls: + description: |- + TLS holds TLS file config details. + Metrics and health endpoints cannot have same port number when metrics is served over HTTPS. + properties: + caFile: + description: CA filename. + type: string + certFile: + description: Client certificate filename. + type: string + keyFile: + description: Client key filename. + type: string + type: object + type: object + policy: + description: Policy specifies default policy applied if not overridden + by the user + properties: + applyToIngress: + description: |- + ApplyToIngress determines if the Policies will apply to ingress objects + Contour's default is false. + type: boolean + requestHeaders: + description: RequestHeadersPolicy defines the request headers + set/removed on all routes + properties: + remove: + items: + type: string + type: array + set: + additionalProperties: + type: string + type: object + type: object + responseHeaders: + description: ResponseHeadersPolicy defines the response headers + set/removed on all routes + properties: + remove: + items: + type: string + type: array + set: + additionalProperties: + type: string + type: object + type: object + type: object + rateLimitService: + description: |- + RateLimitService optionally holds properties of the Rate Limit Service + to be used for global rate limiting. + properties: + defaultGlobalRateLimitPolicy: + description: |- + DefaultGlobalRateLimitPolicy allows setting a default global rate limit policy for every HTTPProxy. + HTTPProxy can overwrite this configuration. + properties: + descriptors: + description: |- + Descriptors defines the list of descriptors that will + be generated and sent to the rate limit service. Each + descriptor contains 1+ key-value pair entries. + items: + description: RateLimitDescriptor defines a list of key-value + pair generators. + properties: + entries: + description: Entries is the list of key-value pair + generators. + items: + description: |- + RateLimitDescriptorEntry is a key-value pair generator. Exactly + one field on this struct must be non-nil. + properties: + genericKey: + description: GenericKey defines a descriptor + entry with a static key and value. + properties: + key: + description: |- + Key defines the key of the descriptor entry. If not set, the + key is set to "generic_key". + type: string + value: + description: Value defines the value of + the descriptor entry. + minLength: 1 + type: string + type: object + remoteAddress: + description: |- + RemoteAddress defines a descriptor entry with a key of "remote_address" + and a value equal to the client's IP address (from x-forwarded-for). + type: object + requestHeader: + description: |- + RequestHeader defines a descriptor entry that's populated only if + a given header is present on the request. The descriptor key is static, + and the descriptor value is equal to the value of the header. + properties: + descriptorKey: + description: DescriptorKey defines the + key to use on the descriptor entry. + minLength: 1 + type: string + headerName: + description: HeaderName defines the name + of the header to look for on the request. + minLength: 1 + type: string + type: object + requestHeaderValueMatch: + description: |- + RequestHeaderValueMatch defines a descriptor entry that's populated + if the request's headers match a set of 1+ match criteria. The + descriptor key is "header_match", and the descriptor value is static. + properties: + expectMatch: + default: true + description: |- + ExpectMatch defines whether the request must positively match the match + criteria in order to generate a descriptor entry (i.e. true), or not + match the match criteria in order to generate a descriptor entry (i.e. false). + The default is true. + type: boolean + headers: + description: |- + Headers is a list of 1+ match criteria to apply against the request + to determine whether to populate the descriptor entry or not. + items: + description: |- + HeaderMatchCondition specifies how to conditionally match against HTTP + headers. The Name field is required, only one of Present, NotPresent, + Contains, NotContains, Exact, NotExact and Regex can be set. + For negative matching rules only (e.g. NotContains or NotExact) you can set + TreatMissingAsEmpty. + IgnoreCase has no effect for Regex. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the header value. + type: string + exact: + description: Exact specifies a string + that the header value must be + equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the header to match against. Name is required. + Header names are case insensitive. + type: string + notcontains: + description: |- + NotContains specifies a substring that must not be present + in the header value. + type: string + notexact: + description: |- + NoExact specifies a string that the header value must not be + equal to. The condition is true if the header has any other value. + type: string + notpresent: + description: |- + NotPresent specifies that condition is true when the named header + is not present. Note that setting NotPresent to false does not + make the condition true if the named header is present. + type: boolean + present: + description: |- + Present specifies that condition is true when the named header + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named header + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the header + value. + type: string + treatMissingAsEmpty: + description: |- + TreatMissingAsEmpty specifies if the header match rule specified header + does not exist, this header value will be treated as empty. Defaults to false. + Unlike the underlying Envoy implementation this is **only** supported for + negative matches (e.g. NotContains, NotExact). + type: boolean + required: + - name + type: object + minItems: 1 + type: array + value: + description: Value defines the value of + the descriptor entry. + minLength: 1 + type: string + type: object + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + disabled: + description: |- + Disabled configures the HTTPProxy to not use + the default global rate limit policy defined by the Contour configuration. + type: boolean + type: object + domain: + description: Domain is passed to the Rate Limit Service. + type: string + enableResourceExhaustedCode: + description: |- + EnableResourceExhaustedCode enables translating error code 429 to + grpc code RESOURCE_EXHAUSTED. When disabled it's translated to UNAVAILABLE + type: boolean + enableXRateLimitHeaders: + description: |- + EnableXRateLimitHeaders defines whether to include the X-RateLimit + headers X-RateLimit-Limit, X-RateLimit-Remaining, and X-RateLimit-Reset + (as defined by the IETF Internet-Draft linked below), on responses + to clients when the Rate Limit Service is consulted for a request. + ref. https://tools.ietf.org/id/draft-polli-ratelimit-headers-03.html + type: boolean + extensionService: + description: ExtensionService identifies the extension service + defining the RLS. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + failOpen: + description: |- + FailOpen defines whether to allow requests to proceed when the + Rate Limit Service fails to respond with a valid rate limit + decision within the timeout defined on the extension service. + type: boolean + required: + - extensionService + type: object + tracing: + description: Tracing defines properties for exporting trace data + to OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with + unique tag name. + items: + description: |- + CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: |- + Literal is a static custom tag value. + Precisely one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: |- + RequestHeaderName indicates which request header + the label value is obtained from. + Precisely one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom + tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: |- + IncludePodDetail defines a flag. + If it is true, contour will add the pod name and namespace to the span of the trace. + the default is true. + Note: The Envoy pods MUST have the HOSTNAME and CONTOUR_NAMESPACE environment variables set for this to work properly. + type: boolean + maxPathTagLength: + description: |- + MaxPathTagLength defines maximum length of the request path + to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: |- + OverallSampling defines the sampling rate of trace data. + contour's default is 100. + type: string + serviceName: + description: |- + ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object + xdsServer: + description: XDSServer contains parameters for the xDS server. + properties: + address: + description: |- + Defines the xDS gRPC API address which Contour will serve. + Contour's default is "0.0.0.0". + minLength: 1 + type: string + port: + description: |- + Defines the xDS gRPC API port which Contour will serve. + Contour's default is 8001. + type: integer + tls: + description: |- + TLS holds TLS file config details. + Contour's default is { caFile: "/certs/ca.crt", certFile: "/certs/tls.cert", keyFile: "/certs/tls.key", insecure: false }. + properties: + caFile: + description: CA filename. + type: string + certFile: + description: Client certificate filename. + type: string + insecure: + description: Allow serving the xDS gRPC API without TLS. + type: boolean + keyFile: + description: Client key filename. + type: string + type: object + type: + description: |- + Defines the XDSServer to use for `contour serve`. + Values: `envoy` (default), `contour (deprecated)`. + Other values will produce an error. + type: string + type: object + type: object + type: object + status: + description: ContourDeploymentStatus defines the observed state of a ContourDeployment + resource. + properties: + conditions: + description: Conditions describe the current conditions of the ContourDeployment + resource. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: extensionservices.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: ExtensionService + listKind: ExtensionServiceList + plural: extensionservices + shortNames: + - extensionservice + - extensionservices + singular: extensionservice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ExtensionService is the schema for the Contour extension services API. + An ExtensionService resource binds a network service to the Contour + API so that Contour API features can be implemented by collaborating + components. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ExtensionServiceSpec defines the desired state of an ExtensionService + resource. + properties: + loadBalancerPolicy: + description: |- + The policy for load balancing GRPC service requests. Note that the + `Cookie` and `RequestHash` load balancing strategies cannot be used + here. + properties: + requestHashPolicies: + description: |- + RequestHashPolicies contains a list of hash policies to apply when the + `RequestHash` load balancing strategy is chosen. If an element of the + supplied list of hash policies is invalid, it will be ignored. If the + list of hash policies is empty after validation, the load balancing + strategy will fall back to the default `RoundRobin`. + items: + description: |- + RequestHashPolicy contains configuration for an individual hash policy + on a request attribute. + properties: + hashSourceIP: + description: |- + HashSourceIP should be set to true when request source IP hash based + load balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + type: boolean + headerHashOptions: + description: |- + HeaderHashOptions should be set when request header hash based load + balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + properties: + headerName: + description: |- + HeaderName is the name of the HTTP request header that will be used to + calculate the hash key. If the header specified is not present on a + request, no hash will be produced. + minLength: 1 + type: string + type: object + queryParameterHashOptions: + description: |- + QueryParameterHashOptions should be set when request query parameter hash based load + balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + properties: + parameterName: + description: |- + ParameterName is the name of the HTTP request query parameter that will be used to + calculate the hash key. If the query parameter specified is not present on a + request, no hash will be produced. + minLength: 1 + type: string + type: object + terminal: + description: |- + Terminal is a flag that allows for short-circuiting computing of a hash + for a given request. If set to true, and the request attribute specified + in the attribute hash options is present, no further hash policies will + be used to calculate a hash for the request. + type: boolean + type: object + type: array + strategy: + description: |- + Strategy specifies the policy used to balance requests + across the pool of backend pods. Valid policy names are + `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, + and `RequestHash`. If an unknown strategy name is specified + or no policy is supplied, the default `RoundRobin` policy + is used. + type: string + type: object + protocol: + description: |- + Protocol may be used to specify (or override) the protocol used to reach this Service. + Values may be h2 or h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + type: string + protocolVersion: + description: |- + This field sets the version of the GRPC protocol that Envoy uses to + send requests to the extension service. Since Contour always uses the + v3 Envoy API, this is currently fixed at "v3". However, other + protocol options will be available in future. + enum: + - v3 + type: string + services: + description: |- + Services specifies the set of Kubernetes Service resources that + receive GRPC extension API requests. + If no weights are specified for any of the entries in + this array, traffic will be spread evenly across all the + services. + Otherwise, traffic is balanced proportionally to the + Weight field in each entry. + items: + description: |- + ExtensionServiceTarget defines an Kubernetes Service to target with + extension service traffic. + properties: + name: + description: |- + Name is the name of Kubernetes service that will accept service + traffic. + type: string + port: + description: Port (defined as Integer) to proxy traffic to since + a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + weight: + description: Weight defines proportion of traffic to balance + to the Kubernetes Service. + format: int32 + type: integer + required: + - name + - port + type: object + minItems: 1 + type: array + timeoutPolicy: + description: The timeout policy for requests to the services. + properties: + idle: + description: |- + Timeout for how long the proxy should wait while there is no activity during single request/response (for HTTP/1.1) or stream (for HTTP/2). + Timeout will not trigger while HTTP/1.1 connection is idle between two consecutive requests. + If not specified, there is no per-route idle timeout, though a connection manager-wide + stream_idle_timeout default of 5m still applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + idleConnection: + description: |- + Timeout for how long connection from the proxy to the upstream service is kept when there are no active requests. + If not supplied, Envoy's default value of 1h applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + response: + description: |- + Timeout for receiving a response from the server after processing a request from client. + If not supplied, Envoy's default value of 15s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + type: object + validation: + description: UpstreamValidation defines how to verify the backend + service's certificate + properties: + caSecret: + description: |- + Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend. + The secret must contain key named ca.crt. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + Max length should be the actual max possible length of a namespaced name (63 + 253 + 1 = 317) + maxLength: 317 + minLength: 1 + type: string + subjectName: + description: |- + Key which is expected to be present in the 'subjectAltName' of the presented certificate. + Deprecated: migrate to using the plural field subjectNames. + maxLength: 250 + minLength: 1 + type: string + subjectNames: + description: |- + List of keys, of which at least one is expected to be present in the 'subjectAltName of the + presented certificate. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + required: + - caSecret + - subjectName + type: object + x-kubernetes-validations: + - message: subjectNames[0] must equal subjectName if set + rule: 'has(self.subjectNames) ? self.subjectNames[0] == self.subjectName + : true' + required: + - services + type: object + status: + description: |- + ExtensionServiceStatus defines the observed state of an + ExtensionService resource. + properties: + conditions: + description: |- + Conditions contains the current status of the ExtensionService resource. + Contour will update a single condition, `Valid`, that is in normal-true polarity. + Contour will not modify any other Conditions set in this block, + in case some other controller wants to add a Condition. + items: + description: |- + DetailedCondition is an extension of the normal Kubernetes conditions, with two extra + fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) + of the condition. + `errors` holds information about sub-conditions which are fatal to that condition and render its state False. + `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. + Remember that Conditions have a type, a status, and a reason. + The type is the type of the condition, the most important one in this CRD set is `Valid`. + `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. + In more detail, `status: true` means that the object is has been ingested into Contour with no errors. + `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` + slice in this case. + `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. + The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` + slice if `status` is `false`. + For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. + When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. + When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. + In either case, there may be entries in the `warnings` slice. + Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason + (if there is one and only one entry in total across both the `errors` and `warnings` slices), or + `MultipleReasons` if there is more than one entry. + properties: + errors: + description: |- + Errors contains a slice of relevant error subconditions for this object. + Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. + An empty slice here indicates no errors. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: |- + Warnings contains a slice of relevant warning subconditions for this object. + Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. + An empty slice here indicates no warnings. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: httpproxies.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: HTTPProxy + listKind: HTTPProxyList + plural: httpproxies + shortNames: + - proxy + - proxies + singular: httpproxy + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Fully qualified domain name + jsonPath: .spec.virtualhost.fqdn + name: FQDN + type: string + - description: Secret with TLS credentials + jsonPath: .spec.virtualhost.tls.secretName + name: TLS Secret + type: string + - description: The current status of the HTTPProxy + jsonPath: .status.currentStatus + name: Status + type: string + - description: Description of the current status + jsonPath: .status.description + name: Status Description + type: string + name: v1 + schema: + openAPIV3Schema: + description: HTTPProxy is an Ingress CRD specification. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: HTTPProxySpec defines the spec of the CRD. + properties: + includes: + description: |- + Includes allow for specific routing configuration to be included from another HTTPProxy, + possibly in another namespace. + items: + description: Include describes a set of policies that can be applied + to an HTTPProxy in a namespace. + properties: + conditions: + description: |- + Conditions are a set of rules that are applied to included HTTPProxies. + In effect, they are added onto the Conditions of included HTTPProxy Route + structs. + When applied, they are merged using AND, with one exception: + There can be only one Prefix MatchCondition per Conditions slice. + More than one Prefix, or contradictory Conditions, will make the + include invalid. Exact and Regex match conditions are not allowed + on includes. + items: + description: |- + MatchCondition are a general holder for matching rules for HTTPProxies. + One of Prefix, Exact, Regex, Header or QueryParameter must be provided. + properties: + exact: + description: |- + Exact defines a exact match for a request. + This field is not allowed in include match conditions. + type: string + header: + description: Header specifies the header condition to + match. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the header value. + type: string + exact: + description: Exact specifies a string that the header + value must be equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the header to match against. Name is required. + Header names are case insensitive. + type: string + notcontains: + description: |- + NotContains specifies a substring that must not be present + in the header value. + type: string + notexact: + description: |- + NoExact specifies a string that the header value must not be + equal to. The condition is true if the header has any other value. + type: string + notpresent: + description: |- + NotPresent specifies that condition is true when the named header + is not present. Note that setting NotPresent to false does not + make the condition true if the named header is present. + type: boolean + present: + description: |- + Present specifies that condition is true when the named header + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named header + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the header + value. + type: string + treatMissingAsEmpty: + description: |- + TreatMissingAsEmpty specifies if the header match rule specified header + does not exist, this header value will be treated as empty. Defaults to false. + Unlike the underlying Envoy implementation this is **only** supported for + negative matches (e.g. NotContains, NotExact). + type: boolean + required: + - name + type: object + prefix: + description: Prefix defines a prefix match for a request. + type: string + queryParameter: + description: QueryParameter specifies the query parameter + condition to match. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the query parameter value. + type: string + exact: + description: Exact specifies a string that the query + parameter value must be equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the query parameter to match against. Name is required. + Query parameter names are case insensitive. + type: string + prefix: + description: Prefix defines a prefix match for the + query parameter value. + type: string + present: + description: |- + Present specifies that condition is true when the named query parameter + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named query parameter + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the query + parameter value. + type: string + suffix: + description: Suffix defines a suffix match for a query + parameter value. + type: string + required: + - name + type: object + regex: + description: |- + Regex defines a regex match for a request. + This field is not allowed in include match conditions. + type: string + type: object + type: array + name: + description: Name of the HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults + to the current namespace if not supplied. + type: string + required: + - name + type: object + type: array + ingressClassName: + description: |- + IngressClassName optionally specifies the ingress class to use for this + HTTPProxy. This replaces the deprecated `kubernetes.io/ingress.class` + annotation. For backwards compatibility, when that annotation is set, it + is given precedence over this field. + type: string + routes: + description: Routes are the ingress routes. If TCPProxy is present, + Routes is ignored. + items: + description: Route contains the set of routes for a virtual host. + properties: + authPolicy: + description: |- + AuthPolicy updates the authorization policy that was set + on the root HTTPProxy object for client requests that + match this route. + properties: + context: + additionalProperties: + type: string + description: |- + Context is a set of key/value pairs that are sent to the + authentication server in the check request. If a context + is provided at an enclosing scope, the entries are merged + such that the inner scope overrides matching keys from the + outer scope. + type: object + disabled: + description: |- + When true, this field disables client request authentication + for the scope of the policy. + type: boolean + type: object + conditions: + description: |- + Conditions are a set of rules that are applied to a Route. + When applied, they are merged using AND, with one exception: + There can be only one Prefix, Exact or Regex MatchCondition + per Conditions slice. More than one of these condition types, + or contradictory Conditions, will make the route invalid. + items: + description: |- + MatchCondition are a general holder for matching rules for HTTPProxies. + One of Prefix, Exact, Regex, Header or QueryParameter must be provided. + properties: + exact: + description: |- + Exact defines a exact match for a request. + This field is not allowed in include match conditions. + type: string + header: + description: Header specifies the header condition to + match. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the header value. + type: string + exact: + description: Exact specifies a string that the header + value must be equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the header to match against. Name is required. + Header names are case insensitive. + type: string + notcontains: + description: |- + NotContains specifies a substring that must not be present + in the header value. + type: string + notexact: + description: |- + NoExact specifies a string that the header value must not be + equal to. The condition is true if the header has any other value. + type: string + notpresent: + description: |- + NotPresent specifies that condition is true when the named header + is not present. Note that setting NotPresent to false does not + make the condition true if the named header is present. + type: boolean + present: + description: |- + Present specifies that condition is true when the named header + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named header + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the header + value. + type: string + treatMissingAsEmpty: + description: |- + TreatMissingAsEmpty specifies if the header match rule specified header + does not exist, this header value will be treated as empty. Defaults to false. + Unlike the underlying Envoy implementation this is **only** supported for + negative matches (e.g. NotContains, NotExact). + type: boolean + required: + - name + type: object + prefix: + description: Prefix defines a prefix match for a request. + type: string + queryParameter: + description: QueryParameter specifies the query parameter + condition to match. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the query parameter value. + type: string + exact: + description: Exact specifies a string that the query + parameter value must be equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the query parameter to match against. Name is required. + Query parameter names are case insensitive. + type: string + prefix: + description: Prefix defines a prefix match for the + query parameter value. + type: string + present: + description: |- + Present specifies that condition is true when the named query parameter + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named query parameter + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the query + parameter value. + type: string + suffix: + description: Suffix defines a suffix match for a query + parameter value. + type: string + required: + - name + type: object + regex: + description: |- + Regex defines a regex match for a request. + This field is not allowed in include match conditions. + type: string + type: object + type: array + cookieRewritePolicies: + description: |- + The policies for rewriting Set-Cookie header attributes. Note that + rewritten cookie names must be unique in this list. Order rewrite + policies are specified in does not matter. + items: + properties: + domainRewrite: + description: |- + DomainRewrite enables rewriting the Set-Cookie Domain element. + If not set, Domain will not be rewritten. + properties: + value: + description: |- + Value is the value to rewrite the Domain attribute to. + For now this is required. + maxLength: 4096 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - value + type: object + name: + description: Name is the name of the cookie for which + attributes will be rewritten. + maxLength: 4096 + minLength: 1 + pattern: ^[^()<>@,;:\\"\/[\]?={} \t\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]+$ + type: string + pathRewrite: + description: |- + PathRewrite enables rewriting the Set-Cookie Path element. + If not set, Path will not be rewritten. + properties: + value: + description: |- + Value is the value to rewrite the Path attribute to. + For now this is required. + maxLength: 4096 + minLength: 1 + pattern: ^[^;\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]+$ + type: string + required: + - value + type: object + sameSite: + description: |- + SameSite enables rewriting the Set-Cookie SameSite element. + If not set, SameSite attribute will not be rewritten. + enum: + - Strict + - Lax + - None + type: string + secure: + description: |- + Secure enables rewriting the Set-Cookie Secure element. + If not set, Secure attribute will not be rewritten. + type: boolean + required: + - name + type: object + type: array + directResponsePolicy: + description: DirectResponsePolicy returns an arbitrary HTTP + response directly. + properties: + body: + description: |- + Body is the content of the response body. + If this setting is omitted, no body is included in the generated response. + Note: Body is not recommended to set too long + otherwise it can have significant resource usage impacts. + type: string + statusCode: + description: StatusCode is the HTTP response status to be + returned. + maximum: 599 + minimum: 200 + type: integer + required: + - statusCode + type: object + enableWebsockets: + description: Enables websocket support for the route. + type: boolean + healthCheckPolicy: + description: The health check policy for this route. + properties: + expectedStatuses: + description: |- + The ranges of HTTP response statuses considered healthy. Follow half-open + semantics, i.e. for each range the start is inclusive and the end is exclusive. + Must be within the range [100,600). If not specified, only a 200 response status + is considered healthy. + items: + properties: + end: + description: The end (exclusive) of a range of HTTP + status codes. + format: int64 + maximum: 600 + minimum: 101 + type: integer + start: + description: The start (inclusive) of a range of HTTP + status codes. + format: int64 + maximum: 599 + minimum: 100 + type: integer + required: + - end + - start + type: object + type: array + healthyThresholdCount: + description: The number of healthy health checks required + before a host is marked healthy + format: int64 + minimum: 0 + type: integer + host: + description: |- + The value of the host header in the HTTP health check request. + If left empty (default value), the name "contour-envoy-healthcheck" + will be used. + type: string + intervalSeconds: + description: The interval (seconds) between health checks + format: int64 + type: integer + path: + description: HTTP endpoint used to perform health checks + on upstream service + type: string + timeoutSeconds: + description: The time to wait (seconds) for a health check + response + format: int64 + type: integer + unhealthyThresholdCount: + description: The number of unhealthy health checks required + before a host is marked unhealthy + format: int64 + minimum: 0 + type: integer + required: + - path + type: object + internalRedirectPolicy: + description: The policy to define when to handle redirects responses + internally. + properties: + allowCrossSchemeRedirect: + default: Never + description: |- + AllowCrossSchemeRedirect Allow internal redirect to follow a target URI with a different scheme + than the value of x-forwarded-proto. + SafeOnly allows same scheme redirect and safe cross scheme redirect, which means if the downstream + scheme is HTTPS, both HTTPS and HTTP redirect targets are allowed, but if the downstream scheme + is HTTP, only HTTP redirect targets are allowed. + enum: + - Always + - Never + - SafeOnly + type: string + denyRepeatedRouteRedirect: + description: |- + If DenyRepeatedRouteRedirect is true, rejects redirect targets that are pointing to a route that has + been followed by a previous redirect from the current route. + type: boolean + maxInternalRedirects: + description: |- + MaxInternalRedirects An internal redirect is not handled, unless the number of previous internal + redirects that a downstream request has encountered is lower than this value. + format: int32 + type: integer + redirectResponseCodes: + description: |- + RedirectResponseCodes If unspecified, only 302 will be treated as internal redirect. + Only 301, 302, 303, 307 and 308 are valid values. + items: + description: RedirectResponseCode is a uint32 type alias + with validation to ensure that the value is valid. + enum: + - 301 + - 302 + - 303 + - 307 + - 308 + format: int32 + type: integer + type: array + type: object + ipAllowPolicy: + description: |- + IPAllowFilterPolicy is a list of ipv4/6 filter rules for which matching + requests should be allowed. All other requests will be denied. + Only one of IPAllowFilterPolicy and IPDenyFilterPolicy can be defined. + The rules defined here override any rules set on the root HTTPProxy. + items: + properties: + cidr: + description: |- + CIDR is a CIDR block of ipv4 or ipv6 addresses to filter on. This can also be + a bare IP address (without a mask) to filter on exactly one address. + type: string + source: + description: |- + Source indicates how to determine the ip address to filter on, and can be + one of two values: + - `Remote` filters on the ip address of the client, accounting for PROXY and + X-Forwarded-For as needed. + - `Peer` filters on the ip of the network request, ignoring PROXY and + X-Forwarded-For. + enum: + - Peer + - Remote + type: string + required: + - cidr + - source + type: object + type: array + ipDenyPolicy: + description: |- + IPDenyFilterPolicy is a list of ipv4/6 filter rules for which matching + requests should be denied. All other requests will be allowed. + Only one of IPAllowFilterPolicy and IPDenyFilterPolicy can be defined. + The rules defined here override any rules set on the root HTTPProxy. + items: + properties: + cidr: + description: |- + CIDR is a CIDR block of ipv4 or ipv6 addresses to filter on. This can also be + a bare IP address (without a mask) to filter on exactly one address. + type: string + source: + description: |- + Source indicates how to determine the ip address to filter on, and can be + one of two values: + - `Remote` filters on the ip address of the client, accounting for PROXY and + X-Forwarded-For as needed. + - `Peer` filters on the ip of the network request, ignoring PROXY and + X-Forwarded-For. + enum: + - Peer + - Remote + type: string + required: + - cidr + - source + type: object + type: array + jwtVerificationPolicy: + description: The policy for verifying JWTs for requests to this + route. + properties: + disabled: + description: |- + Disabled defines whether to disable all JWT verification for this + route. This can be used to opt specific routes out of the default + JWT provider for the HTTPProxy. At most one of this field or the + "require" field can be specified. + type: boolean + require: + description: |- + Require names a specific JWT provider (defined in the virtual host) + to require for the route. If specified, this field overrides the + default provider if one exists. If this field is not specified, + the default provider will be required if one exists. At most one of + this field or the "disabled" field can be specified. + type: string + type: object + loadBalancerPolicy: + description: The load balancing policy for this route. + properties: + requestHashPolicies: + description: |- + RequestHashPolicies contains a list of hash policies to apply when the + `RequestHash` load balancing strategy is chosen. If an element of the + supplied list of hash policies is invalid, it will be ignored. If the + list of hash policies is empty after validation, the load balancing + strategy will fall back to the default `RoundRobin`. + items: + description: |- + RequestHashPolicy contains configuration for an individual hash policy + on a request attribute. + properties: + hashSourceIP: + description: |- + HashSourceIP should be set to true when request source IP hash based + load balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + type: boolean + headerHashOptions: + description: |- + HeaderHashOptions should be set when request header hash based load + balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + properties: + headerName: + description: |- + HeaderName is the name of the HTTP request header that will be used to + calculate the hash key. If the header specified is not present on a + request, no hash will be produced. + minLength: 1 + type: string + type: object + queryParameterHashOptions: + description: |- + QueryParameterHashOptions should be set when request query parameter hash based load + balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + properties: + parameterName: + description: |- + ParameterName is the name of the HTTP request query parameter that will be used to + calculate the hash key. If the query parameter specified is not present on a + request, no hash will be produced. + minLength: 1 + type: string + type: object + terminal: + description: |- + Terminal is a flag that allows for short-circuiting computing of a hash + for a given request. If set to true, and the request attribute specified + in the attribute hash options is present, no further hash policies will + be used to calculate a hash for the request. + type: boolean + type: object + type: array + strategy: + description: |- + Strategy specifies the policy used to balance requests + across the pool of backend pods. Valid policy names are + `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, + and `RequestHash`. If an unknown strategy name is specified + or no policy is supplied, the default `RoundRobin` policy + is used. + type: string + type: object + pathRewritePolicy: + description: |- + The policy for rewriting the path of the request URL + after the request has been routed to a Service. + properties: + replacePrefix: + description: ReplacePrefix describes how the path prefix + should be replaced. + items: + description: ReplacePrefix describes a path prefix replacement. + properties: + prefix: + description: |- + Prefix specifies the URL path prefix to be replaced. + If Prefix is specified, it must exactly match the MatchCondition + prefix that is rendered by the chain of including HTTPProxies + and only that path prefix will be replaced by Replacement. + This allows HTTPProxies that are included through multiple + roots to only replace specific path prefixes, leaving others + unmodified. + If Prefix is not specified, all routing prefixes rendered + by the include chain will be replaced. + minLength: 1 + type: string + replacement: + description: |- + Replacement is the string that the routing path prefix + will be replaced with. This must not be empty. + minLength: 1 + type: string + required: + - replacement + type: object + type: array + type: object + permitInsecure: + description: |- + Allow this path to respond to insecure requests over HTTP which are normally + not permitted when a `virtualhost.tls` block is present. + type: boolean + rateLimitPolicy: + description: The policy for rate limiting on the route. + properties: + global: + description: |- + Global defines global rate limiting parameters, i.e. parameters + defining descriptors that are sent to an external rate limit + service (RLS) for a rate limit decision on each request. + properties: + descriptors: + description: |- + Descriptors defines the list of descriptors that will + be generated and sent to the rate limit service. Each + descriptor contains 1+ key-value pair entries. + items: + description: RateLimitDescriptor defines a list of + key-value pair generators. + properties: + entries: + description: Entries is the list of key-value + pair generators. + items: + description: |- + RateLimitDescriptorEntry is a key-value pair generator. Exactly + one field on this struct must be non-nil. + properties: + genericKey: + description: GenericKey defines a descriptor + entry with a static key and value. + properties: + key: + description: |- + Key defines the key of the descriptor entry. If not set, the + key is set to "generic_key". + type: string + value: + description: Value defines the value + of the descriptor entry. + minLength: 1 + type: string + type: object + remoteAddress: + description: |- + RemoteAddress defines a descriptor entry with a key of "remote_address" + and a value equal to the client's IP address (from x-forwarded-for). + type: object + requestHeader: + description: |- + RequestHeader defines a descriptor entry that's populated only if + a given header is present on the request. The descriptor key is static, + and the descriptor value is equal to the value of the header. + properties: + descriptorKey: + description: DescriptorKey defines the + key to use on the descriptor entry. + minLength: 1 + type: string + headerName: + description: HeaderName defines the + name of the header to look for on + the request. + minLength: 1 + type: string + type: object + requestHeaderValueMatch: + description: |- + RequestHeaderValueMatch defines a descriptor entry that's populated + if the request's headers match a set of 1+ match criteria. The + descriptor key is "header_match", and the descriptor value is static. + properties: + expectMatch: + default: true + description: |- + ExpectMatch defines whether the request must positively match the match + criteria in order to generate a descriptor entry (i.e. true), or not + match the match criteria in order to generate a descriptor entry (i.e. false). + The default is true. + type: boolean + headers: + description: |- + Headers is a list of 1+ match criteria to apply against the request + to determine whether to populate the descriptor entry or not. + items: + description: |- + HeaderMatchCondition specifies how to conditionally match against HTTP + headers. The Name field is required, only one of Present, NotPresent, + Contains, NotContains, Exact, NotExact and Regex can be set. + For negative matching rules only (e.g. NotContains or NotExact) you can set + TreatMissingAsEmpty. + IgnoreCase has no effect for Regex. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the header value. + type: string + exact: + description: Exact specifies a + string that the header value + must be equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the header to match against. Name is required. + Header names are case insensitive. + type: string + notcontains: + description: |- + NotContains specifies a substring that must not be present + in the header value. + type: string + notexact: + description: |- + NoExact specifies a string that the header value must not be + equal to. The condition is true if the header has any other value. + type: string + notpresent: + description: |- + NotPresent specifies that condition is true when the named header + is not present. Note that setting NotPresent to false does not + make the condition true if the named header is present. + type: boolean + present: + description: |- + Present specifies that condition is true when the named header + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named header + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the header + value. + type: string + treatMissingAsEmpty: + description: |- + TreatMissingAsEmpty specifies if the header match rule specified header + does not exist, this header value will be treated as empty. Defaults to false. + Unlike the underlying Envoy implementation this is **only** supported for + negative matches (e.g. NotContains, NotExact). + type: boolean + required: + - name + type: object + minItems: 1 + type: array + value: + description: Value defines the value + of the descriptor entry. + minLength: 1 + type: string + type: object + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + disabled: + description: |- + Disabled configures the HTTPProxy to not use + the default global rate limit policy defined by the Contour configuration. + type: boolean + type: object + local: + description: |- + Local defines local rate limiting parameters, i.e. parameters + for rate limiting that occurs within each Envoy pod as requests + are handled. + properties: + burst: + description: |- + Burst defines the number of requests above the requests per + unit that should be allowed within a short period of time. + format: int32 + type: integer + requests: + description: |- + Requests defines how many requests per unit of time should + be allowed before rate limiting occurs. + format: int32 + minimum: 1 + type: integer + responseHeadersToAdd: + description: |- + ResponseHeadersToAdd is an optional list of response headers to + set when a request is rate-limited. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header + specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + responseStatusCode: + description: |- + ResponseStatusCode is the HTTP status code to use for responses + to rate-limited requests. Codes must be in the 400-599 range + (inclusive). If not specified, the Envoy default of 429 (Too + Many Requests) is used. + format: int32 + maximum: 599 + minimum: 400 + type: integer + unit: + description: |- + Unit defines the period of time within which requests + over the limit will be rate limited. Valid values are + "second", "minute" and "hour". + enum: + - second + - minute + - hour + type: string + required: + - requests + - unit + type: object + type: object + requestHeadersPolicy: + description: |- + The policy for managing request headers during proxying. + You may dynamically rewrite the Host header to be forwarded + upstream to the content of a request header using + the below format "%REQ(X-Header-Name)%". If the value of the header + is empty, it is ignored. + *NOTE: Pay attention to the potential security implications of using this option. + Provided header must come from trusted source. + **NOTE: The header rewrite is only done while forwarding and has no bearing + on the routing decision. + properties: + remove: + description: Remove specifies a list of HTTP header names + to remove. + items: + type: string + type: array + set: + description: |- + Set specifies a list of HTTP header values that will be set in the HTTP header. + If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header + specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + requestRedirectPolicy: + description: RequestRedirectPolicy defines an HTTP redirection. + properties: + hostname: + description: |- + Hostname is the precise hostname to be used in the value of the `Location` + header in the response. + When empty, the hostname of the request is used. + No wildcards are allowed. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path allows for redirection to a different path from the + original on the request. The path must start with a + leading slash. + Note: Only one of Path or Prefix can be defined. + pattern: ^\/.*$ + type: string + port: + description: |- + Port is the port to be used in the value of the `Location` + header in the response. + When empty, port (if specified) of the request is used. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prefix: + description: |- + Prefix defines the value to swap the matched prefix or path with. + The prefix must start with a leading slash. + Note: Only one of Path or Prefix can be defined. + pattern: ^\/.*$ + type: string + scheme: + description: |- + Scheme is the scheme to be used in the value of the `Location` + header in the response. + When empty, the scheme of the request is used. + enum: + - http + - https + type: string + statusCode: + default: 302 + description: StatusCode is the HTTP status code to be used + in response. + enum: + - 301 + - 302 + type: integer + type: object + responseHeadersPolicy: + description: |- + The policy for managing response headers during proxying. + Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header names + to remove. + items: + type: string + type: array + set: + description: |- + Set specifies a list of HTTP header values that will be set in the HTTP header. + If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header + specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + retryPolicy: + description: The retry policy for this route. + properties: + count: + default: 1 + description: |- + NumRetries is maximum allowed number of retries. + If set to -1, then retries are disabled. + If set to 0 or not supplied, the value is set + to the Envoy default of 1. + format: int64 + minimum: -1 + type: integer + perTryTimeout: + description: |- + PerTryTimeout specifies the timeout per retry attempt. + Ignored if NumRetries is not supplied. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + retriableStatusCodes: + description: |- + RetriableStatusCodes specifies the HTTP status codes that should be retried. + This field is only respected when you include `retriable-status-codes` in the `RetryOn` field. + items: + format: int32 + type: integer + type: array + retryOn: + description: |- + RetryOn specifies the conditions on which to retry a request. + Supported [HTTP conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on): + - `5xx` + - `gateway-error` + - `reset` + - `connect-failure` + - `retriable-4xx` + - `refused-stream` + - `retriable-status-codes` + - `retriable-headers` + Supported [gRPC conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-grpc-on): + - `cancelled` + - `deadline-exceeded` + - `internal` + - `resource-exhausted` + - `unavailable` + items: + description: RetryOn is a string type alias with validation + to ensure that the value is valid. + enum: + - 5xx + - gateway-error + - reset + - connect-failure + - retriable-4xx + - refused-stream + - retriable-status-codes + - retriable-headers + - cancelled + - deadline-exceeded + - internal + - resource-exhausted + - unavailable + type: string + type: array + type: object + services: + description: Services are the services to proxy traffic. + items: + description: Service defines an Kubernetes Service to proxy + traffic. + properties: + cookieRewritePolicies: + description: The policies for rewriting Set-Cookie header + attributes. + items: + properties: + domainRewrite: + description: |- + DomainRewrite enables rewriting the Set-Cookie Domain element. + If not set, Domain will not be rewritten. + properties: + value: + description: |- + Value is the value to rewrite the Domain attribute to. + For now this is required. + maxLength: 4096 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - value + type: object + name: + description: Name is the name of the cookie for + which attributes will be rewritten. + maxLength: 4096 + minLength: 1 + pattern: ^[^()<>@,;:\\"\/[\]?={} \t\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]+$ + type: string + pathRewrite: + description: |- + PathRewrite enables rewriting the Set-Cookie Path element. + If not set, Path will not be rewritten. + properties: + value: + description: |- + Value is the value to rewrite the Path attribute to. + For now this is required. + maxLength: 4096 + minLength: 1 + pattern: ^[^;\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]+$ + type: string + required: + - value + type: object + sameSite: + description: |- + SameSite enables rewriting the Set-Cookie SameSite element. + If not set, SameSite attribute will not be rewritten. + enum: + - Strict + - Lax + - None + type: string + secure: + description: |- + Secure enables rewriting the Set-Cookie Secure element. + If not set, Secure attribute will not be rewritten. + type: boolean + required: + - name + type: object + type: array + healthPort: + description: |- + HealthPort is the port for this service healthcheck. + If not specified, Port is used for service healthchecks. + maximum: 65535 + minimum: 1 + type: integer + mirror: + description: |- + If Mirror is true the Service will receive a read only mirror of the traffic for this route. + If Mirror is true, then fractional mirroring can be enabled by optionally setting the Weight + field. Legal values for Weight are 1-100. Omitting the Weight field will result in 100% mirroring. + NOTE: Setting Weight explicitly to 0 will unexpectedly result in 100% traffic mirroring. This + occurs since we cannot distinguish omitted fields from those explicitly set to their default + values + type: boolean + name: + description: |- + Name is the name of Kubernetes service to proxy traffic. + Names defined here will be used to look up corresponding endpoints which contain the ips to route. + type: string + port: + description: Port (defined as Integer) to proxy traffic + to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + protocol: + description: |- + Protocol may be used to specify (or override) the protocol used to reach this Service. + Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + - tls + type: string + requestHeadersPolicy: + description: The policy for managing request headers during + proxying. + properties: + remove: + description: Remove specifies a list of HTTP header + names to remove. + items: + type: string + type: array + set: + description: |- + Set specifies a list of HTTP header values that will be set in the HTTP header. + If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a + header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: |- + The policy for managing response headers during proxying. + Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header + names to remove. + items: + type: string + type: array + set: + description: |- + Set specifies a list of HTTP header values that will be set in the HTTP header. + If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a + header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + slowStartPolicy: + description: Slow start will gradually increase amount + of traffic to a newly added endpoint. + properties: + aggression: + default: "1.0" + description: |- + The speed of traffic increase over the slow start window. + Defaults to 1.0, so that endpoint would get linearly increasing amount of traffic. + When increasing the value for this parameter, the speed of traffic ramp-up increases non-linearly. + The value of aggression parameter should be greater than 0.0. + More info: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/slow_start + pattern: ^([0-9]+([.][0-9]+)?|[.][0-9]+)$ + type: string + minWeightPercent: + default: 10 + description: |- + The minimum or starting percentage of traffic to send to new endpoints. + A non-zero value helps avoid a too small initial weight, which may cause endpoints in slow start mode to receive no traffic in the beginning of the slow start window. + If not specified, the default is 10%. + format: int32 + maximum: 100 + minimum: 0 + type: integer + window: + description: |- + The duration of slow start window. + Duration is expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). + Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+)$ + type: string + required: + - window + type: object + validation: + description: UpstreamValidation defines how to verify + the backend service's certificate + properties: + caSecret: + description: |- + Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend. + The secret must contain key named ca.crt. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + Max length should be the actual max possible length of a namespaced name (63 + 253 + 1 = 317) + maxLength: 317 + minLength: 1 + type: string + subjectName: + description: |- + Key which is expected to be present in the 'subjectAltName' of the presented certificate. + Deprecated: migrate to using the plural field subjectNames. + maxLength: 250 + minLength: 1 + type: string + subjectNames: + description: |- + List of keys, of which at least one is expected to be present in the 'subjectAltName of the + presented certificate. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + required: + - caSecret + - subjectName + type: object + x-kubernetes-validations: + - message: subjectNames[0] must equal subjectName if set + rule: 'has(self.subjectNames) ? self.subjectNames[0] + == self.subjectName : true' + weight: + description: Weight defines percentage of traffic to balance + traffic + format: int64 + minimum: 0 + type: integer + required: + - name + - port + type: object + type: array + timeoutPolicy: + description: The timeout policy for this route. + properties: + idle: + description: |- + Timeout for how long the proxy should wait while there is no activity during single request/response (for HTTP/1.1) or stream (for HTTP/2). + Timeout will not trigger while HTTP/1.1 connection is idle between two consecutive requests. + If not specified, there is no per-route idle timeout, though a connection manager-wide + stream_idle_timeout default of 5m still applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + idleConnection: + description: |- + Timeout for how long connection from the proxy to the upstream service is kept when there are no active requests. + If not supplied, Envoy's default value of 1h applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + response: + description: |- + Timeout for receiving a response from the server after processing a request from client. + If not supplied, Envoy's default value of 15s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + type: object + type: object + type: array + tcpproxy: + description: TCPProxy holds TCP proxy information. + properties: + healthCheckPolicy: + description: The health check policy for this tcp proxy + properties: + healthyThresholdCount: + description: The number of healthy health checks required + before a host is marked healthy + format: int32 + type: integer + intervalSeconds: + description: The interval (seconds) between health checks + format: int64 + type: integer + timeoutSeconds: + description: The time to wait (seconds) for a health check + response + format: int64 + type: integer + unhealthyThresholdCount: + description: The number of unhealthy health checks required + before a host is marked unhealthy + format: int32 + type: integer + type: object + include: + description: Include specifies that this tcpproxy should be delegated + to another HTTPProxy. + properties: + name: + description: Name of the child HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults + to the current namespace if not supplied. + type: string + required: + - name + type: object + includes: + description: |- + IncludesDeprecated allow for specific routing configuration to be appended to another HTTPProxy in another namespace. + Exists due to a mistake when developing HTTPProxy and the field was marked plural + when it should have been singular. This field should stay to not break backwards compatibility to v1 users. + properties: + name: + description: Name of the child HTTPProxy + type: string + namespace: + description: Namespace of the HTTPProxy to include. Defaults + to the current namespace if not supplied. + type: string + required: + - name + type: object + loadBalancerPolicy: + description: |- + The load balancing policy for the backend services. Note that the + `Cookie` and `RequestHash` load balancing strategies cannot be used + here. + properties: + requestHashPolicies: + description: |- + RequestHashPolicies contains a list of hash policies to apply when the + `RequestHash` load balancing strategy is chosen. If an element of the + supplied list of hash policies is invalid, it will be ignored. If the + list of hash policies is empty after validation, the load balancing + strategy will fall back to the default `RoundRobin`. + items: + description: |- + RequestHashPolicy contains configuration for an individual hash policy + on a request attribute. + properties: + hashSourceIP: + description: |- + HashSourceIP should be set to true when request source IP hash based + load balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + type: boolean + headerHashOptions: + description: |- + HeaderHashOptions should be set when request header hash based load + balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + properties: + headerName: + description: |- + HeaderName is the name of the HTTP request header that will be used to + calculate the hash key. If the header specified is not present on a + request, no hash will be produced. + minLength: 1 + type: string + type: object + queryParameterHashOptions: + description: |- + QueryParameterHashOptions should be set when request query parameter hash based load + balancing is desired. It must be the only hash option field set, + otherwise this request hash policy object will be ignored. + properties: + parameterName: + description: |- + ParameterName is the name of the HTTP request query parameter that will be used to + calculate the hash key. If the query parameter specified is not present on a + request, no hash will be produced. + minLength: 1 + type: string + type: object + terminal: + description: |- + Terminal is a flag that allows for short-circuiting computing of a hash + for a given request. If set to true, and the request attribute specified + in the attribute hash options is present, no further hash policies will + be used to calculate a hash for the request. + type: boolean + type: object + type: array + strategy: + description: |- + Strategy specifies the policy used to balance requests + across the pool of backend pods. Valid policy names are + `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, + and `RequestHash`. If an unknown strategy name is specified + or no policy is supplied, the default `RoundRobin` policy + is used. + type: string + type: object + services: + description: Services are the services to proxy traffic + items: + description: Service defines an Kubernetes Service to proxy + traffic. + properties: + cookieRewritePolicies: + description: The policies for rewriting Set-Cookie header + attributes. + items: + properties: + domainRewrite: + description: |- + DomainRewrite enables rewriting the Set-Cookie Domain element. + If not set, Domain will not be rewritten. + properties: + value: + description: |- + Value is the value to rewrite the Domain attribute to. + For now this is required. + maxLength: 4096 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - value + type: object + name: + description: Name is the name of the cookie for which + attributes will be rewritten. + maxLength: 4096 + minLength: 1 + pattern: ^[^()<>@,;:\\"\/[\]?={} \t\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]+$ + type: string + pathRewrite: + description: |- + PathRewrite enables rewriting the Set-Cookie Path element. + If not set, Path will not be rewritten. + properties: + value: + description: |- + Value is the value to rewrite the Path attribute to. + For now this is required. + maxLength: 4096 + minLength: 1 + pattern: ^[^;\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]+$ + type: string + required: + - value + type: object + sameSite: + description: |- + SameSite enables rewriting the Set-Cookie SameSite element. + If not set, SameSite attribute will not be rewritten. + enum: + - Strict + - Lax + - None + type: string + secure: + description: |- + Secure enables rewriting the Set-Cookie Secure element. + If not set, Secure attribute will not be rewritten. + type: boolean + required: + - name + type: object + type: array + healthPort: + description: |- + HealthPort is the port for this service healthcheck. + If not specified, Port is used for service healthchecks. + maximum: 65535 + minimum: 1 + type: integer + mirror: + description: |- + If Mirror is true the Service will receive a read only mirror of the traffic for this route. + If Mirror is true, then fractional mirroring can be enabled by optionally setting the Weight + field. Legal values for Weight are 1-100. Omitting the Weight field will result in 100% mirroring. + NOTE: Setting Weight explicitly to 0 will unexpectedly result in 100% traffic mirroring. This + occurs since we cannot distinguish omitted fields from those explicitly set to their default + values + type: boolean + name: + description: |- + Name is the name of Kubernetes service to proxy traffic. + Names defined here will be used to look up corresponding endpoints which contain the ips to route. + type: string + port: + description: Port (defined as Integer) to proxy traffic + to since a service can have multiple defined. + exclusiveMaximum: true + maximum: 65536 + minimum: 1 + type: integer + protocol: + description: |- + Protocol may be used to specify (or override) the protocol used to reach this Service. + Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. + enum: + - h2 + - h2c + - tls + type: string + requestHeadersPolicy: + description: The policy for managing request headers during + proxying. + properties: + remove: + description: Remove specifies a list of HTTP header + names to remove. + items: + type: string + type: array + set: + description: |- + Set specifies a list of HTTP header values that will be set in the HTTP header. + If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header + specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + responseHeadersPolicy: + description: |- + The policy for managing response headers during proxying. + Rewriting the 'Host' header is not supported. + properties: + remove: + description: Remove specifies a list of HTTP header + names to remove. + items: + type: string + type: array + set: + description: |- + Set specifies a list of HTTP header values that will be set in the HTTP header. + If the header does not exist it will be added, otherwise it will be overwritten with the new value. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header + specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + type: object + slowStartPolicy: + description: Slow start will gradually increase amount of + traffic to a newly added endpoint. + properties: + aggression: + default: "1.0" + description: |- + The speed of traffic increase over the slow start window. + Defaults to 1.0, so that endpoint would get linearly increasing amount of traffic. + When increasing the value for this parameter, the speed of traffic ramp-up increases non-linearly. + The value of aggression parameter should be greater than 0.0. + More info: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/load_balancing/slow_start + pattern: ^([0-9]+([.][0-9]+)?|[.][0-9]+)$ + type: string + minWeightPercent: + default: 10 + description: |- + The minimum or starting percentage of traffic to send to new endpoints. + A non-zero value helps avoid a too small initial weight, which may cause endpoints in slow start mode to receive no traffic in the beginning of the slow start window. + If not specified, the default is 10%. + format: int32 + maximum: 100 + minimum: 0 + type: integer + window: + description: |- + The duration of slow start window. + Duration is expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). + Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+)$ + type: string + required: + - window + type: object + validation: + description: UpstreamValidation defines how to verify the + backend service's certificate + properties: + caSecret: + description: |- + Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend. + The secret must contain key named ca.crt. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + Max length should be the actual max possible length of a namespaced name (63 + 253 + 1 = 317) + maxLength: 317 + minLength: 1 + type: string + subjectName: + description: |- + Key which is expected to be present in the 'subjectAltName' of the presented certificate. + Deprecated: migrate to using the plural field subjectNames. + maxLength: 250 + minLength: 1 + type: string + subjectNames: + description: |- + List of keys, of which at least one is expected to be present in the 'subjectAltName of the + presented certificate. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + required: + - caSecret + - subjectName + type: object + x-kubernetes-validations: + - message: subjectNames[0] must equal subjectName if set + rule: 'has(self.subjectNames) ? self.subjectNames[0] == + self.subjectName : true' + weight: + description: Weight defines percentage of traffic to balance + traffic + format: int64 + minimum: 0 + type: integer + required: + - name + - port + type: object + type: array + type: object + virtualhost: + description: |- + Virtualhost appears at most once. If it is present, the object is considered + to be a "root" HTTPProxy. + properties: + authorization: + description: |- + This field configures an extension service to perform + authorization for this virtual host. Authorization can + only be configured on virtual hosts that have TLS enabled. + If the TLS configuration requires client certificate + validation, the client certificate is always included in the + authentication check request. + properties: + authPolicy: + description: |- + AuthPolicy sets a default authorization policy for client requests. + This policy will be used unless overridden by individual routes. + properties: + context: + additionalProperties: + type: string + description: |- + Context is a set of key/value pairs that are sent to the + authentication server in the check request. If a context + is provided at an enclosing scope, the entries are merged + such that the inner scope overrides matching keys from the + outer scope. + type: object + disabled: + description: |- + When true, this field disables client request authentication + for the scope of the policy. + type: boolean + type: object + extensionRef: + description: ExtensionServiceRef specifies the extension resource + that will authorize client requests. + properties: + apiVersion: + description: |- + API version of the referent. + If this field is not specified, the default "projectcontour.io/v1alpha1" will be used + minLength: 1 + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referent. + If this field is not specifies, the namespace of the resource that targets the referent will be used. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + minLength: 1 + type: string + type: object + failOpen: + description: |- + If FailOpen is true, the client request is forwarded to the upstream service + even if the authorization server fails to respond. This field should not be + set in most cases. It is intended for use only while migrating applications + from internal authorization to Contour external authorization. + type: boolean + responseTimeout: + description: |- + ResponseTimeout configures maximum time to wait for a check response from the authorization server. + Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). + Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + The string "infinity" is also a valid input and specifies no timeout. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ + type: string + withRequestBody: + description: WithRequestBody specifies configuration for sending + the client request's body to authorization server. + properties: + allowPartialMessage: + description: If AllowPartialMessage is true, then Envoy + will buffer the body until MaxRequestBytes are reached. + type: boolean + maxRequestBytes: + default: 1024 + description: MaxRequestBytes sets the maximum size of + message body ExtAuthz filter will hold in-memory. + format: int32 + minimum: 1 + type: integer + packAsBytes: + description: If PackAsBytes is true, the body sent to + Authorization Server is in raw bytes. + type: boolean + type: object + type: object + corsPolicy: + description: Specifies the cross-origin policy to apply to the + VirtualHost. + properties: + allowCredentials: + description: Specifies whether the resource allows credentials. + type: boolean + allowHeaders: + description: AllowHeaders specifies the content for the *access-control-allow-headers* + header. + items: + description: CORSHeaderValue specifies the value of the + string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + minItems: 1 + type: array + allowMethods: + description: AllowMethods specifies the content for the *access-control-allow-methods* + header. + items: + description: CORSHeaderValue specifies the value of the + string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + minItems: 1 + type: array + allowOrigin: + description: |- + AllowOrigin specifies the origins that will be allowed to do CORS requests. + Allowed values include "*" which signifies any origin is allowed, an exact + origin of the form "scheme://host[:port]" (where port is optional), or a valid + regex pattern. + Note that regex patterns are validated and a simple "glob" pattern (e.g. *.foo.com) + will be rejected or produce unexpected matches when applied as a regex. + items: + type: string + minItems: 1 + type: array + allowPrivateNetwork: + description: |- + AllowPrivateNetwork specifies whether to allow private network requests. + See https://developer.chrome.com/blog/private-network-access-preflight. + type: boolean + exposeHeaders: + description: ExposeHeaders Specifies the content for the *access-control-expose-headers* + header. + items: + description: CORSHeaderValue specifies the value of the + string headers returned by a cross-domain request. + pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ + type: string + minItems: 1 + type: array + maxAge: + description: |- + MaxAge indicates for how long the results of a preflight request can be cached. + MaxAge durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). + Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + Only positive values are allowed while 0 disables the cache requiring a preflight OPTIONS + check for all cross-origin requests. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|0)$ + type: string + required: + - allowMethods + - allowOrigin + type: object + fqdn: + description: |- + The fully qualified domain name of the root of the ingress tree + all leaves of the DAG rooted at this object relate to the fqdn. + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + ipAllowPolicy: + description: |- + IPAllowFilterPolicy is a list of ipv4/6 filter rules for which matching + requests should be allowed. All other requests will be denied. + Only one of IPAllowFilterPolicy and IPDenyFilterPolicy can be defined. + The rules defined here may be overridden in a Route. + items: + properties: + cidr: + description: |- + CIDR is a CIDR block of ipv4 or ipv6 addresses to filter on. This can also be + a bare IP address (without a mask) to filter on exactly one address. + type: string + source: + description: |- + Source indicates how to determine the ip address to filter on, and can be + one of two values: + - `Remote` filters on the ip address of the client, accounting for PROXY and + X-Forwarded-For as needed. + - `Peer` filters on the ip of the network request, ignoring PROXY and + X-Forwarded-For. + enum: + - Peer + - Remote + type: string + required: + - cidr + - source + type: object + type: array + ipDenyPolicy: + description: |- + IPDenyFilterPolicy is a list of ipv4/6 filter rules for which matching + requests should be denied. All other requests will be allowed. + Only one of IPAllowFilterPolicy and IPDenyFilterPolicy can be defined. + The rules defined here may be overridden in a Route. + items: + properties: + cidr: + description: |- + CIDR is a CIDR block of ipv4 or ipv6 addresses to filter on. This can also be + a bare IP address (without a mask) to filter on exactly one address. + type: string + source: + description: |- + Source indicates how to determine the ip address to filter on, and can be + one of two values: + - `Remote` filters on the ip address of the client, accounting for PROXY and + X-Forwarded-For as needed. + - `Peer` filters on the ip of the network request, ignoring PROXY and + X-Forwarded-For. + enum: + - Peer + - Remote + type: string + required: + - cidr + - source + type: object + type: array + jwtProviders: + description: Providers to use for verifying JSON Web Tokens (JWTs) + on the virtual host. + items: + description: JWTProvider defines how to verify JWTs on requests. + properties: + audiences: + description: |- + Audiences that JWTs are allowed to have in the "aud" field. + If not provided, JWT audiences are not checked. + items: + type: string + type: array + default: + description: |- + Whether the provider should apply to all + routes in the HTTPProxy/its includes by + default. At most one provider can be marked + as the default. If no provider is marked + as the default, individual routes must explicitly + identify the provider they require. + type: boolean + forwardJWT: + description: |- + Whether the JWT should be forwarded to the backend + service after successful verification. By default, + the JWT is not forwarded. + type: boolean + issuer: + description: |- + Issuer that JWTs are required to have in the "iss" field. + If not provided, JWT issuers are not checked. + type: string + name: + description: Unique name for the provider. + minLength: 1 + type: string + remoteJWKS: + description: Remote JWKS to use for verifying JWT signatures. + properties: + cacheDuration: + description: |- + How long to cache the JWKS locally. If not specified, + Envoy's default of 5m applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+)$ + type: string + dnsLookupFamily: + description: |- + The DNS IP address resolution policy for the JWKS URI. + When configured as "v4", the DNS resolver will only perform a lookup + for addresses in the IPv4 family. If "v6" is configured, the DNS resolver + will only perform a lookup for addresses in the IPv6 family. + If "all" is configured, the DNS resolver + will perform a lookup for addresses in both the IPv4 and IPv6 family. + If "auto" is configured, the DNS resolver will first perform a lookup + for addresses in the IPv6 family and fallback to a lookup for addresses + in the IPv4 family. If not specified, the Contour-wide setting defined + in the config file or ContourConfiguration applies (defaults to "auto"). + See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html#envoy-v3-api-enum-config-cluster-v3-cluster-dnslookupfamily + for more information. + enum: + - auto + - v4 + - v6 + type: string + timeout: + description: |- + How long to wait for a response from the URI. + If not specified, a default of 1s applies. + pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+)$ + type: string + uri: + description: The URI for the JWKS. + minLength: 1 + type: string + validation: + description: UpstreamValidation defines how to verify + the JWKS's TLS certificate. + properties: + caSecret: + description: |- + Name or namespaced name of the Kubernetes secret used to validate the certificate presented by the backend. + The secret must contain key named ca.crt. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + Max length should be the actual max possible length of a namespaced name (63 + 253 + 1 = 317) + maxLength: 317 + minLength: 1 + type: string + subjectName: + description: |- + Key which is expected to be present in the 'subjectAltName' of the presented certificate. + Deprecated: migrate to using the plural field subjectNames. + maxLength: 250 + minLength: 1 + type: string + subjectNames: + description: |- + List of keys, of which at least one is expected to be present in the 'subjectAltName of the + presented certificate. + items: + type: string + maxItems: 8 + minItems: 1 + type: array + required: + - caSecret + - subjectName + type: object + x-kubernetes-validations: + - message: subjectNames[0] must equal subjectName if + set + rule: 'has(self.subjectNames) ? self.subjectNames[0] + == self.subjectName : true' + required: + - uri + type: object + required: + - name + - remoteJWKS + type: object + type: array + rateLimitPolicy: + description: The policy for rate limiting on the virtual host. + properties: + global: + description: |- + Global defines global rate limiting parameters, i.e. parameters + defining descriptors that are sent to an external rate limit + service (RLS) for a rate limit decision on each request. + properties: + descriptors: + description: |- + Descriptors defines the list of descriptors that will + be generated and sent to the rate limit service. Each + descriptor contains 1+ key-value pair entries. + items: + description: RateLimitDescriptor defines a list of key-value + pair generators. + properties: + entries: + description: Entries is the list of key-value pair + generators. + items: + description: |- + RateLimitDescriptorEntry is a key-value pair generator. Exactly + one field on this struct must be non-nil. + properties: + genericKey: + description: GenericKey defines a descriptor + entry with a static key and value. + properties: + key: + description: |- + Key defines the key of the descriptor entry. If not set, the + key is set to "generic_key". + type: string + value: + description: Value defines the value of + the descriptor entry. + minLength: 1 + type: string + type: object + remoteAddress: + description: |- + RemoteAddress defines a descriptor entry with a key of "remote_address" + and a value equal to the client's IP address (from x-forwarded-for). + type: object + requestHeader: + description: |- + RequestHeader defines a descriptor entry that's populated only if + a given header is present on the request. The descriptor key is static, + and the descriptor value is equal to the value of the header. + properties: + descriptorKey: + description: DescriptorKey defines the + key to use on the descriptor entry. + minLength: 1 + type: string + headerName: + description: HeaderName defines the name + of the header to look for on the request. + minLength: 1 + type: string + type: object + requestHeaderValueMatch: + description: |- + RequestHeaderValueMatch defines a descriptor entry that's populated + if the request's headers match a set of 1+ match criteria. The + descriptor key is "header_match", and the descriptor value is static. + properties: + expectMatch: + default: true + description: |- + ExpectMatch defines whether the request must positively match the match + criteria in order to generate a descriptor entry (i.e. true), or not + match the match criteria in order to generate a descriptor entry (i.e. false). + The default is true. + type: boolean + headers: + description: |- + Headers is a list of 1+ match criteria to apply against the request + to determine whether to populate the descriptor entry or not. + items: + description: |- + HeaderMatchCondition specifies how to conditionally match against HTTP + headers. The Name field is required, only one of Present, NotPresent, + Contains, NotContains, Exact, NotExact and Regex can be set. + For negative matching rules only (e.g. NotContains or NotExact) you can set + TreatMissingAsEmpty. + IgnoreCase has no effect for Regex. + properties: + contains: + description: |- + Contains specifies a substring that must be present in + the header value. + type: string + exact: + description: Exact specifies a string + that the header value must be + equal to. + type: string + ignoreCase: + description: |- + IgnoreCase specifies that string matching should be case insensitive. + Note that this has no effect on the Regex parameter. + type: boolean + name: + description: |- + Name is the name of the header to match against. Name is required. + Header names are case insensitive. + type: string + notcontains: + description: |- + NotContains specifies a substring that must not be present + in the header value. + type: string + notexact: + description: |- + NoExact specifies a string that the header value must not be + equal to. The condition is true if the header has any other value. + type: string + notpresent: + description: |- + NotPresent specifies that condition is true when the named header + is not present. Note that setting NotPresent to false does not + make the condition true if the named header is present. + type: boolean + present: + description: |- + Present specifies that condition is true when the named header + is present, regardless of its value. Note that setting Present + to false does not make the condition true if the named header + is absent. + type: boolean + regex: + description: |- + Regex specifies a regular expression pattern that must match the header + value. + type: string + treatMissingAsEmpty: + description: |- + TreatMissingAsEmpty specifies if the header match rule specified header + does not exist, this header value will be treated as empty. Defaults to false. + Unlike the underlying Envoy implementation this is **only** supported for + negative matches (e.g. NotContains, NotExact). + type: boolean + required: + - name + type: object + minItems: 1 + type: array + value: + description: Value defines the value of + the descriptor entry. + minLength: 1 + type: string + type: object + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + disabled: + description: |- + Disabled configures the HTTPProxy to not use + the default global rate limit policy defined by the Contour configuration. + type: boolean + type: object + local: + description: |- + Local defines local rate limiting parameters, i.e. parameters + for rate limiting that occurs within each Envoy pod as requests + are handled. + properties: + burst: + description: |- + Burst defines the number of requests above the requests per + unit that should be allowed within a short period of time. + format: int32 + type: integer + requests: + description: |- + Requests defines how many requests per unit of time should + be allowed before rate limiting occurs. + format: int32 + minimum: 1 + type: integer + responseHeadersToAdd: + description: |- + ResponseHeadersToAdd is an optional list of response headers to + set when a request is rate-limited. + items: + description: HeaderValue represents a header name/value + pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header + specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + responseStatusCode: + description: |- + ResponseStatusCode is the HTTP status code to use for responses + to rate-limited requests. Codes must be in the 400-599 range + (inclusive). If not specified, the Envoy default of 429 (Too + Many Requests) is used. + format: int32 + maximum: 599 + minimum: 400 + type: integer + unit: + description: |- + Unit defines the period of time within which requests + over the limit will be rate limited. Valid values are + "second", "minute" and "hour". + enum: + - second + - minute + - hour + type: string + required: + - requests + - unit + type: object + type: object + tls: + description: |- + If present the fields describes TLS properties of the virtual + host. The SNI names that will be matched on are described in fqdn, + the tls.secretName secret must contain a certificate that itself + contains a name that matches the FQDN. + properties: + clientValidation: + description: |- + ClientValidation defines how to verify the client certificate + when an external client establishes a TLS connection to Envoy. + This setting: + 1. Enables TLS client certificate validation. + 2. Specifies how the client certificate will be validated (i.e. + validation required or skipped). + Note: Setting client certificate validation to be skipped should + be only used in conjunction with an external authorization server that + performs client validation as Contour will ensure client certificates + are passed along. + properties: + caSecret: + description: |- + Name of a Kubernetes secret that contains a CA certificate bundle. + The secret must contain key named ca.crt. + The client certificate must validate against the certificates in the bundle. + If specified and SkipClientCertValidation is true, client certificates will + be required on requests. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + minLength: 1 + type: string + crlOnlyVerifyLeafCert: + description: |- + If this option is set to true, only the certificate at the end of the + certificate chain will be subject to validation by CRL. + type: boolean + crlSecret: + description: |- + Name of a Kubernetes opaque secret that contains a concatenated list of PEM encoded CRLs. + The secret must contain key named crl.pem. + This field will be used to verify that a client certificate has not been revoked. + CRLs must be available from all CAs, unless crlOnlyVerifyLeafCert is true. + Large CRL lists are not supported since individual secrets are limited to 1MiB in size. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + minLength: 1 + type: string + forwardClientCertificate: + description: |- + ForwardClientCertificate adds the selected data from the passed client TLS certificate + to the x-forwarded-client-cert header. + properties: + cert: + description: Client cert in URL encoded PEM format. + type: boolean + chain: + description: Client cert chain (including the leaf + cert) in URL encoded PEM format. + type: boolean + dns: + description: DNS type Subject Alternative Names of + the client cert. + type: boolean + subject: + description: Subject of the client cert. + type: boolean + uri: + description: URI type Subject Alternative Name of + the client cert. + type: boolean + type: object + optionalClientCertificate: + description: |- + OptionalClientCertificate when set to true will request a client certificate + but allow the connection to continue if the client does not provide one. + If a client certificate is sent, it will be verified according to the + other properties, which includes disabling validation if + SkipClientCertValidation is set. Defaults to false. + type: boolean + skipClientCertValidation: + description: |- + SkipClientCertValidation disables downstream client certificate + validation. Defaults to false. This field is intended to be used in + conjunction with external authorization in order to enable the external + authorization server to validate client certificates. When this field + is set to true, client certificates are requested but not verified by + Envoy. If CACertificate is specified, client certificates are required on + requests, but not verified. If external authorization is in use, they are + presented to the external authorization server. + type: boolean + type: object + enableFallbackCertificate: + description: |- + EnableFallbackCertificate defines if the vhost should allow a default certificate to + be applied which handles all requests which don't match the SNI defined in this vhost. + type: boolean + maximumProtocolVersion: + description: |- + MaximumProtocolVersion is the maximum TLS version this vhost should + negotiate. Valid options are `1.2` and `1.3` (default). Any other value + defaults to TLS 1.3. + type: string + minimumProtocolVersion: + description: |- + MinimumProtocolVersion is the minimum TLS version this vhost should + negotiate. Valid options are `1.2` (default) and `1.3`. Any other value + defaults to TLS 1.2. + type: string + passthrough: + description: |- + Passthrough defines whether the encrypted TLS handshake will be + passed through to the backing cluster. Either Passthrough or + SecretName must be specified, but not both. + type: boolean + secretName: + description: |- + SecretName is the name of a TLS secret. + Either SecretName or Passthrough must be specified, but not both. + If specified, the named secret must contain a matching certificate + for the virtual host's FQDN. + The name can be optionally prefixed with namespace "namespace/name". + When cross-namespace reference is used, TLSCertificateDelegation resource must exist in the namespace to grant access to the secret. + type: string + type: object + required: + - fqdn + type: object + type: object + status: + default: + currentStatus: NotReconciled + description: Waiting for controller + description: Status is a container for computed information about the + HTTPProxy. + properties: + conditions: + description: |- + Conditions contains information about the current status of the HTTPProxy, + in an upstream-friendly container. + Contour will update a single condition, `Valid`, that is in normal-true polarity. + That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, + and vice versa. + Contour will leave untouched any other Conditions set in this block, + in case some other controller wants to add a Condition. + If you are another controller owner and wish to add a condition, you *should* + namespace your condition with a label, like `controller.domain.com/ConditionName`. + items: + description: |- + DetailedCondition is an extension of the normal Kubernetes conditions, with two extra + fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) + of the condition. + `errors` holds information about sub-conditions which are fatal to that condition and render its state False. + `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. + Remember that Conditions have a type, a status, and a reason. + The type is the type of the condition, the most important one in this CRD set is `Valid`. + `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. + In more detail, `status: true` means that the object is has been ingested into Contour with no errors. + `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` + slice in this case. + `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. + The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` + slice if `status` is `false`. + For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. + When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. + When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. + In either case, there may be entries in the `warnings` slice. + Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason + (if there is one and only one entry in total across both the `errors` and `warnings` slices), or + `MultipleReasons` if there is more than one entry. + properties: + errors: + description: |- + Errors contains a slice of relevant error subconditions for this object. + Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. + An empty slice here indicates no errors. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: |- + Warnings contains a slice of relevant warning subconditions for this object. + Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. + An empty slice here indicates no warnings. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + currentStatus: + type: string + description: + type: string + loadBalancer: + description: LoadBalancer contains the current status of the load + balancer. + properties: + ingress: + description: |- + Ingress is a list containing ingress points for the load-balancer. + Traffic intended for the service should be sent to these ingress points. + items: + description: |- + LoadBalancerIngress represents the status of a load-balancer ingress point: + traffic intended for the service should be sent to an ingress point. + properties: + hostname: + description: |- + Hostname is set for load-balancer ingress points that are DNS based + (typically AWS load-balancers) + type: string + ip: + description: |- + IP is set for load-balancer ingress points that are IP based + (typically GCE or OpenStack load-balancers) + type: string + ipMode: + description: |- + IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + Setting this to "VIP" indicates that traffic is delivered to the node with + the destination set to the load-balancer's IP and port. + Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + the destination set to the node's IP and node port or the pod's IP and port. + Service implementations may use this information to adjust traffic routing. + type: string + ports: + description: |- + Ports is a list of records of service ports + If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: |- + Error is to record the problem with the service port + The format of the error shall comply with the following rules: + - built-in error values shall be specified in this file and those shall use + CamelCase names + - cloud provider specific error values must have names that comply with the + format foo.example.com/CamelCase. + --- + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: |- + Protocol is the protocol of the service port of which status is recorded here + The supported values are: "TCP", "UDP", "SCTP" + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: tlscertificatedelegations.projectcontour.io +spec: + preserveUnknownFields: false + group: projectcontour.io + names: + kind: TLSCertificateDelegation + listKind: TLSCertificateDelegationList + plural: tlscertificatedelegations + shortNames: + - tlscerts + singular: tlscertificatedelegation + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + TLSCertificateDelegation is an TLS Certificate Delegation CRD specification. + See design/tls-certificate-delegation.md for details. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TLSCertificateDelegationSpec defines the spec of the CRD + properties: + delegations: + items: + description: |- + CertificateDelegation maps the authority to reference a secret + in the current namespace to a set of namespaces. + properties: + secretName: + description: required, the name of a secret in the current namespace. + type: string + targetNamespaces: + description: |- + required, the namespaces the authority to reference the + secret will be delegated to. + If TargetNamespaces is nil or empty, the CertificateDelegation' + is ignored. If the TargetNamespace list contains the character, "*" + the secret will be delegated to all namespaces. + items: + type: string + type: array + required: + - secretName + - targetNamespaces + type: object + type: array + required: + - delegations + type: object + status: + description: |- + TLSCertificateDelegationStatus allows for the status of the delegation + to be presented to the user. + properties: + conditions: + description: |- + Conditions contains information about the current status of the HTTPProxy, + in an upstream-friendly container. + Contour will update a single condition, `Valid`, that is in normal-true polarity. + That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, + and vice versa. + Contour will leave untouched any other Conditions set in this block, + in case some other controller wants to add a Condition. + If you are another controller owner and wish to add a condition, you *should* + namespace your condition with a label, like `controller.domain.com\ConditionName`. + items: + description: |- + DetailedCondition is an extension of the normal Kubernetes conditions, with two extra + fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) + of the condition. + `errors` holds information about sub-conditions which are fatal to that condition and render its state False. + `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. + Remember that Conditions have a type, a status, and a reason. + The type is the type of the condition, the most important one in this CRD set is `Valid`. + `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. + In more detail, `status: true` means that the object is has been ingested into Contour with no errors. + `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` + slice in this case. + `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. + The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` + slice if `status` is `false`. + For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. + When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. + When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. + In either case, there may be entries in the `warnings` slice. + Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason + (if there is one and only one entry in total across both the `errors` and `warnings` slices), or + `MultipleReasons` if there is more than one entry. + properties: + errors: + description: |- + Errors contains a slice of relevant error subconditions for this object. + Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. + An empty slice here indicates no errors. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + warnings: + description: |- + Warnings contains a slice of relevant warning subconditions for this object. + Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. + An empty slice here indicates no warnings. + items: + description: |- + SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. + It contains a subset of the Condition fields. + It is intended for warnings and errors, so `type` names should use abnormal-true polarity, + that is, they should be of the form "ErrorPresent: true". + The expected lifecycle for these errors is that they should only be present when the error or warning is, + and should be removed when they are not relevant. + properties: + message: + description: |- + Message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + reason: + description: |- + Reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. + This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - message + - reason + - status + - type + type: object + type: array + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/controllers/cruisecontroltask_controller_test.go b/controllers/cruisecontroltask_controller_test.go index bc0f9f012..1dff2bcb3 100644 --- a/controllers/cruisecontroltask_controller_test.go +++ b/controllers/cruisecontroltask_controller_test.go @@ -372,7 +372,6 @@ func TestCreateCCOperation(t *testing.T) { } mockCtrl := gomock.NewController(t) - for _, testCase := range testCases { mockClient := mocks.NewMockClient(mockCtrl) mockSubResourceClient := mocks.NewMockSubResourceClient(mockCtrl) diff --git a/controllers/kafkacluster_controller.go b/controllers/kafkacluster_controller.go index 954503766..a48fd299f 100644 --- a/controllers/kafkacluster_controller.go +++ b/controllers/kafkacluster_controller.go @@ -43,6 +43,7 @@ import ( "github.com/banzaicloud/koperator/pkg/kafkaclient" "github.com/banzaicloud/koperator/pkg/pki" "github.com/banzaicloud/koperator/pkg/resources" + "github.com/banzaicloud/koperator/pkg/resources/contouringress" "github.com/banzaicloud/koperator/pkg/resources/cruisecontrol" "github.com/banzaicloud/koperator/pkg/resources/cruisecontrolmonitoring" "github.com/banzaicloud/koperator/pkg/resources/envoy" @@ -51,6 +52,8 @@ import ( "github.com/banzaicloud/koperator/pkg/resources/kafkamonitoring" "github.com/banzaicloud/koperator/pkg/resources/nodeportexternalaccess" "github.com/banzaicloud/koperator/pkg/util" + + contour "github.com/projectcontour/contour/apis/projectcontour/v1" ) var clusterFinalizer = "finalizer.kafkaclusters.kafka.banzaicloud.io" @@ -118,6 +121,7 @@ func (r *KafkaClusterReconciler) Reconcile(ctx context.Context, request ctrl.Req envoy.New(r.Client, instance), istioingress.New(r.Client, instance), nodeportexternalaccess.New(r.Client, instance), + contouringress.New(r.Client, instance), kafkamonitoring.New(r.Client, instance), cruisecontrolmonitoring.New(r.Client, instance), kafka.New(r.Client, r.DirectClient, instance, r.KafkaClientProvider), @@ -365,6 +369,7 @@ func SetupKafkaClusterWithManager(mgr ctrl.Manager) *ctrl.Builder { kafkaWatches(builder) envoyWatches(builder) + contourWatches(builder) cruiseControlWatches(builder) builder.WithEventFilter( @@ -417,6 +422,12 @@ func envoyWatches(builder *ctrl.Builder) *ctrl.Builder { Owns(&corev1.ConfigMap{}) } +func contourWatches(builder *ctrl.Builder) *ctrl.Builder { + return builder. + Owns(&corev1.Service{}). + Owns(&contour.HTTPProxy{}) +} + func cruiseControlWatches(builder *ctrl.Builder) *ctrl.Builder { return builder. Owns(&corev1.Service{}). diff --git a/controllers/tests/clusterregistry/suite_test.go b/controllers/tests/clusterregistry/suite_test.go index 8a2e0a41d..f326c8d55 100644 --- a/controllers/tests/clusterregistry/suite_test.go +++ b/controllers/tests/clusterregistry/suite_test.go @@ -56,6 +56,7 @@ import ( istioclientv1beta1 "github.com/banzaicloud/istio-client-go/pkg/networking/v1beta1" banzaiistiov1alpha1 "github.com/banzaicloud/istio-operator/api/v2/v1alpha1" + contour "github.com/projectcontour/contour/apis/projectcontour/v1" banzaicloudv1alpha1 "github.com/banzaicloud/koperator/api/v1alpha1" banzaicloudv1beta1 "github.com/banzaicloud/koperator/api/v1beta1" @@ -96,6 +97,7 @@ var _ = BeforeSuite(func() { CRDDirectoryPaths: []string{ filepath.Join("..", "..", "..", "config", "base", "crds"), filepath.Join("..", "..", "..", "config", "test", "crd", "cert-manager"), + filepath.Join("..", "..", "..", "config", "test", "crd", "projectcontour"), filepath.Join("..", "..", "..", "config", "test", "crd", "istio"), }, ControlPlaneStopTimeout: stopTimeout, @@ -123,6 +125,7 @@ var _ = BeforeSuite(func() { Expect(banzaicloudv1alpha1.AddToScheme(scheme)).To(Succeed()) Expect(banzaicloudv1beta1.AddToScheme(scheme)).To(Succeed()) Expect(istioclientv1beta1.AddToScheme(scheme)).To(Succeed()) + Expect(contour.AddToScheme(scheme)).To(Succeed()) // +kubebuilder:scaffold:scheme diff --git a/controllers/tests/common_test.go b/controllers/tests/common_test.go index 9357cbc15..3954fbdca 100644 --- a/controllers/tests/common_test.go +++ b/controllers/tests/common_test.go @@ -125,7 +125,7 @@ func createMinimalKafkaClusterCR(name, namespace string) *v1beta1.KafkaCluster { CCJMXExporterConfig: "custom_property: custom_value", }, ReadOnlyConfig: "cruise.control.metrics.topic.auto.create=true", - RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{FailureThreshold: 1, ConcurrentBrokerRestartsAllowed: 1}, + RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{FailureThreshold: 1, ConcurrentBrokerRestartCountPerRack: 1}, }, } } diff --git a/controllers/tests/cruisecontroltask_controller_test.go b/controllers/tests/cruisecontroltask_controller_test.go index 64d7c8b3a..0b830f595 100644 --- a/controllers/tests/cruisecontroltask_controller_test.go +++ b/controllers/tests/cruisecontroltask_controller_test.go @@ -33,7 +33,6 @@ import ( "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" - "github.com/banzaicloud/koperator/pkg/scale" "github.com/banzaicloud/koperator/pkg/util" ) @@ -47,10 +46,10 @@ var _ = Describe("CruiseControlTaskReconciler", func() { operation *v1alpha1.CruiseControlOperation ) - const ( - mountPath = "/kafka-logs-test" - trueStr = "true" - ) + // const ( + // mountPath = "/kafka-logs-test" + // trueStr = "true" + // ) BeforeEach(func(ctx SpecContext) { atomic.AddUint64(&count, 1) @@ -455,18 +454,18 @@ func getScaleMockCCTask1() *mocks.MockCruiseControlScaler { return scaleMock } -func getScaleMockCCTask2(onlineLogDirs []string) *mocks.MockCruiseControlScaler { - mockCtrl := gomock.NewController(GinkgoT()) - scaleMock := mocks.NewMockCruiseControlScaler(mockCtrl) - availableBrokers := []string{"1", "2", "3"} - scaleMock.EXPECT().BrokersWithState(gomock.Any(), gomock.All()).Return(availableBrokers, nil).AnyTimes() - - logDirs := make(map[scale.LogDirState][]string) - logDirsBrokerRet := make(map[string]map[scale.LogDirState][]string) - - logDirs[scale.LogDirStateOnline] = onlineLogDirs - logDirsBrokerRet["0"] = logDirs - logDirsBrokerRet["1"] = logDirs - scaleMock.EXPECT().LogDirsByBroker(gomock.Any()).Return(logDirsBrokerRet, nil).AnyTimes() - return scaleMock -} +// func getScaleMockCCTask2(onlineLogDirs []string) *mocks.MockCruiseControlScaler { +// mockCtrl := gomock.NewController(GinkgoT()) +// scaleMock := mocks.NewMockCruiseControlScaler(mockCtrl) +// availableBrokers := []string{"1", "2", "3"} +// scaleMock.EXPECT().BrokersWithState(gomock.Any(), gomock.All()).Return(availableBrokers, nil).AnyTimes() + +// logDirs := make(map[scale.LogDirState][]string) +// logDirsBrokerRet := make(map[string]map[scale.LogDirState][]string) + +// logDirs[scale.LogDirStateOnline] = onlineLogDirs +// logDirsBrokerRet["0"] = logDirs +// logDirsBrokerRet["1"] = logDirs +// scaleMock.EXPECT().LogDirsByBroker(gomock.Any()).Return(logDirsBrokerRet, nil).AnyTimes() +// return scaleMock +// } diff --git a/controllers/tests/kafkacluster_controller_contour_test.go b/controllers/tests/kafkacluster_controller_contour_test.go new file mode 100644 index 000000000..8965ffe84 --- /dev/null +++ b/controllers/tests/kafkacluster_controller_contour_test.go @@ -0,0 +1,193 @@ +// Copyright © 2020 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "fmt" + "sync/atomic" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "github.com/projectcontour/contour/apis/projectcontour/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/util" + contourutils "github.com/banzaicloud/koperator/pkg/util/contour" + "github.com/banzaicloud/koperator/pkg/util/kafka" +) + +var _ = Describe("KafkaClusterWithContourIngressController", Label("contour"), func() { + var ( + count uint64 = 0 + namespace string + namespaceObj *corev1.Namespace + kafkaCluster *v1beta1.KafkaCluster + ) + + BeforeEach(func() { + atomic.AddUint64(&count, 1) + namespace = fmt.Sprintf("kafkacontourtest-%v", count) + namespaceObj = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + kafkaCluster = createMinimalKafkaClusterCR(fmt.Sprintf("kafkacluster-%d", count), namespace) + kafkaCluster.Spec.IngressController = "contour" + contourListener := kafkaCluster.Spec.ListenersConfig.ExternalListeners[0] + contourListener.AccessMethod = corev1.ServiceTypeClusterIP + contourListener.ExternalStartingPort = -1 + contourListener.AnyCastPort = util.Int32Pointer(8443) + contourListener.Type = "plaintext" + contourListener.Name = "listener1" + contourListener.ServiceAnnotations = map[string]string{ + "kubernetes.io/ingress.class": "contour", + } + contourListener.Config = &v1beta1.Config{ + + DefaultIngressConfig: "", + IngressConfig: map[string]v1beta1.IngressConfig{ + "ingress1": { + IngressServiceSettings: v1beta1.IngressServiceSettings{ + HostnameOverride: "kafka.cluster.local", + }, + ContourIngressConfig: &v1beta1.ContourIngressConfig{ + TLSSecretName: "test-tls-secret", + BrokerFQDNTemplate: "broker-%id.kafka.cluster.local", + }, + }, + }, + } + + kafkaCluster.Spec.ListenersConfig.ExternalListeners[0] = contourListener + kafkaCluster.Spec.Brokers[0].BrokerConfig = &v1beta1.BrokerConfig{BrokerIngressMapping: []string{"ingress1"}} + kafkaCluster.Spec.Brokers[1].BrokerConfig = &v1beta1.BrokerConfig{BrokerIngressMapping: []string{"ingress1"}} + + }) + JustBeforeEach(func(ctx SpecContext) { + By("creating namespace " + namespace) + err := k8sClient.Create(ctx, namespaceObj) + Expect(err).NotTo(HaveOccurred()) + + By("creating kafka cluster object " + kafkaCluster.Name + " in namespace " + namespace) + err = k8sClient.Create(ctx, kafkaCluster) + Expect(err).NotTo(HaveOccurred()) + + waitForClusterRunningState(ctx, kafkaCluster, namespace) + }) + JustAfterEach(func(ctx SpecContext) { + By("deleting Kafka cluster object " + kafkaCluster.Name + " in namespace " + namespace) + err := k8sClient.Delete(ctx, kafkaCluster) + Expect(err).NotTo(HaveOccurred()) + + kafkaCluster = nil + }) + When("configuring Contour ingress expect broker ClusterIp svc", func() { + It("should reconcile object properly", func(ctx SpecContext) { + expectContour(ctx, kafkaCluster) + }) + }) +}) + +func expectContourClusterIpAnycastSvc(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster, eListener v1beta1.ExternalListenerConfig) { + var svc corev1.Service + var ingressConfigName string = "ingress1" + + serviceName := fmt.Sprintf(contourutils.ContourServiceNameWithScope, eListener.Name, ingressConfigName, kafkaCluster.GetName()) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: serviceName}, &svc) + return err + }).Should(Succeed()) + + Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(svc.Spec.Ports).To(HaveLen(1)) + Expect(svc.Spec.Ports[0].Port).To(Equal(*eListener.AnyCastPort)) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(eListener.ContainerPort)))) + Expect(svc.Spec.Ports[0].Name).To(Equal("tcp-all-broker")) + Expect(svc.Spec.Selector).To(HaveKeyWithValue("app", "kafka")) + Expect(svc.Spec.Selector).To(HaveKeyWithValue("kafka_cr", kafkaCluster.GetName())) +} + +func expectContourClusterIpBrokerSvc(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster, eListener v1beta1.ExternalListenerConfig) { + var svc corev1.Service + + for _, broker := range kafkaCluster.Spec.Brokers { + serviceName := fmt.Sprintf(kafka.NodePortServiceTemplate, kafkaCluster.GetName(), broker.Id, eListener.Name) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: serviceName}, &svc) + return err + }).Should(Succeed()) + Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(svc.Spec.Ports).To(HaveLen(1)) + Expect(svc.Spec.Ports[0].Port).To(Equal(*eListener.AnyCastPort)) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(eListener.ContainerPort)))) + Expect(svc.Spec.Ports[0].Name).To(Equal(fmt.Sprintf("broker-%d", broker.Id))) + Expect(svc.Spec.Selector).To(HaveKeyWithValue("app", "kafka")) + Expect(svc.Spec.Selector).To(HaveKeyWithValue(v1beta1.BrokerIdLabelKey, fmt.Sprintf("%d", broker.Id))) + Expect(svc.Spec.Selector).To(HaveKeyWithValue("kafka_cr", kafkaCluster.GetName())) + } +} + +func expectContourAnycastHttpProxy(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster, eListener v1beta1.ExternalListenerConfig) { + var proxy v1.HTTPProxy + var proxyName string = "kafka.cluster.local" + var ingressConfigName string = "ingress1" + serviceName := fmt.Sprintf(contourutils.ContourServiceNameWithScope, eListener.Name, ingressConfigName, kafkaCluster.GetName()) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: proxyName}, &proxy) + return err + }).Should(Succeed()) + Expect(proxy.Spec.VirtualHost.Fqdn).To(Equal(proxyName)) + Expect(proxy.Spec.TCPProxy.Services).To(HaveLen(1)) + Expect(proxy.Spec.TCPProxy.Services[0].Name).To(Equal(serviceName)) + Expect(proxy.Spec.TCPProxy.Services[0].Port).To(Equal(int(*eListener.AnyCastPort))) + for k, v := range eListener.GetServiceAnnotations() { + Expect(proxy.GetAnnotations()).To(HaveKeyWithValue(k, v)) + } +} + +func expectContourBrokerHttpProxy(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster, eListener v1beta1.ExternalListenerConfig) { + var proxy v1.HTTPProxy + for _, broker := range kafkaCluster.Spec.Brokers { + proxyName := fmt.Sprintf("broker-%d.kafka.cluster.local", broker.Id) + serviceName := fmt.Sprintf(kafka.NodePortServiceTemplate, kafkaCluster.GetName(), broker.Id, eListener.Name) + Eventually(ctx, func() error { + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: kafkaCluster.Namespace, Name: proxyName}, &proxy) + return err + }).Should(Succeed()) + Expect(proxy.Spec.VirtualHost.Fqdn).To(Equal(proxyName)) + Expect(proxy.Spec.TCPProxy.Services).To(HaveLen(1)) + Expect(proxy.Spec.TCPProxy.Services[0].Name).To(Equal(serviceName)) + Expect(proxy.Spec.TCPProxy.Services[0].Port).To(Equal(int(*eListener.AnyCastPort))) + for k, v := range eListener.GetServiceAnnotations() { + Expect(proxy.GetAnnotations()).To(HaveKeyWithValue(k, v)) + } + } +} + +func expectContour(ctx context.Context, kafkaCluster *v1beta1.KafkaCluster) { + for _, eListenerName := range kafkaCluster.Spec.ListenersConfig.ExternalListeners { + expectContourClusterIpAnycastSvc(ctx, kafkaCluster, eListenerName) + expectContourClusterIpBrokerSvc(ctx, kafkaCluster, eListenerName) + expectContourAnycastHttpProxy(ctx, kafkaCluster, eListenerName) + expectContourBrokerHttpProxy(ctx, kafkaCluster, eListenerName) + } +} diff --git a/controllers/tests/kafkacluster_controller_test.go b/controllers/tests/kafkacluster_controller_test.go index bcd630d98..0afc1fe6a 100644 --- a/controllers/tests/kafkacluster_controller_test.go +++ b/controllers/tests/kafkacluster_controller_test.go @@ -436,6 +436,7 @@ var _ = Describe("KafkaCluster with two config external listener", func() { kafkaClusterKRaft.Spec.Brokers = createMinimalKRaftBrokers() kafkaClusterKRaft.Name = fmt.Sprintf("kafkacluster-kraft-%d", count) }) + JustBeforeEach(func(ctx SpecContext) { By("creating namespace " + namespace) err := k8sClient.Create(ctx, namespaceObj) @@ -516,6 +517,15 @@ var _ = Describe("KafkaCluster with two config external listener", func() { waitForClusterRunningState(ctx, kafkaClusterKRaft, namespace) }) + JustAfterEach(func(ctx SpecContext) { + // in the tests the CC topic might not get deleted + + By("deleting Kafka cluster object " + kafkaCluster.Name + " in namespace " + namespace) + err := k8sClient.Delete(ctx, kafkaCluster) + Expect(err).NotTo(HaveOccurred()) + + kafkaCluster = nil + }) When("configuring two ingress envoy controller config inside the external listener using both as bindings", func() { BeforeEach(func() { @@ -599,7 +609,15 @@ var _ = Describe("KafkaCluster with two config external listener and tls", func( waitForClusterRunningState(ctx, kafkaCluster, namespace) }) + JustAfterEach(func(ctx SpecContext) { + // in the tests the CC topic might not get deleted + By("deleting Kafka cluster object " + kafkaCluster.Name + " in namespace " + namespace) + err := k8sClient.Delete(ctx, kafkaCluster) + Expect(err).NotTo(HaveOccurred()) + + kafkaCluster = nil + }) When("configuring two ingress envoy controller config inside the external listener using both as bindings", func() { BeforeEach(func() { kafkaCluster.Spec.Brokers[0].BrokerConfig = &v1beta1.BrokerConfig{BrokerIngressMapping: []string{"az1"}} diff --git a/controllers/tests/suite_test.go b/controllers/tests/suite_test.go index d7faf203e..1c77ab978 100644 --- a/controllers/tests/suite_test.go +++ b/controllers/tests/suite_test.go @@ -57,6 +57,7 @@ import ( istioclientv1beta1 "github.com/banzaicloud/istio-client-go/pkg/networking/v1beta1" banzaiistiov1alpha1 "github.com/banzaicloud/istio-operator/api/v2/v1alpha1" + contour "github.com/projectcontour/contour/apis/projectcontour/v1" banzaicloudv1alpha1 "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -94,6 +95,7 @@ var _ = BeforeSuite(func(ctx SpecContext) { CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "base", "crds"), filepath.Join("..", "..", "config", "test", "crd", "cert-manager"), + filepath.Join("..", "..", "config", "test", "crd", "projectcontour"), filepath.Join("..", "..", "config", "test", "crd", "istio"), }, ControlPlaneStartTimeout: timeout, @@ -122,6 +124,7 @@ var _ = BeforeSuite(func(ctx SpecContext) { Expect(banzaicloudv1alpha1.AddToScheme(scheme)).To(Succeed()) Expect(banzaicloudv1beta1.AddToScheme(scheme)).To(Succeed()) Expect(istioclientv1beta1.AddToScheme(scheme)).To(Succeed()) + Expect(contour.AddToScheme(scheme)).To(Succeed()) // +kubebuilder:scaffold:scheme diff --git a/docker/kafka/Dockerfile b/docker/kafka/Dockerfile index 07b0b797f..b24f13034 100644 --- a/docker/kafka/Dockerfile +++ b/docker/kafka/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:latest AS kafka_dist ARG scala_version=2.13 -ARG kafka_version=3.6.1 +ARG kafka_version=3.7.0 ARG kafka_distro_base_url=https://downloads.apache.org/kafka ENV kafka_distro=kafka_$scala_version-$kafka_version.tgz @@ -26,7 +26,7 @@ RUN rm -r kafka_$scala_version-$kafka_version/bin/windows FROM debian:bullseye-slim ARG scala_version=2.13 -ARG kafka_version=3.6.1 +ARG kafka_version=3.7.0 RUN set -eux; \ diff --git a/go.mod b/go.mod index c8dd9f5c5..ab25567f6 100644 --- a/go.mod +++ b/go.mod @@ -15,15 +15,16 @@ require ( github.com/banzaicloud/koperator/properties v0.4.1 github.com/cert-manager/cert-manager v1.13.2 github.com/cisco-open/cluster-registry-controller/api v0.2.12 - github.com/envoyproxy/go-control-plane v0.11.1 + github.com/envoyproxy/go-control-plane v0.11.2-0.20231019082134-6e4589f570e1 github.com/ghodss/yaml v1.0.1-0.20220118164431-d8423dcdf344 github.com/go-logr/logr v1.3.0 github.com/onsi/ginkgo/v2 v2.13.1 github.com/onsi/gomega v1.30.0 github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0 + github.com/projectcontour/contour v1.27.0 github.com/prometheus/common v0.45.0 github.com/stretchr/testify v1.8.4 - go.uber.org/mock v0.4.0 + go.uber.org/mock v0.3.0 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa google.golang.org/protobuf v1.31.0 @@ -40,12 +41,12 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/stretchr/objx v0.5.0 // indirect - golang.org/x/tools v0.18.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect + golang.org/x/tools v0.21.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect ) @@ -79,7 +80,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.4.0 github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -105,7 +106,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/shopspring/decimal v1.3.1 // indirect @@ -116,16 +117,16 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.19.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect istio.io/api v1.19.0-alpha.1 // indirect @@ -133,7 +134,7 @@ require ( k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect - sigs.k8s.io/gateway-api v0.8.0 // indirect + sigs.k8s.io/gateway-api v0.8.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index 6e973bf7e..e5fae886b 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= -github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/go-control-plane v0.11.2-0.20231019082134-6e4589f570e1 h1:i/XN+pZrv2iM+Czc4o4tP6UzUJoOxjNI9gQdE1vIjd0= +github.com/envoyproxy/go-control-plane v0.11.2-0.20231019082134-6e4589f570e1/go.mod h1:3X10o7QcAVxP4y/hnTLgkXLwuZV2DxAEh6uaYD5PoxI= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= @@ -136,8 +136,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -158,8 +158,8 @@ github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6 github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -248,11 +248,13 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/projectcontour/contour v1.27.0 h1:F6VjI+rMojroZBfi3KxMXX+KHFspSsOTZiRe/yeyHO0= +github.com/projectcontour/contour v1.27.0/go.mod h1:o4r7+DcM6RUCjD1sm0U9yK7lH59SHG1lQwJSDQQxx+o= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= @@ -305,8 +307,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= +go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -319,8 +321,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= @@ -353,11 +355,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -365,8 +367,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -390,22 +392,22 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -420,8 +422,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -435,10 +437,10 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 h1:o4LtQxebKIJ4vkzyhtD2rfUNZ20Zf0ik5YVP5E7G7VE= google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -468,7 +470,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -502,8 +503,8 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSn k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= -sigs.k8s.io/gateway-api v0.8.0 h1:isQQ3Jx2qFP7vaA3ls0846F0Amp9Eq14P08xbSwVbQg= -sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM= +sigs.k8s.io/gateway-api v0.8.1 h1:Bo4NMAQFYkQZnHXOfufbYwbPW7b3Ic5NjpbeW6EJxuU= +sigs.k8s.io/gateway-api v0.8.1/go.mod h1:0PteDrsrgkRmr13nDqFWnev8tOysAVrwnvfFM55tSVg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/main.go b/main.go index 4aa93c4e9..4f51ebac4 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,7 @@ package main import ( "context" "flag" + "fmt" "os" "strings" @@ -50,6 +51,8 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + contour "github.com/projectcontour/contour/apis/projectcontour/v1" + banzaicloudv1alpha1 "github.com/banzaicloud/koperator/api/v1alpha1" banzaicloudv1beta1 "github.com/banzaicloud/koperator/api/v1beta1" "github.com/banzaicloud/koperator/controllers" @@ -76,6 +79,8 @@ func init() { _ = banzaiistiov1alpha1.AddToScheme(scheme) _ = istioclientv1beta1.AddToScheme(scheme) + + _ = contour.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } @@ -123,15 +128,23 @@ func main() { watchedNamespaces := make(map[string]cache.Config) if namespaces != "" { namespaceList = strings.Split(namespaces, ",") - for i := range namespaceList { - watchedNamespaces[strings.TrimSpace(namespaceList[i])] = cache.Config{} - } + } else { + namespaces = os.Getenv("WATCH_NAMESPACE") + namespaceList = strings.Split(namespaces, ",") + } + for i := range namespaceList { + watchedNamespaces[strings.TrimSpace(namespaceList[i])] = cache.Config{} } + // hash the watched namespaces to allow for more than one operator deployment per namespace + // same watched namespaces will return the same hash so only one operator will be active + leaderElectionID := fmt.Sprintf("%s-%x", "controller-leader-election-helper", util.GetMD5Hash(namespaces)) + setupLog.Info("Using leader electrion id", "LeaderElectionID", leaderElectionID, "watched namespaces", namespaceList) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, LeaderElection: enableLeaderElection, - LeaderElectionID: "controller-leader-election-helper", + LeaderElectionID: leaderElectionID, WebhookServer: webhook.NewServer(webhook.Options{ Port: webhookServerPort, CertDir: webhookCertDir, diff --git a/pkg/resources/contouringress/contour.go b/pkg/resources/contouringress/contour.go new file mode 100644 index 000000000..18e4eaf3e --- /dev/null +++ b/pkg/resources/contouringress/contour.go @@ -0,0 +1,248 @@ +// Copyright © 2020 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package contouringress + +import ( + "context" + "fmt" + "reflect" + "strings" + + "emperror.dev/errors" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + contour "github.com/projectcontour/contour/apis/projectcontour/v1" + + apiutil "github.com/banzaicloud/koperator/api/util" + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/k8sutil" + "github.com/banzaicloud/koperator/pkg/resources" + "github.com/banzaicloud/koperator/pkg/resources/templates" + "github.com/banzaicloud/koperator/pkg/util" + contourutils "github.com/banzaicloud/koperator/pkg/util/contour" + "github.com/banzaicloud/koperator/pkg/util/kafka" +) + +const ( + componentName = "clusterIpExternalAccess" +) + +// Reconciler implements the Component Reconciler +type Reconciler struct { + resources.Reconciler +} + +// New creates a new reconciler for NodePort based external access +func New(client client.Client, cluster *v1beta1.KafkaCluster) *Reconciler { + return &Reconciler{ + Reconciler: resources.Reconciler{ + Client: client, + KafkaCluster: cluster, + }, + } +} + +// Reconcile implements the reconcile logic for NodePort based external access +func (r *Reconciler) Reconcile(log logr.Logger) error { + log = log.WithValues("component", componentName) + log.V(1).Info("Reconciling") + var reconcileObjects []runtime.Object + // create ClusterIP services for discovery service and brokers + for _, eListener := range r.KafkaCluster.Spec.ListenersConfig.ExternalListeners { + if r.KafkaCluster.Spec.GetIngressController() == contourutils.IngressControllerName && eListener.GetAccessMethod() == corev1.ServiceTypeClusterIP { + // create per ingressConfig services ClusterIP + ingressConfigs, defaultControllerName, err := util.GetIngressConfigs(r.KafkaCluster.Spec, eListener) + if err != nil { + return err + } + for name, ingressConfig := range ingressConfigs { + if !util.IsIngressConfigInUse(name, defaultControllerName, r.KafkaCluster, log) { + continue + } + + clusterService := r.clusterService(log, eListener, ingressConfig, name, defaultControllerName) + reconcileObjects = append(reconcileObjects, clusterService) + + // make sure the HostnameOverride is set otherwise the fqdn will be empty and HTTPProxy creation will fail. + fqdn := ingressConfig.HostnameOverride + ingressRoute := r.httpProxy(log, eListener, fqdn, ingressConfig, clusterService) + reconcileObjects = append(reconcileObjects, ingressRoute) + + // create per broker services ClusterIP + for _, broker := range r.KafkaCluster.Spec.Brokers { + brokerService := r.brokerService(log, broker.Id, eListener) + reconcileObjects = append(reconcileObjects, brokerService) + + fqdn := ingressConfig.ContourIngressConfig.GetBrokerFqdn(broker.Id) + ingressRoute := r.httpProxy(log, eListener, fqdn, ingressConfig, brokerService) + reconcileObjects = append(reconcileObjects, ingressRoute) + } + } + + for _, obj := range reconcileObjects { + err = k8sutil.Reconcile(log, r.Client, obj, r.KafkaCluster) + if err != nil { + return err + } + } + } else if r.KafkaCluster.Spec.RemoveUnusedIngressResources { + // Cleaning up unused contour resources when ingress controller is not contour or externalListener access method is not ClusterIP + deletionCounter := 0 + ctx := context.Background() + contourResourcesGVK := []schema.GroupVersionKind{ + { + Version: corev1.SchemeGroupVersion.Version, + Group: corev1.SchemeGroupVersion.Group, + Kind: reflect.TypeOf(corev1.Service{}).Name(), + }, + { + Version: corev1.SchemeGroupVersion.Version, + Group: corev1.SchemeGroupVersion.Group, + Kind: reflect.TypeOf(contour.HTTPProxy{}).Name(), + }, + } + var contourResources unstructured.UnstructuredList + for _, gvk := range contourResourcesGVK { + contourResources.SetGroupVersionKind(gvk) + + if err := r.List(ctx, &contourResources, client.InNamespace(r.KafkaCluster.GetNamespace()), + client.MatchingLabels(labelsForContourIngressWithoutEListenerName(r.KafkaCluster.Name))); err != nil { + return errors.Wrap(err, "error when getting list of envoy ingress resources for deletion") + } + + for _, removeObject := range contourResources.Items { + if !strings.Contains(removeObject.GetLabels()[util.ExternalListenerLabelNameKey], eListener.Name) || + util.ObjectManagedByClusterRegistry(&removeObject) || + !removeObject.GetDeletionTimestamp().IsZero() { + continue + } + if err := r.Delete(ctx, &removeObject); client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "error when removing contour ingress resources") + } + log.V(1).Info(fmt.Sprintf("Deleted contour ingress '%s' resource '%s' for externalListener '%s'", gvk.Kind, removeObject.GetName(), eListener.Name)) + deletionCounter++ + } + } + if deletionCounter > 0 { + log.Info(fmt.Sprintf("Removed '%d' resources for contour ingress", deletionCounter)) + } + } + } + + log.V(1).Info("Reconciled") + + return nil +} + +// generate service for broker +func (r *Reconciler) brokerService(_ logr.Logger, id int32, extListener v1beta1.ExternalListenerConfig) runtime.Object { + service := &corev1.Service{ + ObjectMeta: templates.ObjectMetaWithAnnotations( + fmt.Sprintf(kafka.NodePortServiceTemplate, r.KafkaCluster.GetName(), id, extListener.Name), + apiutil.MergeLabels( + apiutil.LabelsForKafka(r.KafkaCluster.Name), + map[string]string{v1beta1.BrokerIdLabelKey: fmt.Sprintf("%d", id)}, + labelsForContourIngress(r.KafkaCluster.Name, extListener.Name)), + extListener.GetServiceAnnotations(), r.KafkaCluster), + Spec: corev1.ServiceSpec{ + Selector: apiutil.MergeLabels(apiutil.LabelsForKafka(r.KafkaCluster.Name), + map[string]string{v1beta1.BrokerIdLabelKey: fmt.Sprintf("%d", id)}), + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{ + Name: fmt.Sprintf("broker-%d", id), + Port: *extListener.AnyCastPort, + TargetPort: intstr.FromInt(int(extListener.ContainerPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + ExternalTrafficPolicy: extListener.ExternalTrafficPolicy, + }, + } + + return service +} + +// generate service for anycast port +func (r *Reconciler) clusterService(_ logr.Logger, extListener v1beta1.ExternalListenerConfig, + ingressConfig v1beta1.IngressConfig, ingressConfigName, _ string) runtime.Object { + var serviceName string = util.GenerateEnvoyResourceName(contourutils.ContourServiceName, contourutils.ContourServiceNameWithScope, + extListener, ingressConfig, ingressConfigName, r.KafkaCluster.GetName()) + + service := &corev1.Service{ + ObjectMeta: templates.ObjectMetaWithAnnotations( + serviceName, + apiutil.MergeLabels( + apiutil.LabelsForKafka(r.KafkaCluster.Name), + labelsForContourIngress(r.KafkaCluster.Name, extListener.Name)), + extListener.GetServiceAnnotations(), r.KafkaCluster), + Spec: corev1.ServiceSpec{ + Selector: apiutil.MergeLabels(apiutil.LabelsForKafka(r.KafkaCluster.Name)), + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{ + Name: "tcp-all-broker", + Port: *extListener.AnyCastPort, + TargetPort: intstr.FromInt(int(extListener.ContainerPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + ExternalTrafficPolicy: extListener.ExternalTrafficPolicy, + }, + } + + return service +} + +// generate httproxy resource for contour ingress +func (r *Reconciler) httpProxy(_ logr.Logger, extListener v1beta1.ExternalListenerConfig, fqdn string, + ingressConfig v1beta1.IngressConfig, service runtime.Object) runtime.Object { + svc := service.(*corev1.Service) + ingressRoute := &contour.HTTPProxy{ + ObjectMeta: templates.ObjectMetaWithAnnotations(fqdn, + apiutil.MergeLabels( + apiutil.LabelsForKafka(r.KafkaCluster.Name), + labelsForContourIngress(r.KafkaCluster.Name, extListener.Name)), + extListener.GetServiceAnnotations(), r.KafkaCluster), + Spec: contour.HTTPProxySpec{ + VirtualHost: &contour.VirtualHost{ + Fqdn: fqdn, + TLS: &contour.TLS{ + SecretName: ingressConfig.ContourIngressConfig.TLSSecretName, + }, + }, + TCPProxy: &contour.TCPProxy{ + Services: []contour.Service{{ + Name: svc.GetName(), + Port: int(svc.Spec.Ports[0].Port), + }}, + }, + }, + } + + return ingressRoute +} + +func labelsForContourIngress(crName, eLName string) map[string]string { + return apiutil.MergeLabels(labelsForContourIngressWithoutEListenerName(crName), map[string]string{util.ExternalListenerLabelNameKey: eLName}) +} + +func labelsForContourIngressWithoutEListenerName(crName string) map[string]string { + return map[string]string{v1beta1.AppLabelKey: "contouringress", v1beta1.KafkaCRLabelKey: crName} +} diff --git a/pkg/resources/kafka/configmap.go b/pkg/resources/kafka/configmap.go index 85d67eefe..dc4c29e2d 100644 --- a/pkg/resources/kafka/configmap.go +++ b/pkg/resources/kafka/configmap.go @@ -17,6 +17,7 @@ package kafka import ( "context" "fmt" + "maps" "sort" "strings" @@ -39,37 +40,41 @@ import ( properties "github.com/banzaicloud/koperator/properties/pkg" ) -func (r *Reconciler) getConfigProperties(bConfig *v1beta1.BrokerConfig, id int32, quorumVoters []string, +func (r *Reconciler) getConfigProperties(bConfig *v1beta1.BrokerConfig, broker v1beta1.Broker, quorumVoters []string, extListenerStatuses, intListenerStatuses, controllerIntListenerStatuses map[string]v1beta1.ListenerStatusList, serverPasses map[string]string, clientPass string, superUsers []string, log logr.Logger) *properties.Properties { config := properties.NewProperties() - bootstrapServers, err := kafkautils.GetBootstrapServersService(r.KafkaCluster) - if err != nil { - log.Error(err, "getting Kafka bootstrap servers for Cruise Control failed") - } + // bootstrapServers, err := kafkautils.GetBootstrapServersService(r.KafkaCluster) + // if err != nil { + // log.Error(err, "getting Kafka bootstrap servers for Cruise Control failed") + // } + + // Add listener configuration + listenerConf, _ := generateListenerSpecificConfig(&r.KafkaCluster.Spec, serverPasses, log) + config.Merge(listenerConf) // Cruise Control metrics reporter configuration - configCCMetricsReporter(r.KafkaCluster, config, clientPass, bootstrapServers, log) + r.configCCMetricsReporter(broker, config, clientPass, log) // Kafka Broker configurations if r.KafkaCluster.Spec.KRaftMode { - configureBrokerKRaftMode(bConfig, id, r.KafkaCluster, config, quorumVoters, serverPasses, extListenerStatuses, intListenerStatuses, log) + configureBrokerKRaftMode(bConfig, broker.Id, r.KafkaCluster, config, quorumVoters, serverPasses, extListenerStatuses, intListenerStatuses, log) } else { - configureBrokerZKMode(id, r.KafkaCluster, config, serverPasses, extListenerStatuses, intListenerStatuses, controllerIntListenerStatuses, log) + configureBrokerZKMode(broker.Id, r.KafkaCluster, config, serverPasses, extListenerStatuses, intListenerStatuses, controllerIntListenerStatuses, log) } // This logic prevents the removal of the mountPath from the broker configmap - brokerConfigMapName := fmt.Sprintf(brokerConfigTemplate+"-%d", r.KafkaCluster.Name, id) + brokerConfigMapName := fmt.Sprintf(brokerConfigTemplate+"-"+"%d", r.KafkaCluster.Name, broker.Id) var brokerConfigMapOld v1.ConfigMap - err = r.Client.Get(context.Background(), client.ObjectKey{Name: brokerConfigMapName, Namespace: r.KafkaCluster.GetNamespace()}, &brokerConfigMapOld) + err := r.Client.Get(context.Background(), client.ObjectKey{Name: brokerConfigMapName, Namespace: r.KafkaCluster.GetNamespace()}, &brokerConfigMapOld) if err != nil && !apierrors.IsNotFound(err) { log.Error(err, "getting broker configmap from the Kubernetes API server resulted an error") } mountPathsOld, err := getMountPathsFromBrokerConfigMap(&brokerConfigMapOld) if err != nil { - log.Error(err, "could not get mountPaths from broker configmap", v1beta1.BrokerIdLabelKey, id) + log.Error(err, "could not get mountPaths from broker configmap", v1beta1.BrokerIdLabelKey, broker.Id) } mountPathsNew := generateStorageConfig(bConfig.StorageConfigs) @@ -77,7 +82,7 @@ func (r *Reconciler) getConfigProperties(bConfig *v1beta1.BrokerConfig, id int32 if isMountPathRemoved { log.Error(errors.New("removed storage is found in the KafkaCluster CR"), - "removing storage from broker is not supported", v1beta1.BrokerIdLabelKey, id, "mountPaths", + "removing storage from broker is not supported", v1beta1.BrokerIdLabelKey, broker.Id, "mountPaths", mountPathsOld, "mountPaths in kafkaCluster CR ", mountPathsNew) } @@ -97,10 +102,10 @@ func (r *Reconciler) getConfigProperties(bConfig *v1beta1.BrokerConfig, id int32 return config } -func configCCMetricsReporter(kafkaCluster *v1beta1.KafkaCluster, config *properties.Properties, clientPass, bootstrapServers string, log logr.Logger) { +func (r *Reconciler) configCCMetricsReporter(broker v1beta1.Broker, config *properties.Properties, clientPass string, log logr.Logger) { // Add Cruise Control Metrics Reporter SSL configuration - if util.IsSSLEnabledForInternalCommunication(kafkaCluster.Spec.ListenersConfig.InternalListeners) { - if !kafkaCluster.Spec.IsClientSSLSecretPresent() { + if util.IsSSLEnabledForInternalCommunication(r.KafkaCluster.Spec.ListenersConfig.InternalListeners) { + if !r.KafkaCluster.Spec.IsClientSSLSecretPresent() { log.Error(errors.New("cruise control metrics reporter needs ssl but client certificate hasn't specified"), "") } @@ -122,9 +127,17 @@ func configCCMetricsReporter(kafkaCluster *v1beta1.KafkaCluster, config *propert } } - // Add Cruise Control Metrics Reporter configuration - if err := config.Set(kafkautils.CruiseControlConfigMetricsReporters, kafkautils.CruiseControlConfigMetricsReportersVal); err != nil { - log.Error(err, fmt.Sprintf(kafkautils.BrokerConfigErrorMsgTemplate, kafkautils.CruiseControlConfigMetricsReporters)) + // Add Cruise Control Metrics Reporter configuration. + // When "security.inter.broker.protocol" (e.g. inter broker communication is secure) is configured, the operator disables the reporter. + _, isSecurityInterBrokerProtocolConfigured := getBrokerReadOnlyConfig(broker, r.KafkaCluster, log).Get(kafkautils.KafkaConfigSecurityInterBrokerProtocol) + if !isSecurityInterBrokerProtocolConfigured { + if err := config.Set(kafkautils.CruiseControlConfigMetricsReporters, kafkautils.CruiseControlConfigMetricsReportersVal); err != nil { + log.Error(err, fmt.Sprintf(kafkautils.BrokerConfigErrorMsgTemplate, kafkautils.CruiseControlConfigMetricsReporters)) + } + } + bootstrapServers, err := kafkautils.GetBootstrapServersService(r.KafkaCluster) + if err != nil { + log.Error(err, "getting Kafka bootstrap servers for Cruise Control failed") } if err := config.Set(kafkautils.CruiseControlConfigMetricsReportersBootstrapServers, bootstrapServers); err != nil { log.Error(err, fmt.Sprintf(kafkautils.BrokerConfigErrorMsgTemplate, kafkautils.CruiseControlConfigMetricsReportersBootstrapServers)) @@ -164,7 +177,7 @@ func configureBrokerKRaftMode(bConfig *v1beta1.BrokerConfig, brokerID int32, kaf } // Add listener configuration - listenerConf, listenerConfig := generateListenerSpecificConfig(&kafkaCluster.Spec.ListenersConfig, serverPasses, log) + listenerConf, listenerConfig := generateListenerSpecificConfig(&kafkaCluster.Spec, serverPasses, log) config.Merge(listenerConf) var advertisedListenerConf []string @@ -209,7 +222,7 @@ func configureBrokerZKMode(brokerID int32, kafkaCluster *v1beta1.KafkaCluster, c } // Add listener configuration - listenerConf, _ := generateListenerSpecificConfig(&kafkaCluster.Spec.ListenersConfig, serverPasses, log) + listenerConf, _ := generateListenerSpecificConfig(&kafkaCluster.Spec, serverPasses, log) config.Merge(listenerConf) // Add advertised listener configuration @@ -274,7 +287,7 @@ func (r *Reconciler) configMap(broker v1beta1.Broker, brokerConfig *v1beta1.Brok serverPasses map[string]string, clientPass string, superUsers []string, log logr.Logger) *corev1.ConfigMap { brokerConf := &corev1.ConfigMap{ ObjectMeta: templates.ObjectMeta( - fmt.Sprintf(brokerConfigTemplate+"-%d", r.KafkaCluster.Name, broker.Id), + fmt.Sprintf(brokerConfigTemplate+"-"+"%d", r.KafkaCluster.Name, broker.Id), //nolint:goconst apiutil.MergeLabels( apiutil.LabelsForKafka(r.KafkaCluster.Name), map[string]string{v1beta1.BrokerIdLabelKey: fmt.Sprintf("%d", broker.Id)}, @@ -306,7 +319,7 @@ func appendListenerConfigs(advertisedListenerConfig []string, id int32, listenerStatusList map[string]v1beta1.ListenerStatusList) []string { for listenerName, statuses := range listenerStatusList { for _, status := range statuses { - if status.Name == fmt.Sprintf("broker-%d", id) { + if status.Name == fmt.Sprintf("broker-"+"%d", id) { advertisedListenerConfig = append(advertisedListenerConfig, fmt.Sprintf("%s://%s", strings.ToUpper(listenerName), status.Address)) break @@ -355,10 +368,13 @@ func generateControlPlaneListener(iListeners []v1beta1.InternalListenerConfig) s return controlPlaneListener } -func generateListenerSpecificConfig(l *v1beta1.ListenersConfig, serverPasses map[string]string, log logr.Logger) (*properties.Properties, []string) { +func generateListenerSpecificConfig(kcs *v1beta1.KafkaClusterSpec, serverPasses map[string]string, log logr.Logger) (*properties.Properties, []string) { config := properties.NewProperties() - interBrokerListenerName, securityProtocolMapConfig, listenerConfig, internalListenerSSLConfig, externalListenerSSLConfig := getListenerSpecificConfig(l, serverPasses, log) + l := kcs.ListenersConfig + r := kcs.ReadOnlyConfig + + interBrokerListenerName, securityProtocolMapConfig, listenerConfig, internalListenerSSLConfig, externalListenerSSLConfig := getListenerSpecificConfig(&l, serverPasses, log) for k, v := range internalListenerSSLConfig { if err := config.Set(k, v); err != nil { @@ -375,9 +391,13 @@ func generateListenerSpecificConfig(l *v1beta1.ListenersConfig, serverPasses map if err := config.Set(kafkautils.KafkaConfigListenerSecurityProtocolMap, securityProtocolMapConfig); err != nil { log.Error(err, fmt.Sprintf("setting '%s' parameter in broker configuration resulted an error", kafkautils.KafkaConfigListenerSecurityProtocolMap)) } - if err := config.Set(kafkautils.KafkaConfigInterBrokerListenerName, interBrokerListenerName); err != nil { - log.Error(err, fmt.Sprintf("setting '%s' parameter in broker configuration resulted an error", kafkautils.KafkaConfigInterBrokerListenerName)) + + if !strings.Contains(r, kafkautils.KafkaConfigSecurityInterBrokerProtocol+"=") { + if err := config.Set(kafkautils.KafkaConfigInterBrokerListenerName, interBrokerListenerName); err != nil { + log.Error(err, fmt.Sprintf("setting '%s' parameter in broker configuration resulted an error", kafkautils.KafkaConfigInterBrokerListenerName)) + } } + if err := config.Set(kafkautils.KafkaConfigListeners, listenerConfig); err != nil { log.Error(err, fmt.Sprintf("setting '%s' parameter in broker configuration resulted an error", kafkautils.KafkaConfigListeners)) } @@ -394,6 +414,9 @@ func getListenerSpecificConfig(l *v1beta1.ListenersConfig, serverPasses map[stri externalListenerSSLConfig map[string]string ) + internalListenerSSLConfig = make(map[string]string) + externalListenerSSLConfig = make(map[string]string) + for _, iListener := range l.InternalListeners { if iListener.UsedForInnerBrokerCommunication { if interBrokerListenerName == "" { @@ -409,7 +432,7 @@ func getListenerSpecificConfig(l *v1beta1.ListenersConfig, serverPasses map[stri // Add internal listeners SSL configuration if iListener.Type == v1beta1.SecurityProtocolSSL { - internalListenerSSLConfig = generateListenerSSLConfig(iListener.Name, iListener.SSLClientAuth, serverPasses[iListener.Name]) + maps.Copy(internalListenerSSLConfig, generateListenerSSLConfig(iListener.Name, iListener.SSLClientAuth, serverPasses[iListener.Name])) } } @@ -420,7 +443,7 @@ func getListenerSpecificConfig(l *v1beta1.ListenersConfig, serverPasses map[stri listenerConfig = append(listenerConfig, fmt.Sprintf("%s://:%d", upperedListenerName, eListener.ContainerPort)) // Add external listeners SSL configuration if eListener.Type == v1beta1.SecurityProtocolSSL { - externalListenerSSLConfig = generateListenerSSLConfig(eListener.Name, eListener.SSLClientAuth, serverPasses[eListener.Name]) + maps.Copy(externalListenerSSLConfig, generateListenerSSLConfig(eListener.Name, eListener.SSLClientAuth, serverPasses[eListener.Name])) } } @@ -498,7 +521,7 @@ func (r Reconciler) generateBrokerConfig(broker v1beta1.Broker, brokerConfig *v1 finalBrokerConfig := getBrokerReadOnlyConfig(broker, r.KafkaCluster, log) // Get operator generated configuration - opGenConf := r.getConfigProperties(brokerConfig, broker.Id, quorumVoters, extListenerStatuses, intListenerStatuses, + opGenConf := r.getConfigProperties(brokerConfig, broker, quorumVoters, extListenerStatuses, intListenerStatuses, controllerIntListenerStatuses, serverPasses, clientPass, superUsers, log) // Merge operator generated configuration to the final one diff --git a/pkg/resources/kafka/configmap_test.go b/pkg/resources/kafka/configmap_test.go index 88b4aa824..c44a97c22 100644 --- a/pkg/resources/kafka/configmap_test.go +++ b/pkg/resources/kafka/configmap_test.go @@ -602,6 +602,26 @@ listener.security.protocol.map=INTERNAL:SSL listeners=INTERNAL://:9092 metric.reporters=com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter super.users=User:CN=kafka-headless.kafka.svc.cluster.local +zookeeper.connect=example.zk:2181/`, + }, + { + testName: "security_inter_broker_protocol_Set", + readOnlyConfig: `security.inter.broker.protocol=SASL_SSL`, + zkAddresses: []string{"example.zk:2181"}, + zkPath: ``, + kubernetesClusterDomain: ``, + clusterWideConfig: ``, + perBrokerConfig: ``, + perBrokerReadOnlyConfig: ``, + advertisedListenerAddress: `kafka-0.kafka.svc.cluster.local:9092`, + listenerType: "plaintext", + expectedConfig: `advertised.listeners=INTERNAL://kafka-0.kafka.svc.cluster.local:9092 +broker.id=0 +cruise.control.metrics.reporter.bootstrap.servers=kafka-all-broker.kafka.svc.cluster.local:9092 +cruise.control.metrics.reporter.kubernetes.mode=true +listener.security.protocol.map=INTERNAL:PLAINTEXT +listeners=INTERNAL://:9092 +security.inter.broker.protocol=SASL_SSL zookeeper.connect=example.zk:2181/`, }, } @@ -1022,3 +1042,328 @@ process.roles=broker,controller }) } } + +func TestGenerateBrokerConfigKRaftModeSSL(t *testing.T) { + testCases := []struct { + testName string + brokers []v1beta1.Broker + listenersConfig v1beta1.ListenersConfig + internalListenerStatuses map[string]v1beta1.ListenerStatusList + controllerListenerStatus map[string]v1beta1.ListenerStatusList + expectedBrokerConfigs []string + }{ + { + testName: "a Kafka cluster with a mix of broker-only and controller-only nodes; broker-only nodes with multiple mount paths; two internal ssl listeners", + brokers: []v1beta1.Broker{ + { + Id: 0, + BrokerConfig: &v1beta1.BrokerConfig{ + Roles: []string{"broker"}, + StorageConfigs: []v1beta1.StorageConfig{ + { + MountPath: "/test-kafka-logs", + }, + { + MountPath: "/test-kafka-logs-0", + }, + }, + }, + }, + { + Id: 500, + BrokerConfig: &v1beta1.BrokerConfig{ + Roles: []string{"controller"}, + StorageConfigs: []v1beta1.StorageConfig{ + { + MountPath: "/test-kafka-logs", + }, + }, + }, + }, + }, + listenersConfig: v1beta1.ListenersConfig{ + InternalListeners: []v1beta1.InternalListenerConfig{ + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Type: v1beta1.SecurityProtocolSSL, + Name: "internal", + ContainerPort: 9092, + ServerSSLCertSecret: &v1.LocalObjectReference{ + Name: "server-secret", + }, + SSLClientAuth: "none", + UsedForInnerBrokerCommunication: true, + }, + }, + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Type: v1beta1.SecurityProtocolSSL, + Name: "controller", + ContainerPort: 9093, + ServerSSLCertSecret: &v1.LocalObjectReference{ + Name: "server-secret", + }, + SSLClientAuth: "none", + }, + UsedForControllerCommunication: true, + }, + }, + }, + internalListenerStatuses: map[string]v1beta1.ListenerStatusList{ + "internal": { + { + Name: "broker-0", + Address: "kafka-0.kafka.svc.cluster.local:9092", + }, + { + Name: "broker-500", + Address: "kafka-500.kafka.svc.cluster.local:9092", + }, + }, + }, + controllerListenerStatus: map[string]v1beta1.ListenerStatusList{ + "controller": { + { + Name: "broker-0", + Address: "kafka-0.kafka.svc.cluster.local:9093", + }, + { + Name: "broker-500", + Address: "kafka-500.kafka.svc.cluster.local:9093", + }, + }, + }, + expectedBrokerConfigs: []string{ + `advertised.listeners=INTERNAL://kafka-0.kafka.svc.cluster.local:9092 +controller.listener.names=CONTROLLER +controller.quorum.voters=500@kafka-500.kafka.svc.cluster.local:9093 +cruise.control.metrics.reporter.bootstrap.servers=kafka-all-broker.kafka.svc.cluster.local:9092 +cruise.control.metrics.reporter.kubernetes.mode=true +cruise.control.metrics.reporter.security.protocol=SSL +cruise.control.metrics.reporter.ssl.keystore.location=/var/run/secrets/java.io/keystores/client/keystore.jks +cruise.control.metrics.reporter.ssl.keystore.password= +cruise.control.metrics.reporter.ssl.truststore.location=/var/run/secrets/java.io/keystores/client/truststore.jks +cruise.control.metrics.reporter.ssl.truststore.password= +inter.broker.listener.name=INTERNAL +listener.name.controller.ssl.client.auth=none +listener.name.controller.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/controller/keystore.jks +listener.name.controller.ssl.keystore.password= +listener.name.controller.ssl.keystore.type=JKS +listener.name.controller.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/controller/truststore.jks +listener.name.controller.ssl.truststore.password= +listener.name.controller.ssl.truststore.type=JKS +listener.name.internal.ssl.client.auth=none +listener.name.internal.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/internal/keystore.jks +listener.name.internal.ssl.keystore.password= +listener.name.internal.ssl.keystore.type=JKS +listener.name.internal.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/internal/truststore.jks +listener.name.internal.ssl.truststore.password= +listener.name.internal.ssl.truststore.type=JKS +listener.security.protocol.map=INTERNAL:SSL,CONTROLLER:SSL +listeners=INTERNAL://:9092 +log.dirs=/test-kafka-logs/kafka,/test-kafka-logs-0/kafka +metric.reporters=com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter +node.id=0 +process.roles=broker +`, + `controller.listener.names=CONTROLLER +controller.quorum.voters=500@kafka-500.kafka.svc.cluster.local:9093 +cruise.control.metrics.reporter.bootstrap.servers=kafka-all-broker.kafka.svc.cluster.local:9092 +cruise.control.metrics.reporter.kubernetes.mode=true +cruise.control.metrics.reporter.security.protocol=SSL +cruise.control.metrics.reporter.ssl.keystore.location=/var/run/secrets/java.io/keystores/client/keystore.jks +cruise.control.metrics.reporter.ssl.keystore.password= +cruise.control.metrics.reporter.ssl.truststore.location=/var/run/secrets/java.io/keystores/client/truststore.jks +cruise.control.metrics.reporter.ssl.truststore.password= +inter.broker.listener.name=INTERNAL +listener.name.controller.ssl.client.auth=none +listener.name.controller.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/controller/keystore.jks +listener.name.controller.ssl.keystore.password= +listener.name.controller.ssl.keystore.type=JKS +listener.name.controller.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/controller/truststore.jks +listener.name.controller.ssl.truststore.password= +listener.name.controller.ssl.truststore.type=JKS +listener.name.internal.ssl.client.auth=none +listener.name.internal.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/internal/keystore.jks +listener.name.internal.ssl.keystore.password= +listener.name.internal.ssl.keystore.type=JKS +listener.name.internal.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/internal/truststore.jks +listener.name.internal.ssl.truststore.password= +listener.name.internal.ssl.truststore.type=JKS +listener.security.protocol.map=INTERNAL:SSL,CONTROLLER:SSL +listeners=CONTROLLER://:9093 +log.dirs=/test-kafka-logs/kafka +metric.reporters=com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter +node.id=500 +process.roles=controller +`}, + }, + { + testName: "a Kafka cluster with a mix of broker-only and controller-only nodes; broker-only nodes with multiple mount paths; two external ssl listeners", + brokers: []v1beta1.Broker{ + { + Id: 0, + BrokerConfig: &v1beta1.BrokerConfig{ + Roles: []string{"broker"}, + StorageConfigs: []v1beta1.StorageConfig{ + { + MountPath: "/test-kafka-logs", + }, + { + MountPath: "/test-kafka-logs-0", + }, + }, + }, + }, + { + Id: 500, + BrokerConfig: &v1beta1.BrokerConfig{ + Roles: []string{"controller"}, + StorageConfigs: []v1beta1.StorageConfig{ + { + MountPath: "/test-kafka-logs", + }, + }, + }, + }, + }, + listenersConfig: v1beta1.ListenersConfig{ + ExternalListeners: []v1beta1.ExternalListenerConfig{ + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Type: v1beta1.SecurityProtocolSSL, + Name: "external", + ContainerPort: 9092, + ServerSSLCertSecret: &v1.LocalObjectReference{ + Name: "server-secret", + }, + SSLClientAuth: "none", + UsedForInnerBrokerCommunication: true, + }, + }, + { + CommonListenerSpec: v1beta1.CommonListenerSpec{ + Type: v1beta1.SecurityProtocolSSL, + Name: "controller", + ContainerPort: 9093, + ServerSSLCertSecret: &v1.LocalObjectReference{ + Name: "server-secret", + }, + SSLClientAuth: "none", + }, + }, + }, + }, + internalListenerStatuses: map[string]v1beta1.ListenerStatusList{ + "internal": { + { + Name: "broker-0", + Address: "kafka-0.kafka.svc.cluster.local:9092", + }, + { + Name: "broker-500", + Address: "kafka-500.kafka.svc.cluster.local:9092", + }, + }, + }, + controllerListenerStatus: map[string]v1beta1.ListenerStatusList{ + "controller": { + { + Name: "broker-0", + Address: "kafka-0.kafka.svc.cluster.local:9093", + }, + { + Name: "broker-500", + Address: "kafka-500.kafka.svc.cluster.local:9093", + }, + }, + }, + expectedBrokerConfigs: []string{ + `advertised.listeners=INTERNAL://kafka-0.kafka.svc.cluster.local:9092 +controller.quorum.voters=500@kafka-500.kafka.svc.cluster.local:9093 +cruise.control.metrics.reporter.bootstrap.servers=kafka-all-broker.kafka.svc.cluster.local:9092 +cruise.control.metrics.reporter.kubernetes.mode=true +inter.broker.listener.name= +listener.name.controller.ssl.client.auth=none +listener.name.controller.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/controller/keystore.jks +listener.name.controller.ssl.keystore.password= +listener.name.controller.ssl.keystore.type=JKS +listener.name.controller.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/controller/truststore.jks +listener.name.controller.ssl.truststore.password= +listener.name.controller.ssl.truststore.type=JKS +listener.name.external.ssl.client.auth=none +listener.name.external.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/external/keystore.jks +listener.name.external.ssl.keystore.password= +listener.name.external.ssl.keystore.type=JKS +listener.name.external.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/external/truststore.jks +listener.name.external.ssl.truststore.password= +listener.name.external.ssl.truststore.type=JKS +listener.security.protocol.map=EXTERNAL:SSL,CONTROLLER:SSL +listeners= +log.dirs=/test-kafka-logs/kafka,/test-kafka-logs-0/kafka +metric.reporters=com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter +node.id=0 +process.roles=broker +`, + `controller.quorum.voters=500@kafka-500.kafka.svc.cluster.local:9093 +cruise.control.metrics.reporter.bootstrap.servers=kafka-all-broker.kafka.svc.cluster.local:9092 +cruise.control.metrics.reporter.kubernetes.mode=true +inter.broker.listener.name= +listener.name.controller.ssl.client.auth=none +listener.name.controller.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/controller/keystore.jks +listener.name.controller.ssl.keystore.password= +listener.name.controller.ssl.keystore.type=JKS +listener.name.controller.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/controller/truststore.jks +listener.name.controller.ssl.truststore.password= +listener.name.controller.ssl.truststore.type=JKS +listener.name.external.ssl.client.auth=none +listener.name.external.ssl.keystore.location=/var/run/secrets/java.io/keystores/server/external/keystore.jks +listener.name.external.ssl.keystore.password= +listener.name.external.ssl.keystore.type=JKS +listener.name.external.ssl.truststore.location=/var/run/secrets/java.io/keystores/server/external/truststore.jks +listener.name.external.ssl.truststore.password= +listener.name.external.ssl.truststore.type=JKS +listener.security.protocol.map=EXTERNAL:SSL,CONTROLLER:SSL +listeners=EXTERNAL://:9092 +log.dirs=/test-kafka-logs/kafka +metric.reporters=com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter +node.id=500 +process.roles=controller +`}, + }, + } + t.Parallel() + mockClient := mocks.NewMockClient(gomock.NewController(t)) + mockClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + for _, test := range testCases { + test := test + t.Run(test.testName, func(t *testing.T) { + r := Reconciler{ + Reconciler: resources.Reconciler{ + Client: mockClient, + KafkaCluster: &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kafka", + Namespace: "kafka", + }, + Spec: v1beta1.KafkaClusterSpec{ + KRaftMode: true, + ListenersConfig: test.listenersConfig, + Brokers: test.brokers, + }, + }, + }, + } + for i, b := range test.brokers { + quorumVoters, err := generateQuorumVoters(r.KafkaCluster, test.controllerListenerStatus) + if err != nil { + t.Error(err) + } + generatedConfig := r.generateBrokerConfig(b, b.BrokerConfig, quorumVoters, map[string]v1beta1.ListenerStatusList{}, + test.internalListenerStatuses, test.controllerListenerStatus, nil, "", nil, logr.Discard()) + + require.Equal(t, test.expectedBrokerConfigs[i], generatedConfig) + } + }) + } +} diff --git a/pkg/resources/kafka/kafka.go b/pkg/resources/kafka/kafka.go index e75edc56c..1284ff0e8 100644 --- a/pkg/resources/kafka/kafka.go +++ b/pkg/resources/kafka/kafka.go @@ -18,13 +18,14 @@ import ( "context" "fmt" "reflect" - "regexp" "sort" "strconv" "strings" "emperror.dev/errors" + ccTypes "github.com/banzaicloud/go-cruise-control/pkg/types" "github.com/go-logr/logr" + "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +34,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" + properties "github.com/banzaicloud/koperator/properties/pkg" + apiutil "github.com/banzaicloud/koperator/api/util" "github.com/banzaicloud/k8s-objectmatcher/patch" @@ -50,6 +53,7 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" "github.com/banzaicloud/koperator/pkg/util" certutil "github.com/banzaicloud/koperator/pkg/util/cert" + contourutils "github.com/banzaicloud/koperator/pkg/util/contour" envoyutils "github.com/banzaicloud/koperator/pkg/util/envoy" istioingressutils "github.com/banzaicloud/koperator/pkg/util/istioingress" "github.com/banzaicloud/koperator/pkg/util/kafka" @@ -89,24 +93,13 @@ const ( nonControllerBrokerReconcilePriority // controllerBrokerReconcilePriority the priority used for controller broker used to define its priority in the reconciliation order controllerBrokerReconcilePriority - - // defaultConcurrentBrokerRestartsAllowed the default number of brokers that can be restarted in parallel - defaultConcurrentBrokerRestartsAllowed = 1 -) - -var ( - // kafkaConfigBrokerRackRegex the regex to parse the "broker.rack" Kafka property used in read-only configs - kafkaConfigBrokerRackRegex = regexp.MustCompile(`broker\.rack\s*=\s*([\w-]+)`) ) // Reconciler implements the Component Reconciler type Reconciler struct { resources.Reconciler - // kafkaClientProvider is used to create a new KafkaClient - kafkaClientProvider kafkaclient.Provider - // kafkaBrokerAvailabilityZoneMap is a map of broker id to availability zone used in concurrent broker restarts logic - kafkaBrokerAvailabilityZoneMap map[int32]string - CruiseControlScalerFactory func(ctx context.Context, kafkaCluster *banzaiv1beta1.KafkaCluster) (scale.CruiseControlScaler, error) + kafkaClientProvider kafkaclient.Provider + CruiseControlScalerFactory func(ctx context.Context, kafkaCluster *banzaiv1beta1.KafkaCluster) (scale.CruiseControlScaler, error) } // New creates a new reconciler for Kafka @@ -117,17 +110,20 @@ func New(client client.Client, directClient client.Reader, cluster *v1beta1.Kafk DirectClient: directClient, KafkaCluster: cluster, }, - kafkaClientProvider: kafkaClientProvider, - kafkaBrokerAvailabilityZoneMap: getBrokerAzMap(cluster), + kafkaClientProvider: kafkaClientProvider, + CruiseControlScalerFactory: scale.ScaleFactoryFn(), } } func getBrokerAzMap(cluster *v1beta1.KafkaCluster) map[int32]string { brokerAzMap := make(map[int32]string) for _, broker := range cluster.Spec.Brokers { - brokerRack := getBrokerRack(broker.ReadOnlyConfig) - if brokerRack != "" { - brokerAzMap[broker.Id] = brokerRack + readOnlyConfigs, err := properties.NewFromString(broker.ReadOnlyConfig) + if err == nil { + brokerRack, brokerRackConfigFound := readOnlyConfigs.Get("broker.rack") + if brokerRackConfigFound { + brokerAzMap[broker.Id] = brokerRack.Value() + } } } // if incomplete broker AZ information, consider all brokers as being in different AZs @@ -139,17 +135,6 @@ func getBrokerAzMap(cluster *v1beta1.KafkaCluster) map[int32]string { return brokerAzMap } -func getBrokerRack(readOnlyConfig string) string { - if readOnlyConfig == "" { - return "" - } - match := kafkaConfigBrokerRackRegex.FindStringSubmatch(readOnlyConfig) - if len(match) == 2 { - return match[1] - } - return "" -} - func getCreatedPvcForBroker( ctx context.Context, c client.Reader, @@ -221,7 +206,7 @@ func (r *Reconciler) Reconcile(log logr.Logger) error { log.V(1).Info("Reconciling") - log.Info("broker rack map", "kafkaBrokerAvailabilityZoneMap", r.kafkaBrokerAvailabilityZoneMap) + log.Info("broker rack map", "kafkaBrokerAvailabilityZoneMap", getBrokerAzMap(r.KafkaCluster)) ctx := context.Background() if err := k8sutil.UpdateBrokerConfigurationBackup(r.Client, r.KafkaCluster); err != nil { @@ -917,6 +902,7 @@ func (r *Reconciler) updateStatusWithDockerImageAndVersion(brokerId int32, broke return nil } +//gocyclo:ignore func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPod *corev1.Pod, desiredType reflect.Type) error { // Since toleration does not support patchStrategy:"merge,retainKeys", // we need to add all toleration from the current pod if the toleration is set in the CR @@ -974,21 +960,25 @@ func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPo if err != nil { return errors.WrapIf(err, "failed to reconcile resource") } + // Check that all pods are present as in spec, before checking for terminating or pending pods, as we can have absent pods if len(podList.Items) < len(r.KafkaCluster.Spec.Brokers) { return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New("pod count differs from brokers spec"), "rolling upgrade in progress") } // Check if we support multiple broker restarts and restart only in same AZ, otherwise restart only 1 broker at once - concurrentBrokerRestartsAllowed := r.getConcurrentBrokerRestartsAllowed() terminatingOrPendingPods := getPodsInTerminatingOrPendingState(podList.Items) - if len(terminatingOrPendingPods) > 0 { - log.Info("terminating or pending pods", "terminatingOrPendingPods", terminatingOrPendingPods) + if len(terminatingOrPendingPods) >= r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartCountPerRack { + return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New(strconv.Itoa(r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartCountPerRack)+" pod(s) is still terminating or creating"), "rolling upgrade in progress") } - if len(terminatingOrPendingPods) >= concurrentBrokerRestartsAllowed { - return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New(strconv.Itoa(concurrentBrokerRestartsAllowed)+" pod(s) is still terminating or creating"), "rolling upgrade in progress") + if r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartCountPerRack > 1 && len(terminatingOrPendingPods) > 0 { + err = r.checkCCRackAwareDistributionGoal() + if err != nil { + return err + } } - currentPodAz := r.getBrokerAz(currentPod) - if concurrentBrokerRestartsAllowed > 1 && r.existsTerminatingPodFromAnotherAz(currentPodAz, terminatingOrPendingPods) { + kafkaBrokerAvailabilityZoneMap := getBrokerAzMap(r.KafkaCluster) + currentPodAz, _ := r.getBrokerAz(currentPod, kafkaBrokerAvailabilityZoneMap) + if r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartCountPerRack > 1 && r.existsTerminatingPodFromAnotherAz(currentPodAz, terminatingOrPendingPods, kafkaBrokerAvailabilityZoneMap) { return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New("pod is still terminating or creating from another AZ"), "rolling upgrade in progress") } @@ -1026,8 +1016,8 @@ func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPo } // If multiple concurrent restarts and broker failures allowed, restart only brokers from the same AZ - if concurrentBrokerRestartsAllowed > 1 && r.KafkaCluster.Spec.RollingUpgradeConfig.FailureThreshold > 1 { - if r.existsFailedBrokerFromAnotherRack(currentPodAz, impactedReplicas) { + if r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartCountPerRack > 1 && r.KafkaCluster.Spec.RollingUpgradeConfig.FailureThreshold > 1 { + if r.existsFailedBrokerFromAnotherRack(currentPodAz, impactedReplicas, kafkaBrokerAvailabilityZoneMap) { return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New("broker is not healthy from another AZ"), "rolling upgrade in progress") } } @@ -1052,31 +1042,46 @@ func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPo return nil } -func (r *Reconciler) existsFailedBrokerFromAnotherRack(currentPodAz string, impactedReplicas map[int32]struct{}) bool { +func (r *Reconciler) checkCCRackAwareDistributionGoal() error { + cruiseControlURL := scale.CruiseControlURLFromKafkaCluster(r.KafkaCluster) + cc, err := r.CruiseControlScalerFactory(context.TODO(), r.KafkaCluster) + if err != nil { + return errorfactory.New(errorfactory.CruiseControlNotReady{}, err, "failed to initialize Cruise Control", "cruise control url", cruiseControlURL) + } + status, err := cc.Status(context.Background()) + if err != nil { + return errorfactory.New(errorfactory.CruiseControlNotReady{}, errors.New("failed to get status from Cruise Control"), "rolling upgrade in progress") + } + if !slices.Contains(status.State.AnalyzerState.ReadyGoals, ccTypes.RackAwareDistributionGoal) { + return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New("RackAwareDistributionGoal is not ready"), "rolling upgrade in progress") + } + for _, anomaly := range status.State.AnomalyDetectorState.RecentGoalViolations { + if slices.Contains(anomaly.FixableViolatedGoals, ccTypes.RackAwareDistributionGoal) || slices.Contains(anomaly.UnfixableViolatedGoals, ccTypes.RackAwareDistributionGoal) { + return errorfactory.New(errorfactory.ReconcileRollingUpgrade{}, errors.New("RackAwareDistributionGoal is violated"), "rolling upgrade in progress") + } + } + return nil +} + +func (r *Reconciler) existsFailedBrokerFromAnotherRack(currentPodAz string, impactedReplicas map[int32]struct{}, kafkaBrokerAvailabilityZoneMap map[int32]string) bool { if currentPodAz == "" && len(impactedReplicas) > 0 { return true } for brokerWithFailure := range impactedReplicas { - if currentPodAz != r.kafkaBrokerAvailabilityZoneMap[brokerWithFailure] { + if currentPodAz != kafkaBrokerAvailabilityZoneMap[brokerWithFailure] { return true } } return false } -func (r *Reconciler) getConcurrentBrokerRestartsAllowed() int { - if r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartsAllowed > 1 { - return r.KafkaCluster.Spec.RollingUpgradeConfig.ConcurrentBrokerRestartsAllowed - } - return defaultConcurrentBrokerRestartsAllowed -} - -func (r *Reconciler) existsTerminatingPodFromAnotherAz(currentPodAz string, terminatingOrPendingPods []corev1.Pod) bool { +func (r *Reconciler) existsTerminatingPodFromAnotherAz(currentPodAz string, terminatingOrPendingPods []corev1.Pod, kafkaBrokerAvailabilityZoneMap map[int32]string) bool { if currentPodAz == "" && len(terminatingOrPendingPods) > 0 { return true } for _, terminatingOrPendingPod := range terminatingOrPendingPods { - if currentPodAz != r.getBrokerAz(&terminatingOrPendingPod) { + terminatingOrPendingPodAz, err := r.getBrokerAz(&terminatingOrPendingPod, kafkaBrokerAvailabilityZoneMap) + if err != nil || currentPodAz != terminatingOrPendingPodAz { return true } } @@ -1306,7 +1311,8 @@ func (r *Reconciler) getBrokerHost(log logr.Logger, defaultHost string, broker v brokerHost := defaultHost portNumber := eListener.GetBrokerPort(broker.Id) - if eListener.GetAccessMethod() != corev1.ServiceTypeLoadBalancer { + switch eListener.GetAccessMethod() { + case corev1.ServiceTypeNodePort: bConfig, err := broker.GetBrokerConfig(r.KafkaCluster.Spec) if err != nil { return "", err @@ -1337,12 +1343,22 @@ func (r *Reconciler) getBrokerHost(log logr.Logger, defaultHost string, broker v } else { brokerHost = fmt.Sprintf("%s-%d-%s.%s%s", r.KafkaCluster.Name, broker.Id, eListener.Name, r.KafkaCluster.Namespace, brokerHost) } - } - if eListener.TLSEnabled() { - brokerHost = iConfig.EnvoyConfig.GetBrokerHostname(broker.Id) + case corev1.ServiceTypeClusterIP: + brokerHost = iConfig.ContourIngressConfig.GetBrokerFqdn(broker.Id) if brokerHost == "" { return "", errors.New("brokerHostnameTemplate is not set in the ingress service settings") } + // TODO understand why this is not needed. Tests are failing when this is added + // portNumber = eListener.ContainerPort + case corev1.ServiceTypeLoadBalancer: + if eListener.TLSEnabled() { + brokerHost = iConfig.EnvoyConfig.GetBrokerHostname(broker.Id) + if brokerHost == "" { + return "", errors.New("brokerHostnameTemplate is not set in the ingress service settings") + } + } + case corev1.ServiceTypeExternalName: + return ":", errors.New("unsupported external listener access method") } return fmt.Sprintf("%s:%d", brokerHost, portNumber), nil } @@ -1537,12 +1553,12 @@ func getPodsInTerminatingOrPendingState(items []corev1.Pod) []corev1.Pod { return pods } -func (r *Reconciler) getBrokerAz(pod *corev1.Pod) string { +func (r *Reconciler) getBrokerAz(pod *corev1.Pod, kafkaBrokerAvailabilityZoneMap map[int32]string) (string, error) { brokerId, err := strconv.ParseInt(pod.Labels[v1beta1.BrokerIdLabelKey], 10, 32) if err != nil { - return "" + return "", err } - return r.kafkaBrokerAvailabilityZoneMap[int32(brokerId)] + return kafkaBrokerAvailabilityZoneMap[int32(brokerId)], nil } func getServiceFromExternalListener(client client.Client, cluster *v1beta1.KafkaCluster, @@ -1553,14 +1569,26 @@ func getServiceFromExternalListener(client client.Client, cluster *v1beta1.Kafka case istioingressutils.IngressControllerName: if ingressConfigName == util.IngressConfigGlobalName { iControllerServiceName = fmt.Sprintf(istioingressutils.MeshGatewayNameTemplate, eListenerName, cluster.GetName()) + iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") } else { iControllerServiceName = fmt.Sprintf(istioingressutils.MeshGatewayNameTemplateWithScope, eListenerName, ingressConfigName, cluster.GetName()) + iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") } case envoyutils.IngressControllerName: if ingressConfigName == util.IngressConfigGlobalName { iControllerServiceName = fmt.Sprintf(envoyutils.EnvoyServiceName, eListenerName, cluster.GetName()) + iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") } else { iControllerServiceName = fmt.Sprintf(envoyutils.EnvoyServiceNameWithScope, eListenerName, ingressConfigName, cluster.GetName()) + iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") + } + case contourutils.IngressControllerName: + if ingressConfigName == util.IngressConfigGlobalName { + iControllerServiceName = fmt.Sprintf(contourutils.ContourServiceName, eListenerName, cluster.GetName()) + iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") + } else { + iControllerServiceName = fmt.Sprintf(contourutils.ContourServiceNameWithScope, eListenerName, ingressConfigName, cluster.GetName()) + iControllerServiceName = strings.ReplaceAll(iControllerServiceName, "_", "-") } } @@ -1636,7 +1664,7 @@ func generateServicePortForIListeners(listeners []v1beta1.InternalListenerConfig var usedPorts []corev1.ServicePort for _, iListener := range listeners { usedPorts = append(usedPorts, corev1.ServicePort{ - Name: strings.ReplaceAll(iListener.GetListenerServiceName(), "_", ""), + Name: strings.ReplaceAll(iListener.GetListenerServiceName(), "_", "-"), Port: iListener.ContainerPort, TargetPort: intstr.FromInt(int(iListener.ContainerPort)), Protocol: corev1.ProtocolTCP, @@ -1649,7 +1677,7 @@ func generateServicePortForEListeners(listeners []v1beta1.ExternalListenerConfig var usedPorts []corev1.ServicePort for _, eListener := range listeners { usedPorts = append(usedPorts, corev1.ServicePort{ - Name: eListener.GetListenerServiceName(), + Name: strings.ReplaceAll(eListener.GetListenerServiceName(), "_", "-"), Protocol: corev1.ProtocolTCP, Port: eListener.ContainerPort, TargetPort: intstr.FromInt(int(eListener.ContainerPort)), diff --git a/pkg/resources/kafka/kafka_test.go b/pkg/resources/kafka/kafka_test.go index e6c891f15..3390b6898 100644 --- a/pkg/resources/kafka/kafka_test.go +++ b/pkg/resources/kafka/kafka_test.go @@ -21,6 +21,7 @@ import ( "time" "emperror.dev/errors" + ccTypes "github.com/banzaicloud/go-cruise-control/pkg/types" "github.com/go-logr/logr" "github.com/onsi/gomega" "github.com/stretchr/testify/assert" @@ -28,16 +29,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/banzaicloud/koperator/pkg/kafkaclient" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" + controllerMocks "github.com/banzaicloud/koperator/controllers/tests/mocks" + "github.com/banzaicloud/koperator/pkg/kafkaclient" "github.com/banzaicloud/koperator/pkg/resources" "github.com/banzaicloud/koperator/pkg/resources/kafka/mocks" + "github.com/banzaicloud/koperator/pkg/scale" ) func TestGetBrokersWithPendingOrRunningCCTask(t *testing.T) { @@ -1235,6 +1237,7 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { pods []corev1.Pod allOfflineReplicas []int32 outOfSyncReplicas []int32 + ccStatus *scale.StatusTaskResult errorExpected bool }{ { @@ -1269,9 +1272,9 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { desiredPod: &corev1.Pod{}, currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-201"}}, pods: []corev1.Pod{ - {ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, - {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}}, - {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301"}}, }, errorExpected: true, }, @@ -1285,7 +1288,7 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { Spec: v1beta1.KafkaClusterSpec{ Brokers: []v1beta1.Broker{{Id: 101}, {Id: 201}, {Id: 301}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - ConcurrentBrokerRestartsAllowed: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1293,9 +1296,9 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { desiredPod: &corev1.Pod{}, currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-301"}}, pods: []corev1.Pod{ - {ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, - {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}, DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, - {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301"}}, }, errorExpected: true, }, @@ -1309,7 +1312,7 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { Spec: v1beta1.KafkaClusterSpec{ Brokers: []v1beta1.Broker{{Id: 101}, {Id: 102}, {Id: 201}, {Id: 102}, {Id: 301}, {Id: 302}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - ConcurrentBrokerRestartsAllowed: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1324,6 +1327,14 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, + ccStatus: &scale.StatusTaskResult{ + State: &ccTypes.StateResult{ + AnalyzerState: ccTypes.AnalyzerState{ReadyGoals: []ccTypes.Goal{ccTypes.RackAwareDistributionGoal}}, + AnomalyDetectorState: ccTypes.AnomalyDetectorState{ + RecentGoalViolations: []ccTypes.AnomalyDetails{{UnfixableViolatedGoals: []ccTypes.Goal{}, FixableViolatedGoals: []ccTypes.Goal{}}}, + }, + }, + }, errorExpected: true, }, { @@ -1342,7 +1353,7 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, {Id: 302, ReadOnlyConfig: ""}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - ConcurrentBrokerRestartsAllowed: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1357,6 +1368,14 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, + ccStatus: &scale.StatusTaskResult{ + State: &ccTypes.StateResult{ + AnalyzerState: ccTypes.AnalyzerState{ReadyGoals: []ccTypes.Goal{ccTypes.RackAwareDistributionGoal}}, + AnomalyDetectorState: ccTypes.AnomalyDetectorState{ + RecentGoalViolations: []ccTypes.AnomalyDetails{{UnfixableViolatedGoals: []ccTypes.Goal{}, FixableViolatedGoals: []ccTypes.Goal{}}}, + }, + }, + }, errorExpected: true, }, { @@ -1390,12 +1409,10 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, - outOfSyncReplicas: []int32{101}, - allOfflineReplicas: []int32{}, - errorExpected: true, + errorExpected: true, }, { - testName: "Pod is deleted if allowed concurrent restarts is not specified and failure threshold is not reached", + testName: "Pod is deleted if allowed concurrent restarts is default and failure threshold is not reached", kafkaCluster: v1beta1.KafkaCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "kafka", @@ -1410,7 +1427,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, {Id: 302, ReadOnlyConfig: "broker.rack=az3"}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 1, + FailureThreshold: 1, + ConcurrentBrokerRestartCountPerRack: 1, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1425,9 +1443,9 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, - errorExpected: false, allOfflineReplicas: []int32{}, outOfSyncReplicas: []int32{}, + errorExpected: false, }, { testName: "Pod is not deleted if pod is restarting in another AZ, even if allowed concurrent restarts is not reached", @@ -1445,8 +1463,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, {Id: 302, ReadOnlyConfig: "broker.rack=az3"}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1461,6 +1479,14 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, + ccStatus: &scale.StatusTaskResult{ + State: &ccTypes.StateResult{ + AnalyzerState: ccTypes.AnalyzerState{ReadyGoals: []ccTypes.Goal{ccTypes.RackAwareDistributionGoal}}, + AnomalyDetectorState: ccTypes.AnomalyDetectorState{ + RecentGoalViolations: []ccTypes.AnomalyDetails{{UnfixableViolatedGoals: []ccTypes.Goal{}, FixableViolatedGoals: []ccTypes.Goal{}}}, + }, + }, + }, errorExpected: true, }, { @@ -1479,8 +1505,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, {Id: 302, ReadOnlyConfig: "broker.rack=az3"}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1495,10 +1521,46 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, - outOfSyncReplicas: []int32{201}, allOfflineReplicas: []int32{}, + outOfSyncReplicas: []int32{201}, errorExpected: true, }, + { + testName: "Pod is deleted if all pods are running and CC RackAwareDistributionGoal is not ready and allowed concurrent restarts is not reached", + kafkaCluster: v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kafka", + Namespace: "kafka", + }, + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 101, ReadOnlyConfig: "broker.rack=az1"}, + {Id: 102, ReadOnlyConfig: "broker.rack=az1"}, + {Id: 201, ReadOnlyConfig: "broker.rack=az2"}, + {Id: 202, ReadOnlyConfig: "broker.rack=az2"}, + {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, + {Id: 302, ReadOnlyConfig: "broker.rack=az3"}}, + RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, + }, + }, + Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, + }, + desiredPod: &corev1.Pod{}, + currentPod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "kafka-102", Labels: map[string]string{"brokerId": "102"}}}, + pods: []corev1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-101", Labels: map[string]string{"brokerId": "101"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-102", Labels: map[string]string{"brokerId": "102"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-202", Labels: map[string]string{"brokerId": "202"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, + }, + allOfflineReplicas: []int32{}, + outOfSyncReplicas: []int32{101}, + errorExpected: false, + }, { testName: "Pod is deleted if failure is in same AZ and allowed concurrent restarts is not reached", kafkaCluster: v1beta1.KafkaCluster{ @@ -1515,8 +1577,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, {Id: 302, ReadOnlyConfig: "broker.rack=az3"}}, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1531,9 +1593,17 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-302", Labels: map[string]string{"brokerId": "302"}}}, }, - outOfSyncReplicas: []int32{101}, allOfflineReplicas: []int32{}, - errorExpected: false, + outOfSyncReplicas: []int32{101}, + ccStatus: &scale.StatusTaskResult{ + State: &ccTypes.StateResult{ + AnalyzerState: ccTypes.AnalyzerState{ReadyGoals: []ccTypes.Goal{ccTypes.RackAwareDistributionGoal}}, + AnomalyDetectorState: ccTypes.AnomalyDetectorState{ + RecentGoalViolations: []ccTypes.AnomalyDetails{{UnfixableViolatedGoals: []ccTypes.Goal{}, FixableViolatedGoals: []ccTypes.Goal{}}}, + }, + }, + }, + errorExpected: false, }, { testName: "Pod is not deleted if pod is restarting in another AZ, if brokers per AZ < tolerated failures", @@ -1549,8 +1619,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, }, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1562,6 +1632,14 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, }, + ccStatus: &scale.StatusTaskResult{ + State: &ccTypes.StateResult{ + AnalyzerState: ccTypes.AnalyzerState{ReadyGoals: []ccTypes.Goal{ccTypes.RackAwareDistributionGoal}}, + AnomalyDetectorState: ccTypes.AnomalyDetectorState{ + RecentGoalViolations: []ccTypes.AnomalyDetails{{UnfixableViolatedGoals: []ccTypes.Goal{}, FixableViolatedGoals: []ccTypes.Goal{}}}, + }, + }, + }, errorExpected: true, }, { @@ -1578,8 +1656,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, }, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1591,8 +1669,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, }, - outOfSyncReplicas: []int32{101}, allOfflineReplicas: []int32{}, + outOfSyncReplicas: []int32{101}, errorExpected: true, }, { @@ -1609,8 +1687,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, }, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1640,8 +1718,8 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {Id: 301, ReadOnlyConfig: "broker.rack=az-3"}, }, RollingUpgradeConfig: v1beta1.RollingUpgradeConfig{ - FailureThreshold: 2, - ConcurrentBrokerRestartsAllowed: 2, + FailureThreshold: 2, + ConcurrentBrokerRestartCountPerRack: 2, }, }, Status: v1beta1.KafkaClusterStatus{State: v1beta1.KafkaClusterRollingUpgrading}, @@ -1653,6 +1731,14 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "kafka-201", Labels: map[string]string{"brokerId": "201"}}}, {ObjectMeta: metav1.ObjectMeta{Name: "kafka-301", Labels: map[string]string{"brokerId": "301"}}}, }, + ccStatus: &scale.StatusTaskResult{ + State: &ccTypes.StateResult{ + AnalyzerState: ccTypes.AnalyzerState{ReadyGoals: []ccTypes.Goal{ccTypes.RackAwareDistributionGoal}}, + AnomalyDetectorState: ccTypes.AnomalyDetectorState{ + RecentGoalViolations: []ccTypes.AnomalyDetails{{UnfixableViolatedGoals: []ccTypes.Goal{}, FixableViolatedGoals: []ccTypes.Goal{}}}, + }, + }, + }, errorExpected: true, }, } @@ -1689,6 +1775,13 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { } mockKafkaClientProvider.On("NewFromCluster", mockClient, &test.kafkaCluster).Return(mockedKafkaClient, func() {}, nil) + // Mock Cruise Control client + mockCruiseControl := controllerMocks.NewMockCruiseControlScaler(mockCtrl) + if test.ccStatus != nil { + mockCruiseControl.EXPECT().Status(context.Background()).Return(*test.ccStatus, nil) + } + r.CruiseControlScalerFactory = controllerMocks.NewMockScaleFactory(mockCruiseControl) + // Call the handleRollingUpgrade function with the provided test.desiredPod and test.currentPod err := r.handleRollingUpgrade(logf.Log, test.desiredPod, test.currentPod, reflect.TypeOf(test.desiredPod)) @@ -1701,3 +1794,85 @@ func TestReconcileConcurrentBrokerRestartsAllowed(t *testing.T) { }) } } + +func TestGetBrokerAzMap(t *testing.T) { + t.Parallel() + testCases := []struct { + testName string + kafkaCluster v1beta1.KafkaCluster + expectedAzMap map[int32]string + }{ + { + testName: "Brokers have different AZs if no broker rack value is set", + kafkaCluster: v1beta1.KafkaCluster{ + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 101, ReadOnlyConfig: ""}, + {Id: 201, ReadOnlyConfig: ""}, + {Id: 301, ReadOnlyConfig: ""}, + }, + }, + }, + expectedAzMap: map[int32]string{101: "101", 201: "201", 301: "301"}, + }, + { + testName: "Brokers have different AZs if one broker has no broker rack value set", + kafkaCluster: v1beta1.KafkaCluster{ + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 101, ReadOnlyConfig: "broker.rack=az1"}, + {Id: 102, ReadOnlyConfig: ""}, + {Id: 201, ReadOnlyConfig: "broker.rack=az2"}, + {Id: 202, ReadOnlyConfig: "broker.rack=az2"}, + {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, + {Id: 302, ReadOnlyConfig: "broker.rack=az3"}, + }, + }, + }, + expectedAzMap: map[int32]string{101: "101", 102: "102", 201: "201", 202: "202", 301: "301", 302: "302"}, + }, + { + testName: "Brokers have different AZs if read only configs is a corrupted string for one broker", + kafkaCluster: v1beta1.KafkaCluster{ + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 101, ReadOnlyConfig: "broker.rack;az1"}, + {Id: 201, ReadOnlyConfig: "broker.rack=az2"}, + {Id: 301, ReadOnlyConfig: "broker.rack=az3"}, + }, + }, + }, + expectedAzMap: map[int32]string{101: "101", 201: "201", 301: "301"}, + }, + { + testName: "Brokers have correct AZs if read only configs is valid for all brokers", + kafkaCluster: v1beta1.KafkaCluster{ + Spec: v1beta1.KafkaClusterSpec{ + Brokers: []v1beta1.Broker{ + {Id: 101, ReadOnlyConfig: "broker.rack=az-1"}, + {Id: 102, ReadOnlyConfig: "broker.rack=az-1"}, + {Id: 201, ReadOnlyConfig: "broker.rack=az-2"}, + {Id: 202, ReadOnlyConfig: "broker.rack=az-2"}, + {Id: 301, ReadOnlyConfig: "broker.rack=az-3"}, + {Id: 302, ReadOnlyConfig: "broker.rack=az-3"}, + }, + }, + }, + expectedAzMap: map[int32]string{ + 101: "az-1", + 102: "az-1", + 201: "az-2", + 202: "az-2", + 301: "az-3", + 302: "az-3", + }, + }, + } + + for _, test := range testCases { + t.Run(test.testName, func(t *testing.T) { + azMap := getBrokerAzMap(&test.kafkaCluster) + assert.Equal(t, test.expectedAzMap, azMap) + }) + } +} diff --git a/pkg/resources/kafka/pod.go b/pkg/resources/kafka/pod.go index 9ccb1492e..cc97ad4d8 100644 --- a/pkg/resources/kafka/pod.go +++ b/pkg/resources/kafka/pod.go @@ -301,7 +301,7 @@ func getVolumes(brokerConfigVolumes, dataVolume []corev1.Volume, kafkaClusterSpe Name: brokerConfigMapVolumeMount, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf(brokerConfigTemplate+"-%d", kafkaClusterName, id)}, + LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf(brokerConfigTemplate+"-"+"%d", kafkaClusterName, id)}, DefaultMode: util.Int32Pointer(0644), }, }, @@ -383,7 +383,7 @@ func generateDataVolumeAndVolumeMount(pvcs []corev1.PersistentVolumeClaim, stora // PVC volume mount (pvcs are already sorted by pvc name in 'getCreatedPvcForBroker' function for i, pvc := range pvcs { volumes = append(volumes, corev1.Volume{ - Name: fmt.Sprintf(kafkaDataVolumeMount+"-%d", i), + Name: fmt.Sprintf(kafkaDataVolumeMount+"-"+"%d", i), VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvc.Name, @@ -391,7 +391,7 @@ func generateDataVolumeAndVolumeMount(pvcs []corev1.PersistentVolumeClaim, stora }, }) volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: fmt.Sprintf(kafkaDataVolumeMount+"-%d", i), + Name: fmt.Sprintf(kafkaDataVolumeMount+"-"+"%d", i), MountPath: pvc.Annotations["mountPath"], }) } @@ -412,13 +412,13 @@ func generateDataVolumeAndVolumeMount(pvcs []corev1.PersistentVolumeClaim, stora for i := range emptyDirs { j := i + offset volumes = append(volumes, corev1.Volume{ - Name: fmt.Sprintf(kafkaDataVolumeMount+"-%d", j), + Name: fmt.Sprintf(kafkaDataVolumeMount+"-"+"%d", j), VolumeSource: corev1.VolumeSource{ EmptyDir: emptyDirs[i].EmptyDir, }, }) volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: fmt.Sprintf(kafkaDataVolumeMount+"-%d", j), + Name: fmt.Sprintf(kafkaDataVolumeMount+"-"+"%d", j), MountPath: emptyDirs[i].MountPath, }) } diff --git a/pkg/scale/scale.go b/pkg/scale/scale.go index cb71fd4f1..6c1d589c2 100644 --- a/pkg/scale/scale.go +++ b/pkg/scale/scale.go @@ -141,6 +141,7 @@ func (cc *cruiseControlScaler) Status(ctx context.Context) (StatusTaskResult, er State: v1beta1.CruiseControlTaskActive, }, Status: &status, + State: resp.Result, }, nil } diff --git a/pkg/scale/types.go b/pkg/scale/types.go index 92b9d8f63..ae1311acb 100644 --- a/pkg/scale/types.go +++ b/pkg/scale/types.go @@ -65,6 +65,7 @@ const ( type StatusTaskResult struct { TaskResult *Result Status *CruiseControlStatus + State *types.StateResult } // CruiseControlStatus struct is used to describe internal state of Cruise Control. diff --git a/pkg/util/client/common.go b/pkg/util/client/common.go index 360091f83..76057bff8 100644 --- a/pkg/util/client/common.go +++ b/pkg/util/client/common.go @@ -37,11 +37,18 @@ func UseSSL(cluster *v1beta1.KafkaCluster) bool { func getContainerPortForInnerCom(internalListeners []v1beta1.InternalListenerConfig, extListeners []v1beta1.ExternalListenerConfig) int32 { for _, val := range internalListeners { + if val.UsedForKafkaAdminCommunication { // Optional override to return a port from a different listener. Needed if b2b communication is on an external listener and and you want the koperator to interact with kafka over a different port. + return val.ContainerPort + } if val.UsedForInnerBrokerCommunication { return val.ContainerPort } } + for _, val := range extListeners { + if val.UsedForKafkaAdminCommunication { + return val.ContainerPort + } if val.UsedForInnerBrokerCommunication { return val.ContainerPort } diff --git a/pkg/util/contour/common.go b/pkg/util/contour/common.go new file mode 100644 index 000000000..f94a0c8ed --- /dev/null +++ b/pkg/util/contour/common.go @@ -0,0 +1,25 @@ +// Copyright © 2019 Cisco Systems, Inc. and/or its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package contour + +const ( + + // EnvoyServiceName name for loadbalancer service + ContourServiceName = "contour-svc-%s-%s" + // EnvoyServiceNameWithScope name for loadbalancer service + ContourServiceNameWithScope = "countour-svc-%s-%s-%s" + // IngressControllerName name for contour ingress service + IngressControllerName = "contour" +) diff --git a/pkg/util/kafka/common.go b/pkg/util/kafka/common.go index 2e1ffe4ea..98c2adbaa 100644 --- a/pkg/util/kafka/common.go +++ b/pkg/util/kafka/common.go @@ -193,8 +193,11 @@ func GetBootstrapServersService(cluster *v1beta1.KafkaCluster) (string, error) { // GetBrokerContainerPort return broker container port func GetBrokerContainerPort(cluster *v1beta1.KafkaCluster) (int32, error) { containerPort := int32(0) - for _, lc := range cluster.Spec.ListenersConfig.InternalListeners { + if lc.UsedForKafkaAdminCommunication { // Optional override to return a port from a different listener. Needed if b2b communication is on an external listener and and you want the koperator to interact with kafka over a different port. + containerPort = lc.ContainerPort + break + } if lc.UsedForInnerBrokerCommunication && !lc.UsedForControllerCommunication { containerPort = lc.ContainerPort break @@ -202,6 +205,10 @@ func GetBrokerContainerPort(cluster *v1beta1.KafkaCluster) (int32, error) { } for _, lc := range cluster.Spec.ListenersConfig.ExternalListeners { + if lc.UsedForKafkaAdminCommunication { + containerPort = lc.ContainerPort + break + } if lc.UsedForInnerBrokerCommunication { containerPort = lc.ContainerPort break diff --git a/pkg/util/kafka/const.go b/pkg/util/kafka/const.go index 980fcc45c..ffc06ec28 100644 --- a/pkg/util/kafka/const.go +++ b/pkg/util/kafka/const.go @@ -37,6 +37,7 @@ const ( KafkaConfigListenerName = "listener.name" KafkaConfigListenerSecurityProtocolMap = "listener.security.protocol.map" KafkaConfigInterBrokerListenerName = "inter.broker.listener.name" + KafkaConfigSecurityInterBrokerProtocol = "security.inter.broker.protocol" KafkaConfigAdvertisedListeners = "advertised.listeners" KafkaConfigControlPlaneListener = "control.plane.listener.name" diff --git a/pkg/util/util.go b/pkg/util/util.go index 60871e1ba..152499e85 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -18,9 +18,11 @@ import ( "bytes" "compress/gzip" "context" + "crypto/md5" "crypto/tls" "crypto/x509" "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "io" @@ -54,6 +56,7 @@ import ( "github.com/banzaicloud/koperator/api/v1beta1" "github.com/banzaicloud/koperator/pkg/errorfactory" "github.com/banzaicloud/koperator/pkg/util/cert" + "github.com/banzaicloud/koperator/pkg/util/contour" envoyutils "github.com/banzaicloud/koperator/pkg/util/envoy" "github.com/banzaicloud/koperator/pkg/util/istioingress" properties "github.com/banzaicloud/koperator/properties/pkg" @@ -254,11 +257,12 @@ func IsIngressConfigInUse(iConfigName, defaultConfigName string, cluster *v1beta // ConstructEListenerLabelName construct an eListener label name based on ingress config name and listener name func ConstructEListenerLabelName(ingressConfigName, eListenerName string) string { + externalListenerName := strings.ReplaceAll(eListenerName, "_", "-") if ingressConfigName == IngressConfigGlobalName { - return eListenerName + return externalListenerName } - return fmt.Sprintf(ExternalListenerLabelNameTemplate, eListenerName, ingressConfigName) + return fmt.Sprintf(ExternalListenerLabelNameTemplate, externalListenerName, ingressConfigName) } // ShouldIncludeBroker returns true if the broker should be included as a resource on external listener resources @@ -341,6 +345,34 @@ func GetIngressConfigs(kafkaClusterSpec v1beta1.KafkaClusterSpec, }, } } + case contour.IngressControllerName: + if eListenerConfig.Config != nil { + defaultIngressConfigName = eListenerConfig.Config.DefaultIngressConfig + ingressConfigs = make(map[string]v1beta1.IngressConfig, len(eListenerConfig.Config.IngressConfig)) + for k, iConf := range eListenerConfig.Config.IngressConfig { + if iConf.ContourIngressConfig != nil { + err := mergo.Merge(iConf.ContourIngressConfig, kafkaClusterSpec.ContourIngressConfig) + if err != nil { + return nil, "", errors.WrapWithDetails(err, + "could not merge global envoy config with local one", "envoyConfig", k) + } + err = mergo.Merge(&iConf.IngressServiceSettings, eListenerConfig.IngressServiceSettings) + if err != nil { + return nil, "", errors.WrapWithDetails(err, + "could not merge global loadbalancer config with local one", + "externalListenerName", eListenerConfig.Name) + } + ingressConfigs[k] = iConf + } + } + } else { + ingressConfigs = map[string]v1beta1.IngressConfig{ + IngressConfigGlobalName: { + IngressServiceSettings: eListenerConfig.IngressServiceSettings, + ContourIngressConfig: &kafkaClusterSpec.ContourIngressConfig, + }, + } + } default: return nil, "", errors.NewWithDetails("not supported ingress type", "name", kafkaClusterSpec.GetIngressController()) } @@ -427,10 +459,11 @@ func ConvertConfigEntryListToProperties(config []sarama.ConfigEntry) (*propertie func GenerateEnvoyResourceName(resourceNameFormat string, resourceNameWithScopeFormat string, extListener v1beta1.ExternalListenerConfig, ingressConfig v1beta1.IngressConfig, ingressConfigName, clusterName string) string { var resourceName string + externalListenerName := strings.ReplaceAll(extListener.Name, "_", "-") if ingressConfigName == IngressConfigGlobalName { - resourceName = fmt.Sprintf(resourceNameFormat, extListener.Name, clusterName) + resourceName = fmt.Sprintf(resourceNameFormat, externalListenerName, clusterName) } else { - resourceName = fmt.Sprintf(resourceNameWithScopeFormat, extListener.Name, ingressConfigName, clusterName) + resourceName = fmt.Sprintf(resourceNameWithScopeFormat, externalListenerName, ingressConfigName, clusterName) } return resourceName @@ -572,3 +605,9 @@ func RetryOnConflict(backoff wait.Backoff, fn func() error) error { func GetExternalPortForBroker(externalStartingPort, brokerId int32) int32 { return externalStartingPort + brokerId } + +// Generage MD5 hash for a given string +func GetMD5Hash(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 4462f09ac..bde76f785 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -820,3 +820,112 @@ func TestFilterControllerOnlyNodes(t *testing.T) { }) } } + +func TestConstructEListenerLabelName(t *testing.T) { + tests := []struct { + ingressConfigName string + eListenerName string + expected string + }{ + {"globalConfig", "example_listener_name", "example-listener-name"}, + {"globalConfig", "no_underscores", "no-underscores"}, + {"globalConfig", "multiple___underscores", "multiple---underscores"}, + {"globalConfig", "noUnderscoresHere", "noUnderscoresHere"}, + {"nonGlobalConfig", "example_listener_name", "example-listener-name-nonGlobalConfig"}, + } + + for _, test := range tests { + result := ConstructEListenerLabelName(test.ingressConfigName, test.eListenerName) + if result != test.expected { + t.Errorf("ConstructEListenerLabelName(%q, %q) = %q; want %q", test.ingressConfigName, test.eListenerName, result, test.expected) + } + } +} + +func TestGenerateEnvoyResourceName(t *testing.T) { + testCases := []struct { + resourceNameFormat string + resourceNameWithScopeFormat string + extListener v1beta1.ExternalListenerConfig + ingressConfig v1beta1.IngressConfig + ingressConfigName, clusterName, expected string + }{ + { + resourceNameFormat: "%s-%s", + resourceNameWithScopeFormat: "%s-%s-%s", + extListener: v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{Name: "noUnderscores"}, + }, + ingressConfig: v1beta1.IngressConfig{}, + ingressConfigName: "globalConfig", + clusterName: "clusterName", + expected: "noUnderscores-clusterName", + }, + { + resourceNameFormat: "%s-%s", + resourceNameWithScopeFormat: "%s-%s-%s", + extListener: v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{Name: "under_scores"}, + }, + ingressConfig: v1beta1.IngressConfig{}, + ingressConfigName: "globalConfig", + clusterName: "clusterName", + expected: "under-scores-clusterName", + }, + { + resourceNameFormat: "%s-%s", + resourceNameWithScopeFormat: "%s-%s-%s", + extListener: v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{Name: "noUnderscores"}, + }, + ingressConfig: v1beta1.IngressConfig{}, + ingressConfigName: "nonGlobalConfig", + clusterName: "clusterName", + expected: "noUnderscores-nonGlobalConfig-clusterName", + }, + { + resourceNameFormat: "%s-%s", + resourceNameWithScopeFormat: "%s-%s-%s", + extListener: v1beta1.ExternalListenerConfig{ + CommonListenerSpec: v1beta1.CommonListenerSpec{Name: "under_scores"}, + }, + ingressConfig: v1beta1.IngressConfig{}, + ingressConfigName: "nonGlobalConfig", + clusterName: "clusterName", + expected: "under-scores-nonGlobalConfig-clusterName", + }, + } + for _, test := range testCases { + hash := GenerateEnvoyResourceName(test.resourceNameFormat, test.resourceNameWithScopeFormat, test.extListener, test.ingressConfig, + test.ingressConfigName, test.clusterName) + if hash != test.expected { + t.Errorf("Expected: %s Got: %s", test.expected, hash) + } + } +} + +func TestGetMD5Hash(t *testing.T) { + testCases := []struct { + testName string + input string + expected string + }{ + { + testName: "empty string", + input: "", + expected: "d41d8cd98f00b204e9800998ecf8427e", + }, + { + testName: "non-empty string", + input: "test", + expected: "098f6bcd4621d373cade4e832627b4f6", + }, + } + + for _, test := range testCases { + hash := GetMD5Hash(test.input) + if hash != test.expected { + t.Errorf("Expected: %s Got: %s", test.expected, hash) + } + } +} diff --git a/run-e2e.sh b/run-e2e.sh new file mode 100755 index 000000000..161038856 --- /dev/null +++ b/run-e2e.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export IMG_E2E=koperator_e2e_test:latest + +kind delete clusters e2e-kind +kind create cluster --config=/Users/dvaseeka/Documents/adobe/kRaft-migration/pipeline-kraft-migration/koperators/koperator/tests/e2e/platforms/kind/kind_config.yaml --name=e2e-kind +kubectl label node e2e-kind-control-plane node.kubernetes.io/exclude-from-external-load-balancers- +docker build . -t koperator_e2e_test +kind load docker-image koperator_e2e_test:latest --name e2e-kind +kind load docker-image docker-pipeline-upstream-mirror.dr-uw2.adobeitc.com/adobe/kafka:2.13-3.7.0 --name e2e-kind +kind load docker-image docker-pipeline-upstream-mirror.dr-uw2.adobeitc.com/adobe/cruise-control:2.5.133-adbe-20240313 --name e2e-kind + +go test ./tests/e2e -v -timeout 20m -tags e2e --ginkgo.show-node-events --ginkgo.trace --ginkgo.v diff --git a/run-local.sh b/run-local.sh new file mode 100755 index 000000000..e7f95105c --- /dev/null +++ b/run-local.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# RUN KOPERATOR LOCALLY ON KIND +### Create kind cluster +kind delete clusters e2e-kind +kind create cluster --config=/Users/dvaseeka/Documents/adobe/kRaft-migration/pipeline-kraft-migration/koperators/koperator/tests/e2e/platforms/kind/kind_config.yaml --name=e2e-kind + +### Build/Load images +kind load docker-image docker-pipeline-upstream-mirror.dr-uw2.adobeitc.com/adobe/cruise-control:2.5.133-adbe-20240313 --name e2e-kind +kind load docker-image docker-pipeline-upstream-mirror.dr-uw2.adobeitc.com/adobe/kafka:2.13-3.7.0 --name e2e-kind +docker build . -t koperator_e2e_test +kind load docker-image koperator_e2e_test:latest --name e2e-kind + +### Install Helm Charts and CRDs +#### project contour +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install contour bitnami/contour --namespace projectcontour --create-namespace + +#### cert-manager +helm repo add jetstack https://charts.jetstack.io --force-update +helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.16.2 --set crds.enabled=true + +#### zookeeper-operator +helm repo add pravega https://charts.pravega.io +helm install zookeeper-operator pravega/zookeeper-operator --version 0.2.15 --namespace zookeeper --create-namespace --set crd.create=true + +#### prometheus +helm repo add prometheus https://prometheus-community.github.io/helm-charts +helm install prometheus prometheus/kube-prometheus-stack --version 54.1.0 --namespace prometheus --create-namespace + +#### koperator +helm install kafka-operator charts/kafka-operator --set operator.image.repository=koperator_e2e_test --set operator.image.tag=latest --namespace kafka --create-namespace +kubectl apply -f charts/kafka-operator/crds/ + +### Initialize Kafka Cluster +kubectl apply -f config/samples/kraft/simplekafkacluster_kraft.yaml -n kafka +kubectl config set-context --current --namespace kafka \ No newline at end of file diff --git a/tests/e2e/const.go b/tests/e2e/const.go index ab9888543..20bb6d984 100644 --- a/tests/e2e/const.go +++ b/tests/e2e/const.go @@ -40,7 +40,7 @@ const ( testExternalTopicName = "topic-test-external" testInternalTopicName = "topic-test-internal" - defaultTLSSecretName = "test-secret" + defaultTLSSecretName = "kafka-ca-certificate" kcatName = "kcat" zookeeperKind = "zookeeperclusters.zookeeper.pravega.io" zookeeperClusterName = "zookeeper-server" @@ -75,6 +75,7 @@ func apiGroupKoperatorDependencies() map[string]string { "cert-manager": "cert-manager.io", "zookeeper": "zookeeper.pravega.io", "prometheus": "monitoring.coreos.com", + "contour": "projectcontour.io", } } diff --git a/tests/e2e/global.go b/tests/e2e/global.go index 4c187073f..adfd86f86 100644 --- a/tests/e2e/global.go +++ b/tests/e2e/global.go @@ -34,6 +34,17 @@ var ( }, RemoteCRDPathVersionTemplate: "https://github.com/jetstack/cert-manager/releases/download/v%s/cert-manager.crds.yaml", } + // contour ingress controller + contourIngressControllerHelmDescriptor = helmDescriptor{ + Repository: "https://charts.bitnami.com/bitnami", + ChartName: "contour", + ChartVersion: "15.4.0", + ReleaseName: "contour", + Namespace: "projectcontour", + SetValues: map[string]string{ + "installCRDs": "true", + }, + } // koperatorLocalHelmDescriptor describes the Koperator Helm component with // a local chart and version. diff --git a/tests/e2e/koperator_suite_test.go b/tests/e2e/koperator_suite_test.go index a7ee2f8c3..8e8144150 100644 --- a/tests/e2e/koperator_suite_test.go +++ b/tests/e2e/koperator_suite_test.go @@ -65,6 +65,9 @@ var _ = When("Testing e2e test altogether", Ordered, func() { testProduceConsumeInternalSSL(defaultTLSSecretName) testUninstallKafkaCluster() testUninstallZookeeperCluster() + // testInstallKafkaCluster("../../config/samples/kraft/simplekafkacluster_kraft.yaml") + // testProduceConsumeInternal() + // testUninstallKafkaCluster() testUninstall() snapshotClusterAndCompare(snapshottedInfo) }) diff --git a/tests/e2e/test_install.go b/tests/e2e/test_install.go index 185065644..faed78a0f 100644 --- a/tests/e2e/test_install.go +++ b/tests/e2e/test_install.go @@ -37,6 +37,13 @@ func testInstall() bool { }) }) + When("Installing contour ingress controller", func() { + It("Installing contour Helm chart", func() { + err = contourIngressControllerHelmDescriptor.installHelmChart(kubectlOptions) + Expect(err).NotTo(HaveOccurred()) + }) + }) + When("Installing zookeeper-operator", func() { It("Installing zookeeper-operator Helm chart", func() { err = zookeeperOperatorHelmDescriptor.installHelmChart(kubectlOptions) diff --git a/tests/e2e/test_uninstall.go b/tests/e2e/test_uninstall.go index 71f12de9e..ad3ca7516 100644 --- a/tests/e2e/test_uninstall.go +++ b/tests/e2e/test_uninstall.go @@ -57,5 +57,10 @@ func testUninstall() bool { ConfigPath: kubectlOptions.ConfigPath, Namespace: certManagerHelmDescriptor.Namespace, }) + requireUninstallingContour(k8s.KubectlOptions{ + ContextName: kubectlOptions.ContextName, + ConfigPath: kubectlOptions.ConfigPath, + Namespace: contourIngressControllerHelmDescriptor.Namespace, + }) }) } diff --git a/tests/e2e/types.go b/tests/e2e/types.go index 31565e1ec..6d64ef7e5 100644 --- a/tests/e2e/types.go +++ b/tests/e2e/types.go @@ -24,6 +24,7 @@ type dependencyCRDsType struct { zookeeper []string prometheus []string certManager []string + contour []string } func (c *dependencyCRDsType) Zookeeper() []string { @@ -35,6 +36,9 @@ func (c *dependencyCRDsType) Prometheus() []string { func (c *dependencyCRDsType) CertManager() []string { return c.certManager } +func (c *dependencyCRDsType) Contour() []string { + return c.contour +} func (c *dependencyCRDsType) Initialize(kubectlOptions k8s.KubectlOptions) error { var err error @@ -42,6 +46,10 @@ func (c *dependencyCRDsType) Initialize(kubectlOptions k8s.KubectlOptions) error if err != nil { return fmt.Errorf("initialize Cert-manager CRDs error: %w", err) } + c.contour, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["contour"]) + if err != nil { + return fmt.Errorf("initialize Contour Ingress Controller CRDs error: %w", err) + } c.prometheus, err = listK8sResourceKinds(kubectlOptions, apiGroupKoperatorDependencies()["prometheus"]) if err != nil { return fmt.Errorf("initialize Prometheus CRDs error: %w", err) diff --git a/tests/e2e/uninstall.go b/tests/e2e/uninstall.go index fccaff087..f41b338ff 100644 --- a/tests/e2e/uninstall.go +++ b/tests/e2e/uninstall.go @@ -206,6 +206,49 @@ func requireRemoveCertManagerCRDs(kubectlOptions k8s.KubectlOptions) { } }) } +func requireUninstallingContour(kubectlOptions k8s.KubectlOptions) { + When("Uninstalling zookeeper-operator", func() { + requireUninstallingContourHelmChart(kubectlOptions) + requireRemoveContourCRDs(kubectlOptions) + requireRemoveNamespace(kubectlOptions, contourIngressControllerHelmDescriptor.Namespace) + }) +} + +// requireUninstallingCertManagerHelmChart uninstalls cert-manager helm chart +// and checks the success of that operation. +func requireUninstallingContourHelmChart(kubectlOptions k8s.KubectlOptions) { + It("Uninstalling Project Contour Helm chart", func() { + err := contourIngressControllerHelmDescriptor.uninstallHelmChart(kubectlOptions, true) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Project Contour helm chart resources cleanup") + + k8sResourceKinds, err := listK8sResourceKinds(kubectlOptions, "") + Expect(err).ShouldNot(HaveOccurred()) + + contourAvailableResourceKinds := stringSlicesInstersect(dependencyCRDs.Contour(), k8sResourceKinds) + contourAvailableResourceKinds = append(contourAvailableResourceKinds, basicK8sResourceKinds()...) + + remainedResources, err := getK8sResources(kubectlOptions, + contourAvailableResourceKinds, + fmt.Sprintf(managedByHelmLabelTemplate, contourIngressControllerHelmDescriptor.ReleaseName), + "", + kubectlArgGoTemplateKindNameNamespace, + "--all-namespaces") + Expect(err).ShouldNot(HaveOccurred()) + + Expect(remainedResources).Should(BeEmpty()) + }) +} + +func requireRemoveContourCRDs(kubectlOptions k8s.KubectlOptions) { + It("Removing Contour Ingress Controller CRDs", func() { + for _, crd := range dependencyCRDs.Contour() { + err := deleteK8sResourceNoErrNotFound(kubectlOptions, defaultDeletionTimeout, crdKind, crd) + Expect(err).ShouldNot(HaveOccurred()) + } + }) +} // requireRemoveNamespace deletes the indicated namespace object func requireRemoveNamespace(kubectlOptions k8s.KubectlOptions, namespace string) {