From 91fa555360d5d0050b6c764cce128bab0ead2463 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Fri, 1 Jul 2022 14:59:51 +0530 Subject: [PATCH 01/66] Added spectro manifests --- config/crd/kustomization.yaml | 12 +- spectro/base/kustomization.yaml | 32 + spectro/base/patch_healthcheck.yaml | 6 + spectro/base/patch_service_account.yaml | 2 + spectro/generated/core-base.yaml | 67 + spectro/generated/core-global.yaml | 11125 ++++++++++++++++ spectro/global/kustomization.yaml | 135 + .../global/patch_crd_webhook_namespace.yaml | 3 + spectro/global/patch_namespace.yaml | 3 + .../patch_nmi_cluster_role_binding.yaml | 6 + spectro/global/patch_service_account.yaml | 2 + spectro/run.sh | 6 + 12 files changed, 11393 insertions(+), 6 deletions(-) create mode 100644 spectro/base/kustomization.yaml create mode 100644 spectro/base/patch_healthcheck.yaml create mode 100644 spectro/base/patch_service_account.yaml create mode 100644 spectro/generated/core-base.yaml create mode 100644 spectro/generated/core-global.yaml create mode 100644 spectro/global/kustomization.yaml create mode 100644 spectro/global/patch_crd_webhook_namespace.yaml create mode 100644 spectro/global/patch_namespace.yaml create mode 100644 spectro/global/patch_nmi_cluster_role_binding.yaml create mode 100644 spectro/global/patch_service_account.yaml create mode 100755 spectro/run.sh diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index f41ea1f58f7..38f11946501 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -28,9 +28,9 @@ patchesStrategicMerge: - patches/webhook_in_azuremachinetemplates.yaml - patches/webhook_in_azuremachinepools.yaml - patches/webhook_in_azuremachinepoolmachines.yaml - # - patches/webhook_in_azuremanagedmachinepools.yaml - # - patches/webhook_in_azuremanagedclusters.yaml - # - patches/webhook_in_azuremanagedcontrolplanes.yaml + - patches/webhook_in_azuremanagedmachinepools.yaml + - patches/webhook_in_azuremanagedclusters.yaml + - patches/webhook_in_azuremanagedcontrolplanes.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -42,9 +42,9 @@ patchesStrategicMerge: - patches/cainjection_in_azuremachinetemplates.yaml - patches/cainjection_in_azuremachinepools.yaml - patches/cainjection_in_azuremachinepoolmachines.yaml - # - patches/cainjection_in_azuremanagedmachinepools.yaml - # - patches/cainjection_in_azuremanagedclusters.yaml - # - patches/cainjection_in_azuremanagedcontrolplanes.yaml + - patches/cainjection_in_azuremanagedmachinepools.yaml + - patches/cainjection_in_azuremanagedclusters.yaml + - patches/cainjection_in_azuremanagedcontrolplanes.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/spectro/base/kustomization.yaml b/spectro/base/kustomization.yaml new file mode 100644 index 00000000000..ee221b52f51 --- /dev/null +++ b/spectro/base/kustomization.yaml @@ -0,0 +1,32 @@ +namespace: capz-system + +namePrefix: capz- + +# Labels to add to all resources and selectors. +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-azure" + +bases: + - ../../config/manager + + +patchesStrategicMerge: + - ../../config/default/manager_image_patch.yaml + - ../../config/default/manager_pull_policy.yaml + - ../../config/default/manager_credentials_patch.yaml + +patchesJson6902: + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_service_account.yaml + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_healthcheck.yaml \ No newline at end of file diff --git a/spectro/base/patch_healthcheck.yaml b/spectro/base/patch_healthcheck.yaml new file mode 100644 index 00000000000..1279812a6d2 --- /dev/null +++ b/spectro/base/patch_healthcheck.yaml @@ -0,0 +1,6 @@ +- op: remove + path: "/spec/template/spec/containers/0/ports" +- op: remove + path: "/spec/template/spec/containers/0/livenessProbe" +- op: remove + path: "/spec/template/spec/containers/0/readinessProbe" \ No newline at end of file diff --git a/spectro/base/patch_service_account.yaml b/spectro/base/patch_service_account.yaml new file mode 100644 index 00000000000..99d7e697b56 --- /dev/null +++ b/spectro/base/patch_service_account.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/serviceAccountName" \ No newline at end of file diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml new file mode 100644 index 00000000000..4fad77dd22a --- /dev/null +++ b/spectro/generated/core-base.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + name: capz-controller-manager + namespace: capz-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-logs-container: manager + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + spec: + containers: + - args: + - --leader-elect + - --metrics-bind-addr=localhost:8080 + - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false} + - --v=0 + env: + - name: AZURE_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: subscription-id + name: manager-bootstrap-credentials + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + key: tenant-id + name: manager-bootstrap-credentials + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + key: client-id + name: manager-bootstrap-credentials + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client-secret + name: manager-bootstrap-credentials + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + imagePullPolicy: Always + name: manager + terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml new file mode 100644 index 00000000000..6aff8846340 --- /dev/null +++ b/spectro/generated/core-global.yaml @@ -0,0 +1,11125 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capi-webhook-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azureclusteridentities.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureClusterIdentity + listKind: AzureClusterIdentityList + plural: azureclusteridentities + singular: azureclusteridentity + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity. + properties: + allowedNamespaces: + description: "AllowedNamespaces is an array of namespaces that AzureClusters + can use this Identity from. \n An empty list (default) indicates + that AzureClusters can use this Identity from any namespace. This + field is intentionally not a pointer because the nil behavior (no + namespaces) is undesirable here." + items: + type: string + type: array + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + enum: + - ServicePrincipal + - UserAssignedMSI + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity. + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify the namespaces + the clusters are allowed to use the identity from. Namespaces can + be selected either using an array of namespaces or with label selector. + An empty allowedNamespaces object indicates that AzureClusters can + use this identity from any namespace. If this object is nil, no + namespaces will be allowed (default behaviour, if this field is + not provided) A namespace should be either in the NamespaceList + or match with Selector to use the identity. + nullable: true + properties: + list: + description: A nil or empty list indicates that AzureCluster cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: "Selector is a selector of namespaces that AzureCluster + can use this Identity from. This is a standard Kubernetes LabelSelector, + a label query over a set of resources. The result of matchLabels + and matchExpressions are ANDed. \n A nil or empty selector indicates + that AzureCluster cannot use this AzureClusterIdentity from + any namespace." + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + enum: + - ServicePrincipal + - UserAssignedMSI + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity. + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify the namespaces + the clusters are allowed to use the identity from. Namespaces can + be selected either using an array of namespaces or with label selector. + An empty allowedNamespaces object indicates that AzureClusters can + use this identity from any namespace. If this object is nil, no + namespaces will be allowed (default behaviour, if this field is + not provided) A namespace should be either in the NamespaceList + or match with Selector to use the identity. + nullable: true + properties: + list: + description: A nil or empty list indicates that AzureCluster cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: "Selector is a selector of namespaces that AzureCluster + can use this Identity from. This is a standard Kubernetes LabelSelector, + a label query over a set of resources. The result of matchLabels + and matchExpressions are ANDed. \n A nil or empty selector indicates + that AzureCluster cannot use this AzureClusterIdentity from + any namespace." + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + clientID: + description: ClientID is the service principal client ID. Both User + Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: ResourceID is the Azure resource ID for the User Assigned + MSI resource. Not currently supported. + type: string + tenantID: + description: TenantID is the service principal primary tenant id. + type: string + type: + description: Type is the type of Azure Identity used. ServicePrincipal, + ServicePrincipalCertificate, or ManualServicePrincipal. + enum: + - ServicePrincipal + - ManualServicePrincipal + - ServicePrincipalCertificate + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity. + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azureclusters.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureCluster + listKind: AzureClusterList + plural: azureclusters + singular: azurecluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this AzureCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .spec.resourceGroup + name: Resource Group + priority: 1 + type: string + - jsonPath: .spec.subscriptionID + name: SubscriptionID + priority: 1 + type: string + - jsonPath: .spec.location + name: Location + priority: 1 + type: string + - description: Control Plane Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureCluster is the Schema for the azureclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterSpec defines the desired state of AzureCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + identityRef: + description: IdentityRef is a reference to a AzureIdentity to be used + when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to Azure + network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + id: + type: string + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlock: + description: 'CidrBlock is the CIDR block to be used when + the provider creates a managed Vnet. Deprecated: Use CIDRBlocks + instead' + type: string + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID defines a unique identifier to reference + this resource. + type: string + internalLBIPAddress: + description: 'InternalLBIPAddress is the IP address that + will be used as the internal LB private IP. For the control + plane subnet only. Deprecated: Use LoadBalancer private + IP instead' + type: string + name: + description: Name defines a name for the subnet resource. + type: string + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + type: string + name: + type: string + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + type: string + ingressRule: + description: IngressRules is a slice of Azure ingress + rules for security groups. + items: + description: IngressRule defines an Azure ingress + rule for security groups. + properties: + description: + type: string + destination: + description: Destination - The destination address + prefix. CIDR or destination IP range. Asterix + '*' can also be used to match all source IPs. + Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts - The destination + port or range. Integer or range between 0 and + 65535. Asterix '*' can also be used to match + all ports. + type: string + name: + type: string + priority: + description: Priority - A number between 100 and + 4096. Each rule should have a unique value for + priority. Rules are processed in priority order, + with lower numbers processed before higher numbers. + Once traffic matches a rule, processing stops. + format: int32 + type: integer + protocol: + description: SecurityGroupProtocol defines the + protocol type for a security group rule. + type: string + source: + description: Source - The CIDR or source IP range. + Asterix '*' can also be used to match all source + IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. If this is an ingress rule, specifies + where network traffic originates from. + type: string + sourcePorts: + description: SourcePorts - The source port or + range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all ports. + type: string + required: + - description + - name + - protocol + type: object + type: array + name: + type: string + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - name + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual network. + properties: + cidrBlock: + description: 'CidrBlock is the CIDR block to be used when + the provider creates a managed virtual network. Deprecated: + Use CIDRBlocks instead' + type: string + cidrBlocks: + description: CIDRBlocks defines the virtual network's address + space, specified as one or more address prefixes in CIDR + notation. + items: + type: string + type: array + id: + description: ID is the identifier of the virtual network this + provider should use to create resources. + type: string + name: + description: Name defines a name for the virtual network resource. + type: string + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + required: + - name + type: object + type: object + resourceGroup: + type: string + subscriptionID: + type: string + required: + - location + type: object + status: + description: AzureClusterStatus defines the observed state of AzureCluster. + properties: + conditions: + description: Conditions defines current service state of the AzureCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: 'FailureDomains specifies the list of unique failure + domains for the location/region of the cluster. A FailureDomain + maps to Availability Zone with an Azure Region (if the region support + them). An Availability Zone is a separate data center within a region + and they can be used to ensure the cluster is more resilient to + failure. See: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview + This list will be used by Cluster API to try and spread the machines + across the failure domains.' + type: object + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AzureCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + priority: 1 + type: string + - jsonPath: .spec.resourceGroup + name: Resource Group + priority: 1 + type: string + - jsonPath: .spec.subscriptionID + name: SubscriptionID + priority: 1 + type: string + - jsonPath: .spec.location + name: Location + priority: 1 + type: string + - description: Control Plane Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureCluster is the Schema for the azureclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterSpec defines the desired state of AzureCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud to be + used. The default value that would be used by most users is "AzurePublicCloud", + other values are: - ChinaCloud: "AzureChinaCloud" - GermanCloud: + "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: + "AzureUSGovernmentCloud"' + type: string + bastionSpec: + description: BastionSpec encapsulates all things related to the Bastions + in the cluster. + properties: + azureBastion: + description: AzureBastion specifies how the Azure Bastion cloud + component should be configured. + properties: + name: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create an + Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + subnet: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID defines a unique identifier to reference + this resource. + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + type: string + name: + type: string + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination IP range. + Asterix '*' can also be used to match all + source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. + type: string + destinationPorts: + description: DestinationPorts specifies the + destination port or range. Integer or range + between 0 and 65535. Asterix '*' can also + be used to match all ports. + type: string + direction: + description: Direction indicates whether the + rule applies to inbound, or outbound traffic. + "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before + higher numbers. Once traffic matches a rule, + processing stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol + type. "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to + match all source IPs. Default tags such as + 'VirtualNetwork', 'AzureLoadBalancer' and + 'Internet' can also be used. If this is an + ingress rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all + ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - name + type: object + type: object + type: object + cloudProviderConfigOverrides: + description: 'CloudProviderConfigOverrides is an optional set of configuration + values that can be overridden in azure cloud provider config. This + is only a subset of options that are available in azure cloud provider + config. Some values for the cloud provider config are inferred from + other parts of cluster api provider azure spec, and may not be available + for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs + Note: All cloud provider config values can be customized by creating + the secret beforehand. CloudProviderConfigOverrides is only used + when the secret is managed by the Azure Provider.' + properties: + backOffs: + description: BackOffConfig indicates the back-off config options. + properties: + cloudProviderBackoff: + type: boolean + cloudProviderBackoffDuration: + type: integer + cloudProviderBackoffExponent: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffJitter: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffRetries: + type: integer + type: object + rateLimits: + items: + description: 'RateLimitSpec represents the rate limit configuration + for a particular kind of resource. Eg. loadBalancerRateLimit + is used to configure rate limits for load balancers. This + eventually gets converted to CloudProviderRateLimitConfig + that cloud-provider-azure expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 + We cannot use CloudProviderRateLimitConfig directly because + floating point values are not supported in controller-tools. + See: https://github.com/kubernetes-sigs/controller-tools/issues/245' + properties: + config: + description: RateLimitConfig indicates the rate limit config + options. + properties: + cloudProviderRateLimit: + type: boolean + cloudProviderRateLimitBucket: + type: integer + cloudProviderRateLimitBucketWrite: + type: integer + cloudProviderRateLimitQPS: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderRateLimitQPSWrite: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: Name is the name of the rate limit spec. + enum: + - defaultRateLimit + - routeRateLimit + - subnetsRateLimit + - interfaceRateLimit + - routeTableRateLimit + - loadBalancerRateLimit + - publicIPAddressRateLimit + - securityGroupRateLimit + - virtualMachineRateLimit + - storageAccountRateLimit + - diskRateLimit + - snapshotRateLimit + - virtualMachineScaleSetRateLimit + - virtualMachineSizesRateLimit + - availabilitySetRateLimit + type: string + type: object + type: array + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + identityRef: + description: IdentityRef is a reference to an AzureIdentity to be + used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to Azure + network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + controlPlaneOutboundLB: + description: ControlPlaneOutboundLB is the configuration for the + control-plane outbound load balancer. This is different from + APIServerLB, and is used only in private clusters (optionally) + for enabling outbound traffic. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + nodeOutboundLB: + description: NodeOutboundLB is the configuration for the node + outbound load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + privateDNSZoneName: + description: PrivateDNSZoneName defines the zone name for the + Azure Private DNS. + type: string + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID defines a unique identifier to reference + this resource. + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + type: string + name: + type: string + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination address + prefix. CIDR or destination IP range. Asterix + '*' can also be used to match all source IPs. + Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies the destination + port or range. Integer or range between 0 and + 65535. Asterix '*' can also be used to match + all ports. + type: string + direction: + description: Direction indicates whether the rule + applies to inbound, or outbound traffic. "Inbound" + or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before higher + numbers. Once traffic matches a rule, processing + stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol type. + "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to match + all source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. If this is an ingress rule, specifies + where network traffic originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - name + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual network. + properties: + cidrBlocks: + description: CIDRBlocks defines the virtual network's address + space, specified as one or more address prefixes in CIDR + notation. + items: + type: string + type: array + id: + description: ID is the identifier of the virtual network this + provider should use to create resources. + type: string + name: + description: Name defines a name for the virtual network resource. + type: string + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + required: + - name + type: object + type: object + resourceGroup: + type: string + subscriptionID: + type: string + required: + - location + type: object + status: + description: AzureClusterStatus defines the observed state of AzureCluster. + properties: + conditions: + description: Conditions defines current service state of the AzureCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: 'FailureDomains specifies the list of unique failure + domains for the location/region of the cluster. A FailureDomain + maps to Availability Zone with an Azure Region (if the region support + them). An Availability Zone is a separate data center within a region + and they can be used to ensure the cluster is more resilient to + failure. See: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview + This list will be used by Cluster API to try and spread the machines + across the failure domains.' + type: object + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AzureCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + priority: 1 + type: string + - jsonPath: .spec.resourceGroup + name: Resource Group + priority: 1 + type: string + - jsonPath: .spec.subscriptionID + name: SubscriptionID + priority: 1 + type: string + - jsonPath: .spec.location + name: Location + priority: 1 + type: string + - description: Control Plane Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureCluster is the Schema for the azureclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterSpec defines the desired state of AzureCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud to be + used. The default value that would be used by most users is "AzurePublicCloud", + other values are: - ChinaCloud: "AzureChinaCloud" - GermanCloud: + "AzureGermanCloud" - PublicCloud: "AzurePublicCloud" - USGovernmentCloud: + "AzureUSGovernmentCloud"' + type: string + bastionSpec: + description: BastionSpec encapsulates all things related to the Bastions + in the cluster. + properties: + azureBastion: + description: AzureBastion specifies how the Azure Bastion cloud + component should be configured. + properties: + name: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create an + Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + subnet: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID is the Azure resource ID of the subnet. + READ-ONLY + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + description: ID is the Azure resource ID of the NAT + gateway. READ-ONLY + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + enum: + - node + - control-plane + - bastion + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the route + table. READ-ONLY + type: string + name: + type: string + required: + - name + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the security + group. READ-ONLY + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination IP range. + Asterix '*' can also be used to match all + source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. + type: string + destinationPorts: + description: DestinationPorts specifies the + destination port or range. Integer or range + between 0 and 65535. Asterix '*' can also + be used to match all ports. + type: string + direction: + description: Direction indicates whether the + rule applies to inbound, or outbound traffic. + "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before + higher numbers. Once traffic matches a rule, + processing stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol + type. "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to + match all source IPs. Default tags such as + 'VirtualNetwork', 'AzureLoadBalancer' and + 'Internet' can also be used. If this is an + ingress rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all + ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + required: + - name + type: object + required: + - name + - role + type: object + type: object + type: object + cloudProviderConfigOverrides: + description: 'CloudProviderConfigOverrides is an optional set of configuration + values that can be overridden in azure cloud provider config. This + is only a subset of options that are available in azure cloud provider + config. Some values for the cloud provider config are inferred from + other parts of cluster api provider azure spec, and may not be available + for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs + Note: All cloud provider config values can be customized by creating + the secret beforehand. CloudProviderConfigOverrides is only used + when the secret is managed by the Azure Provider.' + properties: + backOffs: + description: BackOffConfig indicates the back-off config options. + properties: + cloudProviderBackoff: + type: boolean + cloudProviderBackoffDuration: + type: integer + cloudProviderBackoffExponent: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffJitter: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffRetries: + type: integer + type: object + rateLimits: + items: + description: 'RateLimitSpec represents the rate limit configuration + for a particular kind of resource. Eg. loadBalancerRateLimit + is used to configure rate limits for load balancers. This + eventually gets converted to CloudProviderRateLimitConfig + that cloud-provider-azure expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 + We cannot use CloudProviderRateLimitConfig directly because + floating point values are not supported in controller-tools. + See: https://github.com/kubernetes-sigs/controller-tools/issues/245' + properties: + config: + description: RateLimitConfig indicates the rate limit config + options. + properties: + cloudProviderRateLimit: + type: boolean + cloudProviderRateLimitBucket: + type: integer + cloudProviderRateLimitBucketWrite: + type: integer + cloudProviderRateLimitQPS: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderRateLimitQPSWrite: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: Name is the name of the rate limit spec. + enum: + - defaultRateLimit + - routeRateLimit + - subnetsRateLimit + - interfaceRateLimit + - routeTableRateLimit + - loadBalancerRateLimit + - publicIPAddressRateLimit + - securityGroupRateLimit + - virtualMachineRateLimit + - storageAccountRateLimit + - diskRateLimit + - snapshotRateLimit + - virtualMachineScaleSetRateLimit + - virtualMachineSizesRateLimit + - availabilitySetRateLimit + type: string + required: + - name + type: object + type: array + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. It is not recommended to set + this when creating an AzureCluster as CAPZ will set this for you. + However, if it is set, CAPZ will not change it. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + identityRef: + description: IdentityRef is a reference to an AzureIdentity to be + used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to Azure + network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + description: ID is the Azure resource ID of the load balancer. + READ-ONLY + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + controlPlaneOutboundLB: + description: ControlPlaneOutboundLB is the configuration for the + control-plane outbound load balancer. This is different from + APIServerLB, and is used only in private clusters (optionally) + for enabling outbound traffic. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + description: ID is the Azure resource ID of the load balancer. + READ-ONLY + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + nodeOutboundLB: + description: NodeOutboundLB is the configuration for the node + outbound load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + frontendIPsCount: + description: FrontendIPsCount specifies the number of frontend + IP addresses for the load balancer. + format: int32 + type: integer + id: + description: ID is the Azure resource ID of the load balancer. + READ-ONLY + type: string + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout for + the TCP idle connection. + format: int32 + type: integer + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object + privateDNSZoneName: + description: PrivateDNSZoneName defines the zone name for the + Azure Private DNS. + type: string + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + description: SubnetSpec configures an Azure subnet. + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address space, + specified as one or more address prefixes in CIDR notation. + items: + type: string + type: array + id: + description: ID is the Azure resource ID of the subnet. + READ-ONLY + type: string + name: + description: Name defines a name for the subnet resource. + type: string + natGateway: + description: NatGateway associated with this subnet. + properties: + id: + description: ID is the Azure resource ID of the NAT + gateway. READ-ONLY + type: string + ip: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. Node, ControlPlane) + enum: + - node + - control-plane + - bastion + type: string + routeTable: + description: RouteTable defines the route table that should + be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the route + table. READ-ONLY + type: string + name: + type: string + required: + - name + type: object + securityGroup: + description: SecurityGroup defines the NSG (network security + group) that should be attached to this subnet. + properties: + id: + description: ID is the Azure resource ID of the security + group. READ-ONLY + type: string + name: + type: string + securityRules: + description: SecurityRules is a slice of Azure security + rules for security groups. + items: + description: SecurityRule defines an Azure security + rule for security groups. + properties: + description: + description: A description for this rule. Restricted + to 140 chars. + type: string + destination: + description: Destination is the destination address + prefix. CIDR or destination IP range. Asterix + '*' can also be used to match all source IPs. + Default tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies the destination + port or range. Integer or range between 0 and + 65535. Asterix '*' can also be used to match + all ports. + type: string + direction: + description: Direction indicates whether the rule + applies to inbound, or outbound traffic. "Inbound" + or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within the + network security group. + type: string + priority: + description: Priority is a number between 100 + and 4096. Each rule should have a unique value + for priority. Rules are processed in priority + order, with lower numbers processed before higher + numbers. Once traffic matches a rule, processing + stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol type. + "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR or source + IP range. Asterix '*' can also be used to match + all source IPs. Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can also + be used. If this is an ingress rule, specifies + where network traffic originates from. + type: string + sourcePorts: + description: SourcePorts specifies source port + or range. Integer or range between 0 and 65535. + Asterix '*' can also be used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + required: + - name + type: object + required: + - name + - role + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual network. + properties: + cidrBlocks: + description: CIDRBlocks defines the virtual network's address + space, specified as one or more address prefixes in CIDR + notation. + items: + type: string + type: array + id: + description: ID is the Azure resource ID of the virtual network. + READ-ONLY + type: string + name: + description: Name defines a name for the virtual network resource. + type: string + peerings: + description: Peerings defines a list of peerings of the newly + created virtual network with existing virtual networks. + items: + description: VnetPeeringSpec specifies an existing remote + virtual network to peer with the AzureCluster's virtual + network. + properties: + remoteVnetName: + description: RemoteVnetName defines name of the remote + virtual network. + type: string + resourceGroup: + description: ResourceGroup is the resource group name + of the remote virtual network. + type: string + required: + - remoteVnetName + type: object + type: array + resourceGroup: + description: ResourceGroup is the name of the resource group + of the existing virtual network or the resource group where + a managed virtual network should be created. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + required: + - name + type: object + type: object + resourceGroup: + type: string + subscriptionID: + type: string + required: + - location + type: object + status: + description: AzureClusterStatus defines the observed state of AzureCluster. + properties: + conditions: + description: Conditions defines current service state of the AzureCluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: 'FailureDomains specifies the list of unique failure + domains for the location/region of the cluster. A FailureDomain + maps to Availability Zone with an Azure Region (if the region support + them). An Availability Zone is a separate data center within a region + and they can be used to ensure the cluster is more resilient to + failure. See: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview + This list will be used by Cluster API to try and spread the machines + across the failure domains.' + type: object + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azureclustertemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureClusterTemplate + listKind: AzureClusterTemplateList + plural: azureclustertemplates + singular: azureclustertemplate + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureClusterTemplate is the Schema for the azureclustertemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterTemplateSpec defines the desired state of AzureClusterTemplate. + properties: + template: + description: AzureClusterTemplateResource describes the data needed + to create an AzureCluster from a template. + properties: + spec: + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to Azure resources managed by the Azure provider, in + addition to the ones added by default. + type: object + azureEnvironment: + description: 'AzureEnvironment is the name of the AzureCloud + to be used. The default value that would be used by most + users is "AzurePublicCloud", other values are: - ChinaCloud: + "AzureChinaCloud" - GermanCloud: "AzureGermanCloud" - PublicCloud: + "AzurePublicCloud" - USGovernmentCloud: "AzureUSGovernmentCloud"' + type: string + bastionSpec: + description: BastionSpec encapsulates all things related to + the Bastions in the cluster. + properties: + azureBastion: + properties: + subnet: + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address + space, specified as one or more address prefixes + in CIDR notation. + items: + type: string + type: array + natGateway: + description: NatGateway associated with this subnet. + properties: + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. + Node, ControlPlane) + enum: + - node + - control-plane + - bastion + type: string + securityGroup: + description: SecurityGroup defines the NSG (network + security group) that should be attached to this + subnet. + properties: + securityRules: + description: SecurityRules is a slice of Azure + security rules for security groups. + items: + description: SecurityRule defines an Azure + security rule for security groups. + properties: + description: + description: A description for this + rule. Restricted to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination + IP range. Asterix '*' can also be + used to match all source IPs. Default + tags such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies + the destination port or range. Integer + or range between 0 and 65535. Asterix + '*' can also be used to match all + ports. + type: string + direction: + description: Direction indicates whether + the rule applies to inbound, or outbound + traffic. "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within + the network security group. + type: string + priority: + description: Priority is a number between + 100 and 4096. Each rule should have + a unique value for priority. Rules + are processed in priority order, with + lower numbers processed before higher + numbers. Once traffic matches a rule, + processing stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the + protocol type. "Tcp", "Udp", "Icmp", + or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR + or source IP range. Asterix '*' can + also be used to match all source IPs. + Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' + can also be used. If this is an ingress + rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source + port or range. Integer or range between + 0 and 65535. Asterix '*' can also + be used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - role + type: object + type: object + type: object + cloudProviderConfigOverrides: + description: 'CloudProviderConfigOverrides is an optional + set of configuration values that can be overridden in azure + cloud provider config. This is only a subset of options + that are available in azure cloud provider config. Some + values for the cloud provider config are inferred from other + parts of cluster api provider azure spec, and may not be + available for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs + Note: All cloud provider config values can be customized + by creating the secret beforehand. CloudProviderConfigOverrides + is only used when the secret is managed by the Azure Provider.' + properties: + backOffs: + description: BackOffConfig indicates the back-off config + options. + properties: + cloudProviderBackoff: + type: boolean + cloudProviderBackoffDuration: + type: integer + cloudProviderBackoffExponent: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffJitter: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderBackoffRetries: + type: integer + type: object + rateLimits: + items: + description: 'RateLimitSpec represents the rate limit + configuration for a particular kind of resource. Eg. + loadBalancerRateLimit is used to configure rate limits + for load balancers. This eventually gets converted + to CloudProviderRateLimitConfig that cloud-provider-azure + expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 + We cannot use CloudProviderRateLimitConfig directly + because floating point values are not supported in + controller-tools. See: https://github.com/kubernetes-sigs/controller-tools/issues/245' + properties: + config: + description: RateLimitConfig indicates the rate + limit config options. + properties: + cloudProviderRateLimit: + type: boolean + cloudProviderRateLimitBucket: + type: integer + cloudProviderRateLimitBucketWrite: + type: integer + cloudProviderRateLimitQPS: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cloudProviderRateLimitQPSWrite: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + name: + description: Name is the name of the rate limit + spec. + enum: + - defaultRateLimit + - routeRateLimit + - subnetsRateLimit + - interfaceRateLimit + - routeTableRateLimit + - loadBalancerRateLimit + - publicIPAddressRateLimit + - securityGroupRateLimit + - virtualMachineRateLimit + - storageAccountRateLimit + - diskRateLimit + - snapshotRateLimit + - virtualMachineScaleSetRateLimit + - virtualMachineSizesRateLimit + - availabilitySetRateLimit + type: string + required: + - name + type: object + type: array + type: object + identityRef: + description: IdentityRef is a reference to an AzureIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this + pod). This syntax is chosen only to have some well-defined + way of referencing a part of an object. TODO: this design + is not final and this field is subject to change in + the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + location: + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to + Azure network. + properties: + apiServerLB: + description: APIServerLB is the configuration for the + control-plane load balancer. + properties: + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout + for the TCP idle connection. + format: int32 + type: integer + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer + Type. + type: string + type: object + controlPlaneOutboundLB: + description: ControlPlaneOutboundLB is the configuration + for the control-plane outbound load balancer. This is + different from APIServerLB, and is used only in private + clusters (optionally) for enabling outbound traffic. + properties: + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout + for the TCP idle connection. + format: int32 + type: integer + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer + Type. + type: string + type: object + nodeOutboundLB: + description: NodeOutboundLB is the configuration for the + node outbound load balancer. + properties: + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes specifies the timeout + for the TCP idle connection. + format: int32 + type: integer + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer + Type. + type: string + type: object + privateDNSZoneName: + description: PrivateDNSZoneName defines the zone name + for the Azure Private DNS. + type: string + subnets: + description: Subnets is the configuration for the control-plane + subnet and the node subnet. + items: + properties: + cidrBlocks: + description: CIDRBlocks defines the subnet's address + space, specified as one or more address prefixes + in CIDR notation. + items: + type: string + type: array + natGateway: + description: NatGateway associated with this subnet. + properties: + name: + type: string + required: + - name + type: object + role: + description: Role defines the subnet role (eg. Node, + ControlPlane) + enum: + - node + - control-plane + - bastion + type: string + securityGroup: + description: SecurityGroup defines the NSG (network + security group) that should be attached to this + subnet. + properties: + securityRules: + description: SecurityRules is a slice of Azure + security rules for security groups. + items: + description: SecurityRule defines an Azure + security rule for security groups. + properties: + description: + description: A description for this rule. + Restricted to 140 chars. + type: string + destination: + description: Destination is the destination + address prefix. CIDR or destination + IP range. Asterix '*' can also be used + to match all source IPs. Default tags + such as 'VirtualNetwork', 'AzureLoadBalancer' + and 'Internet' can also be used. + type: string + destinationPorts: + description: DestinationPorts specifies + the destination port or range. Integer + or range between 0 and 65535. Asterix + '*' can also be used to match all ports. + type: string + direction: + description: Direction indicates whether + the rule applies to inbound, or outbound + traffic. "Inbound" or "Outbound". + enum: + - Inbound + - Outbound + type: string + name: + description: Name is a unique name within + the network security group. + type: string + priority: + description: Priority is a number between + 100 and 4096. Each rule should have + a unique value for priority. Rules are + processed in priority order, with lower + numbers processed before higher numbers. + Once traffic matches a rule, processing + stops. + format: int32 + type: integer + protocol: + description: Protocol specifies the protocol + type. "Tcp", "Udp", "Icmp", or "*". + enum: + - Tcp + - Udp + - Icmp + - '*' + type: string + source: + description: Source specifies the CIDR + or source IP range. Asterix '*' can + also be used to match all source IPs. + Default tags such as 'VirtualNetwork', + 'AzureLoadBalancer' and 'Internet' can + also be used. If this is an ingress + rule, specifies where network traffic + originates from. + type: string + sourcePorts: + description: SourcePorts specifies source + port or range. Integer or range between + 0 and 65535. Asterix '*' can also be + used to match all ports. + type: string + required: + - description + - direction + - name + - protocol + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags defines a map of tags. + type: object + type: object + required: + - role + type: object + type: array + vnet: + description: Vnet is the configuration for the Azure virtual + network. + properties: + cidrBlocks: + description: CIDRBlocks defines the virtual network's + address space, specified as one or more address + prefixes in CIDR notation. + items: + type: string + type: array + peerings: + description: Peerings defines a list of peerings of + the newly created virtual network with existing + virtual networks. + items: + properties: + remoteVnetName: + description: RemoteVnetName defines name of + the remote virtual network. + type: string + required: + - remoteVnetName + type: object + type: array + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing + the resource. + type: object + type: object + type: object + subscriptionID: + type: string + required: + - location + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved + controller-gen.kubebuilder.io/version: v0.5.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: azureidentities.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + names: + kind: AzureIdentity + listKind: AzureIdentityList + plural: azureidentities + singular: azureidentity + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AzureIdentity is the specification of the identity data structure. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureIdentitySpec describes the credential specifications + of an identity on Azure. + properties: + adEndpoint: + type: string + adResourceID: + description: For service principal. Option param for specifying the AD + details. + type: string + auxiliaryTenantIDs: + description: Service principal auxiliary tenant ids + items: + type: string + nullable: true + type: array + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientPassword: + description: Used for service principal + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + metadata: + type: object + replicas: + format: int32 + nullable: true + type: integer + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + type: integer + type: object + status: + description: AzureIdentityStatus contains the replica status of the resource. + properties: + availableReplicas: + format: int32 + type: integer + metadata: + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved + controller-gen.kubebuilder.io/version: v0.5.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: azureidentitybindings.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + names: + kind: AzureIdentityBinding + listKind: AzureIdentityBindingList + plural: azureidentitybindings + singular: azureidentitybinding + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AzureIdentityBinding brings together the spec of matching pods + and the identity which they can use. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureIdentityBindingSpec matches the pod with the Identity. + Used to indicate the potential matches to look for between the pod/deployment + and the identities present. + properties: + azureIdentity: + type: string + metadata: + type: object + selector: + type: string + weight: + description: Weight is used to figure out which of the matching identities + would be selected. + type: integer + type: object + status: + description: AzureIdentityBindingStatus contains the status of an AzureIdentityBinding. + properties: + availableReplicas: + format: int32 + type: integer + metadata: + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachinepoolmachines.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachinePoolMachine + listKind: AzureMachinePoolMachineList + plural: azuremachinepoolmachines + shortNames: + - ampm + singular: azuremachinepoolmachine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .status.version + name: Version + type: string + - description: Flag indicating infrastructure is successfully provisioned + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS VM provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePoolMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Azure VMSS VM ID + jsonPath: .spec.providerID + name: VMSS VM ID + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachinePoolMachine is the Schema for the azuremachinepoolmachines + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolMachineSpec defines the desired state of + AzureMachinePoolMachine. + properties: + instanceID: + description: InstanceID is the identification of the Machine Instance + within the VMSS + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + required: + - instanceID + - providerID + type: object + status: + description: AzureMachinePoolMachineStatus defines the observed state + of AzureMachinePoolMachine. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n Any transient errors that occur during the reconciliation of + MachinePools can be added as events to the MachinePool object and/or + logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool machine and will + contain a succinct value suitable for machine interpretation. \n + Any transient errors that occur during the reconciliation of MachinePools + can be added as events to the MachinePool object and/or logged in + the controller's output." + type: string + instanceName: + description: InstanceName is the name of the Machine Instance within + the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the image + version the VM is running. If the instance is not running the latest + model, it means the instance may not be running the version of Kubernetes + the Machine Pool has specified and needs to be updated. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + nodeRef: + description: NodeRef will point to the corresponding Node if it exists. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine instance. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + version: + description: Version defines the Kubernetes version for the VM Instance + type: string + required: + - latestModelApplied + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .status.version + name: Version + type: string + - description: Flag indicating infrastructure is successfully provisioned + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS VM provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePoolMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Azure VMSS VM ID + jsonPath: .spec.providerID + name: VMSS VM ID + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachinePoolMachine is the Schema for the azuremachinepoolmachines + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolMachineSpec defines the desired state of + AzureMachinePoolMachine. + properties: + instanceID: + description: InstanceID is the identification of the Machine Instance + within the VMSS + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + required: + - instanceID + - providerID + type: object + status: + description: AzureMachinePoolMachineStatus defines the observed state + of AzureMachinePoolMachine. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n Any transient errors that occur during the reconciliation of + MachinePools can be added as events to the MachinePool object and/or + logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool machine and will + contain a succinct value suitable for machine interpretation. \n + Any transient errors that occur during the reconciliation of MachinePools + can be added as events to the MachinePool object and/or logged in + the controller's output." + type: string + instanceName: + description: InstanceName is the name of the Machine Instance within + the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the image + version the VM is running. If the instance is not running the latest + model, it means the instance may not be running the version of Kubernetes + the Machine Pool has specified and needs to be updated. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + nodeRef: + description: NodeRef will point to the corresponding Node if it exists. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine instance. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + version: + description: Version defines the Kubernetes version for the VM Instance + type: string + required: + - latestModelApplied + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachinepools.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachinePool + listKind: AzureMachinePoolList + plural: azuremachinepools + shortNames: + - amp + singular: azuremachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: AzureMachinePool replicas count + jsonPath: .status.replicas + name: Replicas + type: string + - description: AzureMachinePool replicas count + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePool belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: MachinePool object to which this AzureMachinePool belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="MachinePool")].name + name: MachinePool + priority: 1 + type: string + - description: Azure VMSS ID + jsonPath: .spec.providerID + name: VMSS ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.template.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureMachinePool is the Schema for the azuremachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolSpec defines the desired state of AzureMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + identity: + default: None + description: Identity is the type of identity used for the Virtual + Machine Scale Set. The type 'SystemAssigned' is an implicitly created + identity. The generated identity will be assigned a Subscription + contributor role. The type 'UserAssigned' is a standalone Azure + resource provided by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + location: + description: Location is the Azure region location e.g. westus2 + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + template: + description: Template contains the details used to build a replica + virtual machine within the Machine Pool. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the + requested VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure + will return an error. + type: boolean + dataDisks: + description: DataDisks specifies the list of data disks to be + created for a Virtual Machine + items: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine. + properties: + cachingType: + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the + data disk. This value is used to identify data disks within + the VM and therefore must be unique for each data disk + attached to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to + the machine name to generate the disk name. Each disk + name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + image: + description: Image is used to provide details of an image to use + during Virtual Machine creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, which + is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the + Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify + 'latest' to use the latest version of an image available + at deploy time. Even if you use 'latest', the VM image + will not automatically update after deploy time even + if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + resourceGroup: + description: ResourceGroup specifies the resource group + containing the shared image gallery + minLength: 1 + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk contains the operating system disk information + for a Virtual Machine + properties: + cachingType: + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + osType: + type: string + required: + - diskSizeGB + - managedDisk + - osType + type: object + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine or virtual + machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + description: SSHPublicKey is the SSH public key string base64 + encoded to add to a Virtual Machine + type: string + terminateNotificationTimeout: + description: TerminateNotificationTimeout enables or disables + VMSS scheduled events termination notification with specified + timeout allowed values are between 5 and 15 (mins) + type: integer + vmSize: + description: VMSize is the size of the Virtual Machine to build. + See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachinePool. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + required: + - location + - template + type: object + status: + description: AzureMachinePoolStatus defines the observed state of AzureMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + instances: + description: Instances is the VM instance status for each VM in the + VMSS + items: + description: AzureMachinePoolInstanceStatus provides status information + for each instance in the VMSS. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within the VMSS + type: string + instanceName: + description: InstanceName is the name of the Machine Instance + within the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the + image version the VM is running. If the instance is not running + the latest model, it means the instance may not be running + the version of Kubernetes the Machine Pool has specified and + needs to be updated. + type: boolean + providerID: + description: ProviderID is the provider identification of the + VMSS Instance + type: string + provisioningState: + description: ProvisioningState is the provisioning state of + the Azure virtual machine instance. + type: string + version: + description: Version defines the Kubernetes version for the + VM Instance + type: string + required: + - latestModelApplied + type: object + type: array + longRunningOperationState: + description: LongRunningOperationState saves the state for an Azure + long running operations so it can be continued on the next reconciliation + loop. + properties: + futureData: + description: FutureData is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + type: + description: Type describes the type of future, update, create, + delete, etc. + type: string + required: + - type + type: object + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + version: + description: Version is the Kubernetes version for the current VMSS + model + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AzureMachinePool replicas count + jsonPath: .status.replicas + name: Replicas + type: string + - description: AzureMachinePool replicas count + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePool belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: MachinePool object to which this AzureMachinePool belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="MachinePool")].name + name: MachinePool + priority: 1 + type: string + - description: Azure VMSS ID + jsonPath: .spec.providerID + name: VMSS ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.template.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachinePool is the Schema for the azuremachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolSpec defines the desired state of AzureMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + identity: + default: None + description: Identity is the type of identity used for the Virtual + Machine Scale Set. The type 'SystemAssigned' is an implicitly created + identity. The generated identity will be assigned a Subscription + contributor role. The type 'UserAssigned' is a standalone Azure + resource provided by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + location: + description: Location is the Azure region location e.g. westus2 + type: string + nodeDrainTimeout: + description: 'NodeDrainTimeout is the total amount of time that the + controller will spend on draining a node. The default value is 0, + meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`' + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + strategy: + default: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + description: The deployment strategy to use to replace existing AzureMachinePoolMachines + with new ones. + properties: + rollingUpdate: + description: Rolling update config params. Present only if MachineDeploymentStrategyType + = RollingUpdate. + properties: + deletePolicy: + default: Oldest + description: DeletePolicy defines the policy used by the MachineDeployment + to identify nodes to delete when downscaling. Valid values + are "Random, "Newest", "Oldest" When no value is supplied, + the default is Oldest + enum: + - Random + - Newest + - Oldest + type: string + maxSurge: + anyOf: + - type: integer + - type: string + default: 1 + description: 'The maximum number of machines that can be scheduled + above the desired number of machines. Value can be an absolute + number (ex: 5) or a percentage of desired machines (ex: + 10%). This can not be 0 if MaxUnavailable is 0. Absolute + number is calculated from percentage by rounding up. Defaults + to 1. Example: when this is set to 30%, the new MachineSet + can be scaled up immediately when the rolling update starts, + such that the total number of old and new machines do not + exceed 130% of desired machines. Once old machines have + been killed, new MachineSet can be scaled up further, ensuring + that total number of machines running at any time during + the update is at most 130% of desired machines.' + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + default: 0 + description: 'The maximum number of machines that can be unavailable + during the update. Value can be an absolute number (ex: + 5) or a percentage of desired machines (ex: 10%). Absolute + number is calculated from percentage by rounding down. This + can not be 0 if MaxSurge is 0. Defaults to 0. Example: when + this is set to 30%, the old MachineSet can be scaled down + to 70% of desired machines immediately when the rolling + update starts. Once new machines are ready, old MachineSet + can be scaled down further, followed by scaling up the new + MachineSet, ensuring that the total number of machines available + at all times during the update is at least 70% of desired + machines.' + x-kubernetes-int-or-string: true + type: object + type: + default: RollingUpdate + description: Type of deployment. Currently the only supported + strategy is RollingUpdate + enum: + - RollingUpdate + type: string + type: object + template: + description: Template contains the details used to build a replica + virtual machine within the Machine Pool + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the + requested VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure + will return an error. + type: boolean + dataDisks: + description: DataDisks specifies the list of data disks to be + created for a Virtual Machine + items: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the + data disk. This value is used to identify data disks within + the VM and therefore must be unique for each data disk + attached to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to + the machine name to generate the disk name. Each disk + name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + image: + description: Image is used to provide details of an image to use + during VM creation. If image details are omitted the image will + default the Azure Marketplace "capi" offer, which is based on + Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the + Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify + 'latest' to use the latest version of an image available + at deploy time. Even if you use 'latest', the VM image + will not automatically update after deploy time even + if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used to add + a `Plan` in the API request when creating the VM/VMSS + resource. This is needed when the source image from + which this SIG image was built requires the `Plan` to + be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group + containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk contains the operating system disk information + for a Virtual Machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine or virtual + machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + description: SSHPublicKey is the SSH public key string base64 + encoded to add to a Virtual Machine + type: string + subnetName: + description: SubnetName selects the Subnet where the VMSS will + be placed + type: string + terminateNotificationTimeout: + description: TerminateNotificationTimeout enables or disables + VMSS scheduled events termination notification with specified + timeout allowed values are between 5 and 15 (mins) + type: integer + vmSize: + description: VMSize is the size of the Virtual Machine to build. + See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachinePool. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + required: + - location + - template + type: object + status: + description: AzureMachinePoolStatus defines the observed state of AzureMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + image: + description: Image is the current image used in the AzureMachinePool. + When the spec image is nil, this image is populated with the details + of the defaulted Azure Marketplace "capi" offer. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from an Azure + Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + instances: + description: Instances is the VM instance status for each VM in the + VMSS + items: + description: AzureMachinePoolInstanceStatus provides status information + for each instance in the VMSS. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within the VMSS + type: string + instanceName: + description: InstanceName is the name of the Machine Instance + within the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the + image version the VM is running. If the instance is not running + the latest model, it means the instance may not be running + the version of Kubernetes the Machine Pool has specified and + needs to be updated. + type: boolean + providerID: + description: ProviderID is the provider identification of the + VMSS Instance + type: string + provisioningState: + description: ProvisioningState is the provisioning state of + the Azure virtual machine instance. + type: string + version: + description: Version defines the Kubernetes version for the + VM Instance + type: string + required: + - latestModelApplied + type: object + type: array + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + version: + description: Version is the Kubernetes version for the current VMSS + model + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AzureMachinePool replicas count + jsonPath: .status.replicas + name: Replicas + type: string + - description: AzureMachinePool replicas count + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VMSS provisioning state + jsonPath: .status.provisioningState + name: State + type: string + - description: Cluster to which this AzureMachinePool belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: MachinePool object to which this AzureMachinePool belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="MachinePool")].name + name: MachinePool + priority: 1 + type: string + - description: Azure VMSS ID + jsonPath: .spec.providerID + name: VMSS ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.template.vmSize + name: VM Size + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachinePool is the Schema for the azuremachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachinePoolSpec defines the desired state of AzureMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + identity: + default: None + description: Identity is the type of identity used for the Virtual + Machine Scale Set. The type 'SystemAssigned' is an implicitly created + identity. The generated identity will be assigned a Subscription + contributor role. The type 'UserAssigned' is a standalone Azure + resource provided by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + location: + description: Location is the Azure region location e.g. westus2 + type: string + nodeDrainTimeout: + description: 'NodeDrainTimeout is the total amount of time that the + controller will spend on draining a node. The default value is 0, + meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`' + type: string + providerID: + description: ProviderID is the identification ID of the Virtual Machine + Scale Set + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + strategy: + default: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + description: The deployment strategy to use to replace existing AzureMachinePoolMachines + with new ones. + properties: + rollingUpdate: + description: Rolling update config params. Present only if MachineDeploymentStrategyType + = RollingUpdate. + properties: + deletePolicy: + default: Oldest + description: DeletePolicy defines the policy used by the MachineDeployment + to identify nodes to delete when downscaling. Valid values + are "Random, "Newest", "Oldest" When no value is supplied, + the default is Oldest + enum: + - Random + - Newest + - Oldest + type: string + maxSurge: + anyOf: + - type: integer + - type: string + default: 1 + description: 'The maximum number of machines that can be scheduled + above the desired number of machines. Value can be an absolute + number (ex: 5) or a percentage of desired machines (ex: + 10%). This can not be 0 if MaxUnavailable is 0. Absolute + number is calculated from percentage by rounding up. Defaults + to 1. Example: when this is set to 30%, the new MachineSet + can be scaled up immediately when the rolling update starts, + such that the total number of old and new machines do not + exceed 130% of desired machines. Once old machines have + been killed, new MachineSet can be scaled up further, ensuring + that total number of machines running at any time during + the update is at most 130% of desired machines.' + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + default: 0 + description: 'The maximum number of machines that can be unavailable + during the update. Value can be an absolute number (ex: + 5) or a percentage of desired machines (ex: 10%). Absolute + number is calculated from percentage by rounding down. This + can not be 0 if MaxSurge is 0. Defaults to 0. Example: when + this is set to 30%, the old MachineSet can be scaled down + to 70% of desired machines immediately when the rolling + update starts. Once new machines are ready, old MachineSet + can be scaled down further, followed by scaling up the new + MachineSet, ensuring that the total number of machines available + at all times during the update is at least 70% of desired + machines.' + x-kubernetes-int-or-string: true + type: object + type: + default: RollingUpdate + description: Type of deployment. Currently the only supported + strategy is RollingUpdate + enum: + - RollingUpdate + type: string + type: object + template: + description: Template contains the details used to build a replica + virtual machine within the Machine Pool + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the + requested VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure + will return an error. + type: boolean + dataDisks: + description: DataDisks specifies the list of data disks to be + created for a Virtual Machine + items: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the + data disk. This value is used to identify data disks within + the VM and therefore must be unique for each data disk + attached to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to + the machine name to generate the disk name. Each disk + name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + image: + description: Image is used to provide details of an image to use + during VM creation. If image details are omitted the image will + default the Azure Marketplace "capi" offer, which is based on + Ubuntu. + properties: + computeGallery: + description: ComputeGallery specifies an image to use from + the Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource group + containing the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the + Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify + 'latest' to use the latest version of an image available + at deploy time. Even if you use 'latest', the VM image + will not automatically update after deploy time even + if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use from + an Azure Shared Image Gallery Deprecated: use ComputeGallery + instead.' + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used to add + a `Plan` in the API request when creating the VM/VMSS + resource. This is needed when the source image from + which this SIG image was built requires the `Plan` to + be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group + containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. + This is needed when the source image from which this + SIG image was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an image + available at deploy time. Even if you use 'latest', + the VM image will not automatically update after deploy + time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk contains the operating system disk information + for a Virtual Machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the + OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine or virtual + machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + description: SSHPublicKey is the SSH public key string base64 + encoded to add to a Virtual Machine + type: string + subnetName: + description: SubnetName selects the Subnet where the VMSS will + be placed + type: string + terminateNotificationTimeout: + description: TerminateNotificationTimeout enables or disables + VMSS scheduled events termination notification with specified + timeout allowed values are between 5 and 15 (mins) + type: integer + vmSize: + description: VMSize is the size of the Virtual Machine to build. + See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachinePool. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + required: + - location + - template + type: object + status: + description: AzureMachinePoolStatus defines the observed state of AzureMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + image: + description: Image is the current image used in the AzureMachinePool. + When the spec image is nil, this image is populated with the details + of the defaulted Azure Marketplace "capi" offer. + properties: + computeGallery: + description: ComputeGallery specifies an image to use from the + Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource group containing + the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use from an + Azure Shared Image Gallery Deprecated: use ComputeGallery instead.' + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + instances: + description: Instances is the VM instance status for each VM in the + VMSS + items: + description: AzureMachinePoolInstanceStatus provides status information + for each instance in the VMSS. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within the VMSS + type: string + instanceName: + description: InstanceName is the name of the Machine Instance + within the VMSS + type: string + latestModelApplied: + description: LatestModelApplied indicates the instance is running + the most up-to-date VMSS model. A VMSS model describes the + image version the VM is running. If the instance is not running + the latest model, it means the instance may not be running + the version of Kubernetes the Machine Pool has specified and + needs to be updated. + type: boolean + providerID: + description: ProviderID is the provider identification of the + VMSS Instance + type: string + provisioningState: + description: ProvisioningState is the provisioning state of + the Azure virtual machine instance. + type: string + version: + description: Version defines the Kubernetes version for the + VM Instance + type: string + required: + - latestModelApplied + type: object + type: array + longRunningOperationStates: + description: LongRunningOperationStates saves the state for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + provisioningState: + description: ProvisioningState is the provisioning state of the Azure + virtual machine. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + version: + description: Version is the Kubernetes version for the current VMSS + model + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachines.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachine + listKind: AzureMachineList + plural: azuremachines + singular: azuremachine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: AzureMachine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VM provisioning state + jsonPath: .status.vmState + name: State + type: string + - description: Cluster to which this AzureMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Machine object to which this AzureMachine belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + priority: 1 + type: string + - description: Azure VM ID + jsonPath: .spec.providerID + name: VM ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureMachine is the Schema for the azuremachines API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineSpec defines the desired state of AzureMachine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the requested + VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure will + return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create dynamic + public ips for machines where this value is true. + type: boolean + availabilityZone: + description: 'Deprecated: use FailureDomain instead' + properties: + enabled: + type: boolean + id: + type: string + type: object + dataDisks: + description: DataDisk specifies the parameters that are used to add + one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are used to + add one or more data disks to the machine. + properties: + cachingType: + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the data + disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the data + disk. This value is used to identify data disks within the + VM and therefore must be unique for each data disk attached + to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options for + a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to the + machine name to generate the disk name. Each disk name will + be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure which + is required for some CNI's to send traffic from a pods on one machine + to another. This is required for IpV6 with Calico in combination + with User Defined Routes (set by the Azure Cloud Controller manager). + Default is false for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. This + relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the virtual + machine. The type 'SystemAssigned' is an implicitly created identity. + The generated identity will be assigned a Subscription contributor + role. The type 'UserAssigned' is a standalone Azure resource provided + by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image to use during + VM creation. If image details are omitted the image will default + the Azure Marketplace "capi" offer, which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from an Azure + Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + location: + description: 'Deprecated: to support old clients, will be removed + in v1alpha4/v1beta1' + type: string + osDisk: + description: OSDisk specifies the parameters for the operating system + disk of the machine + properties: + cachingType: + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options for + a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + osType: + type: string + required: + - diskSizeGB + - managedDisk + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption should + be enabled or disabled for a virtual machine or virtual machine + scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM. + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachine. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - location + - osDisk + - sshPublicKey + - vmSize + type: object + status: + description: AzureMachineStatus defines the observed state of AzureMachine. + properties: + addresses: + description: Addresses contains the Azure instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AzureMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + vmState: + description: VMState is the provisioning state of the Azure virtual + machine. + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AzureMachine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Azure VM provisioning state + jsonPath: .status.vmState + name: State + type: string + - description: Cluster to which this AzureMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Machine object to which this AzureMachine belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + priority: 1 + type: string + - description: Azure VM ID + jsonPath: .spec.providerID + name: VM ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.vmSize + name: VM Size + priority: 1 + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachine is the Schema for the azuremachines API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineSpec defines the desired state of AzureMachine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the requested + VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure will + return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create dynamic + public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used to add + one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are used to + add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the data + disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the data + disk. This value is used to identify data disks within the + VM and therefore must be unique for each data disk attached + to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to the + machine name to generate the disk name. Each disk name will + be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure which + is required for some CNI's to send traffic from a pods on one machine + to another. This is required for IpV6 with Calico in combination + with User Defined Routes (set by the Azure Cloud Controller manager). + Default is false for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. This + relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the virtual + machine. The type 'SystemAssigned' is an implicitly created identity. + The generated identity will be assigned a Subscription contributor + role. The type 'UserAssigned' is a standalone Azure resource provided + by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image to use during + VM creation. If image details are omitted the image will default + the Azure Marketplace "capi" offer, which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from an Azure + Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating system + disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the OS + disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption should + be enabled or disabled for a virtual machine or virtual machine + scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachine. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + status: + description: AzureMachineStatus defines the observed state of AzureMachine. + properties: + addresses: + description: Addresses contains the Azure instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AzureMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + vmState: + description: VMState is the provisioning state of the Azure virtual + machine. + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + priority: 1 + type: string + - description: Azure VM provisioning state + jsonPath: .status.vmState + name: State + type: string + - description: Cluster to which this AzureMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + priority: 1 + type: string + - description: Machine object to which this AzureMachine belongs + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + priority: 1 + type: string + - description: Azure VM ID + jsonPath: .spec.providerID + name: VM ID + priority: 1 + type: string + - description: Azure VM Size + jsonPath: .spec.vmSize + name: VM Size + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachine is the Schema for the azuremachines API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineSpec defines the desired state of AzureMachine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure accelerated + networking. If omitted, it will be set based on whether the requested + VMSize supports accelerated networking. If AcceleratedNetworking + is set to true with a VMSize that does not support it, Azure will + return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the Azure + provider. If both the AzureCluster and the AzureMachine specify + the same tag name with different values, the AzureMachine's value + takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create dynamic + public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used to add + one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are used to + add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the data + disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of the data + disk. This value is used to identify data disks within the + VM and therefore must be unique for each data disk attached + to a VM. The value must be between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended to the + machine name to generate the disk name. Each disk name will + be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure which + is required for some CNI's to send traffic from a pods on one machine + to another. This is required for IpV6 with Calico in combination + with User Defined Routes (set by the Azure Cloud Controller manager). + Default is false for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. This + relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the virtual + machine. The type 'SystemAssigned' is an implicitly created identity. + The generated identity will be assigned a Subscription contributor + role. The type 'UserAssigned' is a standalone Azure resource provided + by the user and assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image to use during + VM creation. If image details are omitted the image will default + the Azure Marketplace "capi" offer, which is based on Ubuntu. + properties: + computeGallery: + description: ComputeGallery specifies an image to use from the + Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such + as a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource group containing + the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from the Azure + Marketplace + properties: + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization that + created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is published + by a third party publisher and a Plan will be generated + for it. + type: boolean + version: + description: Version specifies the version of an image sku. + The allowed formats are Major.Minor.Build or 'latest'. Major, + Minor, and Build are decimal numbers. Specify 'latest' to + use the latest version of an image available at deploy time. + Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use from an + Azure Shared Image Gallery Deprecated: use ComputeGallery instead.' + properties: + gallery: + description: Gallery specifies the name of the shared image + gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of related + images created by the publisher. For example, UbuntuServer, + WindowsServer This value will be used to add a `Plan` in + the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization that + created the image. This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource group containing + the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, such as + a major release of a distribution. For example, 18.04-LTS, + 2019-Datacenter This value will be used to add a `Plan` + in the API request when creating the VM/VMSS resource. This + is needed when the source image from which this SIG image + was built requires the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the subscription + that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the marketplace + image. The allowed formats are Major.Minor.Build or 'latest'. + Major, Minor, and Build are decimal numbers. Specify 'latest' + to use the latest version of an image available at deploy + time. Even if you use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating system + disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk settings + for the os disk. + properties: + option: + description: Option enables ephemeral OS when set to "Local" + See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to the OS + disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk encryption + options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any valid GUID. + If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile settings + for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption should + be enabled or disabled for a virtual machine or virtual machine + scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the Machine + should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user The lifecycle of a user-assigned + identity is managed separately from the lifecycle of the AzureMachine. + See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + status: + description: AzureMachineStatus defines the observed state of AzureMachine. + properties: + addresses: + description: Addresses contains the Azure instance associated addresses. + items: + description: NodeAddress contains information for the node's address. + properties: + address: + description: The node address. + type: string + type: + description: Node address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AzureMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "ErrorMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "ErrorReason will be set in the event that there is a + terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + vmState: + description: VMState is the provisioning state of the Azure virtual + machine. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremachinetemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureMachineTemplate + listKind: AzureMachineTemplateList + plural: azuremachinetemplates + singular: azuremachinetemplate + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureMachineTemplate is the Schema for the azuremachinetemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineTemplateSpec defines the desired state of AzureMachineTemplate. + properties: + template: + description: AzureMachineTemplateResource describes the data needed + to create an AzureMachine from a template. + properties: + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure + accelerated networking. If omitted, it will be set based + on whether the requested VMSize supports accelerated networking. + If AcceleratedNetworking is set to true with a VMSize that + does not support it, Azure will return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the Azure provider. If both the AzureCluster and the + AzureMachine specify the same tag name with different values, + the AzureMachine's value takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create + dynamic public ips for machines where this value is true. + type: boolean + availabilityZone: + description: 'Deprecated: use FailureDomain instead' + properties: + enabled: + type: boolean + id: + type: string + type: object + dataDisks: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are + used to add one or more data disks to the machine. + properties: + cachingType: + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign + to the data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of + the data disk. This value is used to identify data + disks within the VM and therefore must be unique for + each data disk attached to a VM. The value must be + between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines + disk encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended + to the machine name to generate the disk name. Each + disk name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure + which is required for some CNI's to send traffic from a + pods on one machine to another. This is required for IpV6 + with Calico in combination with User Defined Routes (set + by the Azure Cloud Controller manager). Default is false + for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. This relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the + virtual machine. The type 'SystemAssigned' is an implicitly + created identity. The generated identity will be assigned + a Subscription contributor role. The type 'UserAssigned' + is a standalone Azure resource provided by the user and + assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image + to use during VM creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, + which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from + the Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is + published by a third party publisher and a Plan + will be generated for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an + image available at deploy time. Even if you use + 'latest', the VM image will not automatically update + after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + resourceGroup: + description: ResourceGroup specifies the resource + group containing the shared image gallery + minLength: 1 + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + location: + description: 'Deprecated: to support old clients, will be + removed in v1alpha4/v1beta1' + type: string + osDisk: + description: OSDisk specifies the parameters for the operating + system disk of the machine + properties: + cachingType: + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk + settings for the os disk. + properties: + option: + description: Option enables ephemeral OS when set + to "Local" See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + format: int32 + type: integer + managedDisk: + description: ManagedDisk defines the managed disk options + for a VM. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + required: + - storageAccountType + type: object + osType: + type: string + required: + - diskSizeGB + - managedDisk + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any + valid GUID. If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile + settings for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine + or virtual machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the + Machine should use a Spot VM. + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone + Azure identities provided by the user The lifecycle of a + user-assigned identity is managed separately from the lifecycle + of the AzureMachine. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned + identities provided by the user to be assigned to Azure + resources. + properties: + providerID: + description: 'ProviderID is the identification ID of + the user-assigned Identity, the format of an identity + is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - location + - osDisk + - sshPublicKey + - vmSize + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureMachineTemplate is the Schema for the azuremachinetemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineTemplateSpec defines the desired state of AzureMachineTemplate. + properties: + template: + description: AzureMachineTemplateResource describes the data needed + to create an AzureMachine from a template. + properties: + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure + accelerated networking. If omitted, it will be set based + on whether the requested VMSize supports accelerated networking. + If AcceleratedNetworking is set to true with a VMSize that + does not support it, Azure will return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the Azure provider. If both the AzureCluster and the + AzureMachine specify the same tag name with different values, + the AzureMachine's value takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create + dynamic public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are + used to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign + to the data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of + the data disk. This value is used to identify data + disks within the VM and therefore must be unique for + each data disk attached to a VM. The value must be + between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk + parameters for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines + disk encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended + to the machine name to generate the disk name. Each + disk name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure + which is required for some CNI's to send traffic from a + pods on one machine to another. This is required for IpV6 + with Calico in combination with User Defined Routes (set + by the Azure Cloud Controller manager). Default is false + for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. This relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the + virtual machine. The type 'SystemAssigned' is an implicitly + created identity. The generated identity will be assigned + a Subscription contributor role. The type 'UserAssigned' + is a standalone Azure resource provided by the user and + assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image + to use during VM creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, + which is based on Ubuntu. + properties: + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from + the Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is + published by a third party publisher and a Plan + will be generated for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an + image available at deploy time. Even if you use + 'latest', the VM image will not automatically update + after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: SharedGallery specifies an image to use from + an Azure Shared Image Gallery + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource + group containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating + system disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk + settings for the os disk. + properties: + option: + description: Option enables ephemeral OS when set + to "Local" See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to + the OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any + valid GUID. If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile + settings for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine + or virtual machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the + Machine should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will + be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone + Azure identities provided by the user The lifecycle of a + user-assigned identity is managed separately from the lifecycle + of the AzureMachine. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned + identities provided by the user to be assigned to Azure + resources. + properties: + providerID: + description: 'ProviderID is the identification ID of + the user-assigned Identity, the format of an identity + is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureMachineTemplate is the Schema for the azuremachinetemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureMachineTemplateSpec defines the desired state of AzureMachineTemplate. + properties: + template: + description: AzureMachineTemplateResource describes the data needed + to create an AzureMachine from a template. + properties: + metadata: + description: "ObjectMeta is metadata that all persisted resources + must have, which includes all objects users must create. This + is a copy of customizable fields from metav1.ObjectMeta. \n + ObjectMeta is embedded in `Machine.Spec`, `MachineDeployment.Template` + and `MachineSet.Template`, which are not top-level Kubernetes + objects. Given that metav1.ObjectMeta has lots of special cases + and read-only fields which end up in the generated CRD validation, + having it as a subset simplifies the API and some issues that + can impact user experience. \n During the [upgrade to controller-tools@v2](https://github.com/kubernetes-sigs/cluster-api/pull/1054) + for v1alpha2, we noticed a failure would occur running Cluster + API test suite against the new CRDs, specifically `spec.metadata.creationTimestamp + in body must be of type string: \"null\"`. The investigation + showed that `controller-tools@v2` behaves differently than its + previous version when handling types from [metav1](k8s.io/apimachinery/pkg/apis/meta/v1) + package. \n In more details, we found that embedded (non-top + level) types that embedded `metav1.ObjectMeta` had validation + properties, including for `creationTimestamp` (metav1.Time). + The `metav1.Time` type specifies a custom json marshaller that, + when IsZero() is true, returns `null` which breaks validation + because the field isn't marked as nullable. \n In future versions, + controller-tools@v2 might allow overriding the type and validation + for embedded types. When that happens, this hack should be revisited." + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + acceleratedNetworking: + description: AcceleratedNetworking enables or disables Azure + accelerated networking. If omitted, it will be set based + on whether the requested VMSize supports accelerated networking. + If AcceleratedNetworking is set to true with a VMSize that + does not support it, Azure will return an error. + type: boolean + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the Azure provider. If both the AzureCluster and the + AzureMachine specify the same tag name with different values, + the AzureMachine's value takes precedence. + type: object + allocatePublicIP: + description: AllocatePublicIP allows the ability to create + dynamic public ips for machines where this value is true. + type: boolean + dataDisks: + description: DataDisk specifies the parameters that are used + to add one or more data disks to the machine + items: + description: DataDisk specifies the parameters that are + used to add one or more data disks to the machine. + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diskSizeGB: + description: DiskSizeGB is the size in GB to assign + to the data disk. + format: int32 + type: integer + lun: + description: Lun Specifies the logical unit number of + the data disk. This value is used to identify data + disks within the VM and therefore must be unique for + each data disk attached to a VM. The value must be + between 0 and 63. + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk + parameters for the data disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines + disk encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + nameSuffix: + description: NameSuffix is the suffix to be appended + to the machine name to generate the disk name. Each + disk name will be in format _. + type: string + required: + - diskSizeGB + - nameSuffix + type: object + type: array + enableIPForwarding: + description: EnableIPForwarding enables IP Forwarding in Azure + which is required for some CNI's to send traffic from a + pods on one machine to another. This is required for IpV6 + with Calico in combination with User Defined Routes (set + by the Azure Cloud Controller manager). Default is false + for disabled. + type: boolean + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. This relates to an Azure Availability Zone + type: string + identity: + default: None + description: Identity is the type of identity used for the + virtual machine. The type 'SystemAssigned' is an implicitly + created identity. The generated identity will be assigned + a Subscription contributor role. The type 'UserAssigned' + is a standalone Azure resource provided by the user and + assigned to the VM + enum: + - None + - SystemAssigned + - UserAssigned + type: string + image: + description: Image is used to provide details of an image + to use during VM creation. If image details are omitted + the image will default the Azure Marketplace "capi" offer, + which is based on Ubuntu. + properties: + computeGallery: + description: ComputeGallery specifies an image to use + from the Azure Compute Gallery + properties: + gallery: + description: Gallery specifies the name of the compute + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + plan: + description: Plan contains plan information. + properties: + offer: + description: Offer specifies the name of a group + of related images created by the publisher. + For example, UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For + example, 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + required: + - offer + - publisher + - sku + type: object + resourceGroup: + description: ResourceGroup specifies the resource + group containing the private compute gallery. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the private compute gallery. + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - version + type: object + id: + description: ID specifies an image to use by ID + type: string + marketplace: + description: Marketplace specifies an image to use from + the Azure Marketplace + properties: + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer + minLength: 1 + type: string + publisher: + description: Publisher is the name of the organization + that created the image + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter + minLength: 1 + type: string + thirdPartyImage: + default: false + description: ThirdPartyImage indicates the image is + published by a third party publisher and a Plan + will be generated for it. + type: boolean + version: + description: Version specifies the version of an image + sku. The allowed formats are Major.Minor.Build or + 'latest'. Major, Minor, and Build are decimal numbers. + Specify 'latest' to use the latest version of an + image available at deploy time. Even if you use + 'latest', the VM image will not automatically update + after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - offer + - publisher + - sku + - version + type: object + sharedGallery: + description: 'SharedGallery specifies an image to use + from an Azure Shared Image Gallery Deprecated: use ComputeGallery + instead.' + properties: + gallery: + description: Gallery specifies the name of the shared + image gallery that contains the image + minLength: 1 + type: string + name: + description: Name is the name of the image + minLength: 1 + type: string + offer: + description: Offer specifies the name of a group of + related images created by the publisher. For example, + UbuntuServer, WindowsServer This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + publisher: + description: Publisher is the name of the organization + that created the image. This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + resourceGroup: + description: ResourceGroup specifies the resource + group containing the shared image gallery + minLength: 1 + type: string + sku: + description: SKU specifies an instance of an offer, + such as a major release of a distribution. For example, + 18.04-LTS, 2019-Datacenter This value will be used + to add a `Plan` in the API request when creating + the VM/VMSS resource. This is needed when the source + image from which this SIG image was built requires + the `Plan` to be used. + type: string + subscriptionID: + description: SubscriptionID is the identifier of the + subscription that contains the shared image gallery + minLength: 1 + type: string + version: + description: Version specifies the version of the + marketplace image. The allowed formats are Major.Minor.Build + or 'latest'. Major, Minor, and Build are decimal + numbers. Specify 'latest' to use the latest version + of an image available at deploy time. Even if you + use 'latest', the VM image will not automatically + update after deploy time even if a new version becomes + available. + minLength: 1 + type: string + required: + - gallery + - name + - resourceGroup + - subscriptionID + - version + type: object + type: object + osDisk: + description: OSDisk specifies the parameters for the operating + system disk of the machine + properties: + cachingType: + description: CachingType specifies the caching requirements. + enum: + - None + - ReadOnly + - ReadWrite + type: string + diffDiskSettings: + description: DiffDiskSettings describe ephemeral disk + settings for the os disk. + properties: + option: + description: Option enables ephemeral OS when set + to "Local" See https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks + for full details + enum: + - Local + type: string + required: + - option + type: object + diskSizeGB: + description: DiskSizeGB is the size in GB to assign to + the OS disk. Will have a default of 30GB if not provided + format: int32 + type: integer + managedDisk: + description: ManagedDisk specifies the Managed Disk parameters + for the OS disk. + properties: + diskEncryptionSet: + description: DiskEncryptionSetParameters defines disk + encryption options. + properties: + id: + description: ID defines resourceID for diskEncryptionSet + resource. It must be in the same subscription + type: string + type: object + storageAccountType: + type: string + type: object + osType: + type: string + required: + - osType + type: object + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + roleAssignmentName: + description: RoleAssignmentName is the name of the role assignment + to create for a system assigned identity. It can be any + valid GUID. If not specified, a random GUID will be generated. + type: string + securityProfile: + description: SecurityProfile specifies the Security profile + settings for a virtual machine. + properties: + encryptionAtHost: + description: This field indicates whether Host Encryption + should be enabled or disabled for a virtual machine + or virtual machine scale set. Default is disabled. + type: boolean + type: object + spotVMOptions: + description: SpotVMOptions allows the ability to specify the + Machine should use a Spot VM + properties: + maxPrice: + anyOf: + - type: integer + - type: string + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + sshPublicKey: + type: string + subnetName: + description: SubnetName selects the Subnet where the VM will + be placed + type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone + Azure identities provided by the user The lifecycle of a + user-assigned identity is managed separately from the lifecycle + of the AzureMachine. See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli + items: + description: UserAssignedIdentity defines the user-assigned + identities provided by the user to be assigned to Azure + resources. + properties: + providerID: + description: 'ProviderID is the identification ID of + the user-assigned Identity, the format of an identity + is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array + vmSize: + type: string + required: + - osDisk + - sshPublicKey + - vmSize + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremanagedclusters.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedCluster + listKind: AzureManagedClusterList + plural: azuremanagedclusters + shortNames: + - amc + singular: azuremanagedcluster + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureManagedCluster is the Schema for the azuremanagedclusters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterSpec defines the desired state of AzureManagedCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: AzureManagedClusterStatus defines the observed state of AzureManagedCluster. + properties: + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureManagedCluster is the Schema for the azuremanagedclusters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterSpec defines the desired state of AzureManagedCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: AzureManagedClusterStatus defines the observed state of AzureManagedCluster. + properties: + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedCluster is the Schema for the azuremanagedclusters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedClusterSpec defines the desired state of AzureManagedCluster. + properties: + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + type: object + status: + description: AzureManagedClusterStatus defines the observed state of AzureManagedCluster. + properties: + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedControlPlane + listKind: AzureManagedControlPlaneList + plural: azuremanagedcontrolplanes + shortNames: + - amcp + singular: azuremanagedcontrolplane + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureManagedControlPlane is the Schema for the azuremanagedcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneSpec defines the desired state of + AzureManagedControlPlane. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration to + integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs that will + have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the Kubernetes + DNS service. It must be within the Kubernetes service address range + specified in serviceCidr. + type: string + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer to be + provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical Azure + region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + nodeResourceGroupName: + description: NodeResourceGroupName is the name of the resource group + containing cluster IaaS resources. Will be populated to default + in webhook. + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource group + for this AKS Cluster. + type: string + sshPublicKey: + description: SSHPublicKey is a string literal containing an ssh public + key base64 encoded. + type: string + subscriptionID: + description: SubscriotionID is the GUID of the Azure subscription + to hold this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS cluster. + Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - nodeResourceGroupName + - resourceGroupName + - sshPublicKey + - version + type: object + status: + description: AzureManagedControlPlaneStatus defines the observed state + of AzureManagedControlPlane. + properties: + initialized: + description: Initialized is true when the the control plane is available + for initial contact. This may occur before the control plane is + fully ready. In the AzureManagedControlPlane implementation, these + are identical. + type: boolean + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureManagedControlPlane is the Schema for the azuremanagedcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneSpec defines the desired state of + AzureManagedControlPlane. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration to + integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs that will + have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + apiServerAccessProfile: + description: APIServerAccessProfile is the access profile for AKS + API server. + properties: + authorizedIPRanges: + description: AuthorizedIPRanges - Authorized IP Ranges to kubernetes + API server. + items: + type: string + type: array + enablePrivateCluster: + description: EnablePrivateCluster - Whether to create the cluster + as a private cluster or not. + type: boolean + enablePrivateClusterPublicFQDN: + description: EnablePrivateClusterPublicFQDN - Whether to create + additional public FQDN for private cluster or not. + type: boolean + privateDNSZone: + description: PrivateDNSZone - Private dns zone mode for private + cluster. + enum: + - System + - None + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the Kubernetes + DNS service. It must be within the Kubernetes service address range + specified in serviceCidr. + type: string + identityRef: + description: IdentityRef is a reference to a AzureClusterIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + loadBalancerProfile: + description: LoadBalancerProfile is the profile of the cluster load + balancer. + properties: + allocatedOutboundPorts: + description: AllocatedOutboundPorts - Desired number of allocated + SNAT ports per VM. Allowed values must be in the range of 0 + to 64000 (inclusive). The default value is 0 which results in + Azure dynamically allocating ports. + format: int32 + type: integer + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes - Desired outbound flow idle + timeout in minutes. Allowed values must be in the range of 4 + to 120 (inclusive). The default value is 30 minutes. + format: int32 + type: integer + managedOutboundIPs: + description: ManagedOutboundIPs - Desired managed outbound IPs + for the cluster load balancer. + format: int32 + type: integer + outboundIPPrefixes: + description: OutboundIPPrefixes - Desired outbound IP Prefix resources + for the cluster load balancer. + items: + type: string + type: array + outboundIPs: + description: OutboundIPs - Desired outbound IP resources for the + cluster load balancer. + items: + type: string + type: array + type: object + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer to be + provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical Azure + region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + nodeResourceGroupName: + description: NodeResourceGroupName is the name of the resource group + containing cluster IaaS resources. Will be populated to default + in webhook. + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource group + for this AKS Cluster. + type: string + sku: + description: SKU is the SKU of the AKS to be provisioned. + properties: + tier: + description: Tier - Tier of a managed cluster SKU. + enum: + - Free + - Paid + type: string + required: + - tier + type: object + sshPublicKey: + description: SSHPublicKey is a string literal containing an ssh public + key base64 encoded. + type: string + subscriptionID: + description: SubscriptionID is the GUID of the Azure subscription + to hold this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS cluster. + Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - resourceGroupName + - sshPublicKey + - version + type: object + status: + description: AzureManagedControlPlaneStatus defines the observed state + of AzureManagedControlPlane. + properties: + initialized: + description: Initialized is true when the the control plane is available + for initial contact. This may occur before the control plane is + fully ready. In the AzureManagedControlPlane implementation, these + are identical. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedControlPlane is the Schema for the azuremanagedcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedControlPlaneSpec defines the desired state of + AzureManagedControlPlane. + properties: + aadProfile: + description: AadProfile is Azure Active Directory configuration to + integrate with AKS for aad authentication. + properties: + adminGroupObjectIDs: + description: AdminGroupObjectIDs - AAD group object IDs that will + have admin role of the cluster. + items: + type: string + type: array + managed: + description: Managed - Whether to enable managed AAD. + type: boolean + required: + - adminGroupObjectIDs + - managed + type: object + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to Azure + resources managed by the Azure provider, in addition to the ones + added by default. + type: object + addonProfiles: + description: AddonProfiles are the profiles of managed cluster add-on. + items: + properties: + config: + additionalProperties: + type: string + description: Config - Key-value pairs for configuring an add-on. + type: object + enabled: + description: Enabled - Whether the add-on is enabled or not. + type: boolean + name: + description: Name- The name of managed cluster add-on. + type: string + required: + - enabled + - name + type: object + type: array + apiServerAccessProfile: + description: APIServerAccessProfile is the access profile for AKS + API server. + properties: + authorizedIPRanges: + description: AuthorizedIPRanges - Authorized IP Ranges to kubernetes + API server. + items: + type: string + type: array + enablePrivateCluster: + description: EnablePrivateCluster - Whether to create the cluster + as a private cluster or not. + type: boolean + enablePrivateClusterPublicFQDN: + description: EnablePrivateClusterPublicFQDN - Whether to create + additional public FQDN for private cluster or not. + type: boolean + privateDNSZone: + description: PrivateDNSZone - Private dns zone mode for private + cluster. + enum: + - System + - None + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + dnsServiceIP: + description: DNSServiceIP is an IP address assigned to the Kubernetes + DNS service. It must be within the Kubernetes service address range + specified in serviceCidr. + type: string + identityRef: + description: IdentityRef is a reference to a AzureClusterIdentity + to be used when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + loadBalancerProfile: + description: LoadBalancerProfile is the profile of the cluster load + balancer. + properties: + allocatedOutboundPorts: + description: AllocatedOutboundPorts - Desired number of allocated + SNAT ports per VM. Allowed values must be in the range of 0 + to 64000 (inclusive). The default value is 0 which results in + Azure dynamically allocating ports. + format: int32 + type: integer + idleTimeoutInMinutes: + description: IdleTimeoutInMinutes - Desired outbound flow idle + timeout in minutes. Allowed values must be in the range of 4 + to 120 (inclusive). The default value is 30 minutes. + format: int32 + type: integer + managedOutboundIPs: + description: ManagedOutboundIPs - Desired managed outbound IPs + for the cluster load balancer. + format: int32 + type: integer + outboundIPPrefixes: + description: OutboundIPPrefixes - Desired outbound IP Prefix resources + for the cluster load balancer. + items: + type: string + type: array + outboundIPs: + description: OutboundIPs - Desired outbound IP resources for the + cluster load balancer. + items: + type: string + type: array + type: object + loadBalancerSKU: + description: LoadBalancerSKU is the SKU of the loadBalancer to be + provisioned. + enum: + - Basic + - Standard + type: string + location: + description: 'Location is a string matching one of the canonical Azure + region names. Examples: "westus2", "eastus".' + type: string + networkPlugin: + description: NetworkPlugin used for building Kubernetes network. + enum: + - azure + - kubenet + type: string + networkPolicy: + description: NetworkPolicy used for building Kubernetes network. + enum: + - azure + - calico + type: string + nodeResourceGroupName: + description: NodeResourceGroupName is the name of the resource group + containing cluster IaaS resources. Will be populated to default + in webhook. + type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource group + for this AKS Cluster. + type: string + sku: + description: SKU is the SKU of the AKS to be provisioned. + properties: + tier: + description: Tier - Tier of a managed cluster SKU. + enum: + - Free + - Paid + type: string + required: + - tier + type: object + sshPublicKey: + description: SSHPublicKey is a string literal containing an ssh public + key base64 encoded. + type: string + subscriptionID: + description: SubscriptionID is the GUID of the Azure subscription + to hold this cluster. + type: string + version: + description: Version defines the desired Kubernetes version. + minLength: 2 + type: string + virtualNetwork: + description: VirtualNetwork describes the vnet for the AKS cluster. + Will be created if it does not exist. + properties: + cidrBlock: + type: string + name: + type: string + subnet: + description: ManagedControlPlaneSubnet describes a subnet for + an AKS cluster. + properties: + cidrBlock: + type: string + name: + type: string + required: + - cidrBlock + - name + type: object + required: + - cidrBlock + - name + type: object + required: + - location + - resourceGroupName + - sshPublicKey + - version + type: object + status: + description: AzureManagedControlPlaneStatus defines the observed state + of AzureManagedControlPlane. + properties: + conditions: + description: Conditions defines current service state of the AzureManagedControlPlane. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + initialized: + description: Initialized is true when the control plane is available + for initial contact. This may occur before the control plane is + fully ready. In the AzureManagedControlPlane implementation, these + are identical. + type: boolean + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + controller-gen.kubebuilder.io/version: v0.8.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + cluster.x-k8s.io/v1beta1: v1beta1 + name: azuremanagedmachinepools.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureManagedMachinePool + listKind: AzureManagedMachinePoolList + plural: azuremanagedmachinepools + shortNames: + - ammp + singular: azuremanagedmachinepool + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureManagedMachinePool is the Schema for the azuremanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolSpec defines the desired state of + AzureManagedMachinePool. + properties: + mode: + description: 'Mode - represents mode of an agent pool. Possible values + include: System, User.' + enum: + - System + - User + type: string + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine in this + agent pool. If you specify 0, it will apply the default osDisk size + according to the vmSize specified. + format: int32 + type: integer + providerIDList: + description: ProviderIDList is the unique identifier as specified + by the cloud provider. + items: + type: string + type: array + sku: + description: SKU is the size of the VMs in the node pool. + type: string + required: + - mode + - sku + type: object + status: + description: AzureManagedMachinePoolStatus defines the observed state + of AzureManagedMachinePool. + properties: + errorMessage: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + errorReason: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AzureManagedMachinePool is the Schema for the azuremanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolSpec defines the desired state of + AzureManagedMachinePool. + properties: + mode: + description: 'Mode - represents mode of an agent pool. Possible values + include: System, User.' + enum: + - System + - User + type: string + name: + description: Name - name of the agent pool. If not specified, CAPZ + uses the name of the CR as the agent pool name. + type: string + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine in this + agent pool. If you specify 0, it will apply the default osDisk size + according to the vmSize specified. + format: int32 + type: integer + providerIDList: + description: ProviderIDList is the unique identifier as specified + by the cloud provider. + items: + type: string + type: array + sku: + description: SKU is the size of the VMs in the node pool. + type: string + required: + - mode + - sku + type: object + status: + description: AzureManagedMachinePoolStatus defines the observed state + of AzureManagedMachinePool. + properties: + errorMessage: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + errorReason: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.mode + name: Mode + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AzureManagedMachinePool is the Schema for the azuremanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureManagedMachinePoolSpec defines the desired state of + AzureManagedMachinePool. + properties: + availabilityZones: + description: AvailabilityZones - Availability zones for nodes. Must + use VirtualMachineScaleSets AgentPoolType. + items: + type: string + type: array + enableUltraSSD: + description: EnableUltraSSD enables the storage type UltraSSD_LRS + for the agent pool. + type: boolean + maxPods: + description: MaxPods specifies the kubelet --max-pods configuration + for the node pool. + format: int32 + type: integer + mode: + description: 'Mode - represents mode of an agent pool. Possible values + include: System, User.' + enum: + - System + - User + type: string + name: + description: Name - name of the agent pool. If not specified, CAPZ + uses the name of the CR as the agent pool name. + type: string + nodeLabels: + additionalProperties: + type: string + description: Node labels - labels for all of the nodes present in + node pool + type: object + osDiskSizeGB: + description: OSDiskSizeGB is the disk size for every machine in this + agent pool. If you specify 0, it will apply the default osDisk size + according to the vmSize specified. + format: int32 + type: integer + osDiskType: + default: Managed + description: OsDiskType specifies the OS disk type for each node in + the pool. Allowed values are 'Ephemeral' and 'Managed'. + enum: + - Ephemeral + - Managed + type: string + providerIDList: + description: ProviderIDList is the unique identifier as specified + by the cloud provider. + items: + type: string + type: array + scaling: + description: Scaling specifies the autoscaling parameters for the + node pool. + properties: + maxSize: + format: int32 + type: integer + minSize: + format: int32 + type: integer + type: object + sku: + description: SKU is the size of the VMs in the node pool. + type: string + taints: + description: Taints specifies the taints for nodes present in this + agent pool. + items: + properties: + effect: + description: Effect specifies the effect for the taint + enum: + - NoSchedule + - NoExecute + - PreferNoSchedule + type: string + key: + description: Key is the key of the taint + type: string + value: + description: Value is the value of the taint + type: string + required: + - effect + - key + - value + type: object + type: array + required: + - mode + - sku + type: object + status: + description: AzureManagedMachinePoolStatus defines the observed state + of AzureManagedMachinePool. + properties: + conditions: + description: Conditions defines current service state of the AzureManagedControlPlane. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + errorMessage: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + errorReason: + description: Any transient errors that occur during the reconciliation + of Machines can be added as events to the Machine object and/or + logged in the controller's output. + type: string + longRunningOperationStates: + description: LongRunningOperationStates saves the states for Azure + long-running operations so they can be continued on the next reconciliation + loop. + items: + description: Future contains the data needed for an Azure long-running + operation to continue across reconcile loops. + properties: + data: + description: Data is the base64 url encoded json Azure AutoRest + Future. + type: string + name: + description: Name is the name of the Azure resource. Together + with the service name, this forms the unique identifier for + the future. + type: string + resourceGroup: + description: ResourceGroup is the Azure resource group for the + resource. + type: string + serviceName: + description: ServiceName is the name of the Azure service. Together + with the name of the resource, this forms the unique identifier + for the future. + type: string + type: + description: Type describes the type of future, such as update, + create, delete, etc. + type: string + required: + - data + - name + - serviceName + - type + type: object + type: array + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved + controller-gen.kubebuilder.io/version: v0.5.0 + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: azurepodidentityexceptions.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + names: + kind: AzurePodIdentityException + listKind: AzurePodIdentityExceptionList + plural: azurepodidentityexceptions + singular: azurepodidentityexception + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: AzurePodIdentityException contains the pod selectors for all + pods that don't require NMI to process and request token on their behalf. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzurePodIdentityExceptionSpec matches pods with the selector + defined. If request originates from a pod that matches the selector, + nmi will proxy the request and send response back without any validation. + properties: + metadata: + type: object + podLabels: + additionalProperties: + type: string + type: object + type: object + status: + description: AzurePodIdentityExceptionStatus contains the status of an + AzurePodIdentityException. + properties: + metadata: + type: object + status: + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-aad-pod-id-nmi-role +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiGroups: + - aadpodidentity.k8s.io + resources: + - azureidentitybindings + - azureidentities + - azurepodidentityexceptions + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + k8s-app: capz-aad-pod-id-nmi-binding + name: capz-aad-pod-id-nmi-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capz-aad-pod-id-nmi-role +subjects: +- kind: ServiceAccount + name: default + namespace: capi-webhook-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-webhook-service + namespace: capi-webhook-system +spec: + ports: + - port: 443 + targetPort: webhook-server + selector: + cluster.x-k8s.io/provider: infrastructure-azure +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + name: capz-controller-manager + namespace: capi-webhook-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-logs-container: manager + labels: + aadpodidbinding: capz-controller-aadpodidentity-selector + cluster.x-k8s.io/provider: infrastructure-azure + control-plane: capz-controller-manager + spec: + containers: + - args: + - --leader-elect + - --metrics-bind-addr=localhost:8080 + - --feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false} + - --v=0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: capz-webhook-service-cert +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + component: nmi + k8s-app: aad-pod-id + tier: node + name: capz-nmi + namespace: capi-webhook-system +spec: + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-azure + component: nmi + tier: node + template: + metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + component: nmi + tier: node + spec: + containers: + - args: + - --node=$(NODE_NAME) + - --operation-mode=managed + - --forceNamespaced + - --http-probe-port=8085 + env: + - name: FORCENAMESPACED + value: "true" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LOG_LEVEL + value: DEBUG + image: mcr.microsoft.com/oss/azure/aad-pod-identity/nmi:v1.8.6 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8085 + initialDelaySeconds: 10 + periodSeconds: 5 + name: nmi + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + securityContext: + capabilities: + add: + - DAC_READ_SEARCH + - NET_ADMIN + - NET_RAW + drop: + - ALL + runAsUser: 0 + volumeMounts: + - mountPath: /run/xtables.lock + name: iptableslock + - mountPath: /etc/default/kubelet + name: kubelet-config + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + volumes: + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: iptableslock + - hostPath: + path: /etc/default/kubelet + type: FileOrCreate + name: kubelet-config + updateStrategy: + type: RollingUpdate +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-serving-cert + namespace: capi-webhook-system +spec: + dnsNames: + - capz-webhook-service.capi-webhook-system.svc + - capz-webhook-service.capi-webhook-system.svc.cluster.local + issuerRef: + kind: Issuer + name: capz-selfsigned-issuer + secretName: capz-webhook-service-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-selfsigned-issuer + namespace: capi-webhook-system +spec: + selfSigned: {} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azurecluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azureclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool + failurePolicy: Fail + name: default.azuremachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepools + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane + failurePolicy: Fail + name: default.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedmachinepools + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azurecluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azureclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool + failurePolicy: Fail + name: validation.azuremachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepools + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepoolmachine + failurePolicy: Fail + name: azuremachinepoolmachine.kb.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepoolmachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcluster + failurePolicy: Fail + name: validation.azuremanagedclusters.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + resources: + - azuremanagedclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane + failurePolicy: Fail + name: validation.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - UPDATE + - DELETE + resources: + - azuremanagedmachinepools + sideEffects: None diff --git a/spectro/global/kustomization.yaml b/spectro/global/kustomization.yaml new file mode 100644 index 00000000000..d0b54f2f58a --- /dev/null +++ b/spectro/global/kustomization.yaml @@ -0,0 +1,135 @@ +namespace: capi-webhook-system + +namePrefix: capz- + +# Labels to add to all resources and selectors. +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-azure" + +resources: + - ../../config/default/namespace.yaml + - ../../config/default/aad-pod-identity-deployment.yaml + +bases: + - ../../config/crd + - ../../config/webhook + - ../../config/manager + - ../../config/certmanager + +patchesStrategicMerge: + - ../../config/default/manager_image_patch.yaml + - ../../config/default/manager_pull_policy.yaml + - ../../config/default/manager_webhook_patch.yaml + - ../../config/default/webhookcainjection_patch.yaml + +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service + +configurations: + - ../../config/default/kustomizeconfig.yaml + +patchesJson6902: + - target: + kind: Namespace + name: system + version: v1 + path: patch_namespace.yaml + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_service_account.yaml + - target: + group: apps + kind: DaemonSet + name: capz-nmi + namespace: capi-webhook-system + version: v1 + path: patch_service_account.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azureclusteridentities.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azureclusters.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremachines.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremachinetemplates.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremachinepools.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremachinepoolmachines.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremanagedclusters.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml +# - target: +# group: apiextensions.k8s.io +# version: v1 +# kind: CustomResourceDefinition +# name: azuremanagedmachinepools.infrastructure.cluster.x-k8s.io +# path: patch_crd_webhook_namespace.yaml + - target: + group: rbac.authorization.k8s.io + version: v1 + kind: ClusterRoleBinding + name: capz-aad-pod-id-nmi-binding + path: patch_nmi_cluster_role_binding.yaml \ No newline at end of file diff --git a/spectro/global/patch_crd_webhook_namespace.yaml b/spectro/global/patch_crd_webhook_namespace.yaml new file mode 100644 index 00000000000..e40df94ba2c --- /dev/null +++ b/spectro/global/patch_crd_webhook_namespace.yaml @@ -0,0 +1,3 @@ +- op: replace + path: "/spec/conversion/webhook/clientConfig/service/namespace" + value: capi-webhook-system \ No newline at end of file diff --git a/spectro/global/patch_namespace.yaml b/spectro/global/patch_namespace.yaml new file mode 100644 index 00000000000..0de288003c7 --- /dev/null +++ b/spectro/global/patch_namespace.yaml @@ -0,0 +1,3 @@ +- op: replace + path: "/metadata/name" + value: capi-webhook-system \ No newline at end of file diff --git a/spectro/global/patch_nmi_cluster_role_binding.yaml b/spectro/global/patch_nmi_cluster_role_binding.yaml new file mode 100644 index 00000000000..e854a54e527 --- /dev/null +++ b/spectro/global/patch_nmi_cluster_role_binding.yaml @@ -0,0 +1,6 @@ +- op: replace + path: "/subjects/0/namespace" + value: capi-webhook-system +- op: replace + path: "/subjects/0/name" + value: default \ No newline at end of file diff --git a/spectro/global/patch_service_account.yaml b/spectro/global/patch_service_account.yaml new file mode 100644 index 00000000000..99d7e697b56 --- /dev/null +++ b/spectro/global/patch_service_account.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/serviceAccountName" \ No newline at end of file diff --git a/spectro/run.sh b/spectro/run.sh new file mode 100755 index 00000000000..31df3583833 --- /dev/null +++ b/spectro/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +rm generated/* + +kustomize build --load-restrictor LoadRestrictionsNone global > ./generated/core-global.yaml +kustomize build --load-restrictor LoadRestrictionsNone base > ./generated/core-base.yaml From 2cadf396487299c24498c973e6694fa35e6a2405 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Fri, 1 Jul 2022 15:33:26 +0530 Subject: [PATCH 02/66] logic to separate webhook and controller as separate pods --- main.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index e6be45517ce..cd3c07bf7ca 100644 --- a/main.go +++ b/main.go @@ -224,7 +224,7 @@ func InitFlags(fs *pflag.FlagSet) { fs.IntVar(&webhookPort, "webhook-port", - 9443, + 0, "Webhook Server port, disabled by default. When enabled, the manager will only work as webhook server, no reconcilers are installed.", ) @@ -309,9 +309,11 @@ func main() { os.Exit(1) } - registerControllers(ctx, mgr) - - registerWebhooks(mgr) + if webhookPort == 0 { + registerControllers(ctx, mgr) + } else { + registerWebhooks(mgr) + } // +kubebuilder:scaffold:builder setupLog.Info("starting manager", "version", version.Get().String()) @@ -326,6 +328,7 @@ func registerControllers(ctx context.Context, mgr manager.Manager) { if err != nil { setupLog.Error(err, "failed to build machineCache ReconcileCache") } + setupLog.V(0).Info("registerControllers") if err := controllers.NewAzureMachineReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("azuremachine-reconciler"), reconcileTimeout, @@ -472,6 +475,7 @@ func registerControllers(ctx context.Context, mgr manager.Manager) { } func registerWebhooks(mgr manager.Manager) { + setupLog.V(0).Info("registerWebhooks") if err := (&infrav1beta1.AzureCluster{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "AzureCluster") os.Exit(1) From b7a0d1bb9e2242d7782ad07b6475f34bb770c295 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Mon, 4 Jul 2022 15:11:33 +0530 Subject: [PATCH 03/66] fix for BET-3719 --- .../azuremanagedmachinepool_webhook.go | 59 ------------------- 1 file changed, 59 deletions(-) diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go index 17259f2ff8d..b02d778defe 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -17,19 +17,16 @@ limitations under the License. package v1beta1 import ( - "context" "fmt" "reflect" "github.com/Azure/go-autorest/autorest/to" - "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/util/maps" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -124,16 +121,6 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c "field is immutable")) } - if m.Spec.Mode != string(NodePoolModeSystem) && old.Spec.Mode == string(NodePoolModeSystem) { - // validate for last system node pool - if err := m.validateLastSystemNodePool(client); err != nil { - allErrs = append(allErrs, field.Invalid( - field.NewPath("Spec", "Mode"), - m.Spec.Mode, - "Last system node pool cannot be mutated to user node pool")) - } - } - if old.Spec.MaxPods != nil { // Prevent MaxPods modification if it was already set to some value if m.Spec.MaxPods == nil { @@ -207,52 +194,6 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. func (m *AzureManagedMachinePool) ValidateDelete(client client.Client) error { - if m.Spec.Mode != string(NodePoolModeSystem) { - return nil - } - - return errors.Wrapf(m.validateLastSystemNodePool(client), "if the delete is triggered via owner MachinePool please refer to trouble shooting section in https://capz.sigs.k8s.io/topics/managedcluster.html") -} - -// validateLastSystemNodePool is used to check if the existing system node pool is the last system node pool. -// If it is a last system node pool it cannot be deleted or mutated to user node pool as AKS expects min 1 system node pool. -func (m *AzureManagedMachinePool) validateLastSystemNodePool(cli client.Client) error { - ctx := context.Background() - - // Fetch the Cluster. - clusterName, ok := m.Labels[clusterv1.ClusterLabelName] - if !ok { - return nil - } - - ownerCluster := &clusterv1.Cluster{} - key := client.ObjectKey{ - Namespace: m.Namespace, - Name: clusterName, - } - - if err := cli.Get(ctx, key, ownerCluster); err != nil { - return err - } - - if !ownerCluster.DeletionTimestamp.IsZero() { - return nil - } - - opt1 := client.InNamespace(m.Namespace) - opt2 := client.MatchingLabels(map[string]string{ - clusterv1.ClusterLabelName: clusterName, - LabelAgentPoolMode: string(NodePoolModeSystem), - }) - - ammpList := &AzureManagedMachinePoolList{} - if err := cli.List(ctx, ammpList, opt1, opt2); err != nil { - return err - } - - if len(ammpList.Items) <= 1 { - return errors.New("AKS Cluster must have at least one system pool") - } return nil } From e9fa2469f807983859360a8739a2647d7bfdf614 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Mon, 4 Jul 2022 15:44:15 +0530 Subject: [PATCH 04/66] fix for BET-3963 --- azure/services/agentpools/agentpools.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 549aba02d0a..9c4b438d940 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -95,12 +95,6 @@ func (s *Service) Reconcile(ctx context.Context) error { return errors.Wrap(err, "failed to create or update agent pool") } } else { - ps := *existingPool.ManagedClusterAgentPoolProfileProperties.ProvisioningState - if ps != string(infrav1alpha4.Canceled) && ps != string(infrav1alpha4.Failed) && ps != string(infrav1alpha4.Succeeded) { - msg := fmt.Sprintf("Unable to update existing agent pool in non terminal state. Agent pool must be in one of the following provisioning states: canceled, failed, or succeeded. Actual state: %s", ps) - log.V(2).Info(msg) - return azure.WithTransientError(errors.New(msg), 20*time.Second) - } // Normalize individual agent pools to diff in case we need to update existingProfile := containerservice.AgentPool{ @@ -129,6 +123,12 @@ func (s *Service) Reconcile(ctx context.Context) error { diff := cmp.Diff(normalizedProfile, existingProfile) if diff != "" { log.V(2).Info(fmt.Sprintf("Update required (+new -old):\n%s", diff)) + ps := *existingPool.ManagedClusterAgentPoolProfileProperties.ProvisioningState + if ps != string(infrav1alpha4.Canceled) && ps != string(infrav1alpha4.Failed) && ps != string(infrav1alpha4.Succeeded) { + msg := fmt.Sprintf("Unable to update existing agent pool in non terminal state. Agent pool must be in one of the following provisioning states: canceled, failed, or succeeded. Actual state: %s", ps) + log.V(2).Info(msg) + return azure.WithTransientError(errors.New(msg), 20*time.Second) + } err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile, customHeaders) if err != nil { From 69373646e2c4f2dc9e2c48295bb10b6d0af63df3 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Tue, 5 Jul 2022 22:08:31 +0530 Subject: [PATCH 05/66] Updated Makefile with spectro changes --- Makefile | 10 +-- config/default/manager_image_patch.yaml | 2 +- spectro/base/kustomization.yaml | 2 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 2 +- spectro/global/kustomization.yaml | 112 ++++++++++++------------ 6 files changed, 64 insertions(+), 66 deletions(-) diff --git a/Makefile b/Makefile index b943699b48b..58ac2858107 100644 --- a/Makefile +++ b/Makefile @@ -120,12 +120,12 @@ KUBE_APISERVER=$(TOOLS_BIN_DIR)/kube-apiserver ETCD=$(TOOLS_BIN_DIR)/etcd # Define Docker related variables. Releases should modify and double check these vars. -REGISTRY ?= gcr.io/$(shell gcloud config get-value project) +REGISTRY ?= gcr.io/spectro-dev-public/snehal/cluster-api-azure STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := registry.k8s.io/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= dev +TAG ?= spectro-v1.3.2-20220704 ARCH ?= $(GOARCH) ALL_ARCH = amd64 arm arm64 ppc64le s390x @@ -339,13 +339,13 @@ docker-pull-prerequisites: ## Pull prerequisites for building controller-manager .PHONY: docker-build docker-build: docker-pull-prerequisites ## Build the docker image for controller-manager. - DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) - $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" + DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG):$(TAG) + $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" .PHONY: docker-push docker-push: ## Push the docker image - docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG) + docker push $(CONTROLLER_IMG):$(TAG) ## -------------------------------------- ## Docker — All ARCH diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 0876a1db40d..0a12a1374be 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220704 name: manager diff --git a/spectro/base/kustomization.yaml b/spectro/base/kustomization.yaml index ee221b52f51..199e3878578 100644 --- a/spectro/base/kustomization.yaml +++ b/spectro/base/kustomization.yaml @@ -6,7 +6,7 @@ namePrefix: capz- commonLabels: cluster.x-k8s.io/provider: "infrastructure-azure" -bases: +resources: - ../../config/manager diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index 4fad77dd22a..40270fa803b 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220704 imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index 6aff8846340..ae330b2c5bb 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -10611,7 +10611,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220704 imagePullPolicy: Always livenessProbe: httpGet: diff --git a/spectro/global/kustomization.yaml b/spectro/global/kustomization.yaml index d0b54f2f58a..55f7288b487 100644 --- a/spectro/global/kustomization.yaml +++ b/spectro/global/kustomization.yaml @@ -9,8 +9,6 @@ commonLabels: resources: - ../../config/default/namespace.yaml - ../../config/default/aad-pod-identity-deployment.yaml - -bases: - ../../config/crd - ../../config/webhook - ../../config/manager @@ -73,63 +71,63 @@ patchesJson6902: namespace: capi-webhook-system version: v1 path: patch_service_account.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azureclusteridentities.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azureclusters.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremachines.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremachinetemplates.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremachinepools.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremachinepoolmachines.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremanagedclusters.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml -# - target: -# group: apiextensions.k8s.io -# version: v1 -# kind: CustomResourceDefinition -# name: azuremanagedmachinepools.infrastructure.cluster.x-k8s.io -# path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azureclusteridentities.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azureclusters.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachines.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachinetemplates.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachinepools.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremachinepoolmachines.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremanagedclusters.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml + - target: + group: apiextensions.k8s.io + version: v1 + kind: CustomResourceDefinition + name: azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + path: patch_crd_webhook_namespace.yaml - target: group: rbac.authorization.k8s.io version: v1 kind: ClusterRoleBinding name: capz-aad-pod-id-nmi-binding - path: patch_nmi_cluster_role_binding.yaml \ No newline at end of file + path: patch_nmi_cluster_role_binding.yaml From beb6bd96fe984a727cbbe6835dc3714c81266c7f Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Wed, 13 Jul 2022 17:21:56 +0530 Subject: [PATCH 06/66] Validation disabled for AzureManagedControlPlane --- .../azuremanagedcontrolplane_webhook.go | 2 +- spectro/generated/core-global.yaml | 15 +++++++++++++ spectro/global/kustomization.yaml | 6 ++++++ spectro/global/patch_cluster_role.yaml | 21 +++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 spectro/global/patch_cluster_role.yaml diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index c0afa5a8ca2..552ec743a6c 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -256,7 +256,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateSSHKey, m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, - m.validateManagedClusterNetwork, + //m.validateManagedClusterNetwork, } var errs []error diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index ae330b2c5bb..c8af9b03ecb 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -10508,6 +10508,21 @@ metadata: cluster.x-k8s.io/provider: infrastructure-azure name: capz-aad-pod-id-nmi-role rules: +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - azuremanagedclusters + - azuremanagedcontrolplanes + verbs: + - '*' - apiGroups: - apiextensions.k8s.io resources: diff --git a/spectro/global/kustomization.yaml b/spectro/global/kustomization.yaml index 55f7288b487..1b70f5d7c4b 100644 --- a/spectro/global/kustomization.yaml +++ b/spectro/global/kustomization.yaml @@ -52,6 +52,12 @@ configurations: - ../../config/default/kustomizeconfig.yaml patchesJson6902: + - target: + group: rbac.authorization.k8s.io + version: v1 + kind: ClusterRole + name: capz-aad-pod-id-nmi-role + path: patch_cluster_role.yaml - target: kind: Namespace name: system diff --git a/spectro/global/patch_cluster_role.yaml b/spectro/global/patch_cluster_role.yaml new file mode 100644 index 00000000000..8d81d5ed7f7 --- /dev/null +++ b/spectro/global/patch_cluster_role.yaml @@ -0,0 +1,21 @@ +- op: add + path: /rules/0 + value: + apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - azuremanagedclusters + - azuremanagedcontrolplanes + verbs: + - '*' +- op: add + path: /rules/0 + value: + apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get + - list + - watch \ No newline at end of file From dc9ef129557a7512842a0e1ae0ce8a8d5679d24e Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Thu, 14 Jul 2022 14:51:33 +0530 Subject: [PATCH 07/66] windows support-added OSType to ammp and amcp --- Makefile | 2 +- azure/converters/managedagentpool.go | 2 +- azure/scope/managedmachinepool.go | 1 + azure/services/agentpools/agentpools.go | 2 + azure/types.go | 3 + ...ter.x-k8s.io_azuremanagedmachinepools.yaml | 7 ++ config/default/manager_image_patch.yaml | 2 +- exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + .../azuremanagedcontrolplane_webhook.go | 21 ++++++ .../azuremanagedcontrolplane_webhook_test.go | 55 +++++++++++++++ .../v1beta1/azuremanagedmachinepool_types.go | 5 ++ .../azuremanagedmachinepool_webhook.go | 46 +++++++++++++ .../azuremanagedmachinepool_webhook_test.go | 69 +++++++++++++++++++ exp/api/v1beta1/zz_generated.deepcopy.go | 5 ++ spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 9 ++- 17 files changed, 228 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 58ac2858107..01b39f617c7 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := registry.k8s.io/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= spectro-v1.3.2-20220704 +TAG ?= spectro-v1.3.2-20220713 ARCH ?= $(GOARCH) ALL_ARCH = amd64 arm arm64 ppc64le s390x diff --git a/azure/converters/managedagentpool.go b/azure/converters/managedagentpool.go index 370ef823cfc..95e9d06756e 100644 --- a/azure/converters/managedagentpool.go +++ b/azure/converters/managedagentpool.go @@ -51,7 +51,7 @@ func AgentPoolToContainerServiceAgentPool(pool azure.AgentPoolSpec) containerser return containerservice.AgentPool{ ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{ VMSize: &pool.SKU, - OsType: containerservice.OSTypeLinux, + OsType: containerservice.OSType(to.String(pool.OSType)), OsDiskSizeGB: &pool.OSDiskSizeGB, Count: &pool.Replicas, Type: containerservice.AgentPoolTypeVirtualMachineScaleSets, diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index 801d609ad01..f5ab0db1415 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -156,6 +156,7 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane MaxPods: managedMachinePool.Spec.MaxPods, AvailabilityZones: managedMachinePool.Spec.AvailabilityZones, OsDiskType: managedMachinePool.Spec.OsDiskType, + OSType: managedMachinePool.Spec.OSType, EnableUltraSSD: managedMachinePool.Spec.EnableUltraSSD, } diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 9c4b438d940..1a4ae4e9fd7 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -105,6 +105,7 @@ func (s *Service) Reconcile(ctx context.Context) error { EnableAutoScaling: existingPool.EnableAutoScaling, MinCount: existingPool.MinCount, MaxCount: existingPool.MaxCount, + OsType: existingPool.OsType, }, } @@ -116,6 +117,7 @@ func (s *Service) Reconcile(ctx context.Context) error { EnableAutoScaling: profile.EnableAutoScaling, MinCount: profile.MinCount, MaxCount: profile.MaxCount, + OsType: profile.OsType, }, } diff --git a/azure/types.go b/azure/types.go index 3e0ad04199e..fd681fdfb2f 100644 --- a/azure/types.go +++ b/azure/types.go @@ -105,6 +105,9 @@ type AgentPoolSpec struct { // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` + + // OSType specifies the operating system for the node pool. Allowed values are 'Linux' and 'Windows' + OSType *string `json:"osType,omitempty"` } // ScaleSetSpec defines the specification for a Scale Set. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index 425b15f688a..ed3eb1dcc92 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -245,6 +245,13 @@ spec: - Ephemeral - Managed type: string + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 0a12a1374be..604e2b1ecbe 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220704 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220713 name: manager diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index af9e278987c..17d9b5165b2 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -886,6 +886,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha3_AzureManagedMac // WARNING: in.Taints requires manual conversion: does not exist in peer-type out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) // WARNING: in.Scaling requires manual conversion: does not exist in peer-type + // WARNING: in.OSType requires manual conversion: does not exist in peer-type // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 0a9bd141e71..86afe9c0ef3 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1191,6 +1191,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac // WARNING: in.Taints requires manual conversion: does not exist in peer-type out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) // WARNING: in.Scaling requires manual conversion: does not exist in peer-type + // WARNING: in.OSType requires manual conversion: does not exist in peer-type // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index 552ec743a6c..b71fe0a686c 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -88,6 +88,14 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client var allErrs field.ErrorList old := oldRaw.(*AzureManagedControlPlane) + if m.Name != old.Name { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Name"), + m.Name, + "field is immutable")) + } + if m.Spec.SubscriptionID != old.Spec.SubscriptionID { allErrs = append(allErrs, field.Invalid( @@ -251,6 +259,7 @@ func (m *AzureManagedControlPlane) ValidateDelete(_ client.Client) error { // Validate the Azure Machine Pool and return an aggregate error. func (m *AzureManagedControlPlane) Validate(cli client.Client) error { validators := []func(client client.Client) error{ + m.validateName, m.validateVersion, m.validateDNSServiceIP, m.validateSSHKey, @@ -280,6 +289,18 @@ func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { return nil } +// validateName validates the AzureManagedControlPlane name +func (m *AzureManagedControlPlane) validateName(_ client.Client) error { + lName := strings.ToLower(m.Name) + if strings.Contains(lName, "microsoft") || + strings.Contains(lName, "windows") { + return field.Invalid(field.NewPath("Name"), m.Name, + "cluster name is invalid because 'MICROSOFT' and 'WINDOWS' can't be used as either a whole word or a substring in the name") + } + + return nil +} + // validateVersion validates the Kubernetes version. func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { if !kubeSemver.MatchString(m.Spec.Version) { diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 2e49570b3b5..25651eeaf00 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -300,6 +300,36 @@ func TestAzureManagedControlPlane_ValidateCreate(t *testing.T) { wantErr: true, errorLen: 1, }, + { + name: "invalid name with microsoft", + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "microsoft-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + SSHPublicKey: generateSSHPublicKey(true), + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.23.5", + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "invalid name with windows", + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-windows-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + SSHPublicKey: generateSSHPublicKey(true), + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.23.5", + }, + }, + wantErr: true, + errorLen: 1, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -722,6 +752,31 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: false, }, + { + name: "AzureManagedControlPlane Name is mutable", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "new-test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + AuthorizedIPRanges: []string{"192.168.0.1/32"}, + }, + }, + }, + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go index cb07ea973ca..d1c75b1bf8d 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_types.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_types.go @@ -76,6 +76,11 @@ type AzureManagedMachinePoolSpec struct { // +optional Scaling *ManagedMachinePoolScaling `json:"scaling,omitempty"` + // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows' + // +kubebuilder:validation:Enum=Linux;Windows + // +optional + OSType *string `json:"osType,omitempty"` + // MaxPods specifies the kubelet --max-pods configuration for the node pool. // +optional MaxPods *int32 `json:"maxPods,omitempty"` diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go index b02d778defe..f916cf40bf2 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -50,6 +50,8 @@ func (m *AzureManagedMachinePool) Default(client client.Client) { func (m *AzureManagedMachinePool) ValidateCreate(client client.Client) error { validators := []func() error{ m.validateMaxPods, + m.validateOSType, + m.validateName, } var errs []error @@ -75,6 +77,25 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c "field is immutable")) } + if old.Spec.OSType != nil { + // Prevent OSType modification if it was already set to some value + if m.Spec.OSType == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OSType"), + m.Spec.OSType, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.OSType != *old.Spec.OSType { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OSType"), + *m.Spec.OSType, + "field is immutable")) + } + } + if old.Spec.OSDiskSizeGB != nil { // Prevent OSDiskSizeGB modification if it was already set to some value if m.Spec.OSDiskSizeGB == nil { @@ -210,6 +231,31 @@ func (m *AzureManagedMachinePool) validateMaxPods() error { return nil } +func (r *AzureManagedMachinePool) validateOSType() error { + if r.Spec.Mode == string(NodePoolModeSystem) { + if r.Spec.OSType != nil && *r.Spec.OSType != azure.LinuxOS { + return field.Forbidden( + field.NewPath("Spec", "OSType"), + "System node pooll must have OSType 'Linux'") + } + } + + return nil +} + +func (r *AzureManagedMachinePool) validateName() error { + if r.Spec.OSType != nil && *r.Spec.OSType == azure.WindowsOS { + if len(r.Name) > 6 { + return field.Invalid( + field.NewPath("Name"), + r.Name, + "Windows agent pool name can not be longer than 6 characters.") + } + } + + return nil +} + func ensureStringSlicesAreEqual(a []string, b []string) bool { if len(a) != len(b) { return false diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go index 006f03926ed..dda97a621c0 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + "sigs.k8s.io/cluster-api-provider-azure/azure" "testing" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" @@ -441,6 +442,26 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { }, wantErr: true, }, + { + name: "Cannot change OSType of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + OSType: to.StringPtr("Linux"), + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + OSType: to.StringPtr("Windows"), + Mode: "System", + SKU: "StandardD2S_V4", + OSDiskSizeGB: to.Int32Ptr(512), + }, + }, + wantErr: true, + }, } var client client.Client for _, tc := range tests { @@ -511,6 +532,54 @@ func TestAzureManagedMachinePool_ValidateCreate(t *testing.T) { wantErr: true, errorLen: 1, }, + { + name: "ostype Windows with System mode not allowed", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: true, + errorLen: 1, + }, + { + name: "ostype windows with User mode", + ammp: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: false, + }, + { + name: "Windows clusters with 6char or less name", + ammp: &AzureManagedMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool0", + }, + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: false, + }, + { + name: "Windows clusters with more than 6char names are not allowed", + ammp: &AzureManagedMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool0-name-too-long", + }, + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + OSType: to.StringPtr(azure.WindowsOS), + }, + }, + wantErr: true, + errorLen: 1, + }, } var client client.Client for _, tc := range tests { diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 9e35f3cf62f..edb1cbeba24 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -824,6 +824,11 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = new(ManagedMachinePoolScaling) (*in).DeepCopyInto(*out) } + if in.OSType != nil { + in, out := &in.OSType, &out.OSType + *out = new(string) + **out = **in + } if in.MaxPods != nil { in, out := &in.MaxPods, &out.MaxPods *out = new(int32) diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index 40270fa803b..a195aa9f3c1 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220704 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220713 imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index c8af9b03ecb..ba7280d85ba 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -10270,6 +10270,13 @@ spec: - Ephemeral - Managed type: string + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. @@ -10626,7 +10633,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220704 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220713 imagePullPolicy: Always livenessProbe: httpGet: From a525b879cebccc243affa96406261da93115e9a9 Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Sat, 2 Jul 2022 01:54:11 +0530 Subject: [PATCH 08/66] Updated the check for updating machinepool when using autoscalar (#35) --- azure/services/agentpools/agentpools.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 1a4ae4e9fd7..2eeed654bd1 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -121,6 +121,13 @@ func (s *Service) Reconcile(ctx context.Context) error { }, } + // When autoscaling is set, the count of the nodes differ based on the autoscalar and should not depend on the + // count present in machinepool, azuremanagedmachinepool, hence we should not make an update api call based + // on difference in count. + if profile.EnableAutoScaling != nil && existingProfile.Count != nil { + normalizedProfile.Count = existingProfile.Count + } + // Diff and check if we require an update diff := cmp.Diff(normalizedProfile, existingProfile) if diff != "" { From d43fcdfdc365c5bd0a33521f6b94d8d199ae6cf4 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Mon, 29 Aug 2022 17:06:31 +0530 Subject: [PATCH 09/66] Suppress warning messages for ServicePrincipal auth --- Makefile | 2 +- config/default/manager_image_patch.yaml | 2 +- controllers/azurejson_machine_controller.go | 5 ++--- controllers/azurejson_machinepool_controller.go | 5 ++--- controllers/azurejson_machinetemplate_controller.go | 5 ++--- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 01b39f617c7..ac53595c003 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := registry.k8s.io/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= spectro-v1.3.2-20220713 +TAG ?= spectro-v1.3.2-$(shell date +%Y%m%d) ARCH ?= $(GOARCH) ALL_ARCH = amd64 arm arm64 ppc64le s390x diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 604e2b1ecbe..1ca95603e55 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220713 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220829 name: manager diff --git a/controllers/azurejson_machine_controller.go b/controllers/azurejson_machine_controller.go index 37348396a94..9839565a061 100644 --- a/controllers/azurejson_machine_controller.go +++ b/controllers/azurejson_machine_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "time" "github.com/go-logr/logr" @@ -195,8 +194,8 @@ func (r *AzureJSONMachineReconciler) Reconcile(ctx context.Context, req ctrl.Req } if azureMachine.Spec.Identity == infrav1.VMIdentityNone { - log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) - r.Recorder.Eventf(azureMachine, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) + //log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) + //r.Recorder.Eventf(azureMachine, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) } newSecret, err := GetCloudProviderSecret( diff --git a/controllers/azurejson_machinepool_controller.go b/controllers/azurejson_machinepool_controller.go index 0e1d1a7db2a..921ef544347 100644 --- a/controllers/azurejson_machinepool_controller.go +++ b/controllers/azurejson_machinepool_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "time" "github.com/pkg/errors" @@ -163,8 +162,8 @@ func (r *AzureJSONMachinePoolReconciler) Reconcile(ctx context.Context, req ctrl } if azureMachinePool.Spec.Identity == infrav1.VMIdentityNone { - log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) - r.Recorder.Eventf(azureMachinePool, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) + //log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) + //r.Recorder.Eventf(azureMachinePool, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) } newSecret, err := GetCloudProviderSecret( diff --git a/controllers/azurejson_machinetemplate_controller.go b/controllers/azurejson_machinetemplate_controller.go index 6b63756e129..291c2fb91d4 100644 --- a/controllers/azurejson_machinetemplate_controller.go +++ b/controllers/azurejson_machinetemplate_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "fmt" "time" "github.com/pkg/errors" @@ -154,8 +153,8 @@ func (r *AzureJSONTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Re } if azureMachineTemplate.Spec.Template.Spec.Identity == infrav1.VMIdentityNone { - log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) - r.Recorder.Eventf(azureMachineTemplate, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) + //log.Info(fmt.Sprintf("WARNING, %s", spIdentityWarning)) + //r.Recorder.Eventf(azureMachineTemplate, corev1.EventTypeWarning, "VMIdentityNone", spIdentityWarning) } newSecret, err := GetCloudProviderSecret( From 9a1abceb9a783371c6e4e446c61e03573dea091f Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Tue, 6 Sep 2022 22:05:44 +0530 Subject: [PATCH 10/66] Updated the check for updating machinepool when using autoscalar (#42) --- azure/const.go | 4 ++ azure/scope/managedmachinepool.go | 52 +++++++++++++++---- azure/services/agentpools/agentpools.go | 36 +++++++++---- .../azuremanagedmachinepool_controller.go | 3 ++ 4 files changed, 74 insertions(+), 21 deletions(-) diff --git a/azure/const.go b/azure/const.go index 9c4ac179ab2..c45b0d1d461 100644 --- a/azure/const.go +++ b/azure/const.go @@ -28,4 +28,8 @@ const ( // See https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ // for annotation formatting rules. RGTagsLastAppliedAnnotation = "sigs.k8s.io/cluster-api-provider-azure-last-applied-tags-rg" + + // ReplicasManagedByAutoscalerAnnotation is set to true in the corresponding capi machine pool + // when an external autoscaler manages the node count of the associated machine pool. + ReplicasManagedByAutoscalerAnnotation = "cluster.x-k8s.io/replicas-managed-by-autoscaler" ) diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index f5ab0db1415..2e40bb80b2d 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -69,21 +69,28 @@ func NewManagedMachinePoolScope(ctx context.Context, params ManagedMachinePoolSc return nil, errors.Wrap(err, "failed to init patch helper") } + capiMachinePoolPatchHelper, err := patch.NewHelper(params.MachinePool, params.Client) + if err != nil { + return nil, errors.Wrap(err, "failed to init patch helper") + } + return &ManagedMachinePoolScope{ - Client: params.Client, - Cluster: params.Cluster, - ControlPlane: params.ControlPlane, - MachinePool: params.MachinePool, - InfraMachinePool: params.InfraMachinePool, - patchHelper: helper, - ManagedClusterScoper: params.ManagedControlPlaneScope, + Client: params.Client, + Cluster: params.Cluster, + ControlPlane: params.ControlPlane, + MachinePool: params.MachinePool, + InfraMachinePool: params.InfraMachinePool, + patchHelper: helper, + capiMachinePoolPatchHelper: capiMachinePoolPatchHelper, + ManagedClusterScoper: params.ManagedControlPlaneScope, }, nil } // ManagedMachinePoolScope defines the basic context for an actuator to operate upon. type ManagedMachinePoolScope struct { - Client client.Client - patchHelper *patch.Helper + Client client.Client + patchHelper *patch.Helper + capiMachinePoolPatchHelper *patch.Helper azure.ManagedClusterScoper Cluster *clusterv1.Cluster @@ -231,7 +238,7 @@ func (s *ManagedMachinePoolScope) UpdateDeleteStatus(condition clusterv1.Conditi } } -// UpdatePutStatus updates a condition on the AzureManagedControlPlane status after a PUT operation. +// UpdatePutStatus updates a condition on the AzureManagedMachinePool status after a PUT operation. func (s *ManagedMachinePoolScope) UpdatePutStatus(condition clusterv1.ConditionType, service string, err error) { switch { case err == nil: @@ -243,7 +250,7 @@ func (s *ManagedMachinePoolScope) UpdatePutStatus(condition clusterv1.ConditionT } } -// UpdatePatchStatus updates a condition on the AzureManagedControlPlane status after a PATCH operation. +// UpdatePatchStatus updates a condition on the AzureManagedMachinePool status after a PATCH operation. func (s *ManagedMachinePoolScope) UpdatePatchStatus(condition clusterv1.ConditionType, service string, err error) { switch { case err == nil: @@ -254,3 +261,26 @@ func (s *ManagedMachinePoolScope) UpdatePatchStatus(condition clusterv1.Conditio conditions.MarkFalse(s.InfraMachinePool, condition, infrav1.FailedReason, clusterv1.ConditionSeverityError, "%s failed to update. err: %s", service, err.Error()) } } + +// PatchCAPIMachinePoolObject persists the capi machinepool configuration and status. +func (s *ManagedMachinePoolScope) PatchCAPIMachinePoolObject(ctx context.Context) error { + return s.capiMachinePoolPatchHelper.Patch( + ctx, + s.MachinePool, + ) +} + +// UpdateCAPIMachinePoolReplicas updates the associated MachinePool replica count. +func (s *ManagedMachinePoolScope) UpdateCAPIMachinePoolReplicas(ctx context.Context, replicas *int32) { + s.MachinePool.Spec.Replicas = replicas +} + +// UpdateCAPIMachinePoolAnnotations updates the associated MachinePool annotation. +func (s *ManagedMachinePoolScope) UpdateCAPIMachinePoolAnnotations(ctx context.Context, key, value string) { + s.MachinePool.Annotations[key] = value +} + +// GetCAPIMachinePoolAnnotations gets the associated MachinePool annotation. +func (s *ManagedMachinePoolScope) GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string { + return s.MachinePool.Annotations[key] +} diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 2eeed654bd1..8953586c921 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -22,9 +22,10 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" + "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" - infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/util/maps" @@ -43,6 +44,9 @@ type ManagedMachinePoolScope interface { SetAgentPoolProviderIDList([]string) SetAgentPoolReplicas(int32) SetAgentPoolReady(bool) + UpdateCAPIMachinePoolReplicas(ctx context.Context, replicas *int32) + UpdateCAPIMachinePoolAnnotations(ctx context.Context, key, value string) + GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string } // Service provides operations on Azure resources. @@ -95,6 +99,12 @@ func (s *Service) Reconcile(ctx context.Context) error { return errors.Wrap(err, "failed to create or update agent pool") } } else { + ps := *existingPool.ManagedClusterAgentPoolProfileProperties.ProvisioningState + if ps != string(infrav1.Canceled) && ps != string(infrav1.Failed) && ps != string(infrav1.Succeeded) { + msg := fmt.Sprintf("Unable to update existing agent pool in non terminal state. Agent pool must be in one of the following provisioning states: canceled, failed, or succeeded. Actual state: %s", ps) + log.V(2).Info(msg) + return azure.WithTransientError(errors.New(msg), 20*time.Second) + } // Normalize individual agent pools to diff in case we need to update existingProfile := containerservice.AgentPool{ @@ -121,23 +131,29 @@ func (s *Service) Reconcile(ctx context.Context) error { }, } - // When autoscaling is set, the count of the nodes differ based on the autoscalar and should not depend on the - // count present in machinepool, azuremanagedmachinepool, hence we should not make an update api call based + // When autoscaling is set, the count of the nodes differ based on the autoscaler and should not depend on the + // count present in MachinePool or AzureManagedMachinePool, hence we should not make an update API call based // on difference in count. - if profile.EnableAutoScaling != nil && existingProfile.Count != nil { + if to.Bool(profile.EnableAutoScaling) && existingProfile.Count != nil { + if to.Bool(profile.EnableAutoScaling) && s.scope.GetCAPIMachinePoolAnnotation(ctx, azure.ReplicasManagedByAutoscalerAnnotation) != "true" { + s.scope.UpdateCAPIMachinePoolAnnotations(ctx, azure.ReplicasManagedByAutoscalerAnnotation, "true") + } + + if to.Int32(existingProfile.Count) != to.Int32(normalizedProfile.Count) { + s.scope.UpdateCAPIMachinePoolReplicas(ctx, existingProfile.Count) + } normalizedProfile.Count = existingProfile.Count } + // set ReplicasManagedByAutoscalerAnnotation to false as it is disabled by the user. + if !to.Bool(profile.EnableAutoScaling) && s.scope.GetCAPIMachinePoolAnnotation(ctx, azure.ReplicasManagedByAutoscalerAnnotation) == "true" { + s.scope.UpdateCAPIMachinePoolAnnotations(ctx, azure.ReplicasManagedByAutoscalerAnnotation, "false") + } + // Diff and check if we require an update diff := cmp.Diff(normalizedProfile, existingProfile) if diff != "" { log.V(2).Info(fmt.Sprintf("Update required (+new -old):\n%s", diff)) - ps := *existingPool.ManagedClusterAgentPoolProfileProperties.ProvisioningState - if ps != string(infrav1alpha4.Canceled) && ps != string(infrav1alpha4.Failed) && ps != string(infrav1alpha4.Succeeded) { - msg := fmt.Sprintf("Unable to update existing agent pool in non terminal state. Agent pool must be in one of the following provisioning states: canceled, failed, or succeeded. Actual state: %s", ps) - log.V(2).Info(msg) - return azure.WithTransientError(errors.New(msg), 20*time.Second) - } err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile, customHeaders) if err != nil { diff --git a/exp/controllers/azuremanagedmachinepool_controller.go b/exp/controllers/azuremanagedmachinepool_controller.go index 9b5f2ad51a0..b3a336cf90d 100644 --- a/exp/controllers/azuremanagedmachinepool_controller.go +++ b/exp/controllers/azuremanagedmachinepool_controller.go @@ -225,6 +225,9 @@ func (ammpr *AzureManagedMachinePoolReconciler) Reconcile(ctx context.Context, r if err := mcpScope.PatchObject(ctx); err != nil && reterr == nil { reterr = err } + if err := mcpScope.PatchCAPIMachinePoolObject(ctx); err != nil && reterr == nil { + reterr = err + } }() // Handle deleted clusters From c7bc2cc86507f49aeaf17188f9fdee7809d7b3f8 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Thu, 8 Sep 2022 11:00:02 +0530 Subject: [PATCH 11/66] AKS static placement for cross RG --- Makefile | 3 ++- azure/scope/managedcontrolplane.go | 8 ++++++-- azure/scope/managedmachinepool.go | 6 +++++- azure/services/agentpools/agentpools.go | 1 + ...uster.x-k8s.io_azuremanagedcontrolplanes.yaml | 8 ++++++++ config/default/manager_image_patch.yaml | 2 +- .../azuremanagedcontrolplane_conversion.go | 9 +++++++++ exp/api/v1alpha3/zz_generated.conversion.go | 16 ++++++---------- .../azuremanagedcontrolplane_conversion.go | 10 ++++++++++ .../v1alpha4/azuremanagedcontrolplane_types.go | 2 ++ exp/api/v1alpha4/zz_generated.conversion.go | 2 ++ .../v1beta1/azuremanagedcontrolplane_types.go | 2 ++ spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 16 +++++++++++++++- 14 files changed, 70 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index ac53595c003..f9f913186dc 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,8 @@ SHELL:=/usr/bin/env bash .DEFAULT_GOAL:=help GOPATH := $(shell go env GOPATH) -GOARCH := $(shell go env GOARCH) +#GOARCH := $(shell go env GOARCH) +GOARCH := amd64 GOOS := $(shell go env GOOS) GOPROXY := $(shell go env GOPROXY) ifeq ($(GOPROXY),) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index a3e8cd4c2dc..7ec1baa5e80 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -197,8 +197,12 @@ func (s *ManagedControlPlaneScope) Close(ctx context.Context) error { // Vnet returns the cluster Vnet. func (s *ManagedControlPlaneScope) Vnet() *infrav1.VnetSpec { + resourceGroupName := s.ControlPlane.Spec.ResourceGroupName + if s.ControlPlane.Spec.VirtualNetwork.ResourceGroupName != "" { + resourceGroupName = s.ControlPlane.Spec.VirtualNetwork.ResourceGroupName + } return &infrav1.VnetSpec{ - ResourceGroup: s.ControlPlane.Spec.ResourceGroupName, + ResourceGroup: resourceGroupName, Name: s.ControlPlane.Spec.VirtualNetwork.Name, VnetClassSpec: infrav1.VnetClassSpec{ CIDRBlocks: []string{s.ControlPlane.Spec.VirtualNetwork.CIDRBlock}, @@ -397,7 +401,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure DNSServiceIP: s.ControlPlane.Spec.DNSServiceIP, VnetSubnetID: azure.SubnetID( s.ControlPlane.Spec.SubscriptionID, - s.ControlPlane.Spec.ResourceGroupName, + s.VNetSpec().ResourceGroupName(), s.ControlPlane.Spec.VirtualNetwork.Name, s.ControlPlane.Spec.VirtualNetwork.Subnet.Name, ), diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index f5ab0db1415..f96eafdbd63 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -139,6 +139,10 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane replicas = *machinePool.Spec.Replicas } + resourceGroupName := managedControlPlane.Spec.ResourceGroupName + if managedControlPlane.Spec.VirtualNetwork.ResourceGroupName != "" { + resourceGroupName = managedControlPlane.Spec.VirtualNetwork.ResourceGroupName + } agentPoolSpec := azure.AgentPoolSpec{ Name: to.String(managedMachinePool.Spec.Name), ResourceGroup: managedControlPlane.Spec.ResourceGroupName, @@ -148,7 +152,7 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane Version: normalizedVersion, VnetSubnetID: azure.SubnetID( managedControlPlane.Spec.SubscriptionID, - managedControlPlane.Spec.ResourceGroupName, + resourceGroupName, managedControlPlane.Spec.VirtualNetwork.Name, managedControlPlane.Spec.VirtualNetwork.Subnet.Name, ), diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 2eeed654bd1..b48bb62cad9 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -127,6 +127,7 @@ func (s *Service) Reconcile(ctx context.Context) error { if profile.EnableAutoScaling != nil && existingProfile.Count != nil { normalizedProfile.Count = existingProfile.Count } + log.V(0).Info("SNEHAL", "normalized", normalizedProfile, "existing", existingProfile) // Diff and check if we require an update diff := cmp.Diff(normalizedProfile, existingProfile) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index f161c796070..c2fc718ade0 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -407,6 +407,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -732,6 +736,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 1ca95603e55..068482199d7 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220829 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220911 name: manager diff --git a/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go b/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go index b0e1f4fd729..75d2e332808 100644 --- a/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go +++ b/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go @@ -81,3 +81,12 @@ func (dst *AzureManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) erro src := srcRaw.(*expv1beta1.AzureManagedControlPlaneList) return Convert_v1beta1_AzureManagedControlPlaneList_To_v1alpha3_AzureManagedControlPlaneList(src, dst, nil) } + +func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in *expv1beta1.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s apiconversion.Scope) error { + out.Name = in.Name + out.Subnet.Name = in.Subnet.Name + out.Subnet.CIDRBlock = in.Subnet.CIDRBlock + out.CIDRBlock = in.CIDRBlock + + return nil +} diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 17d9b5165b2..9ee5ffc8bfc 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -206,11 +206,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.ManagedControlPlaneVirtualNetwork)(nil), (*ManagedControlPlaneVirtualNetwork)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(a.(*v1beta1.ManagedControlPlaneVirtualNetwork), b.(*ManagedControlPlaneVirtualNetwork), scope) - }); err != nil { - return err - } if err := s.AddConversionFunc((*apiv1alpha3.APIEndpoint)(nil), (*apiv1beta1.APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_APIEndpoint_To_v1beta1_APIEndpoint(a.(*apiv1alpha3.APIEndpoint), b.(*apiv1beta1.APIEndpoint), scope) }); err != nil { @@ -276,6 +271,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.ManagedControlPlaneVirtualNetwork)(nil), (*ManagedControlPlaneVirtualNetwork)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(a.(*v1beta1.ManagedControlPlaneVirtualNetwork), b.(*ManagedControlPlaneVirtualNetwork), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*clusterapiproviderazureapiv1beta1.OSDisk)(nil), (*clusterapiproviderazureapiv1alpha3.OSDisk)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_OSDisk_To_v1alpha3_OSDisk(a.(*clusterapiproviderazureapiv1beta1.OSDisk), b.(*clusterapiproviderazureapiv1alpha3.OSDisk), scope) }); err != nil { @@ -958,10 +958,6 @@ func autoConvert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedCo if err := Convert_v1beta1_ManagedControlPlaneSubnet_To_v1alpha3_ManagedControlPlaneSubnet(&in.Subnet, &out.Subnet, s); err != nil { return err } + // WARNING: in.ResourceGroupName requires manual conversion: does not exist in peer-type return nil } - -// Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork is an autogenerated conversion function. -func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in *v1beta1.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s conversion.Scope) error { - return autoConvert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in, out, s) -} diff --git a/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go b/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go index 2f891523b3b..9413ce78138 100644 --- a/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go +++ b/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go @@ -74,3 +74,13 @@ func (dst *AzureManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) erro src := srcRaw.(*expv1beta1.AzureManagedControlPlaneList) return Convert_v1beta1_AzureManagedControlPlaneList_To_v1alpha4_AzureManagedControlPlaneList(src, dst, nil) } + +// +//func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha4_ManagedControlPlaneVirtualNetwork(in *expv1beta1.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s apiconversion.Scope) error { +// out.Name = in.Name +// out.Subnet.Name = in.Subnet.Name +// out.Subnet.CIDRBlock = in.Subnet.CIDRBlock +// out.CIDRBlock = in.CIDRBlock +// +// return nil +//} diff --git a/exp/api/v1alpha4/azuremanagedcontrolplane_types.go b/exp/api/v1alpha4/azuremanagedcontrolplane_types.go index bbc929817ac..cbb167c50fd 100644 --- a/exp/api/v1alpha4/azuremanagedcontrolplane_types.go +++ b/exp/api/v1alpha4/azuremanagedcontrolplane_types.go @@ -176,6 +176,8 @@ type ManagedControlPlaneVirtualNetwork struct { Name string `json:"name"` CIDRBlock string `json:"cidrBlock"` Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` + // ResourceGroupName is the name of the Azure resource group for the VNet and Subnet. + ResourceGroupName string `json:"resourceGroupName,omitempty"` } // ManagedControlPlaneSubnet describes a subnet for an AKS cluster. diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 86afe9c0ef3..cc18230759f 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1301,6 +1301,7 @@ func autoConvert_v1alpha4_ManagedControlPlaneVirtualNetwork_To_v1beta1_ManagedCo if err := Convert_v1alpha4_ManagedControlPlaneSubnet_To_v1beta1_ManagedControlPlaneSubnet(&in.Subnet, &out.Subnet, s); err != nil { return err } + out.ResourceGroupName = in.ResourceGroupName return nil } @@ -1315,6 +1316,7 @@ func autoConvert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha4_ManagedCo if err := Convert_v1beta1_ManagedControlPlaneSubnet_To_v1alpha4_ManagedControlPlaneSubnet(&in.Subnet, &out.Subnet, s); err != nil { return err } + out.ResourceGroupName = in.ResourceGroupName return nil } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index ce698de16b6..5634da24328 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -205,6 +205,8 @@ type ManagedControlPlaneVirtualNetwork struct { CIDRBlock string `json:"cidrBlock"` // +optional Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` + // ResourceGroupName is the name of the Azure resource group for the VNet and Subnet. + ResourceGroupName string `json:"resourceGroupName,omitempty"` } // ManagedControlPlaneSubnet describes a subnet for an AKS cluster. diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index a195aa9f3c1..ef8be0ea7e6 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220713 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220908 imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index ba7280d85ba..70cdd5321f8 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -9284,6 +9284,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -9299,6 +9303,7 @@ spec: required: - cidrBlock - name + - resourceGroupName type: object required: - location @@ -9554,6 +9559,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -9569,6 +9578,7 @@ spec: required: - cidrBlock - name + - resourceGroupName type: object required: - location @@ -9879,6 +9889,10 @@ spec: type: string name: type: string + resourceGroupName: + description: ResourceGroupName is the name of the Azure resource + group for the VNet and Subnet. + type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -10633,7 +10647,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220713 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220908 imagePullPolicy: Always livenessProbe: httpGet: From ba4fc3dc91a88682f6f1f26d4caad90f329af6ff Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Mon, 12 Sep 2022 12:15:08 +0530 Subject: [PATCH 12/66] Fixed osType issue --- ...tructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml | 7 +++++++ exp/api/v1alpha4/azuremanagedmachinepool_types.go | 5 +++++ exp/api/v1alpha4/zz_generated.conversion.go | 3 ++- exp/api/v1alpha4/zz_generated.deepcopy.go | 5 +++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index ed3eb1dcc92..c33a9b5fc22 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -133,6 +133,13 @@ spec: according to the vmSize specified. format: int32 type: integer + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_types.go b/exp/api/v1alpha4/azuremanagedmachinepool_types.go index 646c8a3b859..ee34c72706d 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_types.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_types.go @@ -56,6 +56,11 @@ type AzureManagedMachinePoolSpec struct { // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` + + // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows' + // +kubebuilder:validation:Enum=Linux;Windows + // +optional + OSType *string `json:"osType,omitempty"` } // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool. diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index cc18230759f..ff5f2fab8af 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1173,6 +1173,7 @@ func autoConvert_v1alpha4_AzureManagedMachinePoolSpec_To_v1beta1_AzureManagedMac out.SKU = in.SKU out.OSDiskSizeGB = (*int32)(unsafe.Pointer(in.OSDiskSizeGB)) out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) + out.OSType = (*string)(unsafe.Pointer(in.OSType)) return nil } @@ -1191,7 +1192,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac // WARNING: in.Taints requires manual conversion: does not exist in peer-type out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) // WARNING: in.Scaling requires manual conversion: does not exist in peer-type - // WARNING: in.OSType requires manual conversion: does not exist in peer-type + out.OSType = (*string)(unsafe.Pointer(in.OSType)) // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1alpha4/zz_generated.deepcopy.go b/exp/api/v1alpha4/zz_generated.deepcopy.go index 56fba3899c4..cbf301576c0 100644 --- a/exp/api/v1alpha4/zz_generated.deepcopy.go +++ b/exp/api/v1alpha4/zz_generated.deepcopy.go @@ -766,6 +766,11 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = make([]string, len(*in)) copy(*out, *in) } + if in.OSType != nil { + in, out := &in.OSType, &out.OSType + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolSpec. From 6d045f3c8801d5737271174b6d36ca985a9f5e52 Mon Sep 17 00:00:00 2001 From: Jayesh Srivastava Date: Tue, 13 Sep 2022 22:25:19 +0530 Subject: [PATCH 13/66] PCP-328 fix --- exp/controllers/azuremanagedcontrolplane_controller.go | 2 +- exp/controllers/azuremanagedmachinepool_controller.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/exp/controllers/azuremanagedcontrolplane_controller.go b/exp/controllers/azuremanagedcontrolplane_controller.go index 86c0819b9d8..76b270cddad 100644 --- a/exp/controllers/azuremanagedcontrolplane_controller.go +++ b/exp/controllers/azuremanagedcontrolplane_controller.go @@ -260,7 +260,7 @@ func (amcpr *AzureManagedControlPlaneReconciler) reconcileNormal(ctx context.Con // No errors, so mark us ready so the Cluster API Cluster Controller can pull it scope.ControlPlane.Status.Ready = true scope.ControlPlane.Status.Initialized = true - amcpr.Recorder.Event(scope.ControlPlane, corev1.EventTypeNormal, "AzureManagedControlPlane available", "successfully reconciled") + //amcpr.Recorder.Event(scope.ControlPlane, corev1.EventTypeNormal, "AzureManagedControlPlane available", "successfully reconciled") return reconcile.Result{}, nil } diff --git a/exp/controllers/azuremanagedmachinepool_controller.go b/exp/controllers/azuremanagedmachinepool_controller.go index b3a336cf90d..d309352bc27 100644 --- a/exp/controllers/azuremanagedmachinepool_controller.go +++ b/exp/controllers/azuremanagedmachinepool_controller.go @@ -21,7 +21,6 @@ import ( "time" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" @@ -280,7 +279,7 @@ func (ammpr *AzureManagedMachinePoolReconciler) reconcileNormal(ctx context.Cont // No errors, so mark us ready so the Cluster API Cluster Controller can pull it scope.InfraMachinePool.Status.Ready = true - ammpr.Recorder.Eventf(scope.InfraMachinePool, corev1.EventTypeNormal, "AzureManagedMachinePool available", "agent pool successfully reconciled") + //ammpr.Recorder.Eventf(scope.InfraMachinePool, corev1.EventTypeNormal, "AzureManagedMachinePool available", "agent pool successfully reconciled") return reconcile.Result{}, nil } From dba7ef2005b378958bef1e98f99454b764968b8e Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Thu, 22 Sep 2022 11:07:40 +0530 Subject: [PATCH 14/66] Fixed upgrade issue from v1alpha4 to v1beta1 --- azure/scope/managedmachinepool.go | 6 ++++++ azure/services/agentpools/agentpools.go | 1 - config/default/manager_image_patch.yaml | 2 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 15 ++++++++------- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index e93962b98be..c4ae7cc7e84 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -171,6 +171,12 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane EnableUltraSSD: managedMachinePool.Spec.EnableUltraSSD, } + // Set OSType to Linux by default if not specified + if managedMachinePool.Spec.OSType == nil || *managedMachinePool.Spec.OSType == "" { + osType := azure.LinuxOS + agentPoolSpec.OSType = &osType + } + if managedMachinePool.Spec.OSDiskSizeGB != nil { agentPoolSpec.OSDiskSizeGB = *managedMachinePool.Spec.OSDiskSizeGB } diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index c2aa35f4755..8953586c921 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -144,7 +144,6 @@ func (s *Service) Reconcile(ctx context.Context) error { } normalizedProfile.Count = existingProfile.Count } - log.V(0).Info("SNEHAL", "normalized", normalizedProfile, "existing", existingProfile) // set ReplicasManagedByAutoscalerAnnotation to false as it is disabled by the user. if !to.Bool(profile.EnableAutoScaling) && s.scope.GetCAPIMachinePoolAnnotation(ctx, azure.ReplicasManagedByAutoscalerAnnotation) == "true" { diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 068482199d7..b2be7491aee 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220911 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 name: manager diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index ef8be0ea7e6..b0b2bfb8cc6 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220908 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index 70cdd5321f8..a72e81a202c 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -9284,10 +9284,6 @@ spec: type: string name: type: string - resourceGroupName: - description: ResourceGroupName is the name of the Azure resource - group for the VNet and Subnet. - type: string subnet: description: ManagedControlPlaneSubnet describes a subnet for an AKS cluster. @@ -9303,7 +9299,6 @@ spec: required: - cidrBlock - name - - resourceGroupName type: object required: - location @@ -9578,7 +9573,6 @@ spec: required: - cidrBlock - name - - resourceGroupName type: object required: - location @@ -10172,6 +10166,13 @@ spec: according to the vmSize specified. format: int32 type: integer + osType: + description: 'OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: ''Linux'', ''Windows''' + enum: + - Linux + - Windows + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. @@ -10647,7 +10648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220908 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 imagePullPolicy: Always livenessProbe: httpGet: From 5c7f37247704a387ad97e247cd02f71fd530da5d Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 10 Aug 2022 14:28:48 +0100 Subject: [PATCH 15/66] AKS: enable isVnetManaged, add caching Co-authored-by: Mike Tougeron --- azure/scope/cluster.go | 19 ++++- azure/scope/cluster_test.go | 127 +++++++++++++++++++++++++++++ azure/scope/managedcontrolplane.go | 28 ++++++- azure/services/subnets/subnets.go | 2 +- 4 files changed, 173 insertions(+), 3 deletions(-) diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 4f74556af36..002a852737b 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -54,6 +54,7 @@ type ClusterScopeParams struct { Client client.Client Cluster *clusterv1.Cluster AzureCluster *infrav1.AzureCluster + Cache *ClusterCache } // NewClusterScope creates a new Scope from the supplied parameters. @@ -85,6 +86,10 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc } } + if params.Cache == nil { + params.Cache = &ClusterCache{} + } + helper, err := patch.NewHelper(params.AzureCluster, params.Client) if err != nil { return nil, errors.Errorf("failed to init patch helper: %v", err) @@ -96,6 +101,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc Cluster: params.Cluster, AzureCluster: params.AzureCluster, patchHelper: helper, + cache: params.Cache, }, nil } @@ -103,12 +109,18 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc type ClusterScope struct { Client client.Client patchHelper *patch.Helper + cache *ClusterCache AzureClients Cluster *clusterv1.Cluster AzureCluster *infrav1.AzureCluster } +// ClusterCache stores ClusterCache data locally so we don't have to hit the API multiple times within the same reconcile loop. +type ClusterCache struct { + isVnetManaged *bool +} + // BaseURI returns the Azure ResourceManagerEndpoint. func (s *ClusterScope) BaseURI() string { return s.ResourceManagerEndpoint @@ -477,7 +489,12 @@ func (s *ClusterScope) Vnet() *infrav1.VnetSpec { // IsVnetManaged returns true if the vnet is managed. func (s *ClusterScope) IsVnetManaged() bool { - return s.Vnet().ID == "" || s.Vnet().Tags.HasOwned(s.ClusterName()) + if s.cache.isVnetManaged != nil { + return to.Bool(s.cache.isVnetManaged) + } + isVnetManaged := s.Vnet().ID == "" || s.Vnet().Tags.HasOwned(s.ClusterName()) + s.cache.isVnetManaged = to.BoolPtr(isVnetManaged) + return isVnetManaged } // IsIPv6Enabled returns true if IPv6 is enabled. diff --git a/azure/scope/cluster_test.go b/azure/scope/cluster_test.go index 05ed0c85f89..f407b9b3641 100644 --- a/azure/scope/cluster_test.go +++ b/azure/scope/cluster_test.go @@ -606,6 +606,7 @@ func TestRouteTableSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: nil, }, @@ -641,6 +642,7 @@ func TestRouteTableSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &routetables.RouteTableSpec{ @@ -688,6 +690,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: nil, }, @@ -730,6 +733,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &natgateways.NatGatewaySpec{ @@ -800,6 +804,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &natgateways.NatGatewaySpec{ @@ -869,6 +874,7 @@ func TestNatGatewaySpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &natgateways.NatGatewaySpec{ @@ -946,6 +952,7 @@ func TestNSGSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &securitygroups.NSGSpec{ @@ -991,6 +998,7 @@ func TestSubnetSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{}, }, @@ -1052,6 +1060,7 @@ func TestSubnetSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &subnets.SubnetSpec{ @@ -1154,6 +1163,7 @@ func TestSubnetSpecs(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: []azure.ResourceSpecGetter{ &subnets.SubnetSpec{ @@ -1196,6 +1206,122 @@ func TestSubnetSpecs(t *testing.T) { } } +func TestIsVnetManaged(t *testing.T) { + tests := []struct { + name string + clusterScope ClusterScope + want bool + }{ + { + name: "VNET ID is empty", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Vnet: infrav1.VnetSpec{ + ID: "", + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: true, + }, + { + name: "Wrong tags", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Vnet: infrav1.VnetSpec{ + ID: "my-id", + VnetClassSpec: infrav1.VnetClassSpec{Tags: map[string]string{ + "key": "value", + }}, + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: false, + }, + { + name: "Has owning tags", + clusterScope: ClusterScope{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + Vnet: infrav1.VnetSpec{ + ID: "my-id", + VnetClassSpec: infrav1.VnetClassSpec{Tags: map[string]string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": "owned", + }}, + }, + }, + }, + }, + cache: &ClusterCache{}, + }, + want: true, + }, + { + name: "Has cached value of false", + clusterScope: ClusterScope{ + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{}, + }, + cache: &ClusterCache{ + isVnetManaged: to.BoolPtr(false), + }, + }, + want: false, + }, + { + name: "Has cached value of true", + clusterScope: ClusterScope{ + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{}, + }, + cache: &ClusterCache{ + isVnetManaged: to.BoolPtr(true), + }, + }, + want: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := tt.clusterScope.IsVnetManaged() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("IsVnetManaged() = \n%t, want \n%t", got, tt.want) + } + if to.Bool(tt.clusterScope.cache.isVnetManaged) != got { + t.Errorf("IsVnetManaged() = \n%t, cache = \n%t", got, to.Bool(tt.clusterScope.cache.isVnetManaged)) + } + }) + } +} + func TestAzureBastionSpec(t *testing.T) { tests := []struct { name string @@ -1302,6 +1428,7 @@ func TestAzureBastionSpec(t *testing.T) { }, }, }, + cache: &ClusterCache{}, }, want: &bastionhosts.AzureBastionSpec{ Name: "fake-azure-bastion-1", diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 7ec1baa5e80..c5cc5f4f85a 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" @@ -51,6 +52,7 @@ type ManagedControlPlaneScopeParams struct { Cluster *clusterv1.Cluster ControlPlane *infrav1exp.AzureManagedControlPlane ManagedMachinePools []ManagedMachinePool + Cache *ManagedControlPlaneCache } // NewManagedControlPlaneScope creates a new Scope from the supplied parameters. @@ -82,6 +84,10 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane } } + if params.Cache == nil { + params.Cache = &ManagedControlPlaneCache{} + } + helper, err := patch.NewHelper(params.ControlPlane, params.Client) if err != nil { return nil, errors.Wrap(err, "failed to init patch helper") @@ -94,6 +100,7 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane ControlPlane: params.ControlPlane, ManagedMachinePools: params.ManagedMachinePools, patchHelper: helper, + cache: params.Cache, }, nil } @@ -102,6 +109,7 @@ type ManagedControlPlaneScope struct { Client client.Client patchHelper *patch.Helper kubeConfigData []byte + cache *ManagedControlPlaneCache AzureClients Cluster *clusterv1.Cluster @@ -109,6 +117,11 @@ type ManagedControlPlaneScope struct { ManagedMachinePools []ManagedMachinePool } +// ManagedControlPlaneCache stores ManagedControlPlane data locally so we don't have to hit the API multiple times within the same reconcile loop. +type ManagedControlPlaneCache struct { + isVnetManaged *bool +} + // ResourceGroup returns the managed control plane's resource group. func (s *ManagedControlPlaneScope) ResourceGroup() string { if s.ControlPlane == nil { @@ -332,7 +345,20 @@ func (s *ManagedControlPlaneScope) IsIPv6Enabled() bool { // IsVnetManaged returns true if the vnet is managed. func (s *ManagedControlPlaneScope) IsVnetManaged() bool { - return true + if s.cache.isVnetManaged != nil { + return to.Bool(s.cache.isVnetManaged) + } + // TODO refactor `IsVnetManaged` so that it is able to use an upstream context + // see https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/2581 + ctx := context.Background() + ctx, log, done := tele.StartSpanWithLogger(ctx, "scope.ManagedControlPlaneScope.IsVnetManaged") + defer done() + isManaged, err := virtualnetworks.New(s).IsManaged(ctx) + if err != nil { + log.Error(err, "Unable to determine if ManagedControlPlaneScope VNET is managed by capz", "AzureManagedCluster", s.ClusterName()) + } + s.cache.isVnetManaged = to.BoolPtr(isManaged) + return isManaged } // APIServerLBName returns the API Server LB spec. diff --git a/azure/services/subnets/subnets.go b/azure/services/subnets/subnets.go index 8a92bb1b968..a427beb947e 100644 --- a/azure/services/subnets/subnets.go +++ b/azure/services/subnets/subnets.go @@ -111,7 +111,7 @@ func (s *Service) Delete(ctx context.Context) error { defer cancel() if managed, err := s.IsManaged(ctx); err == nil && !managed { - log.V(4).Info("Skipping subnets deletion in custom vnet mode") + log.Info("Skipping subnets deletion in custom vnet mode") return nil } else if err != nil { return errors.Wrap(err, "failed to check if subnets are managed") From 17fcbfcd00912e14899ce8d2685c9f65c5b26df1 Mon Sep 17 00:00:00 2001 From: LochanRn Date: Sat, 24 Sep 2022 01:35:00 +0530 Subject: [PATCH 16/66] updated computediff authorized ip range logic for nil check --- azure/services/managedclusters/spec.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 6214c46387e..234055c0871 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -400,12 +400,18 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus propertiesNormalized.APIServerAccessProfile = &containerservice.ManagedClusterAPIServerAccessProfile{ AuthorizedIPRanges: managedCluster.APIServerAccessProfile.AuthorizedIPRanges, } + if managedCluster.APIServerAccessProfile.AuthorizedIPRanges == nil || len(*managedCluster.APIServerAccessProfile.AuthorizedIPRanges) == 0 { + propertiesNormalized.APIServerAccessProfile.AuthorizedIPRanges = nil + } } if existingMC.APIServerAccessProfile != nil { existingMCPropertiesNormalized.APIServerAccessProfile = &containerservice.ManagedClusterAPIServerAccessProfile{ AuthorizedIPRanges: existingMC.APIServerAccessProfile.AuthorizedIPRanges, } + if existingMC.APIServerAccessProfile.AuthorizedIPRanges == nil || len(*existingMC.APIServerAccessProfile.AuthorizedIPRanges) == 0 { + propertiesNormalized.APIServerAccessProfile.AuthorizedIPRanges = nil + } } clusterNormalized := &containerservice.ManagedCluster{ From 8ef8b4ae2ab57cf6b6e1fa3779ad0ad3b26aefdf Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Tue, 4 Oct 2022 12:39:49 +0530 Subject: [PATCH 17/66] Added subnetName spec to azure machine pool. (#50) --- azure/scope/managedmachinepool.go | 8 +++++++- ...ructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml | 4 ++++ exp/api/v1alpha3/azuremanagedmachinepool_conversion.go | 1 + exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/azuremanagedmachinepool_conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + exp/api/v1beta1/azuremanagedmachinepool_types.go | 4 ++++ spectro/generated/core-global.yaml | 4 ++++ 8 files changed, 23 insertions(+), 1 deletion(-) diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index c4ae7cc7e84..ae6ccd85961 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -150,6 +150,12 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane if managedControlPlane.Spec.VirtualNetwork.ResourceGroupName != "" { resourceGroupName = managedControlPlane.Spec.VirtualNetwork.ResourceGroupName } + + subnetName := managedControlPlane.Spec.VirtualNetwork.Subnet.Name + if managedMachinePool.Spec.SubnetName != "" { + subnetName = managedMachinePool.Spec.SubnetName + } + agentPoolSpec := azure.AgentPoolSpec{ Name: to.String(managedMachinePool.Spec.Name), ResourceGroup: managedControlPlane.Spec.ResourceGroupName, @@ -161,7 +167,7 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane managedControlPlane.Spec.SubscriptionID, resourceGroupName, managedControlPlane.Spec.VirtualNetwork.Name, - managedControlPlane.Spec.VirtualNetwork.Subnet.Name, + subnetName, ), Mode: managedMachinePool.Spec.Mode, MaxPods: managedMachinePool.Spec.MaxPods, diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index c33a9b5fc22..3b18a66ad2b 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -279,6 +279,10 @@ spec: sku: description: SKU is the size of the VMs in the node pool. type: string + subnetName: + description: SubnetName selects the Subnet where the MachinePool will + be placed + type: string taints: description: Taints specifies the taints for nodes present in this agent pool. diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go index 503fa8b4bf6..ed1a40c73e0 100644 --- a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go @@ -44,6 +44,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.OsDiskType = restored.Spec.OsDiskType dst.Spec.NodeLabels = restored.Spec.NodeLabels dst.Spec.EnableUltraSSD = restored.Spec.EnableUltraSSD + dst.Spec.SubnetName = restored.Spec.SubnetName dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates dst.Status.Conditions = restored.Status.Conditions diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 9ee5ffc8bfc..697654d6882 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -890,6 +890,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha3_AzureManagedMac // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type + // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go index 8d329b9df95..af9f1ca93ef 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go @@ -44,6 +44,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.OsDiskType = restored.Spec.OsDiskType dst.Spec.NodeLabels = restored.Spec.NodeLabels dst.Spec.EnableUltraSSD = restored.Spec.EnableUltraSSD + dst.Spec.SubnetName = restored.Spec.SubnetName dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates dst.Status.Conditions = restored.Status.Conditions diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index ff5f2fab8af..2f5fcba7d74 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1196,6 +1196,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type // WARNING: in.EnableUltraSSD requires manual conversion: does not exist in peer-type + // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go index d1c75b1bf8d..46efb3dfda2 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_types.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_types.go @@ -94,6 +94,10 @@ type AzureManagedMachinePoolSpec struct { // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. // +optional EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` + + // SubnetName selects the Subnet where the MachinePool will be placed + // +optional + SubnetName string `json:"subnetName,omitempty"` } // ManagedMachinePoolScaling specifies scaling options. diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index a72e81a202c..19efe03b31e 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -10312,6 +10312,10 @@ spec: sku: description: SKU is the size of the VMs in the node pool. type: string + subnetName: + description: SubnetName selects the Subnet where the MachinePool will + be placed + type: string taints: description: Taints specifies the taints for nodes present in this agent pool. From c47c27df7e8ce9a6751decdf9e0c9f873f1c5b81 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Wed, 2 Nov 2022 15:34:38 +0530 Subject: [PATCH 18/66] Use same subnet for cp and workers --- api/v1beta1/azurecluster_default.go | 4 +-- api/v1beta1/azurecluster_validation.go | 32 ++++++++++++------- api/v1beta1/azureclustertemplate_default.go | 4 +-- .../azureclustertemplate_validation.go | 2 +- api/v1beta1/types.go | 13 +++++--- api/v1beta1/types_class.go | 2 +- api/v1beta1/types_template.go | 6 ++-- azure/scope/machine.go | 17 +++++++--- ...ucture.cluster.x-k8s.io_azureclusters.yaml | 2 ++ ...luster.x-k8s.io_azureclustertemplates.yaml | 2 ++ config/default/manager_image_patch.yaml | 2 +- controllers/helpers.go | 2 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 6 +++- 14 files changed, 62 insertions(+), 34 deletions(-) diff --git a/api/v1beta1/azurecluster_default.go b/api/v1beta1/azurecluster_default.go index 71ff0f20282..200368ec44e 100644 --- a/api/v1beta1/azurecluster_default.go +++ b/api/v1beta1/azurecluster_default.go @@ -106,7 +106,7 @@ func (c *AzureCluster) setSubnetDefaults() { var nodeSubnetFound bool var nodeSubnetCounter int for i, subnet := range c.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode { + if subnet.Role == SubnetNode || subnet.Role == SubnetAll { nodeSubnetCounter++ nodeSubnetFound = true if subnet.Name == "" { @@ -202,7 +202,7 @@ func (c *AzureCluster) SetNodeOutboundLBDefaults() { var needsOutboundLB bool for _, subnet := range c.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { needsOutboundLB = true break } diff --git a/api/v1beta1/azurecluster_validation.go b/api/v1beta1/azurecluster_validation.go index e6e5a5254f5..dd7a469e296 100644 --- a/api/v1beta1/azurecluster_validation.go +++ b/api/v1beta1/azurecluster_validation.go @@ -135,7 +135,7 @@ func validateNetworkSpec(networkSpec NetworkSpec, old NetworkSpec, fldPath *fiel var oneSubnetWithoutNatGateway bool for _, subnet := range networkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { oneSubnetWithoutNatGateway = true break } @@ -171,20 +171,26 @@ func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field. "control-plane": false, "node": false, } + subnetAllRoleSpecified := false for i, subnet := range subnets { if err := validateSubnetName(subnet.Name, fldPath.Index(i).Child("name")); err != nil { allErrs = append(allErrs, err) } - if _, ok := subnetNames[subnet.Name]; ok { - allErrs = append(allErrs, field.Duplicate(fldPath, subnet.Name)) - } - subnetNames[subnet.Name] = true - for role := range requiredSubnetRoles { - if role == string(subnet.Role) { - requiredSubnetRoles[role] = true + if subnet.Role == SubnetAll { + subnetAllRoleSpecified = true + } else { + if _, ok := subnetNames[subnet.Name]; ok { + allErrs = append(allErrs, field.Duplicate(fldPath, subnet.Name)) + } + subnetNames[subnet.Name] = true + for role := range requiredSubnetRoles { + if role == string(subnet.Role) { + requiredSubnetRoles[role] = true + } } } + for _, rule := range subnet.SecurityGroup.SecurityRules { if err := validateSecurityRule( rule, @@ -195,10 +201,12 @@ func validateSubnets(subnets Subnets, vnet VnetSpec, fldPath *field.Path) field. } allErrs = append(allErrs, validateSubnetCIDR(subnet.CIDRBlocks, vnet.CIDRBlocks, fldPath.Index(i).Child("cidrBlocks"))...) } - for k, v := range requiredSubnetRoles { - if !v { - allErrs = append(allErrs, field.Required(fldPath, - fmt.Sprintf("required role %s not included in provided subnets", k))) + if !subnetAllRoleSpecified { + for k, v := range requiredSubnetRoles { + if !v { + allErrs = append(allErrs, field.Required(fldPath, + fmt.Sprintf("required role %s not included in provided subnets", k))) + } } } return allErrs diff --git a/api/v1beta1/azureclustertemplate_default.go b/api/v1beta1/azureclustertemplate_default.go index fb5d42f82da..3bf3d207086 100644 --- a/api/v1beta1/azureclustertemplate_default.go +++ b/api/v1beta1/azureclustertemplate_default.go @@ -65,7 +65,7 @@ func (c *AzureClusterTemplate) setSubnetsTemplateDefaults() { var nodeSubnetFound bool var nodeSubnetCounter int for i, subnet := range c.Spec.Template.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode { + if subnet.Role == SubnetNode || subnet.Role == SubnetAll { nodeSubnetCounter++ nodeSubnetFound = true subnet.SubnetClassSpec.setDefaults(fmt.Sprintf(DefaultNodeSubnetCIDRPattern, nodeSubnetCounter)) @@ -93,7 +93,7 @@ func (c *AzureClusterTemplate) setNodeOutboundLBDefaults() { var needsOutboundLB bool for _, subnet := range c.Spec.Template.Spec.NetworkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { needsOutboundLB = true break } diff --git a/api/v1beta1/azureclustertemplate_validation.go b/api/v1beta1/azureclustertemplate_validation.go index aaafc1454b4..661bf408bea 100644 --- a/api/v1beta1/azureclustertemplate_validation.go +++ b/api/v1beta1/azureclustertemplate_validation.go @@ -58,7 +58,7 @@ func (c *AzureClusterTemplate) validateClusterTemplateSpec() field.ErrorList { var oneSubnetWithoutNatGateway bool networkSpec := c.Spec.Template.Spec.NetworkSpec for _, subnet := range networkSpec.Subnets { - if subnet.Role == SubnetNode && !subnet.IsNatGatewayEnabled() { + if (subnet.Role == SubnetNode || subnet.Role == SubnetAll) && !subnet.IsNatGatewayEnabled() { oneSubnetWithoutNatGateway = true break } diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 7907eecf493..62aaa36fe58 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -28,6 +28,8 @@ const ( Node string = "node" // Bastion subnet label. Bastion string = "bastion" + // All subnet label. + All string = "all" ) // Futures is a slice of Future. @@ -552,6 +554,9 @@ const ( // SubnetBastion defines a Bastion subnet role. SubnetBastion = SubnetRole(Bastion) + + // SubnetAll defines a role that can be used for both Kubernetes control plane node and Kubernetes workload node. + SubnetAll = SubnetRole(All) ) // SubnetSpec configures an Azure subnet. @@ -582,17 +587,17 @@ type SubnetSpec struct { // GetControlPlaneSubnet returns the cluster control plane subnet. func (n *NetworkSpec) GetControlPlaneSubnet() (SubnetSpec, error) { for _, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { return sn, nil } } - return SubnetSpec{}, errors.Errorf("no subnet found with role %s", SubnetControlPlane) + return SubnetSpec{}, errors.Errorf("no subnet found with role %s or %s", SubnetControlPlane, SubnetAll) } // UpdateControlPlaneSubnet updates the cluster control plane subnet. func (n *NetworkSpec) UpdateControlPlaneSubnet(subnet SubnetSpec) { for i, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { n.Subnets[i] = subnet } } @@ -601,7 +606,7 @@ func (n *NetworkSpec) UpdateControlPlaneSubnet(subnet SubnetSpec) { // UpdateNodeSubnet updates the cluster node subnet. func (n *NetworkSpec) UpdateNodeSubnet(subnet SubnetSpec) { for i, sn := range n.Subnets { - if sn.Role == SubnetNode { + if sn.Role == SubnetNode || sn.Role == SubnetAll { n.Subnets[i] = subnet } } diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index bace21a7b88..79b0a5b5a44 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -73,7 +73,7 @@ type VnetClassSpec struct { // SubnetClassSpec defines the SubnetSpec properties that may be shared across several Azure clusters. type SubnetClassSpec struct { // Role defines the subnet role (eg. Node, ControlPlane) - // +kubebuilder:validation:Enum=node;control-plane;bastion + // +kubebuilder:validation:Enum=node;control-plane;bastion;all Role SubnetRole `json:"role"` // CIDRBlocks defines the subnet's address space, specified as one or more address prefixes in CIDR notation. diff --git a/api/v1beta1/types_template.go b/api/v1beta1/types_template.go index 1b5a20481a3..128fcba79cb 100644 --- a/api/v1beta1/types_template.go +++ b/api/v1beta1/types_template.go @@ -58,17 +58,17 @@ type NetworkTemplateSpec struct { // GetControlPlaneSubnetTemplate returns the cluster control plane subnet template. func (n *NetworkTemplateSpec) GetControlPlaneSubnetTemplate() (SubnetTemplateSpec, error) { for _, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { return sn, nil } } - return SubnetTemplateSpec{}, errors.Errorf("no subnet template found with role %s", SubnetControlPlane) + return SubnetTemplateSpec{}, errors.Errorf("no subnet template found with role %s or %s", SubnetControlPlane, SubnetAll) } // UpdateControlPlaneSubnet updates the cluster control plane subnet. func (n *NetworkTemplateSpec) UpdateControlPlaneSubnetTemplate(subnet SubnetTemplateSpec) { for i, sn := range n.Subnets { - if sn.Role == SubnetControlPlane { + if sn.Role == SubnetControlPlane || sn.Role == SubnetAll { n.Subnets[i] = subnet } } diff --git a/azure/scope/machine.go b/azure/scope/machine.go index be5b837705e..dcae55eea11 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -348,9 +348,9 @@ func (m *MachineScope) Subnet() infrav1.SubnetSpec { // AvailabilityZone returns the AzureMachine Availability Zone. // Priority for selecting the AZ is -// 1) Machine.Spec.FailureDomain -// 2) AzureMachine.Spec.FailureDomain (This is to support deprecated AZ) -// 3) No AZ +// 1. Machine.Spec.FailureDomain +// 2. AzureMachine.Spec.FailureDomain (This is to support deprecated AZ) +// 3. No AZ func (m *MachineScope) AvailabilityZone() string { if m.Machine.Spec.FailureDomain != nil { return *m.Machine.Spec.FailureDomain @@ -653,14 +653,21 @@ func (m *MachineScope) SetSubnetName() error { subnetName := "" subnets := m.Subnets() var subnetCount int + subnetAllSpecified := false for _, subnet := range subnets { if string(subnet.Role) == m.Role() { subnetCount++ subnetName = subnet.Name } + if subnet.Role == infrav1.SubnetAll { + subnetAllSpecified = true + subnetName = subnet.Name + } } - if subnetCount == 0 || subnetCount > 1 || subnetName == "" { - return errors.New("a subnet name must be specified when no subnets are specified or more than 1 subnet of the same role exist") + if !subnetAllSpecified { + if subnetCount == 0 || subnetCount > 1 || subnetName == "" { + return errors.New("a subnet name must be specified when no subnets are specified or more than 1 subnet of the same role exist") + } } m.AzureMachine.Spec.SubnetName = subnetName diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index c95d2fbcf14..f769603a833 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -1370,6 +1370,7 @@ spec: - node - control-plane - bastion + - all type: string routeTable: description: RouteTable defines the route table that should @@ -1846,6 +1847,7 @@ spec: - node - control-plane - bastion + - all type: string routeTable: description: RouteTable defines the route table that should diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml index 685920a786b..0e857917492 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml @@ -88,6 +88,7 @@ spec: - node - control-plane - bastion + - all type: string securityGroup: description: SecurityGroup defines the NSG (network @@ -409,6 +410,7 @@ spec: - node - control-plane - bastion + - all type: string securityGroup: description: SecurityGroup defines the NSG (network diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index b2be7491aee..24ca5c76f87 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 name: manager diff --git a/controllers/helpers.go b/controllers/helpers.go index d04b63f37cd..ed7478d6006 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -283,7 +283,7 @@ func newCloudProviderConfig(d azure.ClusterScoper) (controlPlaneConfig *CloudPro // getOneNodeSubnet returns one of the subnets for the node role. func getOneNodeSubnet(d azure.ClusterScoper) infrav1.SubnetSpec { for _, subnet := range d.Subnets() { - if subnet.Role == infrav1.SubnetNode { + if subnet.Role == infrav1.SubnetNode || subnet.Role == infrav1.SubnetAll { return subnet } } diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index b0b2bfb8cc6..ab8d2dee686 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index 19efe03b31e..f5215b48c86 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -1908,6 +1908,7 @@ spec: - node - control-plane - bastion + - all type: string routeTable: description: RouteTable defines the route table that should @@ -2384,6 +2385,7 @@ spec: - node - control-plane - bastion + - all type: string routeTable: description: RouteTable defines the route table that should @@ -2784,6 +2786,7 @@ spec: - node - control-plane - bastion + - all type: string securityGroup: description: SecurityGroup defines the NSG (network @@ -3105,6 +3108,7 @@ spec: - node - control-plane - bastion + - all type: string securityGroup: description: SecurityGroup defines the NSG (network @@ -10652,7 +10656,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 + image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 imagePullPolicy: Always livenessProbe: httpGet: From 4d320a9abd916244b0076c7c3af0b758da2eb52e Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Thu, 24 Nov 2022 20:43:54 +0530 Subject: [PATCH 19/66] private cluster change --- Makefile | 3 ++- api/v1beta1/azurecluster_default.go | 12 ++++++++++- api/v1beta1/azurecluster_validation.go | 2 +- api/v1beta1/types_class.go | 4 ++++ azure/scope/cluster.go | 2 ++ azure/services/loadbalancers/loadbalancers.go | 21 +++++++++++++++++-- azure/services/loadbalancers/spec.go | 10 +++++++-- config/default/manager_image_patch.yaml | 2 +- 8 files changed, 48 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f9f913186dc..14fa6728b0d 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,8 @@ IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) TAG ?= spectro-v1.3.2-$(shell date +%Y%m%d) ARCH ?= $(GOARCH) -ALL_ARCH = amd64 arm arm64 ppc64le s390x +#ALL_ARCH = amd64 arm arm64 ppc64le s390x +ALL_ARCH = amd64 # Allow overriding manifest generation destination directory MANIFEST_ROOT ?= config diff --git a/api/v1beta1/azurecluster_default.go b/api/v1beta1/azurecluster_default.go index 200368ec44e..9646e65e3b7 100644 --- a/api/v1beta1/azurecluster_default.go +++ b/api/v1beta1/azurecluster_default.go @@ -18,6 +18,7 @@ package v1beta1 import ( "fmt" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" "k8s.io/utils/pointer" ) @@ -178,6 +179,12 @@ func (c *AzureCluster) setAPIServerLBDefaults() { } } } else if lb.Type == Internal { + var privateIP string + if lb.PrivateIP == "" { + privateIP = DefaultInternalLBIPAddress + } else { + privateIP = lb.PrivateIP + } if lb.Name == "" { lb.Name = generateInternalLBName(c.ObjectMeta.Name) } @@ -186,7 +193,7 @@ func (c *AzureCluster) setAPIServerLBDefaults() { { Name: generateFrontendIPConfigName(lb.Name), FrontendIPClass: FrontendIPClass{ - PrivateIPAddress: DefaultInternalLBIPAddress, + PrivateIPAddress: privateIP, }, }, } @@ -304,6 +311,9 @@ func (lb *LoadBalancerClassSpec) setAPIServerLBDefaults() { if lb.SKU == "" { lb.SKU = SKUStandard } + if lb.IPAllocationMethod == "" { + lb.IPAllocationMethod = string(network.IPAllocationMethodDynamic) + } if lb.IdleTimeoutInMinutes == nil { lb.IdleTimeoutInMinutes = pointer.Int32Ptr(DefaultOutboundRuleIdleTimeoutInMinutes) } diff --git a/api/v1beta1/azurecluster_validation.go b/api/v1beta1/azurecluster_validation.go index dd7a469e296..ade8f0f400d 100644 --- a/api/v1beta1/azurecluster_validation.go +++ b/api/v1beta1/azurecluster_validation.go @@ -347,7 +347,7 @@ func validateAPIServerLB(lb LoadBalancerSpec, old LoadBalancerSpec, cidrs []stri fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP")); err != nil { allErrs = append(allErrs, err) } - if len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { + if lb.IPAllocationMethod == "Static" && len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress == "" && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { allErrs = append(allErrs, field.Forbidden(fldPath.Child("name"), "API Server load balancer private IP should not be modified after AzureCluster creation.")) } } diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index 79b0a5b5a44..ec570898ebf 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -87,6 +87,10 @@ type LoadBalancerClassSpec struct { SKU SKU `json:"sku,omitempty"` // +optional Type LBType `json:"type,omitempty"` + // +optional + IPAllocationMethod string `json:"ipAllocationMethod,omitempty"` + // +optional + PrivateIP string `json:"privateIP,omitempty"` // IdleTimeoutInMinutes specifies the timeout for the TCP idle connection. // +optional IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"` diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 002a852737b..881360fc7f4 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -199,6 +199,8 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter { Type: s.APIServerLB().Type, SKU: infrav1.SKUStandard, Role: infrav1.APIServerRole, + IPAllocationMethod: s.APIServerLB().IPAllocationMethod, + PrivateIP: s.APIServerLB().PrivateIP, BackendPoolName: s.APIServerLBPoolName(s.APIServerLB().Name), IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes, AdditionalTags: s.AdditionalTags(), diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index 49aa430ee78..48dd9be32cb 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -18,7 +18,9 @@ package loadbalancers import ( "context" - + "fmt" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" + "github.com/pkg/errors" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" @@ -78,10 +80,25 @@ func (s *Service) Reconcile(ctx context.Context) error { // Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (i.e. error creating) -> operationNotDoneError (i.e. creating in progress) -> no error (i.e. created) var result error for _, lbSpec := range specs { - if _, err := s.CreateResource(ctx, lbSpec, serviceName); err != nil { + if lb, err := s.CreateResource(ctx, lbSpec, serviceName); err != nil { if !azure.IsOperationNotDoneError(err) || result == nil { result = err } + } else { + fmt.Println("RETURN TYPE ", fmt.Sprintf("%T", lb)) + loadBalancer, ok := lb.(network.LoadBalancer) + if !ok { + // Return out of loop since this would be an unexepcted fatal error + result = errors.Errorf("created resource %T is not a network.LoadBalancer", lb) + break + } + if lbSpec.ResourceName() == s.Scope.APIServerLB().Name { + if len(*loadBalancer.FrontendIPConfigurations) > 0 && *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress != "" { + fmt.Println("LOADBALANCER's PRIVATE IP", (*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress) + s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress + } + } + } } diff --git a/azure/services/loadbalancers/spec.go b/azure/services/loadbalancers/spec.go index 3948b56c0ee..a7e9ccf7532 100644 --- a/azure/services/loadbalancers/spec.go +++ b/azure/services/loadbalancers/spec.go @@ -35,6 +35,8 @@ type LBSpec struct { Role string Type infrav1.LBType SKU infrav1.SKU + IPAllocationMethod string + PrivateIP string VNetName string VNetResourceGroup string SubnetName string @@ -163,13 +165,17 @@ func getFrontendIPConfigs(lbSpec LBSpec) ([]network.FrontendIPConfiguration, []n frontendIDs := make([]network.SubResource, 0) for _, ipConfig := range lbSpec.FrontendIPConfigs { var properties network.FrontendIPConfigurationPropertiesFormat + var privateIPAddress string + if lbSpec.IPAllocationMethod == "Static" { + privateIPAddress = ipConfig.PrivateIPAddress + } if lbSpec.Type == infrav1.Internal { properties = network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.IPAllocationMethodStatic, + PrivateIPAllocationMethod: network.IPAllocationMethod(lbSpec.IPAllocationMethod), Subnet: &network.Subnet{ ID: to.StringPtr(azure.SubnetID(lbSpec.SubscriptionID, lbSpec.VNetResourceGroup, lbSpec.VNetName, lbSpec.SubnetName)), }, - PrivateIPAddress: to.StringPtr(ipConfig.PrivateIPAddress), + PrivateIPAddress: to.StringPtr(privateIPAddress), } } else { properties = network.FrontendIPConfigurationPropertiesFormat{ diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 24ca5c76f87..e0cae691e94 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221124 name: manager From 477542f545aaa892833983f4c9c3c7bc204f5d42 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Fri, 25 Nov 2022 14:34:23 +0530 Subject: [PATCH 20/66] cleanup --- api/v1beta1/azurecluster_default.go | 9 +-------- api/v1beta1/types_class.go | 2 -- azure/scope/cluster.go | 1 - azure/services/loadbalancers/loadbalancers.go | 3 --- azure/services/loadbalancers/spec.go | 1 - config/default/manager_image_patch.yaml | 2 +- 6 files changed, 2 insertions(+), 16 deletions(-) diff --git a/api/v1beta1/azurecluster_default.go b/api/v1beta1/azurecluster_default.go index 9646e65e3b7..a00b963c09d 100644 --- a/api/v1beta1/azurecluster_default.go +++ b/api/v1beta1/azurecluster_default.go @@ -19,7 +19,6 @@ package v1beta1 import ( "fmt" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" - "k8s.io/utils/pointer" ) @@ -179,12 +178,6 @@ func (c *AzureCluster) setAPIServerLBDefaults() { } } } else if lb.Type == Internal { - var privateIP string - if lb.PrivateIP == "" { - privateIP = DefaultInternalLBIPAddress - } else { - privateIP = lb.PrivateIP - } if lb.Name == "" { lb.Name = generateInternalLBName(c.ObjectMeta.Name) } @@ -193,7 +186,7 @@ func (c *AzureCluster) setAPIServerLBDefaults() { { Name: generateFrontendIPConfigName(lb.Name), FrontendIPClass: FrontendIPClass{ - PrivateIPAddress: privateIP, + PrivateIPAddress: DefaultInternalLBIPAddress, }, }, } diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index ec570898ebf..26675a02d74 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -89,8 +89,6 @@ type LoadBalancerClassSpec struct { Type LBType `json:"type,omitempty"` // +optional IPAllocationMethod string `json:"ipAllocationMethod,omitempty"` - // +optional - PrivateIP string `json:"privateIP,omitempty"` // IdleTimeoutInMinutes specifies the timeout for the TCP idle connection. // +optional IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"` diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 881360fc7f4..6a2938bb1df 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -200,7 +200,6 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter { SKU: infrav1.SKUStandard, Role: infrav1.APIServerRole, IPAllocationMethod: s.APIServerLB().IPAllocationMethod, - PrivateIP: s.APIServerLB().PrivateIP, BackendPoolName: s.APIServerLBPoolName(s.APIServerLB().Name), IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes, AdditionalTags: s.AdditionalTags(), diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index 48dd9be32cb..c7208159aea 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -18,7 +18,6 @@ package loadbalancers import ( "context" - "fmt" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" "github.com/pkg/errors" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" @@ -85,7 +84,6 @@ func (s *Service) Reconcile(ctx context.Context) error { result = err } } else { - fmt.Println("RETURN TYPE ", fmt.Sprintf("%T", lb)) loadBalancer, ok := lb.(network.LoadBalancer) if !ok { // Return out of loop since this would be an unexepcted fatal error @@ -94,7 +92,6 @@ func (s *Service) Reconcile(ctx context.Context) error { } if lbSpec.ResourceName() == s.Scope.APIServerLB().Name { if len(*loadBalancer.FrontendIPConfigurations) > 0 && *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress != "" { - fmt.Println("LOADBALANCER's PRIVATE IP", (*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress) s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress } } diff --git a/azure/services/loadbalancers/spec.go b/azure/services/loadbalancers/spec.go index a7e9ccf7532..da3ffcc2a48 100644 --- a/azure/services/loadbalancers/spec.go +++ b/azure/services/loadbalancers/spec.go @@ -36,7 +36,6 @@ type LBSpec struct { Type infrav1.LBType SKU infrav1.SKU IPAllocationMethod string - PrivateIP string VNetName string VNetResourceGroup string SubnetName string diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index e0cae691e94..51d5d8453d1 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221124 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221125 name: manager From 4d3fa1be3b6273eb0e8d8a9a73a23c3dfdf3a459 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Fri, 25 Nov 2022 16:07:37 +0530 Subject: [PATCH 21/66] make file --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 14fa6728b0d..f9f913186dc 100644 --- a/Makefile +++ b/Makefile @@ -128,8 +128,7 @@ IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) TAG ?= spectro-v1.3.2-$(shell date +%Y%m%d) ARCH ?= $(GOARCH) -#ALL_ARCH = amd64 arm arm64 ppc64le s390x -ALL_ARCH = amd64 +ALL_ARCH = amd64 arm arm64 ppc64le s390x # Allow overriding manifest generation destination directory MANIFEST_ROOT ?= config From c91b70d805f19efa10a33c757d471011b90b8beb Mon Sep 17 00:00:00 2001 From: Deepak Sharma Date: Sat, 26 Nov 2022 19:16:46 +0530 Subject: [PATCH 22/66] new crds --- .../infrastructure.cluster.x-k8s.io_azureclusters.yaml | 6 ++++++ ...frastructure.cluster.x-k8s.io_azureclustertemplates.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index f769603a833..e56393c5d16 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -1680,6 +1680,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -1734,6 +1736,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -1786,6 +1790,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml index 0e857917492..59e6c1da99a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml @@ -335,6 +335,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -354,6 +356,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -371,6 +375,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string From 5b6bbc960e966ddc99c4538ab89f9827b7e04465 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Mon, 28 Nov 2022 12:58:22 +0530 Subject: [PATCH 23/66] add check for frontend ip --- azure/services/loadbalancers/loadbalancers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index c7208159aea..937b0befc27 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -91,7 +91,7 @@ func (s *Service) Reconcile(ctx context.Context) error { break } if lbSpec.ResourceName() == s.Scope.APIServerLB().Name { - if len(*loadBalancer.FrontendIPConfigurations) > 0 && *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress != "" { + if (loadBalancer.FrontendIPConfigurations != nil) && len(*loadBalancer.FrontendIPConfigurations) > 0 && *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress != "" { s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress } } From 640dd8e15b80464b1e0a766eaecc57bfa80805a4 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Mon, 28 Nov 2022 17:47:44 +0530 Subject: [PATCH 24/66] PEM-484: palette changes to input privatednszone --- azure/defaults.go | 4 ++-- azure/scope/cluster.go | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/azure/defaults.go b/azure/defaults.go index 53c83781e2c..ca5cac39979 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -164,8 +164,8 @@ func GeneratePrivateDNSZoneName(clusterName string) string { } // GeneratePrivateFQDN generates the FQDN for a private API Server based on the private DNS zone name. -func GeneratePrivateFQDN(zoneName string) string { - return fmt.Sprintf("%s.%s", PrivateAPIServerHostname, zoneName) +func GeneratePrivateFQDN(zoneName, clusterName string) string { + return fmt.Sprintf("%s.%s.%s", PrivateAPIServerHostname, clusterName, zoneName) } // GenerateVNetLinkName generates the name of a virtual network link name based on the vnet name. diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 6a2938bb1df..a910db95f6a 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -441,7 +441,7 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link records := make([]azure.ResourceSpecGetter, 1) records[0] = privatedns.RecordSpec{ Record: infrav1.AddressRecord{ - Hostname: azure.PrivateAPIServerHostname, + Hostname: s.getHostName(), IP: s.APIServerPrivateIP(), }, ZoneName: s.GetPrivateDNSZoneName(), @@ -454,6 +454,13 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link return nil, nil, nil } +func (s *ClusterScope) getHostName() string { + if s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneName != "" { + return fmt.Sprintf("%s.%s", azure.PrivateAPIServerHostname, s.AzureCluster.Name) + } + return azure.PrivateAPIServerHostname +} + // IsAzureBastionEnabled returns true if the azure bastion is enabled. func (s *ClusterScope) IsAzureBastionEnabled() bool { return s.AzureCluster.Spec.BastionSpec.AzureBastion != nil @@ -772,7 +779,7 @@ func (s *ClusterScope) APIServerPort() int32 { // APIServerHost returns the hostname used to reach the API server. func (s *ClusterScope) APIServerHost() string { if s.IsAPIServerPrivate() { - return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName()) + return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName(), s.AzureCluster.Name) } return s.APIServerPublicIP().DNSName } From 3fd8ef18e1683963487dc1f542c4304ad43430a2 Mon Sep 17 00:00:00 2001 From: Deepak Sharma Date: Tue, 29 Nov 2022 18:05:19 +0530 Subject: [PATCH 25/66] more nil checks --- azure/services/loadbalancers/loadbalancers.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index 937b0befc27..a3b832e0688 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -91,8 +91,11 @@ func (s *Service) Reconcile(ctx context.Context) error { break } if lbSpec.ResourceName() == s.Scope.APIServerLB().Name { - if (loadBalancer.FrontendIPConfigurations != nil) && len(*loadBalancer.FrontendIPConfigurations) > 0 && *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress != "" { - s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress + lbIPConfig := loadBalancer.FrontendIPConfigurations + if (lbIPConfig != nil) && len(*lbIPConfig) > 0 && + (*lbIPConfig)[0].PrivateIPAddress != nil && + *(*lbIPConfig)[0].PrivateIPAddress != "" { + s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*lbIPConfig)[0].PrivateIPAddress } } From e174277e5779a7daaf4d0d40c118a99e100139c0 Mon Sep 17 00:00:00 2001 From: Deepak Sharma Date: Tue, 29 Nov 2022 22:45:48 +0530 Subject: [PATCH 26/66] Revert "PEM-483: palette changes to input privatednszone" --- azure/defaults.go | 4 ++-- azure/scope/cluster.go | 11 ++--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/azure/defaults.go b/azure/defaults.go index ca5cac39979..53c83781e2c 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -164,8 +164,8 @@ func GeneratePrivateDNSZoneName(clusterName string) string { } // GeneratePrivateFQDN generates the FQDN for a private API Server based on the private DNS zone name. -func GeneratePrivateFQDN(zoneName, clusterName string) string { - return fmt.Sprintf("%s.%s.%s", PrivateAPIServerHostname, clusterName, zoneName) +func GeneratePrivateFQDN(zoneName string) string { + return fmt.Sprintf("%s.%s", PrivateAPIServerHostname, zoneName) } // GenerateVNetLinkName generates the name of a virtual network link name based on the vnet name. diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index a910db95f6a..6a2938bb1df 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -441,7 +441,7 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link records := make([]azure.ResourceSpecGetter, 1) records[0] = privatedns.RecordSpec{ Record: infrav1.AddressRecord{ - Hostname: s.getHostName(), + Hostname: azure.PrivateAPIServerHostname, IP: s.APIServerPrivateIP(), }, ZoneName: s.GetPrivateDNSZoneName(), @@ -454,13 +454,6 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link return nil, nil, nil } -func (s *ClusterScope) getHostName() string { - if s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneName != "" { - return fmt.Sprintf("%s.%s", azure.PrivateAPIServerHostname, s.AzureCluster.Name) - } - return azure.PrivateAPIServerHostname -} - // IsAzureBastionEnabled returns true if the azure bastion is enabled. func (s *ClusterScope) IsAzureBastionEnabled() bool { return s.AzureCluster.Spec.BastionSpec.AzureBastion != nil @@ -779,7 +772,7 @@ func (s *ClusterScope) APIServerPort() int32 { // APIServerHost returns the hostname used to reach the API server. func (s *ClusterScope) APIServerHost() string { if s.IsAPIServerPrivate() { - return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName(), s.AzureCluster.Name) + return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName()) } return s.APIServerPublicIP().DNSName } From 4617a99cded6ea91ba70d7ba74eaccca0a5d7b87 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Wed, 30 Nov 2022 11:39:00 +0530 Subject: [PATCH 27/66] Revert "Revert "PEM-483: palette changes to input privatednszone"" This reverts commit e174277e5779a7daaf4d0d40c118a99e100139c0. --- azure/defaults.go | 4 ++-- azure/scope/cluster.go | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/azure/defaults.go b/azure/defaults.go index 53c83781e2c..ca5cac39979 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -164,8 +164,8 @@ func GeneratePrivateDNSZoneName(clusterName string) string { } // GeneratePrivateFQDN generates the FQDN for a private API Server based on the private DNS zone name. -func GeneratePrivateFQDN(zoneName string) string { - return fmt.Sprintf("%s.%s", PrivateAPIServerHostname, zoneName) +func GeneratePrivateFQDN(zoneName, clusterName string) string { + return fmt.Sprintf("%s.%s.%s", PrivateAPIServerHostname, clusterName, zoneName) } // GenerateVNetLinkName generates the name of a virtual network link name based on the vnet name. diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 6a2938bb1df..a910db95f6a 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -441,7 +441,7 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link records := make([]azure.ResourceSpecGetter, 1) records[0] = privatedns.RecordSpec{ Record: infrav1.AddressRecord{ - Hostname: azure.PrivateAPIServerHostname, + Hostname: s.getHostName(), IP: s.APIServerPrivateIP(), }, ZoneName: s.GetPrivateDNSZoneName(), @@ -454,6 +454,13 @@ func (s *ClusterScope) PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, link return nil, nil, nil } +func (s *ClusterScope) getHostName() string { + if s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneName != "" { + return fmt.Sprintf("%s.%s", azure.PrivateAPIServerHostname, s.AzureCluster.Name) + } + return azure.PrivateAPIServerHostname +} + // IsAzureBastionEnabled returns true if the azure bastion is enabled. func (s *ClusterScope) IsAzureBastionEnabled() bool { return s.AzureCluster.Spec.BastionSpec.AzureBastion != nil @@ -772,7 +779,7 @@ func (s *ClusterScope) APIServerPort() int32 { // APIServerHost returns the hostname used to reach the API server. func (s *ClusterScope) APIServerHost() string { if s.IsAPIServerPrivate() { - return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName()) + return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName(), s.AzureCluster.Name) } return s.APIServerPublicIP().DNSName } From cf9603987cf1384c9df302a79468cff46c7e0354 Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Wed, 30 Nov 2022 16:56:29 +0530 Subject: [PATCH 28/66] fix private dns issue --- azure/defaults.go | 7 +++++-- azure/scope/cluster.go | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/azure/defaults.go b/azure/defaults.go index ca5cac39979..bfd4b531129 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -164,8 +164,11 @@ func GeneratePrivateDNSZoneName(clusterName string) string { } // GeneratePrivateFQDN generates the FQDN for a private API Server based on the private DNS zone name. -func GeneratePrivateFQDN(zoneName, clusterName string) string { - return fmt.Sprintf("%s.%s.%s", PrivateAPIServerHostname, clusterName, zoneName) +func GeneratePrivateFQDN(zoneName, clusterName string, isPrivateDNSZoneName bool) string { + if isPrivateDNSZoneName { + return fmt.Sprintf("%s.%s.%s", PrivateAPIServerHostname, clusterName, zoneName) + } + return fmt.Sprintf("%s.%s", PrivateAPIServerHostname, zoneName) } // GenerateVNetLinkName generates the name of a virtual network link name based on the vnet name. diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index a910db95f6a..3ba3d131bc7 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -779,7 +779,11 @@ func (s *ClusterScope) APIServerPort() int32 { // APIServerHost returns the hostname used to reach the API server. func (s *ClusterScope) APIServerHost() string { if s.IsAPIServerPrivate() { - return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName(), s.AzureCluster.Name) + var isPrivateDNSZoneName bool + if len(s.AzureCluster.Spec.NetworkSpec.PrivateDNSZoneName) > 0 { + isPrivateDNSZoneName = true + } + return azure.GeneratePrivateFQDN(s.GetPrivateDNSZoneName(), s.AzureCluster.Name, isPrivateDNSZoneName) } return s.APIServerPublicIP().DNSName } From 6a0f49ec3d3749363b8998e086dadaf85d03c1a2 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Wed, 28 Dec 2022 20:06:32 +0530 Subject: [PATCH 29/66] Resolved NIC deletion issue --- azure/services/async/async.go | 4 +++- config/default/manager_image_patch.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/azure/services/async/async.go b/azure/services/async/async.go index 93d10100114..4488a63472d 100644 --- a/azure/services/async/async.go +++ b/azure/services/async/async.go @@ -151,7 +151,9 @@ func (s *Service) DeleteResource(ctx context.Context, spec azure.ResourceSpecGet future := s.Scope.GetLongRunningOperationState(resourceName, serviceName) if future != nil { _, err := processOngoingOperation(ctx, s.Scope, s.Deleter, resourceName, serviceName) - return err + if err != nil || future.Type == infrav1.DeleteFuture { + return err + } } // No long running operation is active, so delete the resource. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index b2be7491aee..e443486e724 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20220912 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221228 name: manager From f77b61e22e02723600cc84a0d641d207b89d7bd2 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 5 Aug 2022 17:35:57 +0100 Subject: [PATCH 30/66] enforce lowercase providerID RG to match cloud-provider-azure --- azure/services/scalesets/scalesets.go | 8 +- azure/services/scalesets/scalesets_test.go | 6 +- .../virtualmachines/virtualmachines.go | 8 +- .../virtualmachines/virtualmachines_test.go | 8 +- .../azuremanagedmachinepool_reconciler.go | 9 +- util/azure/azure.go | 37 ++++++++ util/azure/azure_test.go | 85 +++++++++++++++++++ 7 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 util/azure/azure.go create mode 100644 util/azure/azure_test.go diff --git a/azure/services/scalesets/scalesets.go b/azure/services/scalesets/scalesets.go index dafd7706897..2bddfe7ddb7 100644 --- a/azure/services/scalesets/scalesets.go +++ b/azure/services/scalesets/scalesets.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "sigs.k8s.io/cluster-api-provider-azure/util/generators" "sigs.k8s.io/cluster-api-provider-azure/util/slice" "sigs.k8s.io/cluster-api-provider-azure/util/tele" @@ -104,7 +105,12 @@ func (s *Service) Reconcile(ctx context.Context) (retErr error) { } if fetchedVMSS != nil { - s.Scope.SetProviderID(azure.ProviderIDPrefix + fetchedVMSS.ID) + // Transform the VMSS resource representation to conform to the cloud-provider-azure representation + providerID, err := azureutil.ConvertResourceGroupNameToLower(azure.ProviderIDPrefix + fetchedVMSS.ID) + if err != nil { + log.Error(err, "failed to parse VMSS ID", "ID", fetchedVMSS.ID) + } + s.Scope.SetProviderID(providerID) s.Scope.SetVMSSState(fetchedVMSS) } }() diff --git a/azure/services/scalesets/scalesets_test.go b/azure/services/scalesets/scalesets_test.go index b252c5e77a1..fabe05e48c0 100644 --- a/azure/services/scalesets/scalesets_test.go +++ b/azure/services/scalesets/scalesets_test.go @@ -909,7 +909,7 @@ func newWindowsVMSSSpec() azure.ScaleSetSpec { func newDefaultExistingVMSS(vmSize string) compute.VirtualMachineScaleSet { vmss := newDefaultVMSS(vmSize) - vmss.ID = to.StringPtr("vmss-id") + vmss.ID = to.StringPtr("subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") return vmss } @@ -1153,7 +1153,7 @@ func newDefaultInstances() []compute.VirtualMachineScaleSetVM { } func setupDefaultVMSSInProgressOperationDoneExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder, createdVMSS compute.VirtualMachineScaleSet, instances []compute.VirtualMachineScaleSetVM) { - createdVMSS.ID = to.StringPtr("vmss-id") + createdVMSS.ID = to.StringPtr("subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") createdVMSS.ProvisioningState = to.StringPtr(string(infrav1.Succeeded)) setupDefaultVMSSExpectations(s) future := &infrav1.Future{ @@ -1243,7 +1243,7 @@ func setupVMSSExpectationsWithoutVMImage(s *mock_scalesets.MockScaleSetScopeMock func setupDefaultVMSSUpdateExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder) { setupUpdateVMSSExpectations(s) - s.SetProviderID(azure.ProviderIDPrefix + "vmss-id") + s.SetProviderID(azure.ProviderIDPrefix + "subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.GetLongRunningOperationState(defaultVMSSName, serviceName).Return(nil) s.MaxSurge().Return(1, nil) s.SetVMSSState(gomock.Any()) diff --git a/azure/services/virtualmachines/virtualmachines.go b/azure/services/virtualmachines/virtualmachines.go index 79b0ba5d5d8..c52cd9738f6 100644 --- a/azure/services/virtualmachines/virtualmachines.go +++ b/azure/services/virtualmachines/virtualmachines.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" "sigs.k8s.io/cluster-api-provider-azure/azure/services/networkinterfaces" "sigs.k8s.io/cluster-api-provider-azure/azure/services/publicips" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) @@ -98,7 +99,12 @@ func (s *Service) Reconcile(ctx context.Context) error { if err != nil { return err } - s.Scope.SetProviderID(azure.ProviderIDPrefix + infraVM.ID) + // Transform the VM resource representation to conform to the cloud-provider-azure representation + providerID, err := azureutil.ConvertResourceGroupNameToLower(azure.ProviderIDPrefix + infraVM.ID) + if err != nil { + return errors.Wrapf(err, "failed to parse VM ID %s", infraVM.ID) + } + s.Scope.SetProviderID(providerID) s.Scope.SetAnnotation("cluster-api-provider-azure", "true") // Discover addresses for NICs associated with the VM diff --git a/azure/services/virtualmachines/virtualmachines_test.go b/azure/services/virtualmachines/virtualmachines_test.go index d3bb10a9d74..674cb5678f1 100644 --- a/azure/services/virtualmachines/virtualmachines_test.go +++ b/azure/services/virtualmachines/virtualmachines_test.go @@ -53,7 +53,7 @@ var ( BootstrapData: "fake data", } fakeExistingVM = compute.VirtualMachine{ - ID: to.StringPtr("test-vm-id"), + ID: to.StringPtr("subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm"), Name: to.StringPtr("test-vm-name"), VirtualMachineProperties: &compute.VirtualMachineProperties{ ProvisioningState: to.StringPtr("Succeeded"), @@ -127,7 +127,7 @@ func TestReconcileVM(t *testing.T) { r.CreateResource(gomockinternal.AContext(), &fakeVMSpec, serviceName).Return(fakeExistingVM, nil) s.UpdatePutStatus(infrav1.VMRunningCondition, serviceName, nil) s.UpdatePutStatus(infrav1.DisksReadyCondition, serviceName, nil) - s.SetProviderID("azure://test-vm-id") + s.SetProviderID("azure://subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.SetAnnotation("cluster-api-provider-azure", "true") mnic.Get(gomockinternal.AContext(), &fakeNetworkInterfaceGetterSpec).Return(fakeNetworkInterface, nil) mpip.Get(gomockinternal.AContext(), "test-group", "pip-1").Return(fakePublicIPs, nil) @@ -153,7 +153,7 @@ func TestReconcileVM(t *testing.T) { r.CreateResource(gomockinternal.AContext(), &fakeVMSpec, serviceName).Return(fakeExistingVM, nil) s.UpdatePutStatus(infrav1.VMRunningCondition, serviceName, nil) s.UpdatePutStatus(infrav1.DisksReadyCondition, serviceName, nil) - s.SetProviderID("azure://test-vm-id") + s.SetProviderID("azure://subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.SetAnnotation("cluster-api-provider-azure", "true") mnic.Get(gomockinternal.AContext(), &fakeNetworkInterfaceGetterSpec).Return(network.Interface{}, internalError) }, @@ -166,7 +166,7 @@ func TestReconcileVM(t *testing.T) { r.CreateResource(gomockinternal.AContext(), &fakeVMSpec, serviceName).Return(fakeExistingVM, nil) s.UpdatePutStatus(infrav1.VMRunningCondition, serviceName, nil) s.UpdatePutStatus(infrav1.DisksReadyCondition, serviceName, nil) - s.SetProviderID("azure://test-vm-id") + s.SetProviderID("azure://subscriptions/123/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") s.SetAnnotation("cluster-api-provider-azure", "true") mnic.Get(gomockinternal.AContext(), &fakeNetworkInterfaceGetterSpec).Return(fakeNetworkInterface, nil) mpip.Get(gomockinternal.AContext(), "test-group", "pip-1").Return(network.PublicIPAddress{}, internalError) diff --git a/exp/controllers/azuremanagedmachinepool_reconciler.go b/exp/controllers/azuremanagedmachinepool_reconciler.go index 927a7ec4744..67fd56bbc3e 100644 --- a/exp/controllers/azuremanagedmachinepool_reconciler.go +++ b/exp/controllers/azuremanagedmachinepool_reconciler.go @@ -19,7 +19,6 @@ package controllers import ( "context" "fmt" - "strings" "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" @@ -28,6 +27,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure/scope" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools" "sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets" + azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) @@ -133,7 +133,12 @@ func (s *azureManagedMachinePoolService) Reconcile(ctx context.Context) error { var providerIDs = make([]string, len(instances)) for i := 0; i < len(instances); i++ { - providerIDs[i] = strings.ToLower(azure.ProviderIDPrefix + *instances[i].ID) + // Transform the VMSS instance resource representation to conform to the cloud-provider-azure representation + providerID, err := azureutil.ConvertResourceGroupNameToLower(azure.ProviderIDPrefix + *instances[i].ID) + if err != nil { + return errors.Wrapf(err, "failed to parse instance ID %s", *instances[i].ID) + } + providerIDs[i] = providerID } s.scope.SetAgentPoolProviderIDList(providerIDs) diff --git a/util/azure/azure.go b/util/azure/azure.go new file mode 100644 index 00000000000..79b9f5eedd2 --- /dev/null +++ b/util/azure/azure.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "regexp" + "strings" +) + +var azureResourceGroupNameRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/(?:.*)`) + +// ConvertResourceGroupNameToLower converts the resource group name in the resource ID to be lowered. +// Inspired by https://github.com/kubernetes-sigs/cloud-provider-azure/blob/88c9b89611e7c1fcbd39266928cce8406eb0e728/pkg/provider/azure_wrap.go#L409 +func ConvertResourceGroupNameToLower(resourceID string) (string, error) { + matches := azureResourceGroupNameRE.FindStringSubmatch(resourceID) + if len(matches) != 2 { + return "", fmt.Errorf("%q isn't in Azure resource ID format %q", resourceID, azureResourceGroupNameRE.String()) + } + + resourceGroup := matches[1] + return strings.Replace(resourceID, resourceGroup, strings.ToLower(resourceGroup), 1), nil +} diff --git a/util/azure/azure_test.go b/util/azure/azure_test.go new file mode 100644 index 00000000000..d2a6622c9ab --- /dev/null +++ b/util/azure/azure_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestConvertResourceGroupNameToLower(t *testing.T) { + tests := []struct { + desc string + resourceID string + expected string + expectError bool + }{ + { + desc: "empty string should report error", + resourceID: "", + expectError: true, + }, + { + desc: "resourceID not in Azure format should report error", + resourceID: "invalid-id", + expectError: true, + }, + { + desc: "providerID not in Azure format should report error", + resourceID: "azure://invalid-id", + expectError: true, + }, + { + desc: "resource group name in VM providerID should be converted", + resourceID: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + expected: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + }, + { + desc: "resource group name in VM resourceID should be converted", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + expected: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0", + }, + { + desc: "resource group name in VMSS providerID should be converted", + resourceID: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + expected: "azure:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + }, + { + desc: "resource group name in VMSS resourceID should be converted", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + expected: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroupname/providers/Microsoft.Compute/virtualMachineScaleSets/myScaleSetName/virtualMachines/156", + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + var actual string + var err error + actual, err = ConvertResourceGroupNameToLower(test.resourceID) + if test.expectError { + g.Expect(err).NotTo(BeNil()) + } else { + g.Expect(err).To(BeNil()) + g.Expect(actual).To(Equal(test.expected)) + } + }) + } +} From 7c12f46826632bf70ff8055bb4ffa48afa1fa460 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Wed, 1 Mar 2023 20:23:33 +0530 Subject: [PATCH 31/66] Bumped versions to resolve security issues --- Dockerfile | 2 +- config/default/manager_image_patch.yaml | 2 +- go.mod | 18 +++--- go.sum | 84 +++++-------------------- 4 files changed, 27 insertions(+), 79 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9dffa8d2064..0cafaefaed4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # Build the manager binary -FROM golang:1.17 as builder +FROM golang:1.19.4 as builder WORKDIR /workspace # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index e443486e724..642bad1a520 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221228 + - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20230301 name: manager diff --git a/go.mod b/go.mod index 3596b2c6797..c8143a23d93 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module sigs.k8s.io/cluster-api-provider-azure -go 1.17 +go 1.19 require ( - github.com/Azure/aad-pod-identity v1.8.6 + github.com/Azure/aad-pod-identity v1.8.13 github.com/Azure/azure-sdk-for-go v63.4.0+incompatible github.com/Azure/go-autorest/autorest v0.11.23 github.com/Azure/go-autorest/autorest/adal v0.9.18 @@ -32,8 +32,8 @@ require ( go.opentelemetry.io/otel/sdk v1.4.0 go.opentelemetry.io/otel/sdk/metric v0.27.0 go.opentelemetry.io/otel/trace v1.4.0 - golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 - golang.org/x/mod v0.5.1 + golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 helm.sh/helm/v3 v3.8.1 k8s.io/api v0.23.5 k8s.io/apimachinery v0.23.5 @@ -99,7 +99,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.0.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect @@ -175,12 +175,12 @@ require ( go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 1c34a4a3d8b..614728aeba9 100644 --- a/go.sum +++ b/go.sum @@ -37,7 +37,6 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -48,13 +47,11 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/aad-pod-identity v1.8.6 h1:aa5ybqA5d2e18MPEQyXvIDDGmXMa6bN7ZxK5PKpWQoE= -github.com/Azure/aad-pod-identity v1.8.6/go.mod h1:A+7rb0WOEhBmVaFSl/MtdVCiugoTilY7GpwCnrgzm2w= +github.com/Azure/aad-pod-identity v1.8.13 h1:/gUmacA0z7+lsOlGYAYzkGvAB/KOkUe5Pb6qSeiHD0k= +github.com/Azure/aad-pod-identity v1.8.13/go.mod h1:uxM/lsPo/abzqdk0rwEm4SqO9pavMz0fCmKpYAj4HL8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v57.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v63.4.0+incompatible h1:fle3M5Q7vr8auaiPffKyUQmLbvYeqpw30bKU6PrWJFo= github.com/Azure/azure-sdk-for-go v63.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -109,11 +106,9 @@ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= @@ -128,7 +123,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= @@ -155,14 +149,11 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= @@ -179,7 +170,6 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -217,7 +207,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -347,18 +336,14 @@ github.com/coredns/corefile-migration v1.0.14/go.mod h1:XnhgULOEouimnzgn0t4WPuFD github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.3.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -459,8 +444,6 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.2/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= -github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -530,7 +513,6 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -548,8 +530,9 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -681,7 +664,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= @@ -873,7 +855,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= @@ -970,7 +951,6 @@ github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xA github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -980,7 +960,6 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -994,7 +973,6 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= @@ -1011,7 +989,6 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= @@ -1023,7 +1000,6 @@ github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuI github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1048,10 +1024,8 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= @@ -1062,8 +1036,6 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= -github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1074,10 +1046,8 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -1111,7 +1081,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= @@ -1147,15 +1116,12 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= -github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -1195,7 +1161,6 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= @@ -1208,7 +1173,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -1218,7 +1182,6 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= go.opentelemetry.io/otel v1.4.0 h1:7ESuKPq6zpjRaY5nvVDGiuwK7VAJ8MwkKnmNJ9whNZ4= go.opentelemetry.io/otel v1.4.0/go.mod h1:jeAqMFKy2uLIxCtKxoFj0FAL5zAPKQagc3+GtBWakzk= -go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 h1:j7AwzDdAQBJjcqayAaYbvpYeZzII7cEe5qJTu+De6UY= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= @@ -1257,7 +1220,6 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1267,9 +1229,6 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1292,8 +1251,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1330,8 +1290,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1343,7 +1303,6 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1393,8 +1352,9 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d h1:62NvYBuaanGXR2ZOfwDFkhhl6X1DUgf8qg3GuQvxZsE= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1534,27 +1494,28 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1823,7 +1784,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -1868,7 +1828,6 @@ k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI= k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= -k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4= k8s.io/apiextensions-apiserver v0.23.4/go.mod h1:TWYAKymJx7nLMxWCgWm2RYGXHrGlVZnxIlGnvtfYu+g= k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= @@ -1882,34 +1841,28 @@ k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hr k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4= k8s.io/apiserver v0.23.4/go.mod h1:A6l/ZcNtxGfPSqbFDoxxOjEjSKBaQmE+UTveOmMkpNc= k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE= k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= -k8s.io/cli-runtime v0.23.0/go.mod h1:B5N3YH0KP1iKr6gEuJ/RRmGjO0mJQ/f/JrsmEiPQAlU= k8s.io/cli-runtime v0.23.4 h1:C3AFQmo4TK4dlVPLOI62gtHEHu0OfA2Cp4UVRZ1JXns= k8s.io/cli-runtime v0.23.4/go.mod h1:7KywUNTUibmHPqmpDFuRO1kc9RhsufHv2lkjCm2YZyM= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA= k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0= k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= k8s.io/cluster-bootstrap v0.23.0 h1:8pZuuAWPoygewSNB4IddX3HBwXcQkPDXL/ca7GtGf4o= k8s.io/cluster-bootstrap v0.23.0/go.mod h1:VltEnKWfrRTiKgOXp3ts3vh7yqNlH6KFKFflo9GtCBg= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.4/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI= k8s.io/component-base v0.23.4/go.mod h1:8o3Gg8i2vnUXGPOwciiYlkSaZT+p+7gA9Scoz8y4W4E= k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= -k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJEk/aerg= k8s.io/component-helpers v0.23.4/go.mod h1:1Pl7L4zukZ054ElzRbvmZ1FJIU8roBXFOeRFu8zipa4= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -1929,11 +1882,9 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kubectl v0.23.0/go.mod h1:TfcGEs3u4dkmoC2eku1GYymdGaMtPMcaLLFrX/RB2kI= k8s.io/kubectl v0.23.4 h1:mAa+zEOlyZieecEy+xSrhjkpMcukYyHWzcNdX28dzMY= k8s.io/kubectl v0.23.4/go.mod h1:Dgb0Rvx/8JKS/C2EuvsNiQc6RZnX0SbHJVG3XUzH6ok= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/metrics v0.23.0/go.mod h1:NDiZTwppEtAuKJ1Rxt3S4dhyRzdp6yUcJf0vo023dPo= k8s.io/metrics v0.23.4/go.mod h1:cl6sY9BdVT3DubbpqnkPIKi6mn/F2ltkU4yH1tEJ3Bo= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1947,14 +1898,12 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27/go.mod h1:tq2nT0Kx7W+/f2JVE+zxYtUhdjuELJkVpNz+x/QN5R4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/cluster-api v1.1.1 h1:wU1J+Yyu9xDC5M2uACH4Ii8vrIk37oZiCKeMEfyL0pk= sigs.k8s.io/cluster-api v1.1.1/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY= sigs.k8s.io/cluster-api/test v1.1.4 h1:QJSVvAeGtBSf4ImRiyPe1cTZJAGC7QGHZckEfCjLGco= sigs.k8s.io/cluster-api/test v1.1.4/go.mod h1:10gpXFNXUuYHoFfmGR98KfsJHuTQIWjRkiU8ggmXbrc= -sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= @@ -1971,7 +1920,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 6ff059fefa14cf69433b6fdd5dd408ede3db679f Mon Sep 17 00:00:00 2001 From: Jayesh Srivastava Date: Tue, 25 Apr 2023 15:08:34 +0530 Subject: [PATCH 32/66] Update go in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0cafaefaed4..f89f33b9178 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # Build the manager binary -FROM golang:1.19.4 as builder +FROM golang:1.20.3 as builder WORKDIR /workspace # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy From 83ad3fb334fe577a3a68c0bc63002d2c7364304f Mon Sep 17 00:00:00 2001 From: Jayesh Srivastava Date: Fri, 28 Apr 2023 13:52:15 +0530 Subject: [PATCH 33/66] PSS-2845: Update go --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f89f33b9178..2e5ea4de86d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # Build the manager binary -FROM golang:1.20.3 as builder +FROM golang:1.19.8 as builder WORKDIR /workspace # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy From 0ab3bf180ee4abb81db7f476aead5c74a38a16d8 Mon Sep 17 00:00:00 2001 From: Snehal Amrutkar Date: Fri, 28 Apr 2023 14:42:52 +0530 Subject: [PATCH 34/66] Ignore error for other future types in delete functionality --- azure/services/async/async.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/azure/services/async/async.go b/azure/services/async/async.go index 4488a63472d..e8afd248d78 100644 --- a/azure/services/async/async.go +++ b/azure/services/async/async.go @@ -66,26 +66,31 @@ func processOngoingOperation(ctx context.Context, scope FutureScope, client Futu } isDone, err := client.IsDone(ctx, sdkFuture) - if err != nil { - return nil, errors.Wrap(err, "failed checking if the operation was complete") - } - + // Assume that if isDone is true, then we successfully checked that the + // operation was complete even if err is non-nil. Assume the error in that + // case is unrelated and will be captured in Result below. if !isDone { + if err != nil { + return nil, errors.Wrap(err, "failed checking if the operation was complete") + } // Operation is still in progress, update conditions and requeue. log.V(2).Info("long running operation is still ongoing", "service", serviceName, "resource", resourceName) return nil, azure.WithTransientError(azure.NewOperationNotDoneError(future), retryAfter(sdkFuture)) } + if err != nil { + log.V(2).Error(err, "error checking long running operation status after it finished") + } + + // Once the operation is done, we can delete the long running operation state. + // If the operation failed, this will allow it to be retried during the next reconciliation. + // If the resource is not found, we also reset the long-running operation state so we can attempt to create it again. + // This can happen if the resource was deleted by another process before we could get the result. + scope.DeleteLongRunningOperationState(resourceName, serviceName) // Resource has been created/deleted/updated. log.V(2).Info("long running operation has completed", "service", serviceName, "resource", resourceName) result, err = client.Result(ctx, sdkFuture, future.Type) - if err == nil || azure.ResourceNotFound(err) { - // Once we have the result, we can delete the long running operation state. - // If the resource is not found, we also reset the long-running operation state so we can attempt to create it again. - // This can happen if the resource was deleted by another process before we could get the result. - scope.DeleteLongRunningOperationState(resourceName, serviceName) - } - return result, err + return client.Result(ctx, sdkFuture, future.Type) } // CreateResource implements the logic for creating a resource Asynchronously. @@ -151,7 +156,7 @@ func (s *Service) DeleteResource(ctx context.Context, spec azure.ResourceSpecGet future := s.Scope.GetLongRunningOperationState(resourceName, serviceName) if future != nil { _, err := processOngoingOperation(ctx, s.Scope, s.Deleter, resourceName, serviceName) - if err != nil || future.Type == infrav1.DeleteFuture { + if err != nil && future.Type == infrav1.DeleteFuture { return err } } From 9768006f0b23bd2d0c47702c964a0270641ce005 Mon Sep 17 00:00:00 2001 From: zulfilee Date: Sat, 10 Jun 2023 00:27:59 +0000 Subject: [PATCH 35/66] Spectro FIPS and CICD --- .github/workflows/spectro-release.yaml | 68 +++++++++++++++++++++++++ Dockerfile | 5 ++ Makefile | 24 ++++++--- config/default/manager_image_patch.yaml | 2 +- 4 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/spectro-release.yaml diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml new file mode 100644 index 00000000000..0f530c6922b --- /dev/null +++ b/.github/workflows/spectro-release.yaml @@ -0,0 +1,68 @@ +name: Spectro Release +run-name: Release for Cluster API Azure ${{ github.event.inputs.release_version }} +on: + workflow_dispatch: + inputs: + release_version: + description: 'Cluster API Version to Build' + required: true + default: '0.0.0' +jobs: + builder: + # edge-runner machine group is a bunch of machines in US Datacenter + runs-on: ubuntu-latest + # Initialize all secrets required for the job + # Ensure that the credentials are provided as encrypted secrets + env: + SPECTRO_VERSION: ${{ github.event.inputs.release_version }} + steps: + - + uses: mukunku/tag-exists-action@v1.2.0 + id: checkTag + with: + tag: spectro-v${{ github.event.inputs.release_version }} + - + if: ${{ steps.checkTag.outputs.exists == 'true' }} + run: | + echo "Tag already exists for spectro-v${{ github.event.inputs.release_version }}..." + exit 1 + - + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to private registry + uses: docker/login-action@v1 + with: + registry: ${{ secrets.REGISTRY_URL }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + - + name: Build Image + env: + REGISTRY: gcr.io/spectro-images-public/release/cluster-api-azure + run: | + make docker-build-all + make docker-push-all + - + name: Build Image - FIPS Mode + env: + FIPS_ENABLE: yes + REGISTRY: gcr.io/spectro-images-public/release-fips/cluster-api-azure + run: | + make docker-build-all + make docker-push-all + - + name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: spectro-v${{ github.event.inputs.release_version }} + release_name: Release spectro-v${{ github.event.inputs.release_version }} + body: | + Release version ${{ github.event.inputs.release_version }} + draft: false + prerelease: false diff --git a/Dockerfile b/Dockerfile index 2e5ea4de86d..6449c25d449 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,11 @@ WORKDIR /workspace ARG goproxy=https://proxy.golang.org ENV GOPROXY=$goproxy +# FIPS +ARG CRYPTO_LIB +ENV GOEXPERIMENT=${CRYPTO_LIB:+boringcrypto} + + # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum diff --git a/Makefile b/Makefile index f9f913186dc..40a0ac7be64 100644 --- a/Makefile +++ b/Makefile @@ -121,14 +121,26 @@ KUBE_APISERVER=$(TOOLS_BIN_DIR)/kube-apiserver ETCD=$(TOOLS_BIN_DIR)/etcd # Define Docker related variables. Releases should modify and double check these vars. -REGISTRY ?= gcr.io/spectro-dev-public/snehal/cluster-api-azure +# Fips Flags +FIPS_ENABLE ?= "" + +RELEASE_LOC := release +ifeq ($(FIPS_ENABLE),yes) + RELEASE_LOC := release-fips +endif + +SPECTRO_VERSION ?= 4.0.0-dev +TAG ?= v1.3.2-spectro-${SPECTRO_VERSION} +ARCH ?= amd64 +# ALL_ARCH = amd64 arm arm64 ppc64le s390x +ALL_ARCH = amd64 + +REGISTRY ?= gcr.io/spectro-dev-public/$(USER)/${RELEASE_LOC} + STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api-azure PROD_REGISTRY := registry.k8s.io/cluster-api-azure IMAGE_NAME ?= cluster-api-azure-controller CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) -TAG ?= spectro-v1.3.2-$(shell date +%Y%m%d) -ARCH ?= $(GOARCH) -ALL_ARCH = amd64 arm arm64 ppc64le s390x # Allow overriding manifest generation destination directory MANIFEST_ROOT ?= config @@ -340,13 +352,13 @@ docker-pull-prerequisites: ## Pull prerequisites for building controller-manager .PHONY: docker-build docker-build: docker-pull-prerequisites ## Build the docker image for controller-manager. - DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG):$(TAG) + DOCKER_BUILDKIT=1 docker build --build-arg CRYPTO_LIB=${FIPS_ENABLE} --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" .PHONY: docker-push docker-push: ## Push the docker image - docker push $(CONTROLLER_IMG):$(TAG) + docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG) ## -------------------------------------- ## Docker — All ARCH diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 642bad1a520..f8141cfee55 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20230301 + - image: gcr.io/spectro-dev-public/devop2023/release-fips/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev name: manager From 81586695ee76ba73e586f31c8af7c410d43c84bd Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 16 Jun 2023 11:25:09 -0700 Subject: [PATCH 36/66] adding UserAssignedIdentities to AzureManagedControlPlane --- .gitattributes | 2 +- azure/scope/managedcontrolplane.go | 9 ++++++ azure/services/managedclusters/spec.go | 29 +++++++++++++++++++ ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 15 ++++++++++ exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + .../v1beta1/azuremanagedcontrolplane_types.go | 4 +++ exp/api/v1beta1/zz_generated.deepcopy.go | 5 ++++ 8 files changed, 65 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 980d8c0774d..7ccd8e1ac2b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ # Specify generated cluster templates as generated files -**/cluster-template-*.yaml linguist-generated +**/cluster-template-*.yaml linguist-generated \ No newline at end of file diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index c5cc5f4f85a..c8a208a674c 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -496,6 +496,15 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure } } + if len(s.ControlPlane.Spec.UserAssignedIdentities) > 0 { + var userAssignedIdentities []managedclusters.UserAssignedIdentity + for _, i := range s.ControlPlane.Spec.UserAssignedIdentities { + userAssignedIdentities = append(userAssignedIdentities, managedclusters.UserAssignedIdentity{ + ProviderID: i.ProviderID, + }) + } + } + return &managedClusterSpec } diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 234055c0871..d07b6858ae9 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -95,6 +95,17 @@ type ManagedClusterSpec struct { // Headers is the list of headers to add to the HTTP requests to update this resource. Headers map[string]string + + // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster + UserAssignedIdentities []UserAssignedIdentity +} + +// UserAssignedIdentity defines the user-assigned identities provided +// by the user to be assigned to Azure resources. +type UserAssignedIdentity struct { + // ProviderID is the identification ID of the user-assigned Identity, the format of an identity is: + // 'azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}' + ProviderID string `json:"providerID"` } // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. @@ -303,6 +314,24 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if len(s.UserAssignedIdentities) == 0 { + // system assigned assumed if no user assigned input + managedCluster.Identity = &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(infrav1.VMIdentitySystemAssigned), + } + } else { + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue, len(s.UserAssignedIdentities)) + for _, id := range s.UserAssignedIdentities { + uaIDs[id.ProviderID] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + } + managedCluster.Identity = &containerservice.ManagedClusterIdentity{ + Type: containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned), + UserAssignedIdentities: uaIDs, + } + } + if existing != nil { existingMC, ok := existing.(containerservice.ManagedCluster) if !ok { diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index c2fc718ade0..97fae876fd8 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -724,6 +724,21 @@ spec: description: SubscriptionID is the GUID of the Azure subscription to hold this cluster. type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user to assign the cluster + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array version: description: Version defines the desired Kubernetes version. minLength: 2 diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 697654d6882..4a98d54b555 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -767,6 +767,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo // WARNING: in.SKU requires manual conversion: does not exist in peer-type // WARNING: in.LoadBalancerProfile requires manual conversion: does not exist in peer-type // WARNING: in.APIServerAccessProfile requires manual conversion: does not exist in peer-type + // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 2f5fcba7d74..7a036348b42 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1070,6 +1070,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.SKU = (*SKU)(unsafe.Pointer(in.SKU)) out.LoadBalancerProfile = (*LoadBalancerProfile)(unsafe.Pointer(in.LoadBalancerProfile)) out.APIServerAccessProfile = (*APIServerAccessProfile)(unsafe.Pointer(in.APIServerAccessProfile)) + // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index 5634da24328..791ed6da05c 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -112,6 +112,10 @@ type AzureManagedControlPlaneSpec struct { // APIServerAccessProfile is the access profile for AKS API server. // +optional APIServerAccessProfile *APIServerAccessProfile `json:"apiServerAccessProfile,omitempty"` + + // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster + // +optional + UserAssignedIdentities []infrav1.UserAssignedIdentity `json:"userAssignedIdentities,omitempty"` } // AADProfile - AAD integration managed by AKS. diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index edb1cbeba24..62b8b136c9a 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -686,6 +686,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(APIServerAccessProfile) (*in).DeepCopyInto(*out) } + if in.UserAssignedIdentities != nil { + in, out := &in.UserAssignedIdentities, &out.UserAssignedIdentities + *out = make([]apiv1beta1.UserAssignedIdentity, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. From d045c517b0d711a8436074606f65d332ee3a8c0a Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 16 Jun 2023 13:33:39 -0700 Subject: [PATCH 37/66] adding OutboundType to AzureManagedControlPlane --- azure/scope/managedcontrolplane.go | 1 + azure/services/managedclusters/spec.go | 8 ++++++++ ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 8 ++++++++ .../v1beta1/azuremanagedcontrolplane_types.go | 19 ++++++++++++++++++ .../azuremanagedcontrolplane_webhook.go | 19 ++++++++++++++++++ .../azuremanagedcontrolplane_webhook_test.go | 20 +++++++++++++++++++ exp/api/v1beta1/zz_generated.deepcopy.go | 7 ++++++- 7 files changed, 81 insertions(+), 1 deletion(-) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index c8a208a674c..7c5c5f1b815 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -432,6 +432,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure s.ControlPlane.Spec.VirtualNetwork.Subnet.Name, ), GetAllAgentPools: s.GetAllAgentPoolSpecs, + OutboundType: s.ControlPlane.Spec.OutboundType, } if s.ControlPlane.Spec.NetworkPlugin != nil { diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index d07b6858ae9..47132fdcf26 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -29,6 +29,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" + expinfrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" ) // ManagedClusterSpec contains properties to create a managed cluster. @@ -63,6 +64,9 @@ type ManagedClusterSpec struct { // NetworkPolicy used for building Kubernetes network. Possible values include: 'calico', 'azure'. Defaults to azure. NetworkPolicy string + // OutboundType used for building Kubernetes network. Possible values include: 'loadBalancer', 'managedNATGateway', 'userAssignedNATGateway', 'userDefinedRouting'. + OutboundType *expinfrav1.ManagedControlPlaneOutboundType + // SSHPublicKey is a string literal containing an ssh public key. Will autogenerate and discard if not provided. SSHPublicKey string @@ -314,6 +318,10 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if s.OutboundType != nil { + managedCluster.NetworkProfile.OutboundType = containerservice.OutboundType(*s.OutboundType) + } + if len(s.UserAssignedIdentities) == 0 { // system assigned assumed if no user assigned input managedCluster.Identity = &containerservice.ManagedClusterIdentity{ diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 97fae876fd8..a80f7e978f6 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -700,6 +700,14 @@ spec: containing cluster IaaS resources. Will be populated to default in webhook. type: string + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string resourceGroupName: description: ResourceGroupName is the name of the Azure resource group for this AKS Cluster. diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index 791ed6da05c..ae9ed625e3d 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -31,6 +31,20 @@ const ( PrivateDNSZoneModeNone string = "None" ) +// ManagedControlPlaneOutboundType enumerates the values for the managed control plane OutboundType. +type ManagedControlPlaneOutboundType string + +const ( + // ManagedControlPlaneOutboundTypeLoadBalancer ... + ManagedControlPlaneOutboundTypeLoadBalancer ManagedControlPlaneOutboundType = "loadBalancer" + // ManagedControlPlaneOutboundTypeManagedNATGateway ... + ManagedControlPlaneOutboundTypeManagedNATGateway ManagedControlPlaneOutboundType = "managedNATGateway" + // ManagedControlPlaneOutboundTypeUserAssignedNATGateway ... + ManagedControlPlaneOutboundTypeUserAssignedNATGateway ManagedControlPlaneOutboundType = "userAssignedNATGateway" + // ManagedControlPlaneOutboundTypeUserDefinedRouting ... + ManagedControlPlaneOutboundTypeUserDefinedRouting ManagedControlPlaneOutboundType = "userDefinedRouting" +) + // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { // Version defines the desired Kubernetes version. @@ -76,6 +90,11 @@ type AzureManagedControlPlaneSpec struct { // +optional NetworkPolicy *string `json:"networkPolicy,omitempty"` + // Outbound configuration used by Nodes. + // +kubebuilder:validation:Enum=loadBalancer;managedNATGateway;userAssignedNATGateway;userDefinedRouting + // +optional + OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` + // SSHPublicKey is a string literal containing an ssh public key base64 encoded. SSHPublicKey string `json:"sshPublicKey"` diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index b71fe0a686c..832590906db 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -240,6 +240,25 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if old.Spec.OutboundType != nil { + // Prevent NetworkPolicy modification if it was already set to some value + if m.Spec.OutboundType == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OutboundType"), + m.Spec.OutboundType, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.OutboundType != *old.Spec.OutboundType { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OutboundType"), + *m.Spec.OutboundType, + "field is immutable")) + } + } + if errs := m.validateAPIServerAccessProfileUpdate(old); len(errs) > 0 { allErrs = append(allErrs, errs...) } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 25651eeaf00..dab8bda8bd5 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -777,6 +777,26 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "OutboundType update", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + OutboundType: (*ManagedControlPlaneOutboundType)(to.StringPtr(string(ManagedControlPlaneOutboundTypeUserDefinedRouting))), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + OutboundType: (*ManagedControlPlaneOutboundType)(to.StringPtr(string(ManagedControlPlaneOutboundTypeLoadBalancer))), + }, + }, + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 62b8b136c9a..152cff77c97 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1beta1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" apiv1beta1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" @@ -644,6 +644,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(string) **out = **in } + if in.OutboundType != nil { + in, out := &in.OutboundType, &out.OutboundType + *out = new(ManagedControlPlaneOutboundType) + **out = **in + } if in.DNSServiceIP != nil { in, out := &in.DNSServiceIP, &out.DNSServiceIP *out = new(string) From b335198661b3e3f44565b8a913415a54e209f2f0 Mon Sep 17 00:00:00 2001 From: zulfilee Date: Sat, 1 Jul 2023 11:30:03 +0000 Subject: [PATCH 38/66] Fips changes minor --- .github/workflows/spectro-release.yaml | 10 +++++----- Dockerfile | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index 0f530c6922b..0dbd180d78c 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -20,11 +20,11 @@ jobs: uses: mukunku/tag-exists-action@v1.2.0 id: checkTag with: - tag: spectro-v${{ github.event.inputs.release_version }} + tag: v${{ github.event.inputs.release_version }}-spectro - if: ${{ steps.checkTag.outputs.exists == 'true' }} run: | - echo "Tag already exists for spectro-v${{ github.event.inputs.release_version }}..." + echo "Tag already exists for v${{ github.event.inputs.release_version }}-spectro..." exit 1 - uses: actions/checkout@v3 @@ -60,9 +60,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: spectro-v${{ github.event.inputs.release_version }} - release_name: Release spectro-v${{ github.event.inputs.release_version }} + tag_name: v${{ github.event.inputs.release_version }}-spectro + release_name: Release v${{ github.event.inputs.release_version }}-spectro body: | - Release version ${{ github.event.inputs.release_version }} + Release version v${{ github.event.inputs.release_version }}-spectro draft: false prerelease: false diff --git a/Dockerfile b/Dockerfile index 6449c25d449..c4278aed695 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,18 +15,20 @@ # limitations under the License. # Build the manager binary -FROM golang:1.19.8 as builder +FROM golang:1.19.10-alpine3.18 as builder WORKDIR /workspace # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org ENV GOPROXY=$goproxy +RUN apk update +RUN apk add git gcc g++ curl + # FIPS ARG CRYPTO_LIB ENV GOEXPERIMENT=${CRYPTO_LIB:+boringcrypto} - # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum @@ -52,9 +54,16 @@ ARG ldflags # Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ + if [ ${CRYPTO_LIB} ]; \ + then \ + CGO_ENABLED=1 GOOS=linux GOARCH=${ARCH} \ + go build -ldflags "${ldflags} -linkmode=external -extldflags '-static'" \ + -o manager ${package} ;\ + else \ CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ go build -ldflags "${ldflags} -extldflags '-static'" \ - -o manager ${package} + -o manager ${package} ;\ + fi # Production image FROM gcr.io/distroless/static:nonroot From 460b6e8004212c35d7524d8d27fe0d3590fdfed5 Mon Sep 17 00:00:00 2001 From: LochanRn Date: Tue, 4 Jul 2023 12:12:33 +0000 Subject: [PATCH 39/66] fix for add-on profile upgrade --- azure/services/managedclusters/spec.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 47132fdcf26..0e14ab4710e 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -451,6 +451,10 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus } } + if managedCluster.AddonProfiles != nil { + propertiesNormalized.AddonProfiles = managedCluster.AddonProfiles + } + clusterNormalized := &containerservice.ManagedCluster{ ManagedClusterProperties: propertiesNormalized, Tags: managedCluster.Tags, @@ -460,6 +464,10 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus Tags: existingMC.Tags, } + if existingMC.AddonProfiles != nil { + existingMCClusterNormalized.AddonProfiles = existingMC.AddonProfiles + } + if managedCluster.Sku != nil { clusterNormalized.Sku = managedCluster.Sku } From 23c4d8503b29da4166fc23f28f4370b47398b37a Mon Sep 17 00:00:00 2001 From: LochanRn Date: Wed, 5 Jul 2023 08:55:16 +0000 Subject: [PATCH 40/66] update removal flow --- azure/services/managedclusters/spec.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 0e14ab4710e..054f628b3b4 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -350,6 +350,21 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ return nil, azure.WithTransientError(errors.Errorf("Unable to update existing managed cluster in non-terminal state. Managed cluster must be in one of the following provisioning states: Canceled, Failed, or Succeeded. Actual state: %s", ps), 20*time.Second) } + if managedCluster.AddonProfiles == nil && existingMC.AddonProfiles != nil { + managedCluster.AddonProfiles = map[string]*containerservice.ManagedClusterAddonProfile{} + } + for key, item := range existingMC.AddonProfiles { + if _, ok := managedCluster.AddonProfiles[key]; !ok { + addonProfile := &containerservice.ManagedClusterAddonProfile{ + Enabled: to.BoolPtr(false), + } + if item.Config != nil { + addonProfile.Config = item.Config + } + managedCluster.AddonProfiles[key] = addonProfile + } + existingMC.AddonProfiles[key].Identity = nil + } // Normalize the LoadBalancerProfile so the diff below doesn't get thrown off by AKS added properties. if managedCluster.NetworkProfile.LoadBalancerProfile == nil { // If our LoadBalancerProfile generated by the spec is nil, then don't worry about what AKS has added. From f3470d4dc08879abad89cd209e2fe781ae76264a Mon Sep 17 00:00:00 2001 From: Vishwanath Taykhande Date: Wed, 5 Jul 2023 19:11:52 +0530 Subject: [PATCH 41/66] Fix CodeSmell --- azure/services/loadbalancers/loadbalancers.go | 2 +- controllers/azureidentity_controller.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index a3b832e0688..9de73a60cd9 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -86,7 +86,7 @@ func (s *Service) Reconcile(ctx context.Context) error { } else { loadBalancer, ok := lb.(network.LoadBalancer) if !ok { - // Return out of loop since this would be an unexepcted fatal error + // Return out of loop since this would be an unexpected fatal error result = errors.Errorf("created resource %T is not a network.LoadBalancer", lb) break } diff --git a/controllers/azureidentity_controller.go b/controllers/azureidentity_controller.go index a8b399e6c94..9688d36df69 100644 --- a/controllers/azureidentity_controller.go +++ b/controllers/azureidentity_controller.go @@ -185,7 +185,7 @@ func (r *AzureIdentityReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } - // delete bindings and identites no longer used by a cluster + // delete bindings and identities no longer used by a cluster for _, bindingToDelete := range bindingsToDelete { binding := bindingToDelete identityName := binding.Spec.AzureIdentity From ea2402dee88f3d355344209771ab02924842662f Mon Sep 17 00:00:00 2001 From: Vishwanath Taykhande Date: Wed, 12 Jul 2023 19:07:01 +0530 Subject: [PATCH 42/66] PCP-1467: UserAssignedIdentities Fix --- azure/scope/managedcontrolplane.go | 11 ++++--- azure/services/managedclusters/spec.go | 32 +++++++++++++++++---- exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + exp/api/v1beta1/zz_generated.deepcopy.go | 2 +- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 7c5c5f1b815..b0a158af51f 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -497,12 +497,11 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure } } - if len(s.ControlPlane.Spec.UserAssignedIdentities) > 0 { - var userAssignedIdentities []managedclusters.UserAssignedIdentity - for _, i := range s.ControlPlane.Spec.UserAssignedIdentities { - userAssignedIdentities = append(userAssignedIdentities, managedclusters.UserAssignedIdentity{ - ProviderID: i.ProviderID, - }) + if s.ControlPlane.Spec.UserAssignedIdentities != nil { + managedClusterSpec.UserAssignedIdentities = []managedclusters.UserAssignedIdentity{ + { + ProviderID: s.ControlPlane.Spec.UserAssignedIdentities[0].ProviderID, + }, } } diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 054f628b3b4..0e660a27555 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -322,17 +322,15 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ managedCluster.NetworkProfile.OutboundType = containerservice.OutboundType(*s.OutboundType) } - if len(s.UserAssignedIdentities) == 0 { + if s.UserAssignedIdentities == nil { // system assigned assumed if no user assigned input managedCluster.Identity = &containerservice.ManagedClusterIdentity{ Type: containerservice.ResourceIdentityType(infrav1.VMIdentitySystemAssigned), } } else { - uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue, len(s.UserAssignedIdentities)) - for _, id := range s.UserAssignedIdentities { - uaIDs[id.ProviderID] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ - // intentionally empty - } + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + uaIDs[s.UserAssignedIdentities[0].ProviderID] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty } managedCluster.Identity = &containerservice.ManagedClusterIdentity{ Type: containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned), @@ -365,6 +363,20 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } existingMC.AddonProfiles[key].Identity = nil } + + if s.UserAssignedIdentities != nil { + uai := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + uai[s.UserAssignedIdentities[0].ProviderID] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + if managedCluster.Identity == nil { + managedCluster.Identity = &containerservice.ManagedClusterIdentity{UserAssignedIdentities: uai, Type: containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned)} + } else { + managedCluster.Identity.Type = containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned) + managedCluster.Identity.UserAssignedIdentities = uai + } + } + // Normalize the LoadBalancerProfile so the diff below doesn't get thrown off by AKS added properties. if managedCluster.NetworkProfile.LoadBalancerProfile == nil { // If our LoadBalancerProfile generated by the spec is nil, then don't worry about what AKS has added. @@ -490,6 +502,14 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus existingMCClusterNormalized.Sku = existingMC.Sku } + if managedCluster.Identity != nil { + clusterNormalized.Identity = managedCluster.Identity + } + + if existingMC.Identity != nil { + existingMCClusterNormalized.Identity = existingMC.Identity + } + diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff } diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 4a98d54b555..bbf1c2e6f2c 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -758,6 +758,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo out.AdditionalTags = *(*clusterapiproviderazureapiv1alpha3.Tags)(unsafe.Pointer(&in.AdditionalTags)) out.NetworkPlugin = (*string)(unsafe.Pointer(in.NetworkPlugin)) out.NetworkPolicy = (*string)(unsafe.Pointer(in.NetworkPolicy)) + // WARNING: in.OutboundType requires manual conversion: does not exist in peer-type out.SSHPublicKey = in.SSHPublicKey out.DNSServiceIP = (*string)(unsafe.Pointer(in.DNSServiceIP)) out.LoadBalancerSKU = (*string)(unsafe.Pointer(in.LoadBalancerSKU)) diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 7a036348b42..d459c92926c 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1061,6 +1061,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.AdditionalTags = *(*clusterapiproviderazureapiv1alpha4.Tags)(unsafe.Pointer(&in.AdditionalTags)) out.NetworkPlugin = (*string)(unsafe.Pointer(in.NetworkPlugin)) out.NetworkPolicy = (*string)(unsafe.Pointer(in.NetworkPolicy)) + // WARNING: in.OutboundType requires manual conversion: does not exist in peer-type out.SSHPublicKey = in.SSHPublicKey out.DNSServiceIP = (*string)(unsafe.Pointer(in.DNSServiceIP)) out.LoadBalancerSKU = (*string)(unsafe.Pointer(in.LoadBalancerSKU)) diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 152cff77c97..12f216b6216 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1beta1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" apiv1beta1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" From 40324fe1569b6bf8235ec977bcb51e58c4311977 Mon Sep 17 00:00:00 2001 From: Vishwanath Taykhande Date: Thu, 13 Jul 2023 12:28:29 +0530 Subject: [PATCH 43/66] PCP-1467: PR Comments Resolution I --- azure/services/managedclusters/spec.go | 33 ++++++++++++++------------ 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 0e660a27555..74455f5a3f2 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -364,19 +364,6 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ existingMC.AddonProfiles[key].Identity = nil } - if s.UserAssignedIdentities != nil { - uai := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) - uai[s.UserAssignedIdentities[0].ProviderID] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ - // intentionally empty - } - if managedCluster.Identity == nil { - managedCluster.Identity = &containerservice.ManagedClusterIdentity{UserAssignedIdentities: uai, Type: containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned)} - } else { - managedCluster.Identity.Type = containerservice.ResourceIdentityType(infrav1.VMIdentityUserAssigned) - managedCluster.Identity.UserAssignedIdentities = uai - } - } - // Normalize the LoadBalancerProfile so the diff below doesn't get thrown off by AKS added properties. if managedCluster.NetworkProfile.LoadBalancerProfile == nil { // If our LoadBalancerProfile generated by the spec is nil, then don't worry about what AKS has added. @@ -503,11 +490,27 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus } if managedCluster.Identity != nil { - clusterNormalized.Identity = managedCluster.Identity + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + for key := range managedCluster.Identity.UserAssignedIdentities { + uaIDs[key] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + } + clusterNormalized.Identity = &containerservice.ManagedClusterIdentity{ + UserAssignedIdentities: uaIDs, + } } if existingMC.Identity != nil { - existingMCClusterNormalized.Identity = existingMC.Identity + uaIDs := make(map[string]*containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue) + for key := range existingMC.Identity.UserAssignedIdentities { + uaIDs[key] = &containerservice.ManagedClusterIdentityUserAssignedIdentitiesValue{ + // intentionally empty + } + } + existingMCClusterNormalized.Identity = &containerservice.ManagedClusterIdentity{ + UserAssignedIdentities: uaIDs, + } } diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) From c140ad03d9414e67038da369e971963b9c99b796 Mon Sep 17 00:00:00 2001 From: Vishwanath Taykhande <54012750+vishu2498@users.noreply.github.com> Date: Tue, 25 Jul 2023 21:47:37 +0530 Subject: [PATCH 44/66] PCP-1567: AKS ServiceCIDR Fix (#81) * PCP-1467: AKS ServiceCIDR Fix * PCP-1467: PR comments resolutions I --- azure/services/managedclusters/spec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 74455f5a3f2..e357ee98dfa 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -241,8 +241,8 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } if s.ServiceCIDR != "" { + managedCluster.NetworkProfile.ServiceCidr = &s.ServiceCIDR if s.DNSServiceIP == nil { - managedCluster.NetworkProfile.ServiceCidr = &s.ServiceCIDR ip, _, err := net.ParseCIDR(s.ServiceCIDR) if err != nil { return nil, fmt.Errorf("failed to parse service cidr: %w", err) From 3996ff53d36e7e81f781a8460afe7197752df49c Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Wed, 2 Aug 2023 17:49:10 +0530 Subject: [PATCH 45/66] PCP-1595 and PCP-1596 fix (#83) --- azure/scope/managedcontrolplane.go | 19 +- azure/services/managedclusters/spec.go | 25 ++- ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 24 ++- exp/api/v1alpha3/zz_generated.conversion.go | 3 + exp/api/v1alpha4/zz_generated.conversion.go | 3 + .../v1beta1/azuremanagedcontrolplane_types.go | 13 +- .../azuremanagedcontrolplane_webhook.go | 111 ++++++++++- .../azuremanagedcontrolplane_webhook_test.go | 183 ++++++++++++++++++ exp/api/v1beta1/zz_generated.deepcopy.go | 15 ++ 9 files changed, 381 insertions(+), 15 deletions(-) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index b0a158af51f..9b5efb75a51 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -432,7 +432,24 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure s.ControlPlane.Spec.VirtualNetwork.Subnet.Name, ), GetAllAgentPools: s.GetAllAgentPoolSpecs, - OutboundType: s.ControlPlane.Spec.OutboundType, + } + + if s.ControlPlane.Spec.OutboundType != nil { + managedClusterSpec.OutboundType = s.ControlPlane.Spec.OutboundType + } + + if s.ControlPlane.Spec.DNSPrefix != nil { + managedClusterSpec.DNSPrefix = s.ControlPlane.Spec.DNSPrefix + } else { + managedClusterSpec.DNSPrefix = &s.ControlPlane.Name + } + + if s.ControlPlane.Spec.FqdnSubdomain != nil { + managedClusterSpec.FqdnSubdomain = s.ControlPlane.Spec.FqdnSubdomain + } + + if s.ControlPlane.Spec.DockerBridgeCidr != nil { + managedClusterSpec.DockerBridgeCidr = s.ControlPlane.Spec.DockerBridgeCidr } if s.ControlPlane.Spec.NetworkPlugin != nil { diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index e357ee98dfa..d0aa605dd7e 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -79,9 +79,18 @@ type ManagedClusterSpec struct { // ServiceCIDR is the CIDR block for IP addresses distributed to services ServiceCIDR string + // DockerBridgeCidr - A CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. + DockerBridgeCidr *string `json:"dockerBridgeCidr,omitempty"` + // DNSServiceIP is an IP address assigned to the Kubernetes DNS service DNSServiceIP *string + // DNSPrefix - DNS prefix specified when creating the managed cluster. + DNSPrefix *string + + // FqdnSubdomain - FQDN subdomain specified when creating private cluster with custom private dns zone. + FqdnSubdomain *string + // AddonProfiles are the profiles of managed cluster add-on. AddonProfiles []AddonProfile @@ -208,7 +217,7 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ NodeResourceGroup: &s.NodeResourceGroup, EnableRBAC: to.BoolPtr(true), - DNSPrefix: &s.Name, + DNSPrefix: s.DNSPrefix, KubernetesVersion: &s.Version, LinuxProfile: &containerservice.LinuxProfile{ AdminUsername: to.StringPtr(azure.DefaultAKSUserName), @@ -232,6 +241,10 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ }, } + if s.FqdnSubdomain != nil { + managedCluster.FqdnSubdomain = s.FqdnSubdomain + } + if tags := *to.StringMapPtr(s.Tags); len(tags) != 0 { managedCluster.Tags = tags } @@ -259,6 +272,10 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if s.DockerBridgeCidr != nil { + managedCluster.NetworkProfile.DockerBridgeCidr = s.DockerBridgeCidr + } + if s.AADProfile != nil { managedCluster.AadProfile = &containerservice.ManagedClusterAADProfile{ Managed: &s.AADProfile.Managed, @@ -454,6 +471,9 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus if managedCluster.APIServerAccessProfile.AuthorizedIPRanges == nil || len(*managedCluster.APIServerAccessProfile.AuthorizedIPRanges) == 0 { propertiesNormalized.APIServerAccessProfile.AuthorizedIPRanges = nil } + if managedCluster.APIServerAccessProfile.PrivateDNSZone != nil { + propertiesNormalized.APIServerAccessProfile.PrivateDNSZone = managedCluster.APIServerAccessProfile.PrivateDNSZone + } } if existingMC.APIServerAccessProfile != nil { @@ -463,6 +483,9 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus if existingMC.APIServerAccessProfile.AuthorizedIPRanges == nil || len(*existingMC.APIServerAccessProfile.AuthorizedIPRanges) == 0 { propertiesNormalized.APIServerAccessProfile.AuthorizedIPRanges = nil } + if existingMC.APIServerAccessProfile.PrivateDNSZone != nil { + existingMCPropertiesNormalized.APIServerAccessProfile.PrivateDNSZone = existingMC.APIServerAccessProfile.PrivateDNSZone + } } if managedCluster.AddonProfiles != nil { diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index a80f7e978f6..254468b334a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -575,9 +575,6 @@ spec: privateDNSZone: description: PrivateDNSZone - Private dns zone mode for private cluster. - enum: - - System - - None type: string type: object controlPlaneEndpoint: @@ -595,11 +592,24 @@ spec: - host - port type: object + dnsPrefix: + description: DNSPrefix - DNS prefix specified when creating the managed + cluster. + type: string dnsServiceIP: description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. type: string + dockerBridgeCidr: + description: DockerBridgeCidr - A CIDR notation IP range assigned + to the Docker bridge network. It must not overlap with any Subnet + IP ranges or the Kubernetes service address range. + type: string + fqdnSubdomain: + description: FqdnSubdomain - FQDN subdomain specified when creating + private cluster with custom private dns zone. + type: string identityRef: description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster @@ -703,10 +713,10 @@ spec: outboundType: description: Outbound configuration used by Nodes. enum: - - loadBalancer - - managedNATGateway - - userAssignedNATGateway - - userDefinedRouting + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting type: string resourceGroupName: description: ResourceGroupName is the name of the Azure resource group diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index bbf1c2e6f2c..454de50d771 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -759,6 +759,9 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo out.NetworkPlugin = (*string)(unsafe.Pointer(in.NetworkPlugin)) out.NetworkPolicy = (*string)(unsafe.Pointer(in.NetworkPolicy)) // WARNING: in.OutboundType requires manual conversion: does not exist in peer-type + // WARNING: in.DockerBridgeCidr requires manual conversion: does not exist in peer-type + // WARNING: in.DNSPrefix requires manual conversion: does not exist in peer-type + // WARNING: in.FqdnSubdomain requires manual conversion: does not exist in peer-type out.SSHPublicKey = in.SSHPublicKey out.DNSServiceIP = (*string)(unsafe.Pointer(in.DNSServiceIP)) out.LoadBalancerSKU = (*string)(unsafe.Pointer(in.LoadBalancerSKU)) diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index d459c92926c..c5fc195c2e6 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1062,6 +1062,9 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.NetworkPlugin = (*string)(unsafe.Pointer(in.NetworkPlugin)) out.NetworkPolicy = (*string)(unsafe.Pointer(in.NetworkPolicy)) // WARNING: in.OutboundType requires manual conversion: does not exist in peer-type + // WARNING: in.DockerBridgeCidr requires manual conversion: does not exist in peer-type + // WARNING: in.DNSPrefix requires manual conversion: does not exist in peer-type + // WARNING: in.FqdnSubdomain requires manual conversion: does not exist in peer-type out.SSHPublicKey = in.SSHPublicKey out.DNSServiceIP = (*string)(unsafe.Pointer(in.DNSServiceIP)) out.LoadBalancerSKU = (*string)(unsafe.Pointer(in.LoadBalancerSKU)) diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index ae9ed625e3d..c7582e0484f 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -95,6 +95,18 @@ type AzureManagedControlPlaneSpec struct { // +optional OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` + // DockerBridgeCidr - A CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. + // +optional + DockerBridgeCidr *string `json:"dockerBridgeCidr,omitempty"` + + // DNSPrefix - DNS prefix specified when creating the managed cluster. + // +optional + DNSPrefix *string `json:"dnsPrefix,omitempty"` + + // FqdnSubdomain - FQDN subdomain specified when creating private cluster with custom private dns zone. + // +optional + FqdnSubdomain *string `json:"fqdnSubdomain,omitempty"` + // SSHPublicKey is a string literal containing an ssh public key base64 encoded. SSHPublicKey string `json:"sshPublicKey"` @@ -214,7 +226,6 @@ type APIServerAccessProfile struct { // +optional EnablePrivateCluster *bool `json:"enablePrivateCluster,omitempty"` // PrivateDNSZone - Private dns zone mode for private cluster. - // +kubebuilder:validation:Enum=System;None // +optional PrivateDNSZone *string `json:"privateDNSZone,omitempty"` // EnablePrivateClusterPublicFQDN - Whether to create additional public FQDN for private cluster or not. diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index 832590906db..6cb09a3baa8 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -18,17 +18,19 @@ package v1beta1 import ( "context" - "errors" "fmt" "net" "reflect" "regexp" "strings" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" @@ -241,7 +243,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } if old.Spec.OutboundType != nil { - // Prevent NetworkPolicy modification if it was already set to some value + // Prevent OutboundType modification if it was already set to some value if m.Spec.OutboundType == nil { // unsetting the field is not allowed allErrs = append(allErrs, @@ -259,6 +261,30 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if m.Spec.DNSPrefix != old.Spec.DNSPrefix { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec.DNSPrefix"), + m.Spec.DNSPrefix, + "field is immutable")) + } + + if m.Spec.DockerBridgeCidr != old.Spec.DockerBridgeCidr { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec.DockerBridgeCidr"), + m.Spec.DockerBridgeCidr, + "field is immutable")) + } + + if m.Spec.FqdnSubdomain != old.Spec.FqdnSubdomain { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec.FqdnSubdomain"), + m.Spec.FqdnSubdomain, + "field is immutable")) + } + if errs := m.validateAPIServerAccessProfileUpdate(old); len(errs) > 0 { allErrs = append(allErrs, errs...) } @@ -284,6 +310,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateSSHKey, m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, + m.validateDNSPrefix, //m.validateManagedClusterNetwork, } @@ -297,6 +324,43 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { return kerrors.NewAggregate(errs) } +func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { + + if m.Spec.DNSPrefix == nil { + return nil + } + + // Regex pattern for DNS prefix validation + // 1. Between 1 and 54 characters long: {1,54} + // 2. Alphanumerics and hyphens: [a-zA-Z0-9-] + // 3. Start and end with alphanumeric: ^[a-zA-Z0-9].*[a-zA-Z0-9]$ + regex := regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9-]{0,52}[a-zA-Z0-9]$`) + if regex.MatchString(pointer.StringDeref(m.Spec.DNSPrefix, "")) { + return nil + } + return errors.New("DNSPrefix is invalid") +} + +// func (m *AzureManagedControlPlane) validateFqdnSubdomain(_ client.Client) error { + +// if m.Spec.FqdnSubdomain == nil { +// return nil +// } + +// // Regex pattern for FQDN subdomain validation +// // 1. Between 1 and 63 characters long: {1,63} +// // 2. Alphanumerics and hyphens. +// // 3. Start and end with alphanumeric. +// // 4. Parts separated by dots (.) +// pattern := `^(?i)[a-z0-9]([a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*(?:\.aks\.example)$` + +// regex := regexp.MustCompile(pattern) +// if regex.MatchString(pointer.StringDeref(m.Spec.FqdnSubdomain, "")) { +// return nil +// } +// return errors.Errorf("FqdnSubdomain is invalid %s", pointer.StringDeref(m.Spec.FqdnSubdomain, "")) +// } + // validateDNSServiceIP validates the DNSServiceIP. func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { if m.Spec.DNSServiceIP != nil { @@ -480,14 +544,12 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileUpdate(old *Azu if m.Spec.APIServerAccessProfile != nil { newAPIServerAccessProfileNormalized = &APIServerAccessProfile{ EnablePrivateCluster: m.Spec.APIServerAccessProfile.EnablePrivateCluster, - PrivateDNSZone: m.Spec.APIServerAccessProfile.PrivateDNSZone, EnablePrivateClusterPublicFQDN: m.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, } } if old.Spec.APIServerAccessProfile != nil { oldAPIServerAccessProfileNormalized = &APIServerAccessProfile{ EnablePrivateCluster: old.Spec.APIServerAccessProfile.EnablePrivateCluster, - PrivateDNSZone: old.Spec.APIServerAccessProfile.PrivateDNSZone, EnablePrivateClusterPublicFQDN: old.Spec.APIServerAccessProfile.EnablePrivateClusterPublicFQDN, } } @@ -495,7 +557,46 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileUpdate(old *Azu if !reflect.DeepEqual(newAPIServerAccessProfileNormalized, oldAPIServerAccessProfileNormalized) { allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "APIServerAccessProfile"), - m.Spec.APIServerAccessProfile, "fields (except for AuthorizedIPRanges) are immutable"), + m.Spec.APIServerAccessProfile, "fields EnablePrivateCluster and EnablePrivateClusterPublicFQDN are immutable"), + ) + } + + if errs := m.validateAPIServerAccessProfileDNSZoneUpdate(old); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + + return allErrs +} + +func (m *AzureManagedControlPlane) validateAPIServerAccessProfileDNSZoneUpdate(old *AzureManagedControlPlane) field.ErrorList { + var allErrs field.ErrorList + + // You can only update from byo or system to none. No other combination of update values is supported. + if m.Spec.APIServerAccessProfile != nil && old.Spec.APIServerAccessProfile != nil && + pointer.BoolDeref(m.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + pointer.BoolDeref(old.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + m.Spec.APIServerAccessProfile.PrivateDNSZone != old.Spec.APIServerAccessProfile.PrivateDNSZone && + pointer.StringDeref(m.Spec.APIServerAccessProfile.PrivateDNSZone, "") == "None" { + return nil + } + + newAPIServerAccessProfileNormalized := &APIServerAccessProfile{} + oldAPIServerAccessProfileNormalized := &APIServerAccessProfile{} + if m.Spec.APIServerAccessProfile != nil { + newAPIServerAccessProfileNormalized = &APIServerAccessProfile{ + PrivateDNSZone: m.Spec.APIServerAccessProfile.PrivateDNSZone, + } + } + if old.Spec.APIServerAccessProfile != nil { + oldAPIServerAccessProfileNormalized = &APIServerAccessProfile{ + PrivateDNSZone: old.Spec.APIServerAccessProfile.PrivateDNSZone, + } + } + + if !reflect.DeepEqual(newAPIServerAccessProfileNormalized, oldAPIServerAccessProfileNormalized) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("Spec", "APIServerAccessProfile"), + m.Spec.APIServerAccessProfile, "invalid update operation performed on PrivateDNSZone. You can only update from byo or system to none. No other combination of update values is supported."), ) } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index dab8bda8bd5..6437d832ffc 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -82,6 +82,56 @@ func TestValidatingWebhook(t *testing.T) { amcp AzureManagedControlPlane expectErr bool }{ + { + name: "Testing inValid DNSPrefix for starting with invalid charecters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("-thisi$"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing inValid DNSPrefix with more then 54 characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing inValid DNSPrefix with underscore", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("no_underscore"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing inValid DNSPrefix with special characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("no-dollar$@%"), + Version: "v1.17.8", + }, + }, + expectErr: true, + }, + { + name: "Testing valid DNSPrefix ", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, { name: "Testing valid DNSServiceIP", amcp: AzureManagedControlPlane{ @@ -733,6 +783,139 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "AzureManagedControlPlane DockerBridgeCidr is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane DNSPrefix is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("capz-aks-1"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("capz-aks"), + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane FqdnSubdomain is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + FqdnSubdomain: pointer.StringPtr("capzaks.api"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + FqdnSubdomain: pointer.StringPtr("capzaks.com"), + }, + }, + wantErr: true, + }, + { + name: "Valid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + }, + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + PrivateDNSZone: pointer.StringPtr("None"), + }, + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "Valid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + PrivateDNSZone: pointer.StringPtr("example-resource-id"), + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + PrivateDNSZone: pointer.StringPtr("None"), + }, + }, + }, + wantErr: false, + }, + { + name: "Invalid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + PrivateDNSZone: pointer.StringPtr("example-resource-id"), + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + }, + }, + }, + wantErr: true, + }, + { + name: "Invalid AzureManagedControlPlane PrivateDNSZone update", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + PrivateDNSZone: pointer.StringPtr("example-resource-id"), + }, + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + APIServerAccessProfile: &APIServerAccessProfile{ + EnablePrivateCluster: pointer.Bool(true), + PrivateDNSZone: pointer.StringPtr("example-resource-id-1"), + }, + }, + }, + wantErr: true, + }, { name: "AzureManagedControlPlane AuthorizedIPRanges is mutable", oldAMCP: &AzureManagedControlPlane{ diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 12f216b6216..e90218f9258 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -649,6 +649,21 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(ManagedControlPlaneOutboundType) **out = **in } + if in.DockerBridgeCidr != nil { + in, out := &in.DockerBridgeCidr, &out.DockerBridgeCidr + *out = new(string) + **out = **in + } + if in.DNSPrefix != nil { + in, out := &in.DNSPrefix, &out.DNSPrefix + *out = new(string) + **out = **in + } + if in.FqdnSubdomain != nil { + in, out := &in.FqdnSubdomain, &out.FqdnSubdomain + *out = new(string) + **out = **in + } if in.DNSServiceIP != nil { in, out := &in.DNSServiceIP, &out.DNSServiceIP *out = new(string) From bb7c144da3547489676496f594c0290681009bc0 Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Wed, 2 Aug 2023 20:13:04 +0530 Subject: [PATCH 46/66] fix webhook issue (#85) --- .../azuremanagedcontrolplane_webhook.go | 6 +- .../azuremanagedcontrolplane_webhook_test.go | 63 +++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index 6cb09a3baa8..eaf2f570e12 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -261,7 +261,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } - if m.Spec.DNSPrefix != old.Spec.DNSPrefix { + if pointer.StringDeref(m.Spec.DNSPrefix, "") != pointer.StringDeref(old.Spec.DNSPrefix, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.DNSPrefix"), @@ -269,7 +269,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } - if m.Spec.DockerBridgeCidr != old.Spec.DockerBridgeCidr { + if pointer.StringDeref(m.Spec.DockerBridgeCidr, "") != pointer.StringDeref(old.Spec.DockerBridgeCidr, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.DockerBridgeCidr"), @@ -277,7 +277,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } - if m.Spec.FqdnSubdomain != old.Spec.FqdnSubdomain { + if pointer.StringDeref(m.Spec.FqdnSubdomain, "") != pointer.StringDeref(old.Spec.FqdnSubdomain, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.FqdnSubdomain"), diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 6437d832ffc..07e34697670 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -798,6 +798,37 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "AzureManagedControlPlane DockerBridgeCidr is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane DockerBridgeCidr is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DockerBridgeCidr: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: true, + }, { name: "AzureManagedControlPlane DNSPrefix is immutable", oldAMCP: &AzureManagedControlPlane{ @@ -814,6 +845,22 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "AzureManagedControlPlane DNSPrefix is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("capz-aks"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("capz-aks"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, { name: "AzureManagedControlPlane FqdnSubdomain is immutable", oldAMCP: &AzureManagedControlPlane{ @@ -830,6 +877,22 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "AzureManagedControlPlane FqdnSubdomain is immutable", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + FqdnSubdomain: pointer.StringPtr("capzaks.api"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + FqdnSubdomain: pointer.StringPtr("capzaks.api"), + }, + }, + wantErr: false, + }, { name: "Valid AzureManagedControlPlane PrivateDNSZone update", oldAMCP: &AzureManagedControlPlane{ From 77922538556b5eac5c42c398699eaa56dcf270c8 Mon Sep 17 00:00:00 2001 From: Jayesh Srivastava Date: Thu, 3 Aug 2023 21:13:22 +0530 Subject: [PATCH 47/66] PEM-2613: Fix Cipher Suit issue --- go.mod | 78 +++++++++++++++---------------- go.sum | 142 +++++++++++++++++++++++++++++++++++--------------------- main.go | 61 ++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 93 deletions(-) diff --git a/go.mod b/go.mod index c8143a23d93..1e47d0bae7c 100644 --- a/go.mod +++ b/go.mod @@ -5,24 +5,24 @@ go 1.19 require ( github.com/Azure/aad-pod-identity v1.8.13 github.com/Azure/azure-sdk-for-go v63.4.0+incompatible - github.com/Azure/go-autorest/autorest v0.11.23 - github.com/Azure/go-autorest/autorest/adal v0.9.18 + github.com/Azure/go-autorest/autorest v0.11.27 + github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Azure/go-autorest/tracing v0.6.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/blang/semver v3.5.1+incompatible - github.com/go-logr/logr v1.2.2 + github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.7 + github.com/google/go-cmp v0.5.8 github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/hashicorp/golang-lru v0.5.4 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.17.0 + github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.1 + github.com/prometheus/client_golang v1.12.2 github.com/spf13/pflag v1.0.5 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.27.0 go.opentelemetry.io/otel v1.4.0 @@ -35,16 +35,16 @@ require ( golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 helm.sh/helm/v3 v3.8.1 - k8s.io/api v0.23.5 - k8s.io/apimachinery v0.23.5 - k8s.io/client-go v0.23.5 - k8s.io/component-base v0.23.5 - k8s.io/klog/v2 v2.30.0 - k8s.io/kubectl v0.23.4 - k8s.io/utils v0.0.0-20211116205334-6203023598ed + k8s.io/api v0.25.0 + k8s.io/apimachinery v0.25.0 + k8s.io/client-go v0.25.0 + k8s.io/component-base v0.25.0 + k8s.io/klog/v2 v2.70.1 + k8s.io/kubectl v0.25.0 + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed sigs.k8s.io/cluster-api v1.1.1 sigs.k8s.io/cluster-api/test v1.1.4 - sigs.k8s.io/controller-runtime v0.11.2 + sigs.k8s.io/controller-runtime v0.13.1 sigs.k8s.io/kind v0.14.0 ) @@ -65,11 +65,12 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alessio/shellescape v1.4.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect + github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect github.com/containerd/containerd v1.5.9 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coredns/corefile-migration v1.0.14 // indirect @@ -77,19 +78,20 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/cli v20.10.11+incompatible // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.16+incompatible // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect + github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-errors/errors v1.0.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -103,11 +105,11 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect - github.com/google/cel-go v0.9.0 // indirect + github.com/google/cel-go v0.12.4 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-github/v33 v33.0.0 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect @@ -138,17 +140,17 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect @@ -163,13 +165,12 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.10.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - github.com/stretchr/testify v1.7.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/valyala/fastjson v1.6.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect + github.com/xlab/treeprint v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.0 // indirect go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect @@ -177,33 +178,32 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect - google.golang.org/grpc v1.44.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/grpc v1.47.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/gorp.v1 v1.7.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiextensions-apiserver v0.23.5 // indirect - k8s.io/apiserver v0.23.5 // indirect - k8s.io/cli-runtime v0.23.4 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.25.0 // indirect + k8s.io/apiserver v0.25.0 // indirect + k8s.io/cli-runtime v0.25.0 // indirect k8s.io/cluster-bootstrap v0.23.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect + k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect oras.land/oras-go v1.1.0 // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/kustomize/api v0.10.1 // indirect - sigs.k8s.io/kustomize/kyaml v0.13.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/kustomize/api v0.12.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 614728aeba9..fd34e40b4d1 100644 --- a/go.sum +++ b/go.sum @@ -65,15 +65,15 @@ github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKn github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= -github.com/Azure/go-autorest/autorest v0.11.23 h1:bRQWsW25/YkoxnIqXMPF94JW33qWDcrPMZ3bINaAruU= -github.com/Azure/go-autorest/autorest v0.11.23/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs= +github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= +github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 h1:F9A3Z++TtAoFysBsNOIJILoHuYBaYvhVGsMGEqPtIS8= github.com/Azure/go-autorest/autorest/azure/auth v0.5.10/go.mod h1:zQXYYNX9kXzRMrJNVXWUfNy38oPMF5/2TeZ4Wylc9fE= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= @@ -81,8 +81,9 @@ github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpz github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= @@ -158,14 +159,16 @@ github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVK github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e h1:GCzyKMDDjSGnlpl3clrdAK7I1AaVoaiKDOYkUzChZzg= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed h1:ue9pVfIcP+QMEjfgo/Ez4ZjNZfonGgR6NgjMaJMu1Cg= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= @@ -189,6 +192,8 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= @@ -211,8 +216,9 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -384,8 +390,9 @@ github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZT github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.11+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -416,6 +423,8 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7fo github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -425,6 +434,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -450,8 +460,9 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -474,12 +485,13 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -531,6 +543,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -579,9 +592,12 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.9.0 h1:u1hg7lcZ/XWw2d3aV1jFS30ijQQ6q0/h1C2ZBeBD1gY= github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-go v0.12.4 h1:YINKfuHZ8n72tPOqSPZBwGiDpew2CJS48mdM5W8LZQU= +github.com/google/cel-go v0.12.4/go.mod h1:Av7CU6r6X3YmcHR9GXqVDaEJYfEtSxl6wvIjUQTriCw= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -594,8 +610,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= @@ -638,7 +655,6 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -871,8 +887,9 @@ github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -886,6 +903,7 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -909,9 +927,9 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -920,8 +938,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -979,8 +997,9 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1132,8 +1151,9 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1228,7 +1248,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1252,6 +1272,7 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1344,7 +1365,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1384,8 +1404,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1498,6 +1519,7 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1523,8 +1545,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1597,7 +1620,6 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= @@ -1714,8 +1736,9 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1748,8 +1771,10 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1763,8 +1788,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1805,8 +1831,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -1826,43 +1853,43 @@ k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI= -k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= +k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= +k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= k8s.io/apiextensions-apiserver v0.23.4/go.mod h1:TWYAKymJx7nLMxWCgWm2RYGXHrGlVZnxIlGnvtfYu+g= -k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= -k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= +k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= +k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= +k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/apiserver v0.23.4/go.mod h1:A6l/ZcNtxGfPSqbFDoxxOjEjSKBaQmE+UTveOmMkpNc= -k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE= -k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= -k8s.io/cli-runtime v0.23.4 h1:C3AFQmo4TK4dlVPLOI62gtHEHu0OfA2Cp4UVRZ1JXns= +k8s.io/apiserver v0.25.0 h1:8kl2ifbNffD440MyvHtPaIz1mw4mGKVgWqM0nL+oyu4= +k8s.io/apiserver v0.25.0/go.mod h1:BKwsE+PTC+aZK+6OJQDPr0v6uS91/HWxX7evElAH6xo= k8s.io/cli-runtime v0.23.4/go.mod h1:7KywUNTUibmHPqmpDFuRO1kc9RhsufHv2lkjCm2YZyM= +k8s.io/cli-runtime v0.25.0 h1:XBnTc2Fi+w818jcJGzhiJKQuXl8479sZ4FhtV5hVJ1Q= +k8s.io/cli-runtime v0.25.0/go.mod h1:bHOI5ZZInRHhbq12OdUiYZQN8ml8aKZLwQgt9QlLINw= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0= -k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= -k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= +k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= +k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/cluster-bootstrap v0.23.0 h1:8pZuuAWPoygewSNB4IddX3HBwXcQkPDXL/ca7GtGf4o= k8s.io/cluster-bootstrap v0.23.0/go.mod h1:VltEnKWfrRTiKgOXp3ts3vh7yqNlH6KFKFflo9GtCBg= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/code-generator v0.23.4/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.23.4/go.mod h1:8o3Gg8i2vnUXGPOwciiYlkSaZT+p+7gA9Scoz8y4W4E= -k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= -k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= +k8s.io/component-base v0.25.0 h1:haVKlLkPCFZhkcqB6WCvpVxftrg6+FK5x1ZuaIDaQ5Y= +k8s.io/component-base v0.25.0/go.mod h1:F2Sumv9CnbBlqrpdf7rKZTmmd2meJq0HizeyY/yAFxk= k8s.io/component-helpers v0.23.4/go.mod h1:1Pl7L4zukZ054ElzRbvmZ1FJIU8roBXFOeRFu8zipa4= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -1875,22 +1902,26 @@ k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= +k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kubectl v0.23.4 h1:mAa+zEOlyZieecEy+xSrhjkpMcukYyHWzcNdX28dzMY= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/kubectl v0.23.4/go.mod h1:Dgb0Rvx/8JKS/C2EuvsNiQc6RZnX0SbHJVG3XUzH6ok= +k8s.io/kubectl v0.25.0 h1:/Wn1cFqo8ik3iee1EvpxYre3bkWsGLXzLQI6uCCAkQc= +k8s.io/kubectl v0.25.0/go.mod h1:n16ULWsOl2jmQpzt2o7Dud1t4o0+Y186ICb4O+GwKAU= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/metrics v0.23.4/go.mod h1:cl6sY9BdVT3DubbpqnkPIKi6mn/F2ltkU4yH1tEJ3Bo= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg= oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1899,29 +1930,32 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27/go.mod h1:tq2nT0Kx7W+/f2JVE+zxYtUhdjuELJkVpNz+x/QN5R4= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/cluster-api v1.1.1 h1:wU1J+Yyu9xDC5M2uACH4Ii8vrIk37oZiCKeMEfyL0pk= sigs.k8s.io/cluster-api v1.1.1/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY= sigs.k8s.io/cluster-api/test v1.1.4 h1:QJSVvAeGtBSf4ImRiyPe1cTZJAGC7QGHZckEfCjLGco= sigs.k8s.io/cluster-api/test v1.1.4/go.mod h1:10gpXFNXUuYHoFfmGR98KfsJHuTQIWjRkiU8ggmXbrc= -sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= -sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/controller-runtime v0.13.1 h1:tUsRCSJVM1QQOOeViGeX3GMT3dQF1eePPw6sEE3xSlg= +sigs.k8s.io/controller-runtime v0.13.1/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE= sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg= -sigs.k8s.io/kustomize/api v0.10.1 h1:KgU7hfYoscuqag84kxtzKdEC3mKMb99DPI3a0eaV1d0= sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= +sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= +sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ= sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io= -sigs.k8s.io/kustomize/kyaml v0.13.0 h1:9c+ETyNfSrVhxvphs+K2dzT3dh5oVPPEqPOE/cUpScY= sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= +sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= +sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/main.go b/main.go index cd3c07bf7ca..18388f476f8 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,8 @@ import ( _ "net/http/pprof" "os" "time" + "crypto/tls" + "strings" // +kubebuilder:scaffold:imports aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" @@ -59,11 +61,18 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" + cliflag "k8s.io/component-base/cli/flag" ) +type TLSOptions struct { + TLSMinVersion string + TLSCipherSuites []string +} + var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + tlsOptions = TLSOptions{} ) func init() { @@ -241,6 +250,8 @@ func InitFlags(fs *pflag.FlagSet) { "Enable tracing to the opentelemetry-collector service in the same namespace.", ) + AddTLSOptions(fs, &tlsOptions) + feature.MutableGates.AddFlag(fs) } @@ -268,6 +279,12 @@ func main() { BurstSize: 100, }) + tlsOptionOverrides, err := GetTLSOptionOverrideFuncs(tlsOptions) + if err != nil { + setupLog.Error(err, "unable to add TLS settings to the webhook server") + os.Exit(1) + } + restConfig := ctrl.GetConfigOrDie() restConfig.UserAgent = "cluster-api-provider-azure-manager" mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ @@ -285,6 +302,7 @@ func main() { HealthProbeBindAddress: healthAddr, Port: webhookPort, EventBroadcaster: broadcaster, + TLSOpts: tlsOptionOverrides, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -323,6 +341,49 @@ func main() { } } +// AddTLSOptions adds the webhook server TLS configuration flags +// to the flag set. +func AddTLSOptions(fs *pflag.FlagSet, options *TLSOptions) { + fs.StringVar(&options.TLSMinVersion, "tls-min-version", "VersionTLS12", + "The minimum TLS version in use by the webhook server.\n"+ + fmt.Sprintf("Possible values are %s.", strings.Join(cliflag.TLSPossibleVersions(), ", ")), + ) + + tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames() + tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames() + fs.StringSliceVar(&options.TLSCipherSuites, "tls-cipher-suites", []string{}, + "Comma-separated list of cipher suites for the webhook server. "+ + "If omitted, the default Go cipher suites will be used. \n"+ + "Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+ + "Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".") +} + +// GetTLSOptionOverrideFuncs returns a list of TLS configuration overrides to be used +// by the webhook server. +func GetTLSOptionOverrideFuncs(options TLSOptions) ([]func(*tls.Config), error) { + var tlsOptions []func(config *tls.Config) + tlsVersion, err := cliflag.TLSVersion(options.TLSMinVersion) + if err != nil { + return nil, err + } + tlsOptions = append(tlsOptions, func(cfg *tls.Config) { + cfg.MinVersion = tlsVersion + cfg.CipherSuites = GetDefaultTLSCipherSuits() + cfg.MaxVersion = tlsVersion + }) + + return tlsOptions, nil +} + +func GetDefaultTLSCipherSuits() []uint16 { + return []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + } +} + func registerControllers(ctx context.Context, mgr manager.Manager) { machineCache, err := coalescing.NewRequestCache(debouncingTimer) if err != nil { From 13907cebf00841b3f0275c52952e99f3704d1161 Mon Sep 17 00:00:00 2001 From: Jayesh Srivastava Date: Fri, 4 Aug 2023 12:03:38 +0530 Subject: [PATCH 48/66] Update --- main.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/main.go b/main.go index 18388f476f8..0a0143bb52a 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,6 @@ import ( "os" "time" "crypto/tls" - "strings" // +kubebuilder:scaffold:imports aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" @@ -250,8 +249,6 @@ func InitFlags(fs *pflag.FlagSet) { "Enable tracing to the opentelemetry-collector service in the same namespace.", ) - AddTLSOptions(fs, &tlsOptions) - feature.MutableGates.AddFlag(fs) } @@ -341,23 +338,6 @@ func main() { } } -// AddTLSOptions adds the webhook server TLS configuration flags -// to the flag set. -func AddTLSOptions(fs *pflag.FlagSet, options *TLSOptions) { - fs.StringVar(&options.TLSMinVersion, "tls-min-version", "VersionTLS12", - "The minimum TLS version in use by the webhook server.\n"+ - fmt.Sprintf("Possible values are %s.", strings.Join(cliflag.TLSPossibleVersions(), ", ")), - ) - - tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames() - tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames() - fs.StringSliceVar(&options.TLSCipherSuites, "tls-cipher-suites", []string{}, - "Comma-separated list of cipher suites for the webhook server. "+ - "If omitted, the default Go cipher suites will be used. \n"+ - "Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+ - "Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".") -} - // GetTLSOptionOverrideFuncs returns a list of TLS configuration overrides to be used // by the webhook server. func GetTLSOptionOverrideFuncs(options TLSOptions) ([]func(*tls.Config), error) { From 8ed1d11840c627d0efd66906bbf1ea6e4b518216 Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Fri, 8 Sep 2023 21:16:13 +0530 Subject: [PATCH 49/66] AKS UpgradeChannels (#92) --- azure/scope/managedcontrolplane.go | 6 + azure/scope/managedcontrolplane_test.go | 92 ++++ azure/services/managedclusters/spec.go | 31 +- ...uster.x-k8s.io_azureclusteridentities.yaml | 12 +- ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 16 + config/default/manager_image_patch.yaml | 2 +- exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + .../v1beta1/azuremanagedcontrolplane_types.go | 28 ++ .../azuremanagedmachinepool_webhook_test.go | 3 +- exp/api/v1beta1/zz_generated.deepcopy.go | 20 + go.mod | 21 +- go.sum | 44 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 415 ++++++++++-------- spectro/run.sh | 4 +- 16 files changed, 493 insertions(+), 205 deletions(-) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 9b5efb75a51..ceecb7583c3 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -522,6 +522,12 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure } } + if s.ControlPlane.Spec.AutoUpgradeProfile != nil { + managedClusterSpec.AutoUpgradeProfile = &managedclusters.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel, + } + } + return &managedClusterSpec } diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index ebfebeac241..09ddee24aaf 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -267,3 +267,95 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AutoUpgradeProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.ManagedClusterAutoUpgradeProfile + }{ + { + Name: "Without AutoUpgradeProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AutoUpgradeProfile UpgradeChannelNodeImage", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AutoUpgradeProfile).To(Equal(c.Expected)) + }) + } +} diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index d0aa605dd7e..568ddfc0c79 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -80,7 +80,7 @@ type ManagedClusterSpec struct { ServiceCIDR string // DockerBridgeCidr - A CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. - DockerBridgeCidr *string `json:"dockerBridgeCidr,omitempty"` + DockerBridgeCidr *string // DNSServiceIP is an IP address assigned to the Kubernetes DNS service DNSServiceIP *string @@ -111,6 +111,15 @@ type ManagedClusterSpec struct { // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster UserAssignedIdentities []UserAssignedIdentity + + // AutoUpgradeProfile - Profile of auto upgrade configuration. + AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile +} + +// ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. +type ManagedClusterAutoUpgradeProfile struct { + // UpgradeChannel - upgrade channel for auto upgrade. Possible values include: 'UpgradeChannelRapid', 'UpgradeChannelStable', 'UpgradeChannelPatch', 'UpgradeChannelNodeImage', 'UpgradeChannelNone' + UpgradeChannel expinfrav1.UpgradeChannel } // UserAssignedIdentity defines the user-assigned identities provided @@ -118,7 +127,7 @@ type ManagedClusterSpec struct { type UserAssignedIdentity struct { // ProviderID is the identification ID of the user-assigned Identity, the format of an identity is: // 'azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}' - ProviderID string `json:"providerID"` + ProviderID string } // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. @@ -355,6 +364,12 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if s.AutoUpgradeProfile != nil { + managedCluster.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: containerservice.UpgradeChannel(s.AutoUpgradeProfile.UpgradeChannel), + } + } + if existing != nil { existingMC, ok := existing.(containerservice.ManagedCluster) if !ok { @@ -536,6 +551,18 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus } } + if managedCluster.AutoUpgradeProfile != nil { + clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: managedCluster.AutoUpgradeProfile.UpgradeChannel, + } + } + + if existingMC.AutoUpgradeProfile != nil { + existingMCClusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: existingMC.AutoUpgradeProfile.UpgradeChannel, + } + } + diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml index 22517fe9ac5..ceefbf2f571 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml @@ -56,11 +56,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -233,11 +233,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -411,11 +411,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 254468b334a..1d25914bc45 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -577,6 +577,22 @@ spec: cluster. type: string type: object + autoUpgradeProfile: + description: AutoUpgradeProfile - Profile of auto upgrade configuration. + properties: + upgradeChannel: + description: 'UpgradeChannel - upgrade channel for auto upgrade. + Possible values include: "node-image","none","patch","rapid","stable"' + enum: + - node-image + - none + - patch + - rapid + - stable + type: string + required: + - upgradeChannel + type: object controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index f8141cfee55..4242ef31345 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/devop2023/release-fips/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev + - image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev name: manager diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 454de50d771..dae5ba4ccd3 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -772,6 +772,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo // WARNING: in.LoadBalancerProfile requires manual conversion: does not exist in peer-type // WARNING: in.APIServerAccessProfile requires manual conversion: does not exist in peer-type // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type + // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index c5fc195c2e6..253b14db8e6 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1075,6 +1075,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.LoadBalancerProfile = (*LoadBalancerProfile)(unsafe.Pointer(in.LoadBalancerProfile)) out.APIServerAccessProfile = (*APIServerAccessProfile)(unsafe.Pointer(in.APIServerAccessProfile)) // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type + // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index c7582e0484f..b82366f8833 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -45,6 +45,22 @@ const ( ManagedControlPlaneOutboundTypeUserDefinedRouting ManagedControlPlaneOutboundType = "userDefinedRouting" ) +// UpgradeChannel enumerates the values for upgrade channel. +type UpgradeChannel string + +const ( + // UpgradeChannelNodeImage ... + UpgradeChannelNodeImage UpgradeChannel = "node-image" + // UpgradeChannelNone ... + UpgradeChannelNone UpgradeChannel = "none" + // UpgradeChannelPatch ... + UpgradeChannelPatch UpgradeChannel = "patch" + // UpgradeChannelRapid ... + UpgradeChannelRapid UpgradeChannel = "rapid" + // UpgradeChannelStable ... + UpgradeChannelStable UpgradeChannel = "stable" +) + // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { // Version defines the desired Kubernetes version. @@ -147,6 +163,18 @@ type AzureManagedControlPlaneSpec struct { // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster // +optional UserAssignedIdentities []infrav1.UserAssignedIdentity `json:"userAssignedIdentities,omitempty"` + + // AutoUpgradeProfile - Profile of auto upgrade configuration. + // +optional + AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile `json:"autoUpgradeProfile,omitempty"` +} + +// ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. +type ManagedClusterAutoUpgradeProfile struct { + // UpgradeChannel - upgrade channel for auto upgrade. Possible values include: "node-image","none","patch","rapid","stable" + // +kubebuilder:validation:Enum=node-image;none;patch;rapid;stable + // +kubebuilder:validation:Required + UpgradeChannel UpgradeChannel `json:"upgradeChannel"` } // AADProfile - AAD integration managed by AKS. diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go index dda97a621c0..5c17db43360 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -17,9 +17,10 @@ limitations under the License. package v1beta1 import ( - "sigs.k8s.io/cluster-api-provider-azure/azure" "testing" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index e90218f9258..f2ed4c8fbd6 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -711,6 +711,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = make([]apiv1beta1.UserAssignedIdentity, len(*in)) copy(*out, *in) } + if in.AutoUpgradeProfile != nil { + in, out := &in.AutoUpgradeProfile, &out.AutoUpgradeProfile + *out = new(ManagedClusterAutoUpgradeProfile) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. @@ -983,6 +988,21 @@ func (in *MachineRollingUpdateDeployment) DeepCopy() *MachineRollingUpdateDeploy return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedClusterAutoUpgradeProfile) DeepCopyInto(out *ManagedClusterAutoUpgradeProfile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedClusterAutoUpgradeProfile. +func (in *ManagedClusterAutoUpgradeProfile) DeepCopy() *ManagedClusterAutoUpgradeProfile { + if in == nil { + return nil + } + out := new(ManagedClusterAutoUpgradeProfile) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManagedControlPlaneSubnet) DeepCopyInto(out *ManagedControlPlaneSubnet) { *out = *in diff --git a/go.mod b/go.mod index 1e47d0bae7c..d0b4c6d2e01 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.19 require ( github.com/Azure/aad-pod-identity v1.8.13 - github.com/Azure/azure-sdk-for-go v63.4.0+incompatible + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 @@ -32,8 +34,8 @@ require ( go.opentelemetry.io/otel/sdk v1.4.0 go.opentelemetry.io/otel/sdk/metric v0.27.0 go.opentelemetry.io/otel/trace v1.4.0 - golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 + golang.org/x/crypto v0.12.0 + golang.org/x/mod v0.8.0 helm.sh/helm/v3 v3.8.1 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 @@ -49,12 +51,14 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -102,6 +106,7 @@ require ( github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect @@ -123,6 +128,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.4 // indirect @@ -151,6 +157,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect @@ -176,12 +183,12 @@ require ( go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index fd34e40b4d1..7714df6dbf0 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,14 @@ github.com/Azure/aad-pod-identity v1.8.13 h1:/gUmacA0z7+lsOlGYAYzkGvAB/KOkUe5Pb6 github.com/Azure/aad-pod-identity v1.8.13/go.mod h1:uxM/lsPo/abzqdk0rwEm4SqO9pavMz0fCmKpYAj4HL8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v63.4.0+incompatible h1:fle3M5Q7vr8auaiPffKyUQmLbvYeqpw30bKU6PrWJFo= -github.com/Azure/azure-sdk-for-go v63.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -93,6 +99,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= @@ -385,6 +393,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 h1:DBZ2sN7CK6dgvHVpQsQj4sRMCbWTmd17l+5SUCjnQSY= github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684/go.mod h1:UfCu3YXJJCI+IdnqGgYP82dk2+Joxmv+mUTVBES6wac= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -546,6 +555,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -790,6 +801,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -978,6 +991,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1273,8 +1288,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1311,8 +1326,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1373,8 +1388,8 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1504,6 +1519,7 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1520,13 +1536,13 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1536,8 +1552,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index ab8d2dee686..1ee34adc537 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 + image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index f5215b48c86..c96e85065c7 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -11,6 +11,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -77,11 +78,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -254,11 +255,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -432,11 +433,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -528,6 +529,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -2218,6 +2220,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -2272,6 +2276,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -2324,6 +2330,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -2688,6 +2696,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -3033,6 +3042,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -3052,6 +3063,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -3069,6 +3082,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -3436,6 +3451,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -3924,6 +3940,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -6119,6 +6136,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -7711,6 +7729,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -8952,6 +8971,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -9141,6 +9161,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -9726,10 +9747,23 @@ spec: privateDNSZone: description: PrivateDNSZone - Private dns zone mode for private cluster. + type: string + type: object + autoUpgradeProfile: + description: AutoUpgradeProfile - Profile of auto upgrade configuration. + properties: + upgradeChannel: + description: 'UpgradeChannel - upgrade channel for auto upgrade. + Possible values include: "node-image","none","patch","rapid","stable"' enum: - - System - - None + - node-image + - none + - patch + - rapid + - stable type: string + required: + - upgradeChannel type: object controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to @@ -9746,11 +9780,24 @@ spec: - host - port type: object + dnsPrefix: + description: DNSPrefix - DNS prefix specified when creating the managed + cluster. + type: string dnsServiceIP: description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. type: string + dockerBridgeCidr: + description: DockerBridgeCidr - A CIDR notation IP range assigned + to the Docker bridge network. It must not overlap with any Subnet + IP ranges or the Kubernetes service address range. + type: string + fqdnSubdomain: + description: FqdnSubdomain - FQDN subdomain specified when creating + private cluster with custom private dns zone. + type: string identityRef: description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster @@ -9851,6 +9898,14 @@ spec: containing cluster IaaS resources. Will be populated to default in webhook. type: string + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string resourceGroupName: description: ResourceGroupName is the name of the Azure resource group for this AKS Cluster. @@ -9875,6 +9930,21 @@ spec: description: SubscriptionID is the GUID of the Azure subscription to hold this cluster. type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user to assign the cluster + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array version: description: Version defines the desired Kubernetes version. minLength: 2 @@ -10027,6 +10097,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -10531,6 +10602,169 @@ status: conditions: [] storedVersions: [] --- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azurecluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azureclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool + failurePolicy: Fail + name: default.azuremachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepools + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane + failurePolicy: Fail + name: default.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedmachinepools + sideEffects: None +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -10656,7 +10890,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 + image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev imagePullPolicy: Always livenessProbe: httpGet: @@ -10805,172 +11039,11 @@ spec: selfSigned: {} --- apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - annotations: - cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert - labels: - cluster.x-k8s.io/provider: infrastructure-azure - name: capz-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azurecluster.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azureclusters - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azureclustertemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azureclustertemplates - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azuremachine.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremachines - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azuremachinetemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremachinetemplates - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool - failurePolicy: Fail - name: default.azuremachinepool.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremachinepools - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane - failurePolicy: Fail - name: default.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremanagedcontrolplanes - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremanagedmachinepools - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure name: capz-validating-webhook-configuration diff --git a/spectro/run.sh b/spectro/run.sh index 31df3583833..95acb764911 100755 --- a/spectro/run.sh +++ b/spectro/run.sh @@ -2,5 +2,5 @@ rm generated/* -kustomize build --load-restrictor LoadRestrictionsNone global > ./generated/core-global.yaml -kustomize build --load-restrictor LoadRestrictionsNone base > ./generated/core-base.yaml +kustomize build --load_restrictor none global > ./generated/core-global.yaml +kustomize build --load_restrictor none base > ./generated/core-base.yaml From b6cce3573561911d0372da442ce65e6a69ba5da1 Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Fri, 8 Sep 2023 22:44:59 +0530 Subject: [PATCH 50/66] disableLocal accounts (#91) * disableLocal accounts --- azure/scope/managedcontrolplane.go | 50 +- azure/scope/managedcontrolplane_test.go | 433 ++++++++++++++++++ azure/services/managedclusters/client.go | 18 + .../managedclusters/managedclusters.go | 90 +++- .../managedclusters/managedclusters_test.go | 46 +- .../mock_managedclusters/client_mock.go | 15 + .../managedclusters_mock.go | 90 +++- azure/services/managedclusters/spec.go | 13 + azure/services/token/client.go | 80 ++++ ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 5 + exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + .../v1beta1/azuremanagedcontrolplane_types.go | 4 + .../azuremanagedcontrolplane_webhook.go | 20 + .../azuremanagedcontrolplane_webhook_test.go | 96 +++- exp/api/v1beta1/zz_generated.deepcopy.go | 5 + .../azuremanagedcontrolplane_reconciler.go | 30 +- .../prometheus/resources/bundle.yaml | 6 +- spectro/generated/core-global.yaml | 5 + 19 files changed, 956 insertions(+), 52 deletions(-) create mode 100644 azure/services/token/client.go diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index ceecb7583c3..5cdf1dcca56 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -106,10 +106,11 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane // ManagedControlPlaneScope defines the basic context for an actuator to operate upon. type ManagedControlPlaneScope struct { - Client client.Client - patchHelper *patch.Helper - kubeConfigData []byte - cache *ManagedControlPlaneCache + Client client.Client + patchHelper *patch.Helper + adminKubeConfigData []byte + userKubeConfigData []byte + cache *ManagedControlPlaneCache AzureClients Cluster *clusterv1.Cluster @@ -413,6 +414,22 @@ func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string return s.ControlPlane.Annotations } +func (s *ManagedControlPlaneScope) IsLocalAcountsDisabled() bool { + if s.IsAadEnabled() && + s.ControlPlane.Spec.DisableLocalAccounts != nil && + *s.ControlPlane.Spec.DisableLocalAccounts { + return true + } + return false +} + +func (s *ManagedControlPlaneScope) IsAadEnabled() bool { + if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed { + return true + } + return false +} + // ManagedClusterSpec returns the managed cluster spec. func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure.ResourceSpecGetter { managedClusterSpec := managedclusters.ManagedClusterSpec{ @@ -477,6 +494,9 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure EnableAzureRBAC: s.ControlPlane.Spec.AADProfile.Managed, AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs, } + if s.ControlPlane.Spec.DisableLocalAccounts != nil { + managedClusterSpec.DisableLocalAccounts = s.ControlPlane.Spec.DisableLocalAccounts + } } if s.ControlPlane.Spec.AddonProfiles != nil { @@ -579,14 +599,24 @@ func (s *ManagedControlPlaneScope) MakeEmptyKubeConfigSecret() corev1.Secret { } } -// GetKubeConfigData returns a []byte that contains kubeconfig. -func (s *ManagedControlPlaneScope) GetKubeConfigData() []byte { - return s.kubeConfigData +// GetAdminKubeConfigData returns admin kubeconfig. +func (s *ManagedControlPlaneScope) GetAdminKubeConfigData() []byte { + return s.adminKubeConfigData +} + +// SetAdminKubeConfigData sets adminKubeconfig data. +func (s *ManagedControlPlaneScope) SetAdminKubeConfigData(kubeConfigData []byte) { + s.adminKubeConfigData = kubeConfigData +} + +// GetUserKubeConfigData returns user kubeconfig, required when using AAD with AKS cluster. +func (s *ManagedControlPlaneScope) GetUserKubeConfigData() []byte { + return s.userKubeConfigData } -// SetKubeConfigData sets kubeconfig data. -func (s *ManagedControlPlaneScope) SetKubeConfigData(kubeConfigData []byte) { - s.kubeConfigData = kubeConfigData +// SetUserKubeConfigData sets userKubeconfig data. +func (s *ManagedControlPlaneScope) SetUserKubeConfigData(kubeConfigData []byte) { + s.userKubeConfigData = kubeConfigData } // SetLongRunningOperationState will set the future on the AzureManagedControlPlane status to allow the resource to continue diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 09ddee24aaf..969401d2c23 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" @@ -359,3 +360,435 @@ func TestManagedControlPlaneScope_AutoUpgradeProfile(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AADProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.AADProfile + }{ + { + Name: "Without AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.AADProfile{ + Managed: true, + EnableAzureRBAC: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AADProfile).To(Equal(c.Expected)) + }) + } +} + +func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *bool + }{ + { + Name: "Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "Without AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: pointer.BoolPtr(true), + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.DisableLocalAccounts).To(Equal(c.Expected)) + }) + } +} + +func TestIsAaDEnabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "AAD is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + aadEnabled := s.IsAadEnabled() + g.Expect(aadEnabled).To(Equal(c.Expected)) + }) + } +} + +func TestIsLocalAcountsDisabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "DisbaleLocalAccount is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + localAccountsDisabled := s.IsLocalAcountsDisabled() + g.Expect(localAccountsDisabled).To(Equal(c.Expected)) + }) + } +} diff --git a/azure/services/managedclusters/client.go b/azure/services/managedclusters/client.go index 73ab07b1448..6f6ca0ccb7d 100644 --- a/azure/services/managedclusters/client.go +++ b/azure/services/managedclusters/client.go @@ -33,6 +33,7 @@ import ( // CredentialGetter is a helper interface for getting managed cluster credentials. type CredentialGetter interface { GetCredentials(context.Context, string, string) ([]byte, error) + GetUserCredentials(context.Context, string, string) ([]byte, error) } // azureClient contains the Azure go-sdk Client. @@ -79,6 +80,23 @@ func (ac *azureClient) GetCredentials(ctx context.Context, resourceGroupName, na return *(*credentialList.Kubeconfigs)[0].Value, nil } +// GetUserCredentials fetches the user kubeconfig for a managed cluster. +func (ac *azureClient) GetUserCredentials(ctx context.Context, resourceGroupName, name string) ([]byte, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.azureClient.GetCredentials") + defer done() + + credentialList, err := ac.managedclusters.ListClusterUserCredentials(ctx, resourceGroupName, name, "") + if err != nil { + return nil, err + } + + if credentialList.Kubeconfigs == nil || len(*credentialList.Kubeconfigs) < 1 { + return nil, errors.New("no user kubeconfigs available for the managed cluster") + } + + return *(*credentialList.Kubeconfigs)[0].Value, nil +} + // CreateOrUpdateAsync creates or updates a managed cluster. // It sends a PUT request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing // progress of the operation. diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index b9662600a67..6978968412f 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -23,15 +23,20 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/clientcmd" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/token" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) -const serviceName = "managedcluster" +const ( + serviceName = "managedcluster" + aadResourceId = "6dae42f8-4368-4678-94ff-3960e28e3630" +) // ManagedClusterScope defines the scope interface for a managed cluster. type ManagedClusterScope interface { @@ -40,8 +45,12 @@ type ManagedClusterScope interface { ManagedClusterSpec(context.Context) azure.ResourceSpecGetter SetControlPlaneEndpoint(clusterv1.APIEndpoint) MakeEmptyKubeConfigSecret() corev1.Secret - GetKubeConfigData() []byte - SetKubeConfigData([]byte) + GetAdminKubeConfigData() []byte + SetAdminKubeConfigData([]byte) + GetUserKubeConfigData() []byte + SetUserKubeConfigData([]byte) + IsAadEnabled() bool + IsLocalAcountsDisabled() bool } // Service provides operations on azure resources. @@ -94,11 +103,13 @@ func (s *Service) Reconcile(ctx context.Context) error { // Update kubeconfig data // Always fetch credentials in case of rotation - kubeConfigData, err := s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + adminKubeConfigData, userKubeConfigData, err := s.ReconcileKubeconfig(ctx, managedClusterSpec) if err != nil { - return errors.Wrap(err, "failed to get credentials for managed cluster") + return errors.Wrap(err, "error while reconciling adminKubeConfigData") } - s.Scope.SetKubeConfigData(kubeConfigData) + + s.Scope.SetAdminKubeConfigData(adminKubeConfigData) + s.Scope.SetUserKubeConfigData(userKubeConfigData) } s.Scope.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, resultErr) return resultErr @@ -126,3 +137,70 @@ func (s *Service) Delete(ctx context.Context) error { func (s *Service) IsManaged(ctx context.Context) (bool, error) { return true, nil } + +func (s *Service) ReconcileKubeconfig(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, []byte, error) { + var ( + userKubeConfigData []byte + adminKubeConfigData []byte + err error + ) + + if s.Scope.IsAadEnabled() { + if userKubeConfigData, err = s.GetUserKubeConfigData(ctx, managedClusterSpec); err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig") + } + } + + if s.Scope.IsLocalAcountsDisabled() { + userKubeconfigWithToken, err := s.GetUserKubeConfigWithToken(userKubeConfigData, ctx, managedClusterSpec) + if err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig with token") + } + return userKubeconfigWithToken, userKubeConfigData, nil + } + + adminKubeConfigData, err = s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return adminKubeConfigData, userKubeConfigData, nil +} + +func (s *Service) GetUserKubeConfigData(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + kubeConfigData, err := s.GetUserCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return kubeConfigData, nil +} + +func (s *Service) GetUserKubeConfigWithToken(userKubeConfigData []byte, ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + + tokenClient, err := token.NewClient(s.Scope) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token client") + } + + token, err := tokenClient.GetAzureActiveDirectoryToken(ctx, aadResourceId) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token for user kubeconfig") + } + + return s.CreateUserKubeconfigWithToken(token, userKubeConfigData) +} + +func (s *Service) CreateUserKubeconfigWithToken(token string, userKubeConfigData []byte) ([]byte, error) { + config, err := clientcmd.Load(userKubeConfigData) + if err != nil { + return nil, errors.Wrap(err, "error while trying to unmarshal new user kubeconfig with token") + } + for _, auth := range config.AuthInfos { + auth.Token = token + auth.Exec = nil + } + kubeconfig, err := clientcmd.Write(*config) + if err != nil { + return nil, errors.Wrap(err, "error while trying to marshal new user kubeconfig with token") + } + return kubeconfig, nil +} diff --git a/azure/services/managedclusters/managedclusters_test.go b/azure/services/managedclusters/managedclusters_test.go index af875091dd6..6a185a626ef 100644 --- a/azure/services/managedclusters/managedclusters_test.go +++ b/azure/services/managedclusters/managedclusters_test.go @@ -33,6 +33,14 @@ import ( ) var fakeManagedClusterSpec = &ManagedClusterSpec{Name: "my-managedcluster", ResourceGroup: "my-rg"} +var fakeManagedClusterSpecWithAAD = &ManagedClusterSpec{ + Name: "my-managedcluster", + ResourceGroup: "my-rg", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"000000-000000-000000-000000"}, + }, +} func TestReconcile(t *testing.T) { testcases := []struct { @@ -60,6 +68,7 @@ func TestReconcile(t *testing.T) { name: "create managed cluster succeeds", expectedError: "", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ @@ -71,14 +80,45 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds with user kubeconfig", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpecWithAAD) + r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpecWithAAD, serviceName).Return(containerservice.ManagedCluster{ + ManagedClusterProperties: &containerservice.ManagedClusterProperties{ + Fqdn: pointer.String("my-managedcluster-fqdn"), + ProvisioningState: pointer.String("Succeeded"), + AadProfile: &containerservice.ManagedClusterAADProfile{ + Managed: pointer.Bool(true), + AdminGroupObjectIDs: &[]string{"000000-000000-000000-000000"}, + }, + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(true) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) - s.SetKubeConfigData([]byte("credentials")) + m.GetUserCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials-user"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData([]byte("credentials-user")) s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) }, }, { name: "fail to get managed cluster credentials", - expectedError: "failed to get credentials for managed cluster: internal server error", + expectedError: "error while reconciling adminKubeConfigData: failed to get credentials for managed cluster: internal server error", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ @@ -91,6 +131,8 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte(""), errors.New("internal server error")) }, }, diff --git a/azure/services/managedclusters/mock_managedclusters/client_mock.go b/azure/services/managedclusters/mock_managedclusters/client_mock.go index 735939603c5..77216af037c 100644 --- a/azure/services/managedclusters/mock_managedclusters/client_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/client_mock.go @@ -64,3 +64,18 @@ func (mr *MockCredentialGetterMockRecorder) GetCredentials(arg0, arg1, arg2 inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetCredentials), arg0, arg1, arg2) } + +// GetUserCredentials mocks base method. +func (m *MockCredentialGetter) GetUserCredentials(arg0 context.Context, arg1, arg2 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserCredentials", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUserCredentials indicates an expected call of GetUserCredentials. +func (mr *MockCredentialGetterMockRecorder) GetUserCredentials(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetUserCredentials), arg0, arg1, arg2) +} diff --git a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go index abf3c16817b..0d0126f8866 100644 --- a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go @@ -137,18 +137,18 @@ func (mr *MockManagedClusterScopeMockRecorder) DeleteLongRunningOperationState(a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).DeleteLongRunningOperationState), arg0, arg1) } -// GetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) GetKubeConfigData() []byte { +// GetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetAdminKubeConfigData() []byte { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetKubeConfigData") + ret := m.ctrl.Call(m, "GetAdminKubeConfigData") ret0, _ := ret[0].([]byte) return ret0 } -// GetKubeConfigData indicates an expected call of GetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) GetKubeConfigData() *gomock.Call { +// GetAdminKubeConfigData indicates an expected call of GetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetAdminKubeConfigData() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetKubeConfigData)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetAdminKubeConfigData)) } // GetLongRunningOperationState mocks base method. @@ -165,6 +165,20 @@ func (mr *MockManagedClusterScopeMockRecorder) GetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).GetLongRunningOperationState), arg0, arg1) } +// GetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetUserKubeConfigData() []byte { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserKubeConfigData") + ret0, _ := ret[0].([]byte) + return ret0 +} + +// GetUserKubeConfigData indicates an expected call of GetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetUserKubeConfigData() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetUserKubeConfigData)) +} + // HashKey mocks base method. func (m *MockManagedClusterScope) HashKey() string { m.ctrl.T.Helper() @@ -179,6 +193,34 @@ func (mr *MockManagedClusterScopeMockRecorder) HashKey() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockManagedClusterScope)(nil).HashKey)) } +// IsAadEnabled mocks base method. +func (m *MockManagedClusterScope) IsAadEnabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAadEnabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAadEnabled indicates an expected call of IsAadEnabled. +func (mr *MockManagedClusterScopeMockRecorder) IsAadEnabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAadEnabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsAadEnabled)) +} + +// IsLocalAcountsDisabled mocks base method. +func (m *MockManagedClusterScope) IsLocalAcountsDisabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsLocalAcountsDisabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsLocalAcountsDisabled indicates an expected call of IsLocalAcountsDisabled. +func (mr *MockManagedClusterScopeMockRecorder) IsLocalAcountsDisabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLocalAcountsDisabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsLocalAcountsDisabled)) +} + // MakeEmptyKubeConfigSecret mocks base method. func (m *MockManagedClusterScope) MakeEmptyKubeConfigSecret() v1.Secret { m.ctrl.T.Helper() @@ -207,28 +249,28 @@ func (mr *MockManagedClusterScopeMockRecorder) ManagedClusterSpec(arg0 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManagedClusterSpec", reflect.TypeOf((*MockManagedClusterScope)(nil).ManagedClusterSpec), arg0) } -// SetControlPlaneEndpoint mocks base method. -func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { +// SetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetAdminKubeConfigData(arg0 []byte) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) + m.ctrl.Call(m, "SetAdminKubeConfigData", arg0) } -// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. -func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { +// SetAdminKubeConfigData indicates an expected call of SetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetAdminKubeConfigData(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAdminKubeConfigData), arg0) } -// SetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) SetKubeConfigData(arg0 []byte) { +// SetControlPlaneEndpoint mocks base method. +func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetKubeConfigData", arg0) + m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) } -// SetKubeConfigData indicates an expected call of SetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) SetKubeConfigData(arg0 interface{}) *gomock.Call { +// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. +func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetKubeConfigData), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) } // SetLongRunningOperationState mocks base method. @@ -243,6 +285,18 @@ func (mr *MockManagedClusterScopeMockRecorder) SetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).SetLongRunningOperationState), arg0) } +// SetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetUserKubeConfigData(arg0 []byte) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetUserKubeConfigData", arg0) +} + +// SetUserKubeConfigData indicates an expected call of SetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetUserKubeConfigData(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetUserKubeConfigData), arg0) +} + // SubscriptionID mocks base method. func (m *MockManagedClusterScope) SubscriptionID() string { m.ctrl.T.Helper() diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 568ddfc0c79..1e0e8f1255d 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -112,6 +112,9 @@ type ManagedClusterSpec struct { // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster UserAssignedIdentities []UserAssignedIdentity + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + DisableLocalAccounts *bool + // AutoUpgradeProfile - Profile of auto upgrade configuration. AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile } @@ -291,6 +294,9 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ EnableAzureRBAC: &s.AADProfile.EnableAzureRBAC, AdminGroupObjectIDs: &s.AADProfile.AdminGroupObjectIDs, } + if s.DisableLocalAccounts != nil { + managedCluster.DisableLocalAccounts = s.DisableLocalAccounts + } } for i := range s.AddonProfiles { @@ -562,6 +568,13 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus UpgradeChannel: existingMC.AutoUpgradeProfile.UpgradeChannel, } } + if managedCluster.DisableLocalAccounts != nil { + clusterNormalized.DisableLocalAccounts = managedCluster.DisableLocalAccounts + } + + if existingMC.DisableLocalAccounts != nil { + existingMCClusterNormalized.DisableLocalAccounts = existingMC.DisableLocalAccounts + } diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff diff --git a/azure/services/token/client.go b/azure/services/token/client.go new file mode 100644 index 00000000000..23f6e473cab --- /dev/null +++ b/azure/services/token/client.go @@ -0,0 +1,80 @@ +package token + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + az "github.com/Azure/go-autorest/autorest/azure" + "github.com/pkg/errors" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +const ( + defaultEnvironmentName = "AzurePublicCloud" +) + +type AzureClient struct { + aadToken *azidentity.ClientSecretCredential +} + +// newClient creates a new managed cluster client from an authorizer. +func NewClient(auth azure.Authorizer) (*AzureClient, error) { + aadToken, err := newAzureActiveDirectoryTokenClient(auth.TenantID(), + auth.ClientID(), + auth.ClientSecret(), + auth.CloudEnvironment()) + if err != nil { + return nil, err + } + return &AzureClient{ + aadToken: aadToken, + }, nil +} + +// newAzureActiveDirectoryTokenClient creates a new aad token client from an authorizer. +func newAzureActiveDirectoryTokenClient(tenantId, clientId, clientSecret, envName string) (*azidentity.ClientSecretCredential, error) { + cliOpts, err := getAzureClientOptions(envName) + if err != nil { + return nil, errors.Wrap(err, "error while getting client options") + } + clientOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: cliOpts, + } + cred, err := azidentity.NewClientSecretCredential(tenantId, clientId, clientSecret, clientOptions) + if err != nil { + return nil, errors.Wrap(err, "error while getting az client secret credentials") + } + return cred, nil +} + +func getAzureClientOptions(environment string) (azcore.ClientOptions, error) { + + if environment == "" { + environment = defaultEnvironmentName + } + env, err := az.EnvironmentFromName(environment) + if err != nil { + return azcore.ClientOptions{}, errors.Wrap(err, "error while getting azure env") + } + c := cloud.Configuration{ + ActiveDirectoryAuthorityHost: env.ActiveDirectoryEndpoint, + } + return azcore.ClientOptions{ + Cloud: c, + }, nil +} + +func (ac *AzureClient) GetAzureActiveDirectoryToken(ctx context.Context, resourceId string) (string, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "aadToken.GetToken") + defer done() + + spnAccessToken, err := ac.aadToken.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{resourceId + "/.default"}}) + if err != nil { + return "", errors.Wrap(err, "failed to get token") + } + return spnAccessToken.Token, nil +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 1d25914bc45..79b598017e1 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -608,6 +608,11 @@ spec: - host - port type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean dnsPrefix: description: DNSPrefix - DNS prefix specified when creating the managed cluster. diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index dae5ba4ccd3..fe1ecea044b 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -773,6 +773,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo // WARNING: in.APIServerAccessProfile requires manual conversion: does not exist in peer-type // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type + // WARNING: in.DisableLocalAccounts requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 253b14db8e6..7c316cfa13d 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1076,6 +1076,7 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.APIServerAccessProfile = (*APIServerAccessProfile)(unsafe.Pointer(in.APIServerAccessProfile)) // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type + // WARNING: in.DisableLocalAccounts requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index b82366f8833..f8396523776 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -167,6 +167,10 @@ type AzureManagedControlPlaneSpec struct { // AutoUpgradeProfile - Profile of auto upgrade configuration. // +optional AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile `json:"autoUpgradeProfile,omitempty"` + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + // +optional + DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` } // ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index eaf2f570e12..fd79f3d86b0 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -242,6 +242,16 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if old.Spec.DisableLocalAccounts == nil && + m.Spec.DisableLocalAccounts != nil && + m.Spec.AADProfile == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "DisableLocalAccounts can be set only for AAD enabled clusters")) + } + if old.Spec.OutboundType != nil { // Prevent OutboundType modification if it was already set to some value if m.Spec.OutboundType == nil { @@ -311,6 +321,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, m.validateDNSPrefix, + m.validateDisableLocalAccounts, //m.validateManagedClusterNetwork, } @@ -324,6 +335,15 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { return kerrors.NewAggregate(errs) } +func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) error { + + if m.Spec.DisableLocalAccounts != nil && m.Spec.AADProfile == nil { + return errors.New("DisableLocalAccounts should be set only for AAD enabled clusters") + } + + return nil +} + func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { if m.Spec.DNSPrefix == nil { diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 07e34697670..9f6343d4828 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -83,7 +83,7 @@ func TestValidatingWebhook(t *testing.T) { expectErr bool }{ { - name: "Testing inValid DNSPrefix for starting with invalid charecters", + name: "Testing inValid DNSPrefix for starting with invalid characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ DNSPrefix: pointer.StringPtr("-thisi$"), @@ -122,6 +122,26 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: true, }, + { + name: "Testing Valid DNSPrefix with hyphen characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("hyphen-allowed"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, + { + name: "Testing Valid DNSPrefix with hyphen characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: pointer.StringPtr("palette-test07"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, { name: "Testing valid DNSPrefix ", amcp: AzureManagedControlPlane{ @@ -284,6 +304,30 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: true, }, + { + name: "DisableLocalAccounts cannot be set for non AAD clusters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + expectErr: true, + }, + { + name: "DisableLocalAccounts can be set for AAD clusters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + expectErr: false, + }, } for _, tt := range tests { @@ -1023,6 +1067,56 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "DisableLocalAccounts can be set only for AAD enabled clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: pointer.BoolPtr(true), + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: false, + }, + { + name: "DisableLocalAccounts cannot be set only for non AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + wantErr: true, + }, { name: "OutboundType update", oldAMCP: &AzureManagedControlPlane{ diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index f2ed4c8fbd6..84656a0a7f2 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -716,6 +716,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(ManagedClusterAutoUpgradeProfile) **out = **in } + if in.DisableLocalAccounts != nil { + in, out := &in.DisableLocalAccounts, &out.DisableLocalAccounts + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. diff --git a/exp/controllers/azuremanagedcontrolplane_reconciler.go b/exp/controllers/azuremanagedcontrolplane_reconciler.go index afc0b301d38..26fbba56688 100644 --- a/exp/controllers/azuremanagedcontrolplane_reconciler.go +++ b/exp/controllers/azuremanagedcontrolplane_reconciler.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "fmt" "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-azure/azure" @@ -92,20 +93,25 @@ func (r *azureManagedControlPlaneService) reconcileKubeconfig(ctx context.Contex ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedControlPlaneService.reconcileKubeconfig") defer done() - kubeConfigData := r.scope.GetKubeConfigData() - if kubeConfigData == nil { - return nil - } - kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + kubeConfigs := [][]byte{r.scope.GetAdminKubeConfigData(), r.scope.GetUserKubeConfigData()} - // Always update credentials in case of rotation - if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { - kubeConfigSecret.Data = map[string][]byte{ - secret.KubeconfigDataName: kubeConfigData, + for i, kubeConfigData := range kubeConfigs { + if len(kubeConfigData) == 0 { + continue + } + kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + if i == 1 { + // 2nd kubeconfig is the user kubeconfig + kubeConfigSecret.Name = fmt.Sprintf("%s-user", kubeConfigSecret.Name) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { + kubeConfigSecret.Data = map[string][]byte{ + secret.KubeconfigDataName: kubeConfigData, + } + return nil + }); err != nil { + return errors.Wrap(err, "failed to kubeconfig secret for cluster") } - return nil - }); err != nil { - return errors.Wrap(err, "failed to kubeconfig secret for cluster") } return nil diff --git a/hack/observability/prometheus/resources/bundle.yaml b/hack/observability/prometheus/resources/bundle.yaml index 2fce683d225..e65100bf1f3 100644 --- a/hack/observability/prometheus/resources/bundle.yaml +++ b/hack/observability/prometheus/resources/bundle.yaml @@ -4486,7 +4486,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: @@ -11335,7 +11335,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: @@ -17146,7 +17146,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index c96e85065c7..d2070bba34a 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -9780,6 +9780,11 @@ spec: - host - port type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean dnsPrefix: description: DNSPrefix - DNS prefix specified when creating the managed cluster. From 5a53d6411eddabe70df7749acfc1b631a0126247 Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Wed, 27 Sep 2023 11:15:58 +0530 Subject: [PATCH 51/66] ensure to not downgrade an auto-upgraded cluster (#95) * ensure to not downgrade an auto-upgraded cluster --- azure/scope/managedcontrolplane.go | 42 ++++ azure/scope/managedcontrolplane_test.go | 215 ++++++++++++++++++ .../managedclusters/managedclusters.go | 8 + .../managedclusters/managedclusters_test.go | 29 +++ .../managedclusters_mock.go | 26 +++ azure/services/managedclusters/spec.go | 33 ++- azure/services/managedclusters/spec_test.go | 18 +- ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 5 + exp/api/v1alpha3/zz_generated.conversion.go | 1 + exp/api/v1alpha4/zz_generated.conversion.go | 1 + .../v1beta1/azuremanagedcontrolplane_types.go | 6 + .../azuremanagedcontrolplane_webhook.go | 64 +++--- .../azuremanagedcontrolplane_webhook_test.go | 140 ++++++++++++ go.mod | 4 +- go.sum | 8 +- spectro/generated/core-global.yaml | 5 + util/versions/version.go | 22 ++ util/versions/version_test.go | 86 +++++++ 18 files changed, 673 insertions(+), 40 deletions(-) create mode 100644 util/versions/version.go create mode 100644 util/versions/version_test.go diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 5cdf1dcca56..5b1c3b2d8df 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -414,6 +414,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string return s.ControlPlane.Annotations } +// IsLocalAcountsDisabled checks if local accounts have been disabled. func (s *ManagedControlPlaneScope) IsLocalAcountsDisabled() bool { if s.IsAadEnabled() && s.ControlPlane.Spec.DisableLocalAccounts != nil && @@ -423,6 +424,7 @@ func (s *ManagedControlPlaneScope) IsLocalAcountsDisabled() bool { return false } +// IsAadEnabled checks if aad is enabled. func (s *ManagedControlPlaneScope) IsAadEnabled() bool { if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed { return true @@ -430,6 +432,46 @@ func (s *ManagedControlPlaneScope) IsAadEnabled() bool { return false } +// SetAutoUpgradeVersionStatus sets the auto upgrade version in status +func (s *ManagedControlPlaneScope) SetAutoUpgradeVersionStatus(version string) { + s.ControlPlane.Status.AutoUpgradeVersion = version +} + +// IsManagedVersionUpgrade checks if auto upgrade profile is set and if the upgradeChannel is of the type patch, stable or rapid. +func (s *ManagedControlPlaneScope) IsManagedVersionUpgrade() bool { + return s.IsPatchAutoUpgrade() || s.IsStableAutoUpgrade() || s.IsRapidAutoUpgrade() +} + +// IsAutoUpgradeStable checks if auto upgrade channel is stable. +func (s *ManagedControlPlaneScope) IsStableAutoUpgrade() bool { + if s.ControlPlane.Spec.AutoUpgradeProfile != nil { + if upgradeChannel := s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel; upgradeChannel == infrav1exp.UpgradeChannelStable { + return true + } + } + return false +} + +// IsAutoUpgradeStable checks if auto upgrade channel is rapid. +func (s *ManagedControlPlaneScope) IsRapidAutoUpgrade() bool { + if s.ControlPlane.Spec.AutoUpgradeProfile != nil { + if upgradeChannel := s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel; upgradeChannel == infrav1exp.UpgradeChannelRapid { + return true + } + } + return false +} + +// IsAutoUpgradeStable checks if auto upgrade channel is patch. +func (s *ManagedControlPlaneScope) IsPatchAutoUpgrade() bool { + if s.ControlPlane.Spec.AutoUpgradeProfile != nil { + if upgradeChannel := s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel; upgradeChannel == infrav1exp.UpgradeChannelPatch { + return true + } + } + return false +} + // ManagedClusterSpec returns the managed cluster spec. func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure.ResourceSpecGetter { managedClusterSpec := managedclusters.ManagedClusterSpec{ diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index 969401d2c23..acb384f98a1 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -792,3 +792,218 @@ func TestIsLocalAcountsDisabled(t *testing.T) { }) } } + +func Test_IsManagedVersionUpgrade(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "Upgrade channel not set", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "Upgradechannel is set rapid", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelRapid, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set patch", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelPatch, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set stable", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelStable, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set none", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNone, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "Upgradechannel is set node-image", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + autoUpgrade := s.IsManagedVersionUpgrade() + g.Expect(autoUpgrade).To(Equal(c.Expected)) + }) + } +} diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index 6978968412f..4902558a5e2 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -18,6 +18,7 @@ package managedclusters import ( "context" + "fmt" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" "github.com/Azure/go-autorest/autorest/to" @@ -51,6 +52,8 @@ type ManagedClusterScope interface { SetUserKubeConfigData([]byte) IsAadEnabled() bool IsLocalAcountsDisabled() bool + SetAutoUpgradeVersionStatus(version string) + IsManagedVersionUpgrade() bool } // Service provides operations on azure resources. @@ -108,6 +111,11 @@ func (s *Service) Reconcile(ctx context.Context) error { return errors.Wrap(err, "error while reconciling adminKubeConfigData") } + if s.Scope.IsManagedVersionUpgrade() && managedCluster.KubernetesVersion != nil { + kubernetesVersion := fmt.Sprintf("v%s", *managedCluster.KubernetesVersion) + s.Scope.SetAutoUpgradeVersionStatus(kubernetesVersion) + } + s.Scope.SetAdminKubeConfigData(adminKubeConfigData) s.Scope.SetUserKubeConfigData(userKubeConfigData) } diff --git a/azure/services/managedclusters/managedclusters_test.go b/azure/services/managedclusters/managedclusters_test.go index 6a185a626ef..549e380902f 100644 --- a/azure/services/managedclusters/managedclusters_test.go +++ b/azure/services/managedclusters/managedclusters_test.go @@ -85,6 +85,34 @@ func TestReconcile(t *testing.T) { m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) s.SetAdminKubeConfigData([]byte("credentials")) s.SetUserKubeConfigData(userKubeConfigData) + s.IsManagedVersionUpgrade().Return(false) + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds, update autoupgrade status", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte + s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) + r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ + ManagedClusterProperties: &containerservice.ManagedClusterProperties{ + Fqdn: pointer.String("my-managedcluster-fqdn"), + ProvisioningState: pointer.String("Succeeded"), + KubernetesVersion: pointer.String("1.27.3"), + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.IsManagedVersionUpgrade().Return(true) + s.SetAutoUpgradeVersionStatus("v1.27.3") s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) }, }, @@ -113,6 +141,7 @@ func TestReconcile(t *testing.T) { m.GetUserCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials-user"), nil) s.SetAdminKubeConfigData([]byte("credentials")) s.SetUserKubeConfigData([]byte("credentials-user")) + s.IsManagedVersionUpgrade().Return(false) s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) }, }, diff --git a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go index 0d0126f8866..1badc3806c1 100644 --- a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go @@ -221,6 +221,20 @@ func (mr *MockManagedClusterScopeMockRecorder) IsLocalAcountsDisabled() *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLocalAcountsDisabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsLocalAcountsDisabled)) } +// IsManagedVersionUpgrade mocks base method. +func (m *MockManagedClusterScope) IsManagedVersionUpgrade() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsManagedVersionUpgrade") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsManagedVersionUpgrade indicates an expected call of IsManagedVersionUpgrade. +func (mr *MockManagedClusterScopeMockRecorder) IsManagedVersionUpgrade() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsManagedVersionUpgrade", reflect.TypeOf((*MockManagedClusterScope)(nil).IsManagedVersionUpgrade)) +} + // MakeEmptyKubeConfigSecret mocks base method. func (m *MockManagedClusterScope) MakeEmptyKubeConfigSecret() v1.Secret { m.ctrl.T.Helper() @@ -261,6 +275,18 @@ func (mr *MockManagedClusterScopeMockRecorder) SetAdminKubeConfigData(arg0 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAdminKubeConfigData), arg0) } +// SetAutoUpgradeVersionStatus mocks base method. +func (m *MockManagedClusterScope) SetAutoUpgradeVersionStatus(version string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetAutoUpgradeVersionStatus", version) +} + +// SetAutoUpgradeVersionStatus indicates an expected call of SetAutoUpgradeVersionStatus. +func (mr *MockManagedClusterScopeMockRecorder) SetAutoUpgradeVersionStatus(version interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAutoUpgradeVersionStatus", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAutoUpgradeVersionStatus), version) +} + // SetControlPlaneEndpoint mocks base method. func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { m.ctrl.T.Helper() diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index 1e0e8f1255d..f8b140b2523 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -26,10 +26,12 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" expinfrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" ) // ManagedClusterSpec contains properties to create a managed cluster. @@ -215,6 +217,28 @@ func (s *ManagedClusterSpec) CustomHeaders() map[string]string { return s.Headers } +// GetManagedClusterVersion gets the desired managed k8s version. +// If autoupgrade channels is set to patch, stable or rapid, clusters can be upgraded to higher version by AKS. +// If autoupgrade is triggered, existing kubernetes version will be higher than the user desired kubernetes version. +// CAPZ should honour the upgrade and it should not downgrade to the lower desired version. +func (s *ManagedClusterSpec) GetManagedClusterVersion(existing interface{}) (string, error) { + version := s.Version + if existing != nil && version != "" { + existingMC, ok := existing.(containerservice.ManagedCluster) + if !ok { + return version, fmt.Errorf("%T is not a containerservice.ManagedCluster", existing) + } + if v, err := versions.GetHigherK8sVersion( + version, + ptr.Deref(existingMC.KubernetesVersion, version)); err != nil { + return "", err + } else { + version = v + } + } + return version, nil +} + // Parameters returns the parameters for the managed clusters. func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{}, err error) { decodedSSHPublicKey, err := base64.StdEncoding.DecodeString(s.SSHPublicKey) @@ -230,7 +254,6 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ NodeResourceGroup: &s.NodeResourceGroup, EnableRBAC: to.BoolPtr(true), DNSPrefix: s.DNSPrefix, - KubernetesVersion: &s.Version, LinuxProfile: &containerservice.LinuxProfile{ AdminUsername: to.StringPtr(azure.DefaultAKSUserName), SSH: &containerservice.SSHConfiguration{ @@ -253,6 +276,12 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ }, } + if kubernetesVersion, err := s.GetManagedClusterVersion(existing); err != nil { + return nil, err + } else { + managedCluster.KubernetesVersion = &kubernetesVersion + } + if s.FqdnSubdomain != nil { managedCluster.FqdnSubdomain = s.FqdnSubdomain } @@ -561,6 +590,8 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ UpgradeChannel: managedCluster.AutoUpgradeProfile.UpgradeChannel, } + } else { + clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{} } if existingMC.AutoUpgradeProfile != nil { diff --git a/azure/services/managedclusters/spec_test.go b/azure/services/managedclusters/spec_test.go index e6256e900a0..0347d9f4107 100644 --- a/azure/services/managedclusters/spec_test.go +++ b/azure/services/managedclusters/spec_test.go @@ -110,6 +110,23 @@ func TestParameters(t *testing.T) { g.Expect(result).To(BeNil()) }, }, + { + name: "managedcluster exists, no update needed", + existing: getExistingCluster(), + spec: &ManagedClusterSpec{ + Name: "test-managedcluster", + ResourceGroup: "test-rg", + Location: "test-location", + Tags: map[string]string{ + "test-tag": "test-value", + }, + Version: "v1.21.0", + LoadBalancerSKU: "Standard", + }, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + }, { name: "managedcluster exists and an update is needed", existing: getExistingCluster(), @@ -158,7 +175,6 @@ func getSampleManagedCluster() containerservice.ManagedCluster { return containerservice.ManagedCluster{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ KubernetesVersion: to.StringPtr("v1.22.0"), - DNSPrefix: to.StringPtr("test-managedcluster"), AgentPoolProfiles: &[]containerservice.ManagedClusterAgentPoolProfile{ converters.AgentPoolToManagedClusterAgentPoolProfile(azure.AgentPoolSpec{ Name: "test-agentpool-0", diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 79b598017e1..5ab7c8e8ac0 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -820,6 +820,11 @@ spec: description: AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. properties: + autoUpgradeVersion: + description: AutoUpgradeVersion is the Kubernetes version populated + after autoupgrade based on the upgrade channel. + minLength: 2 + type: string conditions: description: Conditions defines current service state of the AzureManagedControlPlane. items: diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index fe1ecea044b..f2af706016e 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -789,6 +789,7 @@ func Convert_v1alpha3_AzureManagedControlPlaneStatus_To_v1beta1_AzureManagedCont } func autoConvert_v1beta1_AzureManagedControlPlaneStatus_To_v1alpha3_AzureManagedControlPlaneStatus(in *v1beta1.AzureManagedControlPlaneStatus, out *AzureManagedControlPlaneStatus, s conversion.Scope) error { + // WARNING: in.AutoUpgradeVersion requires manual conversion: does not exist in peer-type out.Ready = in.Ready out.Initialized = in.Initialized // WARNING: in.Conditions requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 7c316cfa13d..3ed27aae260 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1093,6 +1093,7 @@ func Convert_v1alpha4_AzureManagedControlPlaneStatus_To_v1beta1_AzureManagedCont } func autoConvert_v1beta1_AzureManagedControlPlaneStatus_To_v1alpha4_AzureManagedControlPlaneStatus(in *v1beta1.AzureManagedControlPlaneStatus, out *AzureManagedControlPlaneStatus, s conversion.Scope) error { + // WARNING: in.AutoUpgradeVersion requires manual conversion: does not exist in peer-type out.Ready = in.Ready out.Initialized = in.Initialized // WARNING: in.Conditions requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index f8396523776..4f8afccfb33 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -283,6 +283,12 @@ type ManagedControlPlaneSubnet struct { // AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. type AzureManagedControlPlaneStatus struct { + + // AutoUpgradeVersion is the Kubernetes version populated after autoupgrade based on the upgrade channel. + // +kubebuilder:validation:MinLength:=2 + // +optional + AutoUpgradeVersion string `json:"autoUpgradeVersion,omitempty"` + // Ready is true when the provider resource is ready. // +optional Ready bool `json:"ready,omitempty"` diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index fd79f3d86b0..434b476d520 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -30,8 +30,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -242,8 +243,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } - if old.Spec.DisableLocalAccounts == nil && - m.Spec.DisableLocalAccounts != nil && + if m.Spec.DisableLocalAccounts != nil && m.Spec.AADProfile == nil { allErrs = append(allErrs, field.Invalid( @@ -252,6 +252,14 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "DisableLocalAccounts can be set only for AAD enabled clusters")) } + if old.Spec.DisableLocalAccounts != nil && m.Spec.DisableLocalAccounts == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "field cannot be disabled")) + } + if old.Spec.OutboundType != nil { // Prevent OutboundType modification if it was already set to some value if m.Spec.OutboundType == nil { @@ -271,7 +279,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } - if pointer.StringDeref(m.Spec.DNSPrefix, "") != pointer.StringDeref(old.Spec.DNSPrefix, "") { + if ptr.Deref[string](m.Spec.DNSPrefix, "") != ptr.Deref[string](old.Spec.DNSPrefix, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.DNSPrefix"), @@ -279,7 +287,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } - if pointer.StringDeref(m.Spec.DockerBridgeCidr, "") != pointer.StringDeref(old.Spec.DockerBridgeCidr, "") { + if ptr.Deref[string](m.Spec.DockerBridgeCidr, "") != ptr.Deref[string](old.Spec.DockerBridgeCidr, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.DockerBridgeCidr"), @@ -287,7 +295,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } - if pointer.StringDeref(m.Spec.FqdnSubdomain, "") != pointer.StringDeref(old.Spec.FqdnSubdomain, "") { + if ptr.Deref[string](m.Spec.FqdnSubdomain, "") != ptr.Deref[string](old.Spec.FqdnSubdomain, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.FqdnSubdomain"), @@ -295,6 +303,20 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } + if hv, _ := versions.GetHigherK8sVersion(m.Spec.Version, old.Spec.Version); hv != m.Spec.Version { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Version"), + m.Spec.Version, "fields version cannot be downgraded"), + ) + } + + if old.Status.AutoUpgradeVersion != "" && m.Spec.Version != old.Spec.Version { + if hv, _ := versions.GetHigherK8sVersion(m.Spec.Version, old.Status.AutoUpgradeVersion); hv != m.Spec.Version { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Version"), + m.Spec.Version, "fields version cannot be downgraded"), + ) + } + } + if errs := m.validateAPIServerAccessProfileUpdate(old); len(errs) > 0 { allErrs = append(allErrs, errs...) } @@ -345,7 +367,6 @@ func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) } func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { - if m.Spec.DNSPrefix == nil { return nil } @@ -355,32 +376,12 @@ func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { // 2. Alphanumerics and hyphens: [a-zA-Z0-9-] // 3. Start and end with alphanumeric: ^[a-zA-Z0-9].*[a-zA-Z0-9]$ regex := regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9-]{0,52}[a-zA-Z0-9]$`) - if regex.MatchString(pointer.StringDeref(m.Spec.DNSPrefix, "")) { + if regex.MatchString(ptr.Deref[string](m.Spec.DNSPrefix, "")) { return nil } return errors.New("DNSPrefix is invalid") } -// func (m *AzureManagedControlPlane) validateFqdnSubdomain(_ client.Client) error { - -// if m.Spec.FqdnSubdomain == nil { -// return nil -// } - -// // Regex pattern for FQDN subdomain validation -// // 1. Between 1 and 63 characters long: {1,63} -// // 2. Alphanumerics and hyphens. -// // 3. Start and end with alphanumeric. -// // 4. Parts separated by dots (.) -// pattern := `^(?i)[a-z0-9]([a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*(?:\.aks\.example)$` - -// regex := regexp.MustCompile(pattern) -// if regex.MatchString(pointer.StringDeref(m.Spec.FqdnSubdomain, "")) { -// return nil -// } -// return errors.Errorf("FqdnSubdomain is invalid %s", pointer.StringDeref(m.Spec.FqdnSubdomain, "")) -// } - // validateDNSServiceIP validates the DNSServiceIP. func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { if m.Spec.DNSServiceIP != nil { @@ -409,7 +410,6 @@ func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { if !kubeSemver.MatchString(m.Spec.Version) { return errors.New("must be a valid semantic version") } - return nil } @@ -593,10 +593,10 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileDNSZoneUpdate(o // You can only update from byo or system to none. No other combination of update values is supported. if m.Spec.APIServerAccessProfile != nil && old.Spec.APIServerAccessProfile != nil && - pointer.BoolDeref(m.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && - pointer.BoolDeref(old.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + ptr.Deref(m.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + ptr.Deref(old.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && m.Spec.APIServerAccessProfile.PrivateDNSZone != old.Spec.APIServerAccessProfile.PrivateDNSZone && - pointer.StringDeref(m.Spec.APIServerAccessProfile.PrivateDNSZone, "") == "None" { + ptr.Deref[string](m.Spec.APIServerAccessProfile.PrivateDNSZone, "") == "None" { return nil } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 9f6343d4828..ec08aa9fa0b 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -471,6 +471,117 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { amcp: createAzureManagedControlPlane("192.168.0.0", "1.999.9", generateSSHPublicKey(true)), wantErr: true, }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.17.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.18.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.1", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.6", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane no version change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane valid version upgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.19.5", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane valid version change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.19.3", + }, + }, + wantErr: false, + }, { name: "AzureManagedControlPlane SubscriptionID is immutable", oldAMCP: &AzureManagedControlPlane{ @@ -1117,6 +1228,35 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "DisableLocalAccounts cannot be disabled AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.BoolPtr(true), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: true, + }, { name: "OutboundType update", oldAMCP: &AzureManagedControlPlane{ diff --git a/go.mod b/go.mod index d0b4c6d2e01..86dda86eed5 100644 --- a/go.mod +++ b/go.mod @@ -41,9 +41,9 @@ require ( k8s.io/apimachinery v0.25.0 k8s.io/client-go v0.25.0 k8s.io/component-base v0.25.0 - k8s.io/klog/v2 v2.70.1 + k8s.io/klog/v2 v2.80.1 k8s.io/kubectl v0.25.0 - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed + k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/cluster-api v1.1.1 sigs.k8s.io/cluster-api/test v1.1.4 sigs.k8s.io/controller-runtime v0.13.1 diff --git a/go.sum b/go.sum index 7714df6dbf0..37d6030d3a1 100644 --- a/go.sum +++ b/go.sum @@ -1919,8 +1919,8 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= @@ -1936,8 +1936,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg= oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index d2070bba34a..18148548248 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -9992,6 +9992,11 @@ spec: description: AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. properties: + autoUpgradeVersion: + description: AutoUpgradeVersion is the Kubernetes version populated + after autoupgrade based on the upgrade channel. + minLength: 2 + type: string conditions: description: Conditions defines current service state of the AzureManagedControlPlane. items: diff --git a/util/versions/version.go b/util/versions/version.go new file mode 100644 index 00000000000..0b83c8acec0 --- /dev/null +++ b/util/versions/version.go @@ -0,0 +1,22 @@ +package versions + +import ( + semverv4 "github.com/blang/semver" + "github.com/pkg/errors" +) + +// GetHigherK8sVersion returns the higher k8s version out of a and b +func GetHigherK8sVersion(a, b string) (string, error) { + v1, err := semverv4.ParseTolerant(a) + if err != nil { + return "", errors.Wrap(err, "error parsing k8s version") + } + v2, err := semverv4.ParseTolerant(b) + if err != nil { + return "", errors.Wrap(err, "error parsing k8s version") + } + if v1.GTE(v2) { + return a, nil + } + return b, nil +} diff --git a/util/versions/version_test.go b/util/versions/version_test.go new file mode 100644 index 00000000000..e18991176bb --- /dev/null +++ b/util/versions/version_test.go @@ -0,0 +1,86 @@ +package versions + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestGetHigherK8sVersion(t *testing.T) { + cases := []struct { + name string + a string + b string + output string + expectErr bool + }{ + { + name: "a is greater than b", + a: "v1.17.8", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "b is greater than a", + a: "v1.18.9", + b: "v1.18.8", + output: "v1.18.9", + expectErr: false, + }, + { + name: "b is greater than a", + a: "v1.18", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "a is equal to b", + a: "v1.18.8", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "a is greater than b and a is major.minor", + a: "v1.18", + b: "v1.17.8", + output: "v1.18", + expectErr: false, + }, + { + name: "a is greater than b and a is major.minor", + a: "1.18", + b: "1.17.8", + output: "1.18", + expectErr: false, + }, + { + name: "a is invalid", + a: "1.18.", + b: "v1.17.8", + output: "", + expectErr: true, + }, + { + name: "b is invalid", + a: "1.18.1", + b: "v1.17.8.", + output: "", + expectErr: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + g := NewWithT(t) + output, err := GetHigherK8sVersion(c.a, c.b) + g.Expect(output).To(Equal(c.output)) + if c.expectErr { + g.Expect(err).NotTo(BeNil()) + } else { + g.Expect(err).To(BeNil()) + } + }) + } +} From 5e16775a3ce67faa15d3beb9fbd0ff5e18fdc162 Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Tue, 3 Oct 2023 17:55:08 +0530 Subject: [PATCH 52/66] Cherry pick PCP-1614 PCP-1615 (#96) (#98) * AKS UpgradeChannels (#92) * disableLocal accounts (#91) * disableLocal accounts * ensure to not downgrade an auto-upgraded cluster (#95) * ensure to not downgrade an auto-upgraded cluster * fix version issue for auto upgraded clusters (#97) --- azure/scope/managedcontrolplane.go | 35 +--- azure/scope/managedcontrolplane_test.go | 17 +- azure/scope/managedmachinepool.go | 29 ++- azure/scope/managedmachinepool_test.go | 130 +++++++++++- azure/services/agentpools/agentpools.go | 17 +- azure/services/managedclusters/spec.go | 10 +- .../azuremanagedcontrolplane_webhook.go | 33 +++- .../azuremanagedcontrolplane_webhook_test.go | 187 ++++++++++++++---- util/versions/version.go | 18 +- util/versions/version_test.go | 18 +- 10 files changed, 388 insertions(+), 106 deletions(-) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 5b1c3b2d8df..b9680ab79e6 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -437,37 +437,16 @@ func (s *ManagedControlPlaneScope) SetAutoUpgradeVersionStatus(version string) { s.ControlPlane.Status.AutoUpgradeVersion = version } -// IsManagedVersionUpgrade checks if auto upgrade profile is set and if the upgradeChannel is of the type patch, stable or rapid. +// IsManagedVersionUpgrade checks if version is auto managed by AKS. func (s *ManagedControlPlaneScope) IsManagedVersionUpgrade() bool { - return s.IsPatchAutoUpgrade() || s.IsStableAutoUpgrade() || s.IsRapidAutoUpgrade() + return isManagedVersionUpgrade(s.ControlPlane) } -// IsAutoUpgradeStable checks if auto upgrade channel is stable. -func (s *ManagedControlPlaneScope) IsStableAutoUpgrade() bool { - if s.ControlPlane.Spec.AutoUpgradeProfile != nil { - if upgradeChannel := s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel; upgradeChannel == infrav1exp.UpgradeChannelStable { - return true - } - } - return false -} - -// IsAutoUpgradeStable checks if auto upgrade channel is rapid. -func (s *ManagedControlPlaneScope) IsRapidAutoUpgrade() bool { - if s.ControlPlane.Spec.AutoUpgradeProfile != nil { - if upgradeChannel := s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel; upgradeChannel == infrav1exp.UpgradeChannelRapid { - return true - } - } - return false -} - -// IsAutoUpgradeStable checks if auto upgrade channel is patch. -func (s *ManagedControlPlaneScope) IsPatchAutoUpgrade() bool { - if s.ControlPlane.Spec.AutoUpgradeProfile != nil { - if upgradeChannel := s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel; upgradeChannel == infrav1exp.UpgradeChannelPatch { - return true - } +func isManagedVersionUpgrade(managedControlPlane *infrav1exp.AzureManagedControlPlane) bool { + if managedControlPlane.Spec.AutoUpgradeProfile != nil && + (managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1exp.UpgradeChannelNone && + managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1exp.UpgradeChannelNodeImage) { + return true } return false } diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index acb384f98a1..b8db1b0d4d5 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -21,11 +21,10 @@ import ( "testing" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/pointer" + pointer "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" @@ -81,6 +80,7 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { Mode: "System", Cluster: "cluster1", VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: pointer.To("Linux"), }, }, }, @@ -119,9 +119,10 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { SKU: "Standard_D2s_v3", Mode: "System", Replicas: 1, - Version: to.StringPtr("1.21.1"), + Version: pointer.To("1.21.1"), Cluster: "cluster1", VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: pointer.To("Linux"), }, }, }, @@ -515,7 +516,7 @@ func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { }, Spec: infrav1.AzureManagedControlPlaneSpec{ SubscriptionID: "00000000-0000-0000-0000-000000000000", - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: pointer.To(true), }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -550,7 +551,7 @@ func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { Managed: true, AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, }, - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: pointer.To(true), }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -560,7 +561,7 @@ func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { }, }, }, - Expected: pointer.BoolPtr(true), + Expected: pointer.To(true), }, } for _, c := range cases { @@ -642,7 +643,7 @@ func TestIsAaDEnabled(t *testing.T) { Managed: true, AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, }, - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: pointer.To(true), }, }, ManagedMachinePools: []ManagedMachinePool{ @@ -766,7 +767,7 @@ func TestIsLocalAcountsDisabled(t *testing.T) { Managed: true, AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, }, - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: pointer.To(true), }, }, ManagedMachinePools: []ManagedMachinePool{ diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index ae6ccd85961..1da1d3070df 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -23,11 +23,13 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/util/futures" "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util/conditions" @@ -135,11 +137,7 @@ func (s *ManagedMachinePoolScope) AgentPoolSpec() azure.AgentPoolSpec { func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane, machinePool *clusterv1exp.MachinePool, managedMachinePool *infrav1exp.AzureManagedMachinePool) azure.AgentPoolSpec { - var normalizedVersion *string - if machinePool.Spec.Template.Spec.Version != nil { - v := strings.TrimPrefix(*machinePool.Spec.Template.Spec.Version, "v") - normalizedVersion = &v - } + normalizedVersion := getManagedMachinePoolVersion(managedControlPlane, machinePool) replicas := int32(1) if machinePool.Spec.Replicas != nil { @@ -300,3 +298,24 @@ func (s *ManagedMachinePoolScope) UpdateCAPIMachinePoolAnnotations(ctx context.C func (s *ManagedMachinePoolScope) GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string { return s.MachinePool.Annotations[key] } + +// IsManagedVersionUpgrade checks if version is auto managed by AKS. +func (s *ManagedMachinePoolScope) IsManagedAutoUpgrade() bool { + return isManagedVersionUpgrade(s.ControlPlane) +} + +func getManagedMachinePoolVersion(managedControlPlane *infrav1exp.AzureManagedControlPlane, + machinePool *clusterv1exp.MachinePool) *string { + var v, av string + if machinePool != nil { + v = ptr.Deref(machinePool.Spec.Template.Spec.Version, "") + } + if managedControlPlane != nil { + av = managedControlPlane.Status.AutoUpgradeVersion + } + higherVersion, err := versions.GetHigherK8sVersion(v, av) + if err != nil { + return nil + } + return ptr.To(strings.TrimPrefix(higherVersion, "v")) +} diff --git a/azure/scope/managedmachinepool_test.go b/azure/scope/managedmachinepool_test.go index 907ca55a31a..dba02a8837c 100644 --- a/azure/scope/managedmachinepool_test.go +++ b/azure/scope/managedmachinepool_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -66,7 +67,7 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -108,6 +109,7 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { MinCount: to.Int32Ptr(2), MaxCount: to.Int32Ptr(10), VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: ptr.To("Linux"), }, }, } @@ -160,7 +162,7 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -195,6 +197,7 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "System", @@ -256,6 +259,7 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -288,6 +292,7 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "System", @@ -347,7 +352,7 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -386,6 +391,7 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "User", @@ -445,6 +451,7 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -477,6 +484,7 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "User", @@ -502,6 +510,122 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { } } +func Test_getManagedMachinePoolVersion(t *testing.T) { + cases := []struct { + Name string + managedControlPlane *infrav1.AzureManagedControlPlane + machinePool *capiv1exp.MachinePool + Expected *string + }{ + { + Name: "Empty configs", + managedControlPlane: nil, + machinePool: nil, + Expected: nil, + }, + { + Name: "Empty mp", + managedControlPlane: &infrav1.AzureManagedControlPlane{}, + machinePool: nil, + Expected: nil, + }, + { + Name: "Only machine pool is available", + managedControlPlane: nil, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.15.0"), + }, + { + Name: "Only machine pool is available and cp is nil", + managedControlPlane: nil, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.15.0"), + }, + { + Name: "mcp.status.autoUpgradeVersion > mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.20.3"), + }, + { + Name: "suffix + mcp.status.autoUpgradeVersion > mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.20.3"), + }, + { + Name: "mcp.status.autoUpgradeVersion < mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.21.0"), + }, + }, + }, + }, + Expected: ptr.To("1.21.0"), + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + v := getManagedMachinePoolVersion(c.managedControlPlane, c.machinePool) + if c.Expected != nil { + g.Expect(*v).To(Equal(*c.Expected)) + } else { + g.Expect(v).To(Equal(c.Expected)) + } + }) + } +} + func getAzureMachinePool(name string, mode infrav1.NodePoolMode) *infrav1.AzureManagedMachinePool { return &infrav1.AzureManagedMachinePool{ ObjectMeta: metav1.ObjectMeta{ diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 8953586c921..4664d80851c 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -25,11 +25,13 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/util/maps" "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" ) const serviceName = "agentpools" @@ -47,6 +49,7 @@ type ManagedMachinePoolScope interface { UpdateCAPIMachinePoolReplicas(ctx context.Context, replicas *int32) UpdateCAPIMachinePoolAnnotations(ctx context.Context, key, value string) GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string + IsManagedAutoUpgrade() bool } // Service provides operations on Azure resources. @@ -78,7 +81,6 @@ func (s *Service) Reconcile(ctx context.Context) error { agentPoolSpec := s.scope.AgentPoolSpec() profile := converters.AgentPoolToContainerServiceAgentPool(agentPoolSpec) - existingPool, err := s.Client.Get(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name) if err != nil && !azure.ResourceNotFound(err) { return errors.Wrap(err, "failed to get existing agent pool") @@ -91,6 +93,9 @@ func (s *Service) Reconcile(ctx context.Context) error { customHeaders := maps.FilterByKeyPrefix(s.scope.AgentPoolAnnotations(), azure.CustomHeaderPrefix) if isCreate := azure.ResourceNotFound(err); isCreate { + if s.scope.IsManagedAutoUpgrade() { + profile.OrchestratorVersion = nil + } err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile, customHeaders) if err != nil && azure.ResourceNotFound(err) { @@ -105,6 +110,14 @@ func (s *Service) Reconcile(ctx context.Context) error { log.V(2).Info(msg) return azure.WithTransientError(errors.New(msg), 20*time.Second) } + // Get the higher version out of the existing and new version + profileOrchestratorVersion, err := versions.GetHigherK8sVersion( + ptr.Deref(existingPool.OrchestratorVersion, ""), + ptr.Deref(profile.OrchestratorVersion, "")) + + if err != nil { + return errors.Wrap(err, "error while calculating k8s version") + } // Normalize individual agent pools to diff in case we need to update existingProfile := containerservice.AgentPool{ @@ -122,7 +135,7 @@ func (s *Service) Reconcile(ctx context.Context) error { normalizedProfile := containerservice.AgentPool{ ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{ Count: profile.Count, - OrchestratorVersion: profile.OrchestratorVersion, + OrchestratorVersion: ptr.To(profileOrchestratorVersion), Mode: profile.Mode, EnableAutoScaling: profile.EnableAutoScaling, MinCount: profile.MinCount, diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index f8b140b2523..1d62a19b765 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -590,13 +590,15 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ UpgradeChannel: managedCluster.AutoUpgradeProfile.UpgradeChannel, } - } else { - clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{} } if existingMC.AutoUpgradeProfile != nil { - existingMCClusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ - UpgradeChannel: existingMC.AutoUpgradeProfile.UpgradeChannel, + if existingMC.AutoUpgradeProfile.UpgradeChannel == "" { + existingMC.AutoUpgradeProfile = nil + } else { + existingMCClusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: existingMC.AutoUpgradeProfile.UpgradeChannel, + } } } if managedCluster.DisableLocalAccounts != nil { diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index 434b476d520..6d397a37821 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -161,6 +161,16 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if old.Spec.AutoUpgradeProfile != nil && m.Spec.AutoUpgradeProfile == nil { + // Prevent AutoUpgradeProfile to be set to nil. + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "AutoUpgradeProfile"), + m.Spec.AutoUpgradeProfile, + "field is cannot be set to nil, to disable auto upgrades set the channel to none.")) + } + if old.Spec.NetworkPlugin != nil { // Prevent NetworkPlugin modification if it was already set to some value if m.Spec.NetworkPlugin == nil { @@ -252,12 +262,23 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "DisableLocalAccounts can be set only for AAD enabled clusters")) } - if old.Spec.DisableLocalAccounts != nil && m.Spec.DisableLocalAccounts == nil { - allErrs = append(allErrs, - field.Invalid( - field.NewPath("Spec", "DisableLocalAccounts"), - m.Spec.DisableLocalAccounts, - "field cannot be disabled")) + if old.Spec.DisableLocalAccounts != nil { + // Prevent DisableLocalAccounts modification if it was already set to some value + if m.Spec.DisableLocalAccounts == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.DisableLocalAccounts != *old.Spec.DisableLocalAccounts { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + *m.Spec.DisableLocalAccounts, + "field is immutable")) + } } if old.Spec.OutboundType != nil { diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index ec08aa9fa0b..c5481550ddf 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestDefaultingWebhook(t *testing.T) { @@ -86,7 +86,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix for starting with invalid characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("-thisi$"), + DNSPrefix: ptr.To("-thisi$"), Version: "v1.17.8", }, }, @@ -96,7 +96,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix with more then 54 characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), + DNSPrefix: ptr.To("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), Version: "v1.17.8", }, }, @@ -106,7 +106,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix with underscore", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("no_underscore"), + DNSPrefix: ptr.To("no_underscore"), Version: "v1.17.8", }, }, @@ -116,7 +116,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix with special characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("no-dollar$@%"), + DNSPrefix: ptr.To("no-dollar$@%"), Version: "v1.17.8", }, }, @@ -126,7 +126,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing Valid DNSPrefix with hyphen characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("hyphen-allowed"), + DNSPrefix: ptr.To("hyphen-allowed"), Version: "v1.17.8", }, }, @@ -136,7 +136,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing Valid DNSPrefix with hyphen characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("palette-test07"), + DNSPrefix: ptr.To("palette-test07"), Version: "v1.17.8", }, }, @@ -146,7 +146,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing valid DNSPrefix ", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), + DNSPrefix: ptr.To("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), Version: "v1.17.8", }, }, @@ -156,7 +156,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing valid DNSServiceIP", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "v1.17.8", }, }, @@ -166,7 +166,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing invalid DNSServiceIP", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0.3"), + DNSServiceIP: ptr.To("192.168.0.0.3"), Version: "v1.17.8", }, }, @@ -176,7 +176,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Invalid Version", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "honk", }, }, @@ -186,7 +186,7 @@ func TestValidatingWebhook(t *testing.T) { name: "not following the Kubernetes Version pattern", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "1.19.0", }, }, @@ -196,7 +196,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Version not set", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "", }, }, @@ -206,7 +206,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Valid Version", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "v1.17.8", }, }, @@ -309,7 +309,7 @@ func TestValidatingWebhook(t *testing.T) { amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ Version: "v1.21.2", - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: ptr.To(true), }, }, expectErr: true, @@ -323,7 +323,7 @@ func TestValidatingWebhook(t *testing.T) { Managed: true, AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, }, - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: ptr.To(true), }, }, expectErr: false, @@ -988,13 +988,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane DNSPrefix is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks-1"), + DNSPrefix: ptr.To("capz-aks-1"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks"), + DNSPrefix: ptr.To("capz-aks"), Version: "v1.18.0", }, }, @@ -1004,13 +1004,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane DNSPrefix is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks"), + DNSPrefix: ptr.To("capz-aks"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks"), + DNSPrefix: ptr.To("capz-aks"), Version: "v1.18.0", }, }, @@ -1020,14 +1020,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane FqdnSubdomain is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - FqdnSubdomain: pointer.StringPtr("capzaks.api"), + FqdnSubdomain: ptr.To("capzaks.api"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", - FqdnSubdomain: pointer.StringPtr("capzaks.com"), + FqdnSubdomain: ptr.To("capzaks.com"), }, }, wantErr: true, @@ -1036,14 +1036,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane FqdnSubdomain is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - FqdnSubdomain: pointer.StringPtr("capzaks.api"), + FqdnSubdomain: ptr.To("capzaks.api"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", - FqdnSubdomain: pointer.StringPtr("capzaks.api"), + FqdnSubdomain: ptr.To("capzaks.api"), }, }, wantErr: false, @@ -1053,7 +1053,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), + EnablePrivateCluster: ptr.To(true), }, Version: "v1.18.0", }, @@ -1061,8 +1061,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("None"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("None"), }, Version: "v1.18.0", }, @@ -1075,8 +1075,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), }, }, }, @@ -1084,8 +1084,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("None"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("None"), }, }, }, @@ -1097,8 +1097,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), }, }, }, @@ -1106,7 +1106,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), + EnablePrivateCluster: ptr.To(true), }, }, }, @@ -1118,8 +1118,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), }, }, }, @@ -1127,8 +1127,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id-1"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id-1"), }, }, }, @@ -1198,7 +1198,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: ptr.To(true), AADProfile: &AADProfile{ Managed: true, AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, @@ -1223,7 +1223,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: ptr.To(true), }, }, wantErr: true, @@ -1240,7 +1240,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Managed: true, AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, }, - DisableLocalAccounts: pointer.BoolPtr(true), + DisableLocalAccounts: ptr.To(true), }, }, amcp: &AzureManagedControlPlane{ @@ -1257,6 +1257,111 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "DisableLocalAccounts cannot change the value AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(false), + }, + }, + wantErr: true, + }, + { + name: "Auto upgrade profile cannot be disabled", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelNone, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "Auto upgrade profile channel can be updated", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelStable, + }, + }, + }, + wantErr: false, + }, + { + name: "Auto upgrade profile channel can remain same", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + wantErr: false, + }, { name: "OutboundType update", oldAMCP: &AzureManagedControlPlane{ diff --git a/util/versions/version.go b/util/versions/version.go index 0b83c8acec0..4646c3c8cfe 100644 --- a/util/versions/version.go +++ b/util/versions/version.go @@ -5,16 +5,20 @@ import ( "github.com/pkg/errors" ) -// GetHigherK8sVersion returns the higher k8s version out of a and b +// GetHigherK8sVersion returns the higher k8s version out of a and b. func GetHigherK8sVersion(a, b string) (string, error) { - v1, err := semverv4.ParseTolerant(a) - if err != nil { - return "", errors.Wrap(err, "error parsing k8s version") + v1, errv1 := semverv4.ParseTolerant(a) + v2, errv2 := semverv4.ParseTolerant(b) + if errv1 != nil && errv2 != nil { + return "", errors.Wrapf(errv1, "error parsing k8s version %s, %v error parsing k8s version %s", a, errv2, b) } - v2, err := semverv4.ParseTolerant(b) - if err != nil { - return "", errors.Wrap(err, "error parsing k8s version") + if errv1 != nil { + return b, nil } + if errv2 != nil { + return a, nil + } + if v1.GTE(v2) { return a, nil } diff --git a/util/versions/version_test.go b/util/versions/version_test.go index e18991176bb..738b39e6ba4 100644 --- a/util/versions/version_test.go +++ b/util/versions/version_test.go @@ -60,13 +60,27 @@ func TestGetHigherK8sVersion(t *testing.T) { name: "a is invalid", a: "1.18.", b: "v1.17.8", - output: "", - expectErr: true, + output: "v1.17.8", + expectErr: false, }, { name: "b is invalid", a: "1.18.1", b: "v1.17.8.", + output: "1.18.1", + expectErr: false, + }, + { + name: "b is invalid", + a: "9.99.9999", + b: "v1.17.8.", + output: "9.99.9999", + expectErr: false, + }, + { + name: "a & b is invalid", + a: "", + b: "v1.17.8.", output: "", expectErr: true, }, From b97d6168b35c599cedcb0f486b9a00b96adb48b4 Mon Sep 17 00:00:00 2001 From: zulfilee Date: Tue, 3 Oct 2023 16:29:17 +0000 Subject: [PATCH 53/66] OPS-2199 Enable ARM and Golang Alpine support --- .github/workflows/spectro-release.yaml | 14 +++++++++--- Dockerfile | 30 ++++++++++++-------------- Makefile | 7 ++++-- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index 0dbd180d78c..3c18181e647 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -7,6 +7,12 @@ on: description: 'Cluster API Version to Build' required: true default: '0.0.0' + rel_type: + type: choice + description: Type of release + options: + - release + - rc jobs: builder: # edge-runner machine group is a bunch of machines in US Datacenter @@ -15,6 +21,7 @@ jobs: # Ensure that the credentials are provided as encrypted secrets env: SPECTRO_VERSION: ${{ github.event.inputs.release_version }} + REGISTRY: gcr.io/spectro-images-public/release/cluster-api-azure steps: - uses: mukunku/tag-exists-action@v1.2.0 @@ -26,6 +33,9 @@ jobs: run: | echo "Tag already exists for v${{ github.event.inputs.release_version }}-spectro..." exit 1 + - + if: ${{ github.event.inputs.rel_type == 'rc' }} + run: echo "REGISTRY=gcr.io/spectro-dev-public/release/cluster-api-azure" >> $GITHUB_ENV - uses: actions/checkout@v3 - @@ -40,8 +50,6 @@ jobs: password: ${{ secrets.REGISTRY_PASSWORD }} - name: Build Image - env: - REGISTRY: gcr.io/spectro-images-public/release/cluster-api-azure run: | make docker-build-all make docker-push-all @@ -49,12 +57,12 @@ jobs: name: Build Image - FIPS Mode env: FIPS_ENABLE: yes - REGISTRY: gcr.io/spectro-images-public/release-fips/cluster-api-azure run: | make docker-build-all make docker-push-all - name: Create Release + if: ${{ github.event.inputs.rel_type == 'release' }} id: create_release uses: actions/create-release@v1 env: diff --git a/Dockerfile b/Dockerfile index c4278aed695..ce3e9b0e51a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -# syntax=docker/dockerfile:1.4 - # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,24 +13,26 @@ # limitations under the License. # Build the manager binary -FROM golang:1.19.10-alpine3.18 as builder -WORKDIR /workspace - +ARG BUILDER_GOLANG_VERSION +# First stage: build the executable. +FROM gcr.io/spectro-images-public/golang:${BUILDER_GOLANG_VERSION}-alpine as toolchain # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org ENV GOPROXY=$goproxy -RUN apk update -RUN apk add git gcc g++ curl - # FIPS ARG CRYPTO_LIB ENV GOEXPERIMENT=${CRYPTO_LIB:+boringcrypto} +FROM toolchain as builder +WORKDIR /workspace + +RUN apk update +RUN apk add git gcc g++ curl + # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum - # Cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN --mount=type=cache,target=/go/pkg/mod \ @@ -49,21 +49,19 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # Build ARG package=. ARG ARCH -ARG ldflags # Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ if [ ${CRYPTO_LIB} ]; \ then \ - CGO_ENABLED=1 GOOS=linux GOARCH=${ARCH} \ - go build -ldflags "${ldflags} -linkmode=external -extldflags '-static'" \ - -o manager ${package} ;\ + GOARCH=${ARCH} go-build-fips.sh -a -o manager ${package};\ else \ - CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ - go build -ldflags "${ldflags} -extldflags '-static'" \ - -o manager ${package} ;\ + GOARCH=${ARCH} go-build-static.sh -a -o manager ${package} ;\ fi +RUN if [ "${CRYPTO_LIB}" ]; then assert-static.sh manager; fi +RUN if [ "${CRYPTO_LIB}" ]; then assert-fips.sh manager; fi +RUN scan-govulncheck.sh manager # Production image FROM gcr.io/distroless/static:nonroot diff --git a/Makefile b/Makefile index 40a0ac7be64..ffaf53c2524 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ # If you update this file, please follow # https://suva.sh/posts/well-documented-makefiles + # Ensure Make is run with bash shell as some syntax below is bash-specific SHELL:=/usr/bin/env bash @@ -123,6 +124,8 @@ ETCD=$(TOOLS_BIN_DIR)/etcd # Define Docker related variables. Releases should modify and double check these vars. # Fips Flags FIPS_ENABLE ?= "" +BUILDER_GOLANG_VERSION ?= 1.21 +BUILD_ARGS = --build-arg CRYPTO_LIB=${FIPS_ENABLE} --build-arg BUILDER_GOLANG_VERSION=${BUILDER_GOLANG_VERSION} RELEASE_LOC := release ifeq ($(FIPS_ENABLE),yes) @@ -133,7 +136,7 @@ SPECTRO_VERSION ?= 4.0.0-dev TAG ?= v1.3.2-spectro-${SPECTRO_VERSION} ARCH ?= amd64 # ALL_ARCH = amd64 arm arm64 ppc64le s390x -ALL_ARCH = amd64 +ALL_ARCH = amd64 arm64 REGISTRY ?= gcr.io/spectro-dev-public/$(USER)/${RELEASE_LOC} @@ -352,7 +355,7 @@ docker-pull-prerequisites: ## Pull prerequisites for building controller-manager .PHONY: docker-build docker-build: docker-pull-prerequisites ## Build the docker image for controller-manager. - DOCKER_BUILDKIT=1 docker build --build-arg CRYPTO_LIB=${FIPS_ENABLE} --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) + docker buildx build --load --platform linux/${ARCH} ${BUILD_ARGS} --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" From 107fb1a83b171d4c88c960eaf38edca020142567 Mon Sep 17 00:00:00 2001 From: zulfilee Date: Tue, 3 Oct 2023 18:17:03 +0000 Subject: [PATCH 54/66] Fix GO build --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ce3e9b0e51a..8dab6395906 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,12 @@ COPY ./ ./ # Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - go build . + if [ ${CRYPTO_LIB} ]; \ + then \ + GOARCH=${ARCH} go-build-fips.sh . ;\ + else \ + GOARCH=${ARCH} go-build-static.sh . ;\ + fi # Build ARG package=. From 162f3e431bb3f21f0f6dee94dbac6b6d25da5b75 Mon Sep 17 00:00:00 2001 From: zulfilee Date: Wed, 4 Oct 2023 10:00:37 +0000 Subject: [PATCH 55/66] Add Platform to Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8dab6395906..6fc368de3de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # Build the manager binary ARG BUILDER_GOLANG_VERSION # First stage: build the executable. -FROM gcr.io/spectro-images-public/golang:${BUILDER_GOLANG_VERSION}-alpine as toolchain +FROM --platform=$TARGETPLATFORM gcr.io/spectro-images-public/golang:${BUILDER_GOLANG_VERSION}-alpine as toolchain # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org ENV GOPROXY=$goproxy From 0d9676d64be8bb5a813d639ba1ed22e27449a55a Mon Sep 17 00:00:00 2001 From: zulfilee Date: Wed, 4 Oct 2023 10:02:46 +0000 Subject: [PATCH 56/66] Add Platform to Dockerfile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ffaf53c2524..849f773d1f7 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ SPECTRO_VERSION ?= 4.0.0-dev TAG ?= v1.3.2-spectro-${SPECTRO_VERSION} ARCH ?= amd64 # ALL_ARCH = amd64 arm arm64 ppc64le s390x -ALL_ARCH = amd64 arm64 +ALL_ARCH = arm64 amd64 REGISTRY ?= gcr.io/spectro-dev-public/$(USER)/${RELEASE_LOC} From eac5a2a40b99edcda779bf0a4d72953fdbe53238 Mon Sep 17 00:00:00 2001 From: zulfilee Date: Wed, 4 Oct 2023 11:48:26 +0000 Subject: [PATCH 57/66] Add Platform to Dockerfile --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6fc368de3de..d1d50111d0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,10 @@ RUN --mount=type=cache,target=/go/pkg/mod \ # Copy the sources COPY ./ ./ +# Build +ARG package=. +ARG ARCH + # Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ @@ -51,9 +55,6 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ GOARCH=${ARCH} go-build-static.sh . ;\ fi -# Build -ARG package=. -ARG ARCH # Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder RUN --mount=type=cache,target=/root/.cache/go-build \ From bb6e07e80f21d680c983cd27ea03559f24dfbb4e Mon Sep 17 00:00:00 2001 From: Zulfi Date: Thu, 5 Oct 2023 13:05:53 +0530 Subject: [PATCH 58/66] Add Remote Builder for ARM --- .github/workflows/spectro-release.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index 3c18181e647..a2650e21042 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -36,6 +36,13 @@ jobs: - if: ${{ github.event.inputs.rel_type == 'rc' }} run: echo "REGISTRY=gcr.io/spectro-dev-public/release/cluster-api-azure" >> $GITHUB_ENV + - + name: Install SSH key for remote docker build + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_KEY }} + known_hosts: ${{ secrets.KNOWN_HOSTS }} + if_key_exists: fail - uses: actions/checkout@v3 - @@ -48,15 +55,22 @@ jobs: registry: ${{ secrets.REGISTRY_URL }} username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} + - + name: Build ARM Image + env: + DOCKER_HOST: 'ssh://builder@'${{ secrets.KNOWN_HOSTS }} + run: | + make docker-build arm64 - name: Build Image run: | - make docker-build-all + make docker-build amd64 make docker-push-all - name: Build Image - FIPS Mode env: FIPS_ENABLE: yes + ALL_ARCH: amd64 run: | make docker-build-all make docker-push-all From 8c431f2ac9bba7b55ad9ad2e59222209cc315e48 Mon Sep 17 00:00:00 2001 From: Zulfi Date: Thu, 5 Oct 2023 14:57:22 +0530 Subject: [PATCH 59/66] Add Remote Builder for ARM --- .github/workflows/spectro-release.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index a2650e21042..f29d068f62d 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -21,7 +21,8 @@ jobs: # Ensure that the credentials are provided as encrypted secrets env: SPECTRO_VERSION: ${{ github.event.inputs.release_version }} - REGISTRY: gcr.io/spectro-images-public/release/cluster-api-azure + LEGACY_REGISTRY: gcr.io/spectro-images-public/release/cluster-api-azure + FIPS_REGISTRY: gcr.io/spectro-images-public/release-fips/cluster-api-azure steps: - uses: mukunku/tag-exists-action@v1.2.0 @@ -35,7 +36,13 @@ jobs: exit 1 - if: ${{ github.event.inputs.rel_type == 'rc' }} - run: echo "REGISTRY=gcr.io/spectro-dev-public/release/cluster-api-azure" >> $GITHUB_ENV + run: | + echo "LEGACY_REGISTRY=gcr.io/spectro-dev-public/release/cluster-api-azure" >> $GITHUB_ENV + echo "FIPS_REGISTRY=gcr.io/spectro-dev-public/release-fips/cluster-api-azure" >> $GITHUB_ENV + - + name: Set ARM Builder Host + run: | + echo "ARM_DOCKER_HOST='ssh://builder@'${{ secrets.KNOWN_HOSTS }}" >> $GITHUB_ENV - name: Install SSH key for remote docker build uses: shimataro/ssh-key-action@v2 @@ -58,17 +65,21 @@ jobs: - name: Build ARM Image env: - DOCKER_HOST: 'ssh://builder@'${{ secrets.KNOWN_HOSTS }} + REGISTRY: ${{ env.LEGACY_REGISTRY }} + DOCKER_HOST: ${{ env.ARM_DOCKER_HOST }} run: | make docker-build arm64 - name: Build Image + env: + REGISTRY: ${{ env.LEGACY_REGISTRY }} run: | make docker-build amd64 make docker-push-all - name: Build Image - FIPS Mode env: + REGISTRY: ${{ env.FIPS_REGISTRY }} FIPS_ENABLE: yes ALL_ARCH: amd64 run: | From bf014642dc36103d1180ed97abec99a2411919d2 Mon Sep 17 00:00:00 2001 From: Zulfi Date: Thu, 5 Oct 2023 15:04:28 +0530 Subject: [PATCH 60/66] Add Remote Builder for ARM --- .github/workflows/spectro-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index f29d068f62d..fba2d92ad26 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -42,7 +42,7 @@ jobs: - name: Set ARM Builder Host run: | - echo "ARM_DOCKER_HOST='ssh://builder@'${{ secrets.KNOWN_HOSTS }}" >> $GITHUB_ENV + echo "ARM_DOCKER_HOST=ssh:\/\/builder\@${{ secrets.KNOWN_HOSTS }}" >> $GITHUB_ENV - name: Install SSH key for remote docker build uses: shimataro/ssh-key-action@v2 From 0cc43fe484d0f69f9c2d592338b2ebee7cd8ba16 Mon Sep 17 00:00:00 2001 From: Zulfi Date: Thu, 5 Oct 2023 15:06:00 +0530 Subject: [PATCH 61/66] Add Remote Builder for ARM --- .github/workflows/spectro-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index fba2d92ad26..82bf6bcaded 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -42,7 +42,7 @@ jobs: - name: Set ARM Builder Host run: | - echo "ARM_DOCKER_HOST=ssh:\/\/builder\@${{ secrets.KNOWN_HOSTS }}" >> $GITHUB_ENV + echo "ARM_DOCKER_HOST=ssh://builder@${{ secrets.KNOWN_HOSTS }}" >> $GITHUB_ENV - name: Install SSH key for remote docker build uses: shimataro/ssh-key-action@v2 From 5292f6fa52ea2f01d778f0792acaceb47f2ce965 Mon Sep 17 00:00:00 2001 From: Zulfi Date: Mon, 9 Oct 2023 08:57:20 +0530 Subject: [PATCH 62/66] Update spectro-release.yaml --- .github/workflows/spectro-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index 82bf6bcaded..48fdf024603 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -42,7 +42,7 @@ jobs: - name: Set ARM Builder Host run: | - echo "ARM_DOCKER_HOST=ssh://builder@${{ secrets.KNOWN_HOSTS }}" >> $GITHUB_ENV + echo "ARM_DOCKER_HOST=ssh://builder@${{ secrets.DOCKER_HOST_ARM }}" >> $GITHUB_ENV - name: Install SSH key for remote docker build uses: shimataro/ssh-key-action@v2 From 8682101285d4d10e31601e5a958e0ebe1491e0ce Mon Sep 17 00:00:00 2001 From: Zulfi Date: Mon, 9 Oct 2023 09:16:48 +0530 Subject: [PATCH 63/66] Update spectro-release.yaml --- .github/workflows/spectro-release.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index 48fdf024603..fd40900008a 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -67,14 +67,16 @@ jobs: env: REGISTRY: ${{ env.LEGACY_REGISTRY }} DOCKER_HOST: ${{ env.ARM_DOCKER_HOST }} + ARCH: arm64 run: | - make docker-build arm64 + make docker-build - name: Build Image env: REGISTRY: ${{ env.LEGACY_REGISTRY }} + ARCH: amd64 run: | - make docker-build amd64 + make docker-build make docker-push-all - name: Build Image - FIPS Mode From 967487b3f3a93735eb2ece3d87fc46315b669e65 Mon Sep 17 00:00:00 2001 From: Zulfi Date: Mon, 9 Oct 2023 12:09:23 +0530 Subject: [PATCH 64/66] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d1d50111d0d..4b2b9b42a38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ FROM toolchain as builder WORKDIR /workspace RUN apk update -RUN apk add git gcc g++ curl +RUN apk add git gcc g++ curl binutils-gold # Copy the Go Modules manifests COPY go.mod go.mod From 64d9eabc9f6dfbc4be2ae2d7fb9aaaa06d576dc3 Mon Sep 17 00:00:00 2001 From: Zulfi Date: Mon, 9 Oct 2023 12:32:12 +0530 Subject: [PATCH 65/66] Update spectro-release.yaml --- .github/workflows/spectro-release.yaml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/spectro-release.yaml b/.github/workflows/spectro-release.yaml index fd40900008a..4830447fa47 100644 --- a/.github/workflows/spectro-release.yaml +++ b/.github/workflows/spectro-release.yaml @@ -39,10 +39,6 @@ jobs: run: | echo "LEGACY_REGISTRY=gcr.io/spectro-dev-public/release/cluster-api-azure" >> $GITHUB_ENV echo "FIPS_REGISTRY=gcr.io/spectro-dev-public/release-fips/cluster-api-azure" >> $GITHUB_ENV - - - name: Set ARM Builder Host - run: | - echo "ARM_DOCKER_HOST=ssh://builder@${{ secrets.DOCKER_HOST_ARM }}" >> $GITHUB_ENV - name: Install SSH key for remote docker build uses: shimataro/ssh-key-action@v2 @@ -62,21 +58,12 @@ jobs: registry: ${{ secrets.REGISTRY_URL }} username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} - - - name: Build ARM Image - env: - REGISTRY: ${{ env.LEGACY_REGISTRY }} - DOCKER_HOST: ${{ env.ARM_DOCKER_HOST }} - ARCH: arm64 - run: | - make docker-build - name: Build Image env: REGISTRY: ${{ env.LEGACY_REGISTRY }} - ARCH: amd64 run: | - make docker-build + make docker-build-all make docker-push-all - name: Build Image - FIPS Mode From 9f81622a4414353d03b97eb279e7dbe67790a32a Mon Sep 17 00:00:00 2001 From: "kedar.jamkhindikar@spectrocloud.com" <79581321+kedar-calsoftinc@users.noreply.github.com> Date: Thu, 19 Oct 2023 19:18:16 +0530 Subject: [PATCH 66/66] PCP-2009 : Add PR validations for CAPZ --- .../bulwark-gitleaks-pr-validation.yaml | 36 +++++++++++++++++++ .github/workflows/bulwark-gosec-pr-scan.yaml | 35 ++++++++++++++++++ .../workflows/golicense-pr-validation.yaml | 31 ++++++++++++++++ .../workflows/govulncheck-pr-validation.yaml | 33 +++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 .github/workflows/bulwark-gitleaks-pr-validation.yaml create mode 100644 .github/workflows/bulwark-gosec-pr-scan.yaml create mode 100644 .github/workflows/golicense-pr-validation.yaml create mode 100644 .github/workflows/govulncheck-pr-validation.yaml diff --git a/.github/workflows/bulwark-gitleaks-pr-validation.yaml b/.github/workflows/bulwark-gitleaks-pr-validation.yaml new file mode 100644 index 00000000000..6e50bc2a1e2 --- /dev/null +++ b/.github/workflows/bulwark-gitleaks-pr-validation.yaml @@ -0,0 +1,36 @@ +name: BulwarkGitLeaks +on: [pull_request] + +concurrency: + group: gitleaks-${{ github.ref }} + cancel-in-progress: true + +jobs: + gitleaks-pr-scan: + runs-on: ubuntu-latest + container: + image: gcr.io/spectro-dev-public/bulwark/gitleaks:latest + env: + REPO: ${{ github.event.repository.name }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_CONFIG: /workspace/config.toml + steps: + + - name: run-bulwark-gitleaks-scan + shell: sh + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + run: /workspace/bulwark -name CodeSASTGitLeaks -target $REPO -tags "branch:$BRANCH,options:--log-opts origin..HEAD" + + - name: check-result + shell: sh + run: | + resultPath=./$REPO/gitleaks.json + cat $resultPath | grep -v \"Match\"\: | grep -v \"Secret\"\: + total_failed_tests=`cat $resultPath | grep \"Fingerprint\"\: | wc -l` + if [ "$total_failed_tests" -gt 0 ]; then + echo "GitLeaks validation check failed with above findings..." + exit 1 + else + echo "GitLeaks validation check passed" + fi \ No newline at end of file diff --git a/.github/workflows/bulwark-gosec-pr-scan.yaml b/.github/workflows/bulwark-gosec-pr-scan.yaml new file mode 100644 index 00000000000..bbc50ca193a --- /dev/null +++ b/.github/workflows/bulwark-gosec-pr-scan.yaml @@ -0,0 +1,35 @@ +name: BulwarkGoSec +on: [pull_request] + +concurrency: + group: gosec-${{ github.ref }} + cancel-in-progress: true + +jobs: + gosec-pr-scan: + runs-on: ubuntu-latest + container: + image: gcr.io/spectro-dev-public/bulwark/gosec:latest + steps: + + - name: run-gosec-scan + shell: sh + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + GO111MODULE: on + run: | + /workspace/bulwark -name CodeSASTGoSec -verbose -target $REPO -tags "branch:$BRANCH,rules:all" + + - name: check-result + shell: sh + run: | + resultPath=$REPO-result.json + issues=$(cat $resultPath | jq -r '.Stats.found') + echo "Found ${issues} issues" + if [ "$issues" -gt 0 ]; then + echo "GoSec SAST scan failed with below findings..." + cat $resultPath + exit 1 + else + echo "GoSec SAST scan passed" + fi diff --git a/.github/workflows/golicense-pr-validation.yaml b/.github/workflows/golicense-pr-validation.yaml new file mode 100644 index 00000000000..129eea9f918 --- /dev/null +++ b/.github/workflows/golicense-pr-validation.yaml @@ -0,0 +1,31 @@ +name: GoLicenses +on: [pull_request] + +concurrency: + group: golicenses-${{ github.ref }} + cancel-in-progress: true + +jobs: + golicense-pr-scan: + runs-on: ubuntu-latest + steps: + - name: install-git + run: sudo apt-get install -y git + + - name: install-golicenses + run: GOBIN=/usr/local/bin go install github.com/google/go-licenses@latest + + - name: checkout + uses: actions/checkout@v3 + + - name: set-github-access + run: | + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf ssh://git@github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf https://github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf git@github + + - name: golicense-scan + run: | + go-licenses check --ignore github.com/spectrocloud ./ +# go-licenses check --ignore github.com/spectrocloud ./hack/tools +# go-licenses check --ignore github.com/spectrocloud ./spate/xk6-spate \ No newline at end of file diff --git a/.github/workflows/govulncheck-pr-validation.yaml b/.github/workflows/govulncheck-pr-validation.yaml new file mode 100644 index 00000000000..711dd704be1 --- /dev/null +++ b/.github/workflows/govulncheck-pr-validation.yaml @@ -0,0 +1,33 @@ +name: GoVulnCheck +on: [pull_request] + +concurrency: + group: govulncheck-${{ github.ref }} + cancel-in-progress: true + +jobs: + govulncheck-pr-scan: + runs-on: ubuntu-latest + container: + image: gcr.io/spectro-images-public/golang:alpine + steps: + - name: install-govulncheck + run: GOBIN=/usr/local/bin go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: checkout + uses: actions/checkout@v3 + + - name: set-github-access + run: | + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf ssh://git@github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf https://github + /usr/bin/git config --global --add url."https://${{ secrets.GH_TOKEN }}:x-oauth-basic@github".insteadOf git@github + + - name: govulncheck-scan + run: | + go version + govulncheck -mode source ./ + govulncheck -mode source ./hack/tools + + +