diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index c70503e59135d..4b24214c8d7de 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -26,14 +26,10 @@ API rule violation: list_type_missing,k8s.io/api/admissionregistration/v1beta1,V API rule violation: list_type_missing,k8s.io/api/admissionregistration/v1beta1,WebhookClientConfig,CABundle API rule violation: list_type_missing,k8s.io/api/apps/v1,ControllerRevisionList,Items API rule violation: list_type_missing,k8s.io/api/apps/v1,DaemonSetList,Items -API rule violation: list_type_missing,k8s.io/api/apps/v1,DaemonSetStatus,Conditions API rule violation: list_type_missing,k8s.io/api/apps/v1,DeploymentList,Items -API rule violation: list_type_missing,k8s.io/api/apps/v1,DeploymentStatus,Conditions API rule violation: list_type_missing,k8s.io/api/apps/v1,ReplicaSetList,Items -API rule violation: list_type_missing,k8s.io/api/apps/v1,ReplicaSetStatus,Conditions API rule violation: list_type_missing,k8s.io/api/apps/v1,StatefulSetList,Items API rule violation: list_type_missing,k8s.io/api/apps/v1,StatefulSetSpec,VolumeClaimTemplates -API rule violation: list_type_missing,k8s.io/api/apps/v1,StatefulSetStatus,Conditions API rule violation: list_type_missing,k8s.io/api/apps/v1beta1,ControllerRevisionList,Items API rule violation: list_type_missing,k8s.io/api/apps/v1beta1,DeploymentList,Items API rule violation: list_type_missing,k8s.io/api/apps/v1beta1,DeploymentStatus,Conditions @@ -107,17 +103,13 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,Capabilities,Add API rule violation: list_type_missing,k8s.io/api/core/v1,Capabilities,Drop API rule violation: list_type_missing,k8s.io/api/core/v1,CephFSPersistentVolumeSource,Monitors API rule violation: list_type_missing,k8s.io/api/core/v1,CephFSVolumeSource,Monitors -API rule violation: list_type_missing,k8s.io/api/core/v1,ComponentStatus,Conditions API rule violation: list_type_missing,k8s.io/api/core/v1,ComponentStatusList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ConfigMapList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ConfigMapProjection,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ConfigMapVolumeSource,Items API rule violation: list_type_missing,k8s.io/api/core/v1,Container,Args API rule violation: list_type_missing,k8s.io/api/core/v1,Container,Command -API rule violation: list_type_missing,k8s.io/api/core/v1,Container,Env API rule violation: list_type_missing,k8s.io/api/core/v1,Container,EnvFrom -API rule violation: list_type_missing,k8s.io/api/core/v1,Container,VolumeDevices -API rule violation: list_type_missing,k8s.io/api/core/v1,Container,VolumeMounts API rule violation: list_type_missing,k8s.io/api/core/v1,ContainerImage,Names API rule violation: list_type_missing,k8s.io/api/core/v1,DownwardAPIProjection,Items API rule violation: list_type_missing,k8s.io/api/core/v1,DownwardAPIVolumeSource,Items @@ -128,12 +120,8 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,Endpoints,Subsets API rule violation: list_type_missing,k8s.io/api/core/v1,EndpointsList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Args API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Command -API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Env API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,EnvFrom API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Ports -API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,VolumeDevices -API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,VolumeMounts -API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainers,EphemeralContainers API rule violation: list_type_missing,k8s.io/api/core/v1,EventList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ExecAction,Command API rule violation: list_type_missing,k8s.io/api/core/v1,FCVolumeSource,TargetWWNs @@ -148,24 +136,19 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,List,Items API rule violation: list_type_missing,k8s.io/api/core/v1,LoadBalancerStatus,Ingress API rule violation: list_type_missing,k8s.io/api/core/v1,NamespaceList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,NamespaceSpec,Finalizers -API rule violation: list_type_missing,k8s.io/api/core/v1,NamespaceStatus,Conditions API rule violation: list_type_missing,k8s.io/api/core/v1,NodeAffinity,PreferredDuringSchedulingIgnoredDuringExecution API rule violation: list_type_missing,k8s.io/api/core/v1,NodeList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,NodeSelector,NodeSelectorTerms API rule violation: list_type_missing,k8s.io/api/core/v1,NodeSelectorRequirement,Values API rule violation: list_type_missing,k8s.io/api/core/v1,NodeSelectorTerm,MatchExpressions API rule violation: list_type_missing,k8s.io/api/core/v1,NodeSelectorTerm,MatchFields -API rule violation: list_type_missing,k8s.io/api/core/v1,NodeSpec,PodCIDRs API rule violation: list_type_missing,k8s.io/api/core/v1,NodeSpec,Taints -API rule violation: list_type_missing,k8s.io/api/core/v1,NodeStatus,Addresses -API rule violation: list_type_missing,k8s.io/api/core/v1,NodeStatus,Conditions API rule violation: list_type_missing,k8s.io/api/core/v1,NodeStatus,Images API rule violation: list_type_missing,k8s.io/api/core/v1,NodeStatus,VolumesAttached API rule violation: list_type_missing,k8s.io/api/core/v1,NodeStatus,VolumesInUse API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeClaimList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeClaimSpec,AccessModes API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeClaimStatus,AccessModes -API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeClaimStatus,Conditions API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeSpec,AccessModes API rule violation: list_type_missing,k8s.io/api/core/v1,PersistentVolumeSpec,MountOptions @@ -182,26 +165,17 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,PodList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,PodPortForwardOptions,Ports API rule violation: list_type_missing,k8s.io/api/core/v1,PodSecurityContext,SupplementalGroups API rule violation: list_type_missing,k8s.io/api/core/v1,PodSecurityContext,Sysctls -API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,Containers -API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,EphemeralContainers -API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,HostAliases -API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,ImagePullSecrets -API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,InitContainers API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,ReadinessGates API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,Tolerations -API rule violation: list_type_missing,k8s.io/api/core/v1,PodSpec,Volumes -API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,Conditions API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,ContainerStatuses API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,EphemeralContainerStatuses API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,InitContainerStatuses -API rule violation: list_type_missing,k8s.io/api/core/v1,PodStatus,PodIPs API rule violation: list_type_missing,k8s.io/api/core/v1,PodTemplateList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ProjectedVolumeSource,Sources API rule violation: list_type_missing,k8s.io/api/core/v1,RBDPersistentVolumeSource,CephMonitors API rule violation: list_type_missing,k8s.io/api/core/v1,RBDVolumeSource,CephMonitors API rule violation: list_type_missing,k8s.io/api/core/v1,RangeAllocation,Data API rule violation: list_type_missing,k8s.io/api/core/v1,ReplicationControllerList,Items -API rule violation: list_type_missing,k8s.io/api/core/v1,ReplicationControllerStatus,Conditions API rule violation: list_type_missing,k8s.io/api/core/v1,ResourceQuotaList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ResourceQuotaSpec,Scopes API rule violation: list_type_missing,k8s.io/api/core/v1,ScopeSelector,MatchExpressions @@ -210,7 +184,6 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,SecretList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,SecretProjection,Items API rule violation: list_type_missing,k8s.io/api/core/v1,SecretVolumeSource,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ServiceAccount,ImagePullSecrets -API rule violation: list_type_missing,k8s.io/api/core/v1,ServiceAccount,Secrets API rule violation: list_type_missing,k8s.io/api/core/v1,ServiceAccountList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ServiceList,Items API rule violation: list_type_missing,k8s.io/api/core/v1,ServiceSpec,ExternalIPs diff --git a/go.mod b/go.mod index fd9a2a24f7885..029157d556884 100644 --- a/go.mod +++ b/go.mod @@ -13,20 +13,19 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.5.0 github.com/Azure/go-autorest/autorest/to v0.2.0 github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534 - github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab - github.com/Microsoft/go-winio v0.4.14 - github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d + github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/Microsoft/go-winio v0.4.14 // indirect + github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d // indirect github.com/PuerkitoBio/purell v1.1.1 github.com/Rican7/retry v0.1.0 // indirect github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7 // indirect github.com/aws/aws-sdk-go v1.28.2 - github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798 - github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89 + github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798 // indirect github.com/blang/semver v3.5.0+incompatible github.com/boltdb/bolt v1.3.1 // indirect github.com/caddyserver/caddy v1.0.3 - github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c + github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c // indirect github.com/checkpoint-restore/go-criu v0.0.0-20181120144056-17b0214f6c48 // indirect github.com/cilium/ebpf v0.0.0-20191025125908-95b36a581eed // indirect github.com/client9/misspell v0.3.4 @@ -53,7 +52,7 @@ require ( github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect github.com/evanphx/json-patch v4.2.0+incompatible github.com/fsnotify/fsnotify v1.4.7 - github.com/go-bindata/go-bindata v3.1.1+incompatible + github.com/go-bindata/go-bindata v3.1.1+incompatible // indirect github.com/go-openapi/analysis v0.19.5 github.com/go-openapi/loads v0.19.4 github.com/go-openapi/spec v0.19.3 @@ -65,7 +64,7 @@ require ( github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 github.com/golang/mock v1.3.1 github.com/google/cadvisor v0.35.0 - github.com/google/go-cmp v0.3.0 + github.com/google/go-cmp v0.5.2 github.com/google/gofuzz v1.1.0 github.com/google/uuid v1.1.1 github.com/googleapis/gnostic v0.1.0 @@ -89,8 +88,8 @@ require ( github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/mvdan/xurls v1.1.0 - github.com/onsi/ginkgo v1.11.0 - github.com/onsi/gomega v1.7.0 + github.com/onsi/ginkgo v1.14.1 + github.com/onsi/gomega v1.10.2 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v1.0.0-rc10 @@ -108,7 +107,7 @@ require ( github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/afero v1.2.2 - github.com/spf13/cobra v0.0.5 + github.com/spf13/cobra v1.1.1 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.3.2 @@ -122,12 +121,11 @@ require ( github.com/vmware/govmomi v0.20.3 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 - golang.org/x/lint v0.0.0-20190409202823-959b441ac422 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 - golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 + golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 gonum.org/v1/gonum v0.6.2 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e // indirect google.golang.org/api v0.6.1-0.20190607001116-5213b8090861 @@ -136,11 +134,11 @@ require ( gopkg.in/square/go-jose.v2 v2.2.2 gopkg.in/yaml.v2 v2.2.8 gotest.tools v2.2.0+incompatible - gotest.tools/gotestsum v0.3.5 - honnef.co/go/tools v0.0.1-2019.2.2 - k8s.io/api v0.0.0 - k8s.io/apiextensions-apiserver v0.0.0 - k8s.io/apimachinery v0.0.0 + gotest.tools/gotestsum v0.3.5 // indirect + honnef.co/go/tools v0.0.1-2019.2.2 // indirect + k8s.io/api v0.20.2 + k8s.io/apiextensions-apiserver v0.20.2 + k8s.io/apimachinery v0.20.2 k8s.io/apiserver v0.0.0 k8s.io/cli-runtime v0.0.0 k8s.io/client-go v0.0.0 @@ -162,10 +160,11 @@ require ( k8s.io/kubelet v0.0.0 k8s.io/legacy-cloud-providers v0.0.0 k8s.io/metrics v0.0.0 - k8s.io/repo-infra v0.0.1-alpha.1 + k8s.io/repo-infra v0.0.1-alpha.1 // indirect k8s.io/sample-apiserver v0.0.0 k8s.io/system-validators v1.0.4 k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 + sigs.k8s.io/controller-tools v0.5.0 // indirect sigs.k8s.io/kustomize v2.0.3+incompatible sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 25f88fe809fd2..618decec3cc51 100644 --- a/go.sum +++ b/go.sum @@ -209,6 +209,8 @@ github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdY github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A= +github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f h1:zlOR3rOlPAVvtfuxGKoghCmop5B0TRyu/ZieziZuGiM= github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= @@ -325,6 +327,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kubernetes-sigs/controller-tools v0.5.0 h1:PRBwp3mEB70SYlUmJs6wnuzqT5QhQ58in4EfDPwd868= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= @@ -587,6 +590,8 @@ gopkg.in/warnings.v0 v0.1.1 h1:XM28wIgFzaBmeZ5dNHIpWLQpt/9DGKxk+rCg/22nnYE= gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v0.3.5 h1:VePOWRsuWFYpfp/G8mbmOZKxO5T3501SEGQRUdvq7h0= @@ -619,6 +624,8 @@ mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskX rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7 h1:uuHDyjllyzRyCIvvn0OBjiRB0SgBZGqHNYAmjR7fO50= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= +sigs.k8s.io/controller-tools v0.5.0 h1:3u2RCwOlp0cjCALAigpOcbAf50pE+kHSdueUosrC/AE= +sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= diff --git a/pkg/controlplane/aggregator.go b/pkg/controlplane/aggregator.go new file mode 100644 index 0000000000000..2c05c5f04bac3 --- /dev/null +++ b/pkg/controlplane/aggregator.go @@ -0,0 +1,326 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app does all of the work necessary to create a Kubernetes +// APIServer by binding together the API, master and APIServer infrastructure. +// It can be configured and called directly or via the hyperkube framework. +package controlplane + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + "sync" + + "k8s.io/klog" + + apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/features" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" + genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/feature" + utilfeature "k8s.io/apiserver/pkg/util/feature" + kubeexternalinformers "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" + v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" + "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" + informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" + "k8s.io/kube-aggregator/pkg/controllers/autoregister" + "k8s.io/kubernetes/pkg/controlplane/options" + "k8s.io/kubernetes/pkg/master/controller/crdregistration" +) + +func createAggregatorConfig( + kubeAPIServerConfig genericapiserver.Config, + commandOptions *options.ServerRunOptions, + externalInformers kubeexternalinformers.SharedInformerFactory, + serviceResolver aggregatorapiserver.ServiceResolver, + proxyTransport *http.Transport, + pluginInitializers []admission.PluginInitializer, +) (*aggregatorapiserver.Config, error) { + // make a shallow copy to let us twiddle a few things + // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator + genericConfig := kubeAPIServerConfig + genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} + genericConfig.RESTOptionsGetter = nil + + // override genericConfig.AdmissionControl with kube-aggregator's scheme, + // because aggregator apiserver should use its own scheme to convert its own resources. + err := commandOptions.Admission.ApplyTo( + &genericConfig, + externalInformers, + genericConfig.LoopbackClientConfig, + feature.DefaultFeatureGate, + pluginInitializers...) + if err != nil { + return nil, err + } + + // copy the etcd options so we don't mutate originals. + etcdOptions := *commandOptions.Etcd + etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) + etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) + genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} + + // override MergedResourceConfig with aggregator defaults and registry + if err := commandOptions.APIEnablement.ApplyTo( + &genericConfig, + aggregatorapiserver.DefaultAPIResourceConfigSource(), + aggregatorscheme.Scheme); err != nil { + return nil, err + } + + var certBytes, keyBytes []byte + if len(commandOptions.ProxyClientCertFile) > 0 && len(commandOptions.ProxyClientKeyFile) > 0 { + certBytes, err = ioutil.ReadFile(commandOptions.ProxyClientCertFile) + if err != nil { + return nil, err + } + keyBytes, err = ioutil.ReadFile(commandOptions.ProxyClientKeyFile) + if err != nil { + return nil, err + } + } + + aggregatorConfig := &aggregatorapiserver.Config{ + GenericConfig: &genericapiserver.RecommendedConfig{ + Config: genericConfig, + SharedInformerFactory: externalInformers, + }, + ExtraConfig: aggregatorapiserver.ExtraConfig{ + ProxyClientCert: certBytes, + ProxyClientKey: keyBytes, + ServiceResolver: serviceResolver, + ProxyTransport: proxyTransport, + }, + } + + // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) + aggregatorConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} + + return aggregatorConfig, nil +} + +func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) { + aggregatorServer, err := aggregatorConfig.Complete().NewWithDelegate(delegateAPIServer) + if err != nil { + return nil, err + } + + // create controllers for auto-registration + apiRegistrationClient, err := apiregistrationclient.NewForConfig(aggregatorConfig.GenericConfig.LoopbackClientConfig) + if err != nil { + return nil, err + } + autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient) + apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController) + crdRegistrationController := crdregistration.NewCRDRegistrationController( + apiExtensionInformers.Apiextensions().V1().CustomResourceDefinitions(), + autoRegistrationController) + + err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error { + go crdRegistrationController.Run(5, context.StopCh) + go func() { + // let the CRD controller process the initial set of CRDs before starting the autoregistration controller. + // this prevents the autoregistration controller's initial sync from deleting APIServices for CRDs that still exist. + // we only need to do this if CRDs are enabled on this server. We can't use discovery because we are the source for discovery. + if aggregatorConfig.GenericConfig.MergedResourceConfig.AnyVersionForGroupEnabled("apiextensions.k8s.io") { + crdRegistrationController.WaitForInitialSync() + } + autoRegistrationController.Run(5, context.StopCh) + }() + return nil + }) + if err != nil { + return nil, err + } + + err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks( + makeAPIServiceAvailableHealthCheck( + "autoregister-completion", + apiServices, + aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), + ), + ) + if err != nil { + return nil, err + } + + return aggregatorServer, nil +} + +func makeAPIService(gv schema.GroupVersion) *v1.APIService { + apiServicePriority, ok := apiVersionPriorities[gv] + if !ok { + // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version + // being permanently stuck in the APIServices list. + klog.Infof("Skipping APIService creation for %v", gv) + return nil + } + return &v1.APIService{ + ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, + Spec: v1.APIServiceSpec{ + Group: gv.Group, + Version: gv.Version, + GroupPriorityMinimum: apiServicePriority.group, + VersionPriority: apiServicePriority.version, + }, + } +} + +// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy +// once all of the specified services have been observed to be available at least once. +func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer informers.APIServiceInformer) healthz.HealthChecker { + // Track the auto-registered API services that have not been observed to be available yet + pendingServiceNamesLock := &sync.RWMutex{} + pendingServiceNames := sets.NewString() + for _, service := range apiServices { + pendingServiceNames.Insert(service.Name) + } + + // When an APIService in the list is seen as available, remove it from the pending list + handleAPIServiceChange := func(service *v1.APIService) { + pendingServiceNamesLock.Lock() + defer pendingServiceNamesLock.Unlock() + if !pendingServiceNames.Has(service.Name) { + return + } + if v1helper.IsAPIServiceConditionTrue(service, v1.Available) { + pendingServiceNames.Delete(service.Name) + } + } + + // Watch add/update events for APIServices + apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) }, + UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) }, + }) + + // Don't return healthy until the pending list is empty + return healthz.NamedCheck(name, func(r *http.Request) error { + pendingServiceNamesLock.RLock() + defer pendingServiceNamesLock.RUnlock() + if pendingServiceNames.Len() > 0 { + return fmt.Errorf("missing APIService: %v", pendingServiceNames.List()) + } + return nil + }) +} + +// priority defines group priority that is used in discovery. This controls +// group position in the kubectl output. +type priority struct { + // group indicates the order of the group relative to other groups. + group int32 + // version indicates the relative order of the version inside of its group. + version int32 +} + +// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers +// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. +// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. +// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. +var apiVersionPriorities = map[schema.GroupVersion]priority{ + {Group: "", Version: "v1"}: {group: 18000, version: 1}, + // extensions is above the rest for CLI compatibility, though the level of unqualified resource compatibility we + // can reasonably expect seems questionable. + {Group: "extensions", Version: "v1beta1"}: {group: 17900, version: 1}, + // to my knowledge, nothing below here collides + {Group: "apps", Version: "v1"}: {group: 17800, version: 15}, + {Group: "events.k8s.io", Version: "v1beta1"}: {group: 17750, version: 5}, + {Group: "authentication.k8s.io", Version: "v1"}: {group: 17700, version: 15}, + {Group: "authentication.k8s.io", Version: "v1beta1"}: {group: 17700, version: 9}, + {Group: "authorization.k8s.io", Version: "v1"}: {group: 17600, version: 15}, + {Group: "authorization.k8s.io", Version: "v1beta1"}: {group: 17600, version: 9}, + {Group: "autoscaling", Version: "v1"}: {group: 17500, version: 15}, + {Group: "autoscaling", Version: "v2beta1"}: {group: 17500, version: 9}, + {Group: "autoscaling", Version: "v2beta2"}: {group: 17500, version: 1}, + {Group: "batch", Version: "v1"}: {group: 17400, version: 15}, + {Group: "batch", Version: "v1beta1"}: {group: 17400, version: 9}, + {Group: "batch", Version: "v2alpha1"}: {group: 17400, version: 9}, + {Group: "certificates.k8s.io", Version: "v1beta1"}: {group: 17300, version: 9}, + {Group: "networking.k8s.io", Version: "v1"}: {group: 17200, version: 15}, + {Group: "networking.k8s.io", Version: "v1beta1"}: {group: 17200, version: 9}, + {Group: "policy", Version: "v1beta1"}: {group: 17100, version: 9}, + {Group: "rbac.authorization.k8s.io", Version: "v1"}: {group: 17000, version: 15}, + {Group: "rbac.authorization.k8s.io", Version: "v1beta1"}: {group: 17000, version: 12}, + {Group: "rbac.authorization.k8s.io", Version: "v1alpha1"}: {group: 17000, version: 9}, + {Group: "settings.k8s.io", Version: "v1alpha1"}: {group: 16900, version: 9}, + {Group: "storage.k8s.io", Version: "v1"}: {group: 16800, version: 15}, + {Group: "storage.k8s.io", Version: "v1beta1"}: {group: 16800, version: 9}, + {Group: "storage.k8s.io", Version: "v1alpha1"}: {group: 16800, version: 1}, + {Group: "apiextensions.k8s.io", Version: "v1"}: {group: 16700, version: 15}, + {Group: "apiextensions.k8s.io", Version: "v1beta1"}: {group: 16700, version: 9}, + {Group: "admissionregistration.k8s.io", Version: "v1"}: {group: 16700, version: 15}, + {Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {group: 16700, version: 12}, + {Group: "scheduling.k8s.io", Version: "v1"}: {group: 16600, version: 15}, + {Group: "scheduling.k8s.io", Version: "v1beta1"}: {group: 16600, version: 12}, + {Group: "scheduling.k8s.io", Version: "v1alpha1"}: {group: 16600, version: 9}, + {Group: "coordination.k8s.io", Version: "v1"}: {group: 16500, version: 15}, + {Group: "coordination.k8s.io", Version: "v1beta1"}: {group: 16500, version: 9}, + {Group: "auditregistration.k8s.io", Version: "v1alpha1"}: {group: 16400, version: 1}, + {Group: "node.k8s.io", Version: "v1alpha1"}: {group: 16300, version: 1}, + {Group: "node.k8s.io", Version: "v1beta1"}: {group: 16300, version: 9}, + {Group: "discovery.k8s.io", Version: "v1beta1"}: {group: 16200, version: 12}, + {Group: "discovery.k8s.io", Version: "v1alpha1"}: {group: 16200, version: 9}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1"}: {group: 16100, version: 9}, + // Append a new group to the end of the list if unsure. + // You can use min(existing group)-100 as the initial value for a group. + // Version can be set to 9 (to have space around) for a new group. +} + +func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService { + apiServices := []*v1.APIService{} + + for _, curr := range delegateAPIServer.ListedPaths() { + if curr == "/api/v1" { + apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"}) + registration.AddAPIServiceToSyncOnStart(apiService) + apiServices = append(apiServices, apiService) + continue + } + + if !strings.HasPrefix(curr, "/apis/") { + continue + } + // this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1 + tokens := strings.Split(curr, "/") + if len(tokens) != 4 { + continue + } + + apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) + if apiService == nil { + continue + } + registration.AddAPIServiceToSyncOnStart(apiService) + apiServices = append(apiServices, apiService) + } + + return apiServices +} diff --git a/pkg/controlplane/server.go b/pkg/controlplane/server.go index 530e66eaa3dbf..d2e19ea07f0d8 100644 --- a/pkg/controlplane/server.go +++ b/pkg/controlplane/server.go @@ -27,11 +27,13 @@ import ( "strings" "time" + "github.com/emicklei/go-restful" extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/union" + "k8s.io/apiserver/pkg/endpoints/discovery" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" genericapiserver "k8s.io/apiserver/pkg/server" @@ -71,12 +73,15 @@ func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) erro return err } - prepared := server.PrepareRun() + prepared, err := server.PrepareRun() + if err != nil { + return err + } return prepared.Run(stopCh) } // CreateServerChain creates the apiservers connected via delegation. -func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{}) (*genericapiserver.GenericAPIServer, error) { +func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{}) (*aggregatorapiserver.APIAggregator, error) { kubeAPIServerConfig, _, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions) if err != nil { return nil, err @@ -98,7 +103,26 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan return nil, err } - return kubeAPIServer.GenericAPIServer, nil + // aggregator comes last in the chain + aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, nil, pluginInitializer) + if err != nil { + return nil, err + } + aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers) + if err != nil { + // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines + return nil, err + } + + kubeAPIServer.GenericAPIServer.Handler.GoRestfulContainer.Filter(func(req *restful.Request, res *restful.Response, chain *restful.FilterChain){ + if discovery.IsAPIContributed(req.Request.URL.Path) { + apiExtensionsServer.GenericAPIServer.Handler.NonGoRestfulMux.ServeHTTP(res.ResponseWriter, req.Request) + } else { + chain.ProcessFilter(req, res) + } + }) + + return aggregatorServer, nil } // CreateKubeAPIServer creates and wires a workable kube-apiserver diff --git a/staging/src/k8s.io/api/apps/v1/doc.go b/staging/src/k8s.io/api/apps/v1/doc.go index 61dc97bde5233..12eb9e4592b71 100644 --- a/staging/src/k8s.io/api/apps/v1/doc.go +++ b/staging/src/k8s.io/api/apps/v1/doc.go @@ -18,4 +18,6 @@ limitations under the License. // +k8s:protobuf-gen=package // +k8s:openapi-gen=true +// +groupName="apps" + package v1 // import "k8s.io/api/apps/v1" diff --git a/staging/src/k8s.io/api/apps/v1/types.go b/staging/src/k8s.io/api/apps/v1/types.go index e003a0c4f7c05..11330442f28ab 100644 --- a/staging/src/k8s.io/api/apps/v1/types.go +++ b/staging/src/k8s.io/api/apps/v1/types.go @@ -35,6 +35,8 @@ const ( // +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas // StatefulSet represents a set of pods with consistent identities. // Identities are defined as: @@ -213,6 +215,8 @@ type StatefulSetStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []StatefulSetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,10,rep,name=conditions"` } @@ -249,6 +253,8 @@ type StatefulSetList struct { // +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas // Deployment enables declarative updates for Pods and ReplicaSets. type Deployment struct { @@ -403,6 +409,8 @@ type DeploymentStatus struct { // Represents the latest available observations of a deployment's current state. // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []DeploymentCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,6,rep,name=conditions"` // Count of hash collisions for the Deployment. The Deployment controller uses this @@ -587,6 +595,8 @@ type DaemonSetStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []DaemonSetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,10,rep,name=conditions"` } @@ -613,6 +623,7 @@ type DaemonSetCondition struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status // DaemonSet represents the configuration of a daemon set. type DaemonSet struct { @@ -661,6 +672,8 @@ type DaemonSetList struct { // +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas // ReplicaSet ensures that a specified number of pod replicas are running at any given time. type ReplicaSet struct { @@ -755,6 +768,8 @@ type ReplicaSetStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []ReplicaSetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,6,rep,name=conditions"` } diff --git a/staging/src/k8s.io/api/core/v1/doc.go b/staging/src/k8s.io/api/core/v1/doc.go index 1bdf0b25b1b07..1a68cd8142aea 100644 --- a/staging/src/k8s.io/api/core/v1/doc.go +++ b/staging/src/k8s.io/api/core/v1/doc.go @@ -18,5 +18,7 @@ limitations under the License. // +k8s:deepcopy-gen=package // +k8s:protobuf-gen=package +// +groupName="" + // Package v1 is the v1 version of the core API. package v1 // import "k8s.io/api/core/v1" diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index b61a86aba160d..537f9cba5343a 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -524,6 +524,8 @@ type PersistentVolumeClaimStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []PersistentVolumeClaimCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,4,rep,name=conditions"` } @@ -2170,6 +2172,8 @@ type Container struct { // +optional // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name Env []EnvVar `json:"env,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,7,rep,name=env"` // Compute Resources required by this container. // Cannot be updated. @@ -2181,10 +2185,14 @@ type Container struct { // +optional // +patchMergeKey=mountPath // +patchStrategy=merge + // +listType=map + // +listMapKey=mountPath VolumeMounts []VolumeMount `json:"volumeMounts,omitempty" patchStrategy:"merge" patchMergeKey:"mountPath" protobuf:"bytes,9,rep,name=volumeMounts"` // volumeDevices is the list of block devices to be used by the container. // +patchMergeKey=devicePath // +patchStrategy=merge + // +listType=map + // +listMapKey=devicePath // +optional VolumeDevices []VolumeDevice `json:"volumeDevices,omitempty" patchStrategy:"merge" patchMergeKey:"devicePath" protobuf:"bytes,21,rep,name=volumeDevices"` // Periodic probe of container liveness. @@ -2838,6 +2846,8 @@ type PodSpec struct { // +optional // +patchMergeKey=name // +patchStrategy=merge,retainKeys + // +listType=map + // +listMapKey=name Volumes []Volume `json:"volumes,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name" protobuf:"bytes,1,rep,name=volumes"` // List of initialization containers belonging to the pod. // Init containers are executed in order prior to containers being started. If any @@ -2854,6 +2864,8 @@ type PodSpec struct { // More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name InitContainers []Container `json:"initContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,20,rep,name=initContainers"` // List of containers belonging to the pod. // Containers cannot currently be added or removed. @@ -2861,6 +2873,8 @@ type PodSpec struct { // Cannot be updated. // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name Containers []Container `json:"containers" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,2,rep,name=containers"` // List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing // pod to perform user-initiated actions such as debugging. This list cannot be specified when @@ -2870,6 +2884,8 @@ type PodSpec struct { // +optional // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name EphemeralContainers []EphemeralContainer `json:"ephemeralContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,34,rep,name=ephemeralContainers"` // Restart policy for all containers within the pod. // One of Always, OnFailure, Never. @@ -2958,6 +2974,8 @@ type PodSpec struct { // +optional // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name ImagePullSecrets []LocalObjectReference `json:"imagePullSecrets,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,15,rep,name=imagePullSecrets"` // Specifies the hostname of the Pod // If not specified, the pod's hostname will be set to a system-defined value. @@ -2982,6 +3000,8 @@ type PodSpec struct { // +optional // +patchMergeKey=ip // +patchStrategy=merge + // +listType=map + // +listMapKey=ip HostAliases []HostAlias `json:"hostAliases,omitempty" patchStrategy:"merge" patchMergeKey:"ip" protobuf:"bytes,23,rep,name=hostAliases"` // If specified, indicates the pod's priority. "system-node-critical" and // "system-cluster-critical" are two special keywords which indicate the @@ -3121,7 +3141,7 @@ const ( // pod's hosts file. type HostAlias struct { // IP address of the host file entry. - IP string `json:"ip,omitempty" protobuf:"bytes,1,opt,name=ip"` + IP string `json:"ip" protobuf:"bytes,1,opt,name=ip"` // Hostnames for the above IP address. Hostnames []string `json:"hostnames,omitempty" protobuf:"bytes,2,rep,name=hostnames"` } @@ -3256,7 +3276,7 @@ type PodDNSConfigOption struct { // IP: An IP address allocated to the pod. Routable at least within the cluster. type PodIP struct { // ip is an IP address (IPv4 or IPv6) assigned to the pod - IP string `json:"ip,omitempty" protobuf:"bytes,1,opt,name=ip"` + IP string `json:"ip" protobuf:"bytes,1,opt,name=ip"` } // EphemeralContainerCommon is a copy of all fields in Container to be inlined in @@ -3311,6 +3331,8 @@ type EphemeralContainerCommon struct { // +optional // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name Env []EnvVar `json:"env,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,7,rep,name=env"` // Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources // already allocated to the pod. @@ -3321,10 +3343,14 @@ type EphemeralContainerCommon struct { // +optional // +patchMergeKey=mountPath // +patchStrategy=merge + // +listType=map + // +listMapKey=mountPath VolumeMounts []VolumeMount `json:"volumeMounts,omitempty" patchStrategy:"merge" patchMergeKey:"mountPath" protobuf:"bytes,9,rep,name=volumeMounts"` // volumeDevices is the list of block devices to be used by the container. // +patchMergeKey=devicePath // +patchStrategy=merge + // +listType=map + // +listMapKey=devicePath // +optional VolumeDevices []VolumeDevice `json:"volumeDevices,omitempty" patchStrategy:"merge" patchMergeKey:"devicePath" protobuf:"bytes,21,rep,name=volumeDevices"` // Probes are not allowed for ephemeral containers. @@ -3448,6 +3474,8 @@ type PodStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []PodCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,2,rep,name=conditions"` // A human readable message indicating details about why the pod is in this condition. // +optional @@ -3480,6 +3508,8 @@ type PodStatus struct { // +optional // +patchStrategy=merge // +patchMergeKey=ip + // +listType=map + // +listMapKey=ip PodIPs []PodIP `json:"podIPs,omitempty" protobuf:"bytes,12,rep,name=podIPs" patchStrategy:"merge" patchMergeKey:"ip"` // RFC 3339 date and time at which the object was acknowledged by the Kubelet. @@ -3531,6 +3561,7 @@ type PodStatusResult struct { // +genclient:method=GetEphemeralContainers,verb=get,subresource=ephemeralcontainers,result=EphemeralContainers // +genclient:method=UpdateEphemeralContainers,verb=update,subresource=ephemeralcontainers,input=EphemeralContainers,result=EphemeralContainers // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status // Pod is a collection of containers that can run on a host. This resource is created // by clients and scheduled onto hosts. @@ -3677,6 +3708,8 @@ type ReplicationControllerStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []ReplicationControllerCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,6,rep,name=conditions"` } @@ -4050,6 +4083,7 @@ type ServicePort struct { // +genclient // +genclient:skipVerbs=deleteCollection // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status // Service is a named abstraction of software service (for example, mysql) consisting of local port // (for example 3306) that the proxy listens on, and the selector that determines which pods @@ -4114,6 +4148,8 @@ type ServiceAccount struct { // +optional // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name Secrets []ObjectReference `json:"secrets,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,2,rep,name=secrets"` // ImagePullSecrets is a list of references to secrets in the same namespace to use for pulling any images @@ -4276,6 +4312,7 @@ type NodeSpec struct { // each of IPv4 and IPv6. // +optional // +patchStrategy=merge + // +listType=set PodCIDRs []string `json:"podCIDRs,omitempty" protobuf:"bytes,7,opt,name=podCIDRs" patchStrategy:"merge"` // ID of the node assigned by the cloud provider in the format: :// @@ -4455,6 +4492,8 @@ type NodeStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []NodeCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,4,rep,name=conditions"` // List of addresses reachable to the node. // Queried from cloud provider, if available. @@ -4465,6 +4504,8 @@ type NodeStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Addresses []NodeAddress `json:"addresses,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,5,rep,name=addresses"` // Endpoints of daemons running on the Node. // +optional @@ -4645,6 +4686,8 @@ type ResourceList map[ResourceName]resource.Quantity // +genclient // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster // Node is a worker node in Kubernetes. // Each node will have a unique identifier in the cache (i.e. in etcd). @@ -4710,6 +4753,8 @@ type NamespaceStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []NamespaceCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,2,rep,name=conditions"` } @@ -4827,6 +4872,8 @@ type EphemeralContainers struct { // or modified. // +patchMergeKey=name // +patchStrategy=merge + // +listType=map + // +listMapKey=name EphemeralContainers []EphemeralContainer `json:"ephemeralContainers" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,2,rep,name=ephemeralContainers"` } @@ -5084,8 +5131,7 @@ type LocalObjectReference struct { // Name of the referent. // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names // TODO: Add other useful fields. apiVersion, kind, uid? - // +optional - Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"` + Name string `json:"name" protobuf:"bytes,1,opt,name=name"` } // TypedLocalObjectReference contains enough information to let you locate the @@ -5706,6 +5752,8 @@ type ComponentStatus struct { // +optional // +patchMergeKey=type // +patchStrategy=merge + // +listType=map + // +listMapKey=type Conditions []ComponentCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,2,rep,name=conditions"` } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go index 82e4e9d15dd43..f15b4b4dca5ab 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go @@ -30,6 +30,7 @@ import ( utilvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -48,9 +49,11 @@ var ( func ValidateCustomResourceDefinition(obj *apiextensions.CustomResourceDefinition, requestGV schema.GroupVersion) field.ErrorList { nameValidationFn := func(name string, prefix bool) []string { ret := genericvalidation.NameIsDNSSubdomain(name, prefix) - requiredName := obj.Spec.Names.Plural + "." + obj.Spec.Group - if name != requiredName { - ret = append(ret, fmt.Sprintf(`must be spec.names.plural+"."+spec.group`)) + if obj.Spec.Group != "" { + requiredName := obj.Spec.Names.Plural + "." + obj.Spec.Group + if name != requiredName { + ret = append(ret, fmt.Sprintf(`must be spec.names.plural+"."+spec.group`)) + } } return ret } @@ -171,12 +174,16 @@ func validateCustomResourceDefinitionVersion(version *apiextensions.CustomResour func validateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefinitionSpec, opts validationOptions, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - if len(spec.Group) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("group"), "")) + // HACK: Relax naming constraints when registering legacy schema resources through CRDs + // for the KCP scenario + if legacyscheme.Scheme.IsGroupRegistered(spec.Group) { + // No error: these are legacy schema kubernetes types + // that are not added in the controlplane schema + // and that we want to move up to the KCP as CRDs } else if errs := utilvalidation.IsDNS1123Subdomain(spec.Group); len(errs) > 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("group"), spec.Group, strings.Join(errs, ","))) + allErrs = append(allErrs, field.Invalid(fldPath.Child("group"), spec.Group, strings.Join(errs, ","))) } else if len(strings.Split(spec.Group, ".")) < 2 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("group"), spec.Group, "should be a domain with at least one dot")) + allErrs = append(allErrs, field.Invalid(fldPath.Child("group"), spec.Group, "should be a domain with at least one dot")) } allErrs = append(allErrs, validateEnumStrings(fldPath.Child("scope"), string(spec.Scope), []string{string(apiextensions.ClusterScoped), string(apiextensions.NamespaceScoped)}, true)...) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 390b5a4471b00..d32b292f9538b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -206,6 +206,8 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler) s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler) + // HACK: Added to allow serving core resources registered through CRDs (for the KCP scenario) + s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/api/v1/", crdHandler) crdController := NewDiscoveryController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), versionDiscoveryHandler, groupDiscoveryHandler) namingController := status.NewNamingConditionController(s.Informers.Apiextensions().V1().CustomResourceDefinitions(), crdClient.ApiextensionsV1()) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go index 23564762c7566..db4ccd1bfbd4f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go @@ -115,6 +115,8 @@ func (c *crConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, valu return label, value, nil case !c.clusterScoped && label == "metadata.namespace": return label, value, nil + case gvk.Kind == "Pod" && label == "spec.nodeName": + return label, value, nil default: return "", "", fmt.Errorf("field label not supported: %s", label) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go index f897abd9ac50a..4f23b51581e8a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/klog" + "k8s.io/kubernetes/pkg/api/legacyscheme" autoscaling "k8s.io/api/autoscaling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,11 +105,18 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { // If there is any Served version, that means the group should show up in discovery foundGroup = true + // HACK: support the case when we add core resources through CRDs (KCP scenario) + groupVersion := crd.Spec.Group + "/" + v.Name + if crd.Spec.Group == "" { + groupVersion = v.Name + } + gv := metav1.GroupVersion{Group: crd.Spec.Group, Version: v.Name} + if !versionsForDiscoveryMap[gv] { versionsForDiscoveryMap[gv] = true apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{ - GroupVersion: crd.Spec.Group + "/" + v.Name, + GroupVersion: groupVersion, Version: v.Name, }) } @@ -167,30 +175,46 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { } } - if !foundGroup { - c.groupHandler.unsetDiscovery(version.Group) - c.versionHandler.unsetDiscovery(version) - return nil - } - sortGroupDiscoveryByKubeAwareVersion(apiVersionsForDiscovery) - apiGroup := metav1.APIGroup{ - Name: version.Group, - Versions: apiVersionsForDiscovery, - // the preferred versions for a group is the first item in - // apiVersionsForDiscovery after it put in the right ordered - PreferredVersion: apiVersionsForDiscovery[0], + resourceListerFunc := discovery.APIResourceListerFunc(func() []metav1.APIResource { + return apiResourcesForDiscovery + }) + + // HACK: if we are adding resources in legacy scheme group through CRDs (KCP scenario) + // then do not expose the CRD `APIResource`s in their own CRD-related group`, + // But instead add them in the existing legacy schema group + if legacyscheme.Scheme.IsGroupRegistered(version.Group) { + if !foundGroup || !foundVersion{ + delete(discovery.ContributedResources, version) + } + + discovery.ContributedResources[version] = resourceListerFunc } - c.groupHandler.setDiscovery(version.Group, discovery.NewAPIGroupHandler(Codecs, apiGroup)) - if !foundVersion { + if !foundGroup { + c.groupHandler.unsetDiscovery(version.Group) c.versionHandler.unsetDiscovery(version) return nil } - c.versionHandler.setDiscovery(version, discovery.NewAPIVersionHandler(Codecs, version, discovery.APIResourceListerFunc(func() []metav1.APIResource { - return apiResourcesForDiscovery - }))) + + if version.Group != "" { + // If we don't add resources in the core API group + apiGroup := metav1.APIGroup{ + Name: version.Group, + Versions: apiVersionsForDiscovery, + // the preferred versions for a group is the first item in + // apiVersionsForDiscovery after it put in the right ordered + PreferredVersion: apiVersionsForDiscovery[0], + } + c.groupHandler.setDiscovery(version.Group, discovery.NewAPIGroupHandler(Codecs, apiGroup)) + + if !foundVersion { + c.versionHandler.unsetDiscovery(version) + return nil + } + c.versionHandler.setDiscovery(version, discovery.NewAPIVersionHandler(Codecs, version, resourceListerFunc)) + } return nil } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index fcaa954c9425c..4d91f6c22b1bd 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -46,6 +46,7 @@ import ( "k8s.io/apiextensions-apiserver/pkg/crdserverscheme" "k8s.io/apiextensions-apiserver/pkg/registry/customresource" "k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor" + "k8s.io/kubernetes/pkg/api/legacyscheme" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -69,6 +70,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/metrics" + "k8s.io/apiserver/pkg/endpoints/openapi" apirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/generic" @@ -257,6 +259,10 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } crdName := requestInfo.Resource + "." + requestInfo.APIGroup + // HACK: support the case when we add core resources through CRDs (KCP scenario) + if requestInfo.APIGroup == "" { + crdName = crdName + "core" + } crd, err := r.crdLister.Get(crdName) if apierrors.IsNotFound(err) { if !r.hasSynced() { @@ -331,6 +337,9 @@ func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { string(types.JSONPatchType), string(types.MergePatchType), } + if legacyscheme.Scheme.IsGroupRegistered(requestInfo.APIGroup) { + supportedTypes = append(supportedTypes, string(types.StrategicMergePatchType)) + } if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) { supportedTypes = append(supportedTypes, string(types.ApplyPatchType)) } @@ -764,12 +773,16 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd table, ) + selfLinkPrefixPrefix := path.Join("apis", crd.Spec.Group, v.Name) + if crd.Spec.Group == "" { + selfLinkPrefixPrefix = path.Join("api", v.Name) + } selfLinkPrefix := "" switch crd.Spec.Scope { case apiextensionsv1.ClusterScoped: - selfLinkPrefix = "/" + path.Join("apis", crd.Spec.Group, v.Name) + "/" + crd.Status.AcceptedNames.Plural + "/" + selfLinkPrefix = "/" + selfLinkPrefixPrefix + "/" + crd.Status.AcceptedNames.Plural + "/" case apiextensionsv1.NamespaceScoped: - selfLinkPrefix = "/" + path.Join("apis", crd.Spec.Group, v.Name, "namespaces") + "/" + selfLinkPrefix = "/" + selfLinkPrefixPrefix + "/namespaces/" } clusterScoped := crd.Spec.Scope == apiextensionsv1.ClusterScoped @@ -791,6 +804,10 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd standardSerializers = append(standardSerializers, s) } + modelsByGKV, err := openapi.GetModelsByGKV(openAPIModels) + if err != nil { + klog.V(2).Infof("The CRD cannot gather openapi models by GKV: %v", err) + } requestScopes[v.Name] = &handlers.RequestScope{ Namer: handlers.ContextBasedNaming{ SelfLinker: meta.NewAccessor(), @@ -822,6 +839,8 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd Authorizer: r.authorizer, MaxRequestBodyBytes: r.maxRequestBodyBytes, + + OpenapiModels: modelsByGKV, } if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) { reqScope := *requestScopes[v.Name] @@ -1247,7 +1266,7 @@ func buildOpenAPIModelsForApply(staticOpenAPISpec *spec.Swagger, crd *apiextensi specs := []*spec.Swagger{} for _, v := range crd.Spec.Versions { - s, err := builder.BuildSwagger(crd, v.Name, builder.Options{V2: false, StripDefaults: true, StripValueValidation: true, StripNullable: true, AllowNonStructural: true}) + s, err := builder.BuildSwagger(crd, v.Name, builder.Options{V2: false, StripDefaults: true, StripValueValidation: true, StripNullable: true, AllowNonStructural: false}) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/goopenapi.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/goopenapi.go index 607489c718a51..7139282e2a335 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/goopenapi.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/goopenapi.go @@ -80,12 +80,22 @@ func (x *Extensions) toGoOpenAPI(ret *spec.Schema) { } if len(x.XListMapKeys) > 0 { ret.VendorExtensible.AddExtension("x-kubernetes-list-map-keys", x.XListMapKeys) + ret.VendorExtensible.AddExtension("x-kubernetes-patch-merge-key", x.XListMapKeys[0]) } if x.XListType != nil { ret.VendorExtensible.AddExtension("x-kubernetes-list-type", *x.XListType) + if *x.XListType == "map" || *x.XListType == "set" { + ret.VendorExtensible.AddExtension("x-kubernetes-patch-strategy", "merge") + } + if *x.XListType == "atomic" { + ret.VendorExtensible.AddExtension("x-kubernetes-patch-strategy", "replace") + } } if x.XMapType != nil { ret.VendorExtensible.AddExtension("x-kubernetes-map-type", *x.XMapType) + if *x.XMapType == "atomic" { + ret.VendorExtensible.AddExtension("x-kubernetes-patch-strategy", "replace") + } } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go index 06dfc7f1add22..e023846e3cae0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder/builder.go @@ -45,6 +45,7 @@ import ( openapibuilder "k8s.io/kube-openapi/pkg/builder" "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/util" + "k8s.io/kubernetes/pkg/api/legacyscheme" ) const ( @@ -104,6 +105,9 @@ func BuildSwagger(crd *apiextensionsv1.CustomResourceDefinition, version string, if opts.AllowNonStructural || len(structuralschema.ValidateStructural(nil, ss)) == 0 { schema = ss + // This adds ValueValidation fields (anyOf, allOf) which may be stripped below if opts.StripValueValidation is true + schema = schema.Unfold() + if opts.StripDefaults { schema = schema.StripDefaults() } @@ -113,8 +117,6 @@ func BuildSwagger(crd *apiextensionsv1.CustomResourceDefinition, version string, if opts.StripNullable { schema = schema.StripNullable() } - - schema = schema.Unfold() } } } @@ -142,11 +144,17 @@ func BuildSwagger(crd *apiextensionsv1.CustomResourceDefinition, version string, scale := &v1.Scale{} routes := make([]*restful.RouteBuilder, 0) - root := fmt.Sprintf("/apis/%s/%s/%s", b.group, b.version, b.plural) + // HACK: support the case when we add core resources through CRDs (KCP scenario) + rootPrefix := fmt.Sprintf("/apis/%s/%s", b.group, b.version) + if b.group == "" { + rootPrefix = fmt.Sprintf("/api/%s", b.version) + } + + root := fmt.Sprintf("%s/%s", rootPrefix, b.plural) if b.namespaced { routes = append(routes, b.buildRoute(root, "", "GET", "list", "list", sampleList).Operation("list"+b.kind+"ForAllNamespaces")) - root = fmt.Sprintf("/apis/%s/%s/namespaces/{namespace}/%s", b.group, b.version, b.plural) + root = fmt.Sprintf("%s/namespaces/{namespace}/%s", rootPrefix, b.plural) } routes = append(routes, b.buildRoute(root, "", "GET", "list", "list", sampleList)) routes = append(routes, b.buildRoute(root, "", "POST", "post", "create", sample).Reads(sample)) @@ -195,9 +203,21 @@ type CRDCanonicalTypeNamer struct { kind string } +// HACK: support the case when we add core or other legacy scheme resources through CRDs (KCP scenario) +func packagePrefix(group string) string { + if !strings.Contains(group, ".") && + legacyscheme.Scheme.IsGroupRegistered(group) { + if group == "" { + group = "core" + } + return "k8s.io/api/" + group + } + return group +} + // OpenAPICanonicalTypeName returns canonical type name for given CRD func (c *CRDCanonicalTypeNamer) OpenAPICanonicalTypeName() string { - return fmt.Sprintf("%s/%s.%s", c.group, c.version, c.kind) + return fmt.Sprintf("%s/%s.%s", packagePrefix(c.group), c.version, c.kind) } // builder contains validation schema and basic naming information for a CRD in @@ -452,7 +472,7 @@ func addTypeMetaProperties(s *spec.Schema) { // buildListSchema builds the list kind schema for the CRD func (b *builder) buildListSchema() *spec.Schema { - name := definitionPrefix + util.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", b.group, b.version, b.kind)) + name := definitionPrefix + util.ToRESTFriendlyName(fmt.Sprintf("%s/%s/%s", packagePrefix(b.group), b.version, b.kind)) doc := fmt.Sprintf("List of %s. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md", b.plural) s := new(spec.Schema).WithDescription(fmt.Sprintf("%s is a list of %s", b.listKind, b.kind)). WithRequired("items"). @@ -489,14 +509,19 @@ func (b *builder) getOpenAPIConfig() *common.Config { GetOperationIDAndTags: openapi.GetOperationIDAndTags, GetDefinitionName: func(name string) (string, spec.Extensions) { buildDefinitions.Do(buildDefinitionsFunc) + // HACK: support the case when we add core or other legacy scheme resources through CRDs (KCP scenario) + parts := strings.Split(name, "/") + if len(parts) == 2 { + name = packagePrefix(parts[0]) + } return namer.GetDefinitionName(name) }, GetDefinitions: func(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { def := generatedopenapi.GetOpenAPIDefinitions(ref) - def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.kind)] = common.OpenAPIDefinition{ + def[fmt.Sprintf("%s/%s.%s", packagePrefix(b.group), b.version, b.kind)] = common.OpenAPIDefinition{ Schema: *b.schema, } - def[fmt.Sprintf("%s/%s.%s", b.group, b.version, b.listKind)] = common.OpenAPIDefinition{ + def[fmt.Sprintf("%s/%s.%s", packagePrefix(b.group), b.version, b.listKind)] = common.OpenAPIDefinition{ Schema: *b.listSchema, } return def @@ -505,6 +530,8 @@ func (b *builder) getOpenAPIConfig() *common.Config { } func newBuilder(crd *apiextensionsv1.CustomResourceDefinition, version string, schema *structuralschema.Structural, v2 bool) *builder { + group := crd.Spec.Group + // HACK: support the case when we add core resources through CRDs (KCP scenario) b := &builder{ schema: &spec.Schema{ SchemaProps: spec.SchemaProps{Type: []string{"object"}}, @@ -512,7 +539,7 @@ func newBuilder(crd *apiextensionsv1.CustomResourceDefinition, version string, s listSchema: &spec.Schema{}, ws: &restful.WebService{}, - group: crd.Spec.Group, + group: group, version: version, kind: crd.Spec.Names.Kind, listKind: crd.Spec.Names.ListKind, diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go index d621f8474888c..822c39874b0af 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status/naming_controller.go @@ -368,6 +368,10 @@ func (c *NamingConditionController) deleteCustomResourceDefinition(obj interface func (c *NamingConditionController) requeueAllOtherGroupCRDs(name string) error { pluralGroup := strings.SplitN(name, ".", 2) + // In case the group is empty because we're adding core resources as CRDs in KCP + if len(pluralGroup) == 1 { + pluralGroup = append(pluralGroup, "") + } list, err := c.crdLister.List(labels.Everything()) if err != nil { return err diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/validator.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/validator.go index 6d89e1bbe24df..27458318e2a98 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/validator.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource/validator.go @@ -131,7 +131,12 @@ func (a customResourceValidator) ValidateTypeMeta(ctx context.Context, obj *unst if typeAccessor.GetKind() != a.kind.Kind { allErrs = append(allErrs, field.Invalid(field.NewPath("kind"), typeAccessor.GetKind(), fmt.Sprintf("must be %v", a.kind.Kind))) } - if typeAccessor.GetAPIVersion() != a.kind.Group+"/"+a.kind.Version { + // HACK: support the case when we add core resources through CRDs (KCP scenario) + expectedAPIVersion := a.kind.Group+"/"+a.kind.Version + if a.kind.Group == "" { + expectedAPIVersion = a.kind.Version + } + if typeAccessor.GetAPIVersion() != expectedAPIVersion { allErrs = append(allErrs, field.Invalid(field.NewPath("apiVersion"), typeAccessor.GetAPIVersion(), fmt.Sprintf("must be %v", a.kind.Group+"/"+a.kind.Version))) } return allErrs diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/version.go b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/version.go index 0976041bff0e0..544992ae45d4b 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/version.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/version.go @@ -18,6 +18,9 @@ package discovery import ( "net/http" + "regexp" + "sort" + "strings" restful "github.com/emicklei/go-restful" @@ -47,6 +50,49 @@ type APIVersionHandler struct { apiResourceLister APIResourceLister } +// HACK: support the case when we can add core or other legacy scheme resources through CRDs (KCP scenario) +var ContributedResources map[schema.GroupVersion]APIResourceLister = map[schema.GroupVersion]APIResourceLister{} + +func withContributedResources(groupVersion schema.GroupVersion, apiResourceLister APIResourceLister) APIResourceLister { + return APIResourceListerFunc(func() []metav1.APIResource { + result := apiResourceLister.ListAPIResources() + if additionalResources := ContributedResources[groupVersion]; additionalResources != nil { + result = append(result, additionalResources.ListAPIResources()...) + } + sort.Slice(result, func(i, j int) bool { + return result[i].Name < result[j].Name + }) + + return result + }) +} + +func IsAPIContributed(path string) bool { + for gv, resourceLister := range ContributedResources { + prefix := gv.Group + if prefix != "" { + prefix = "/apis/" + prefix + "/" + gv.Version + "/" + } else { + prefix = "/api/" + gv.Version + "/" + } + if !strings.HasPrefix(path, prefix) { + return false + } + + for _, resource := range resourceLister.ListAPIResources() { + if strings.HasPrefix(path, prefix+resource.Name) { + return true + } + if resource.Namespaced { + if matched, _ := regexp.MatchString(prefix+"namespaces/[^/][^/]*/"+resource.Name+"(/[^/].*)?", path); matched { + return true + } + } + } + } + return false +} + func NewAPIVersionHandler(serializer runtime.NegotiatedSerializer, groupVersion schema.GroupVersion, apiResourceLister APIResourceLister) *APIVersionHandler { if keepUnversioned(groupVersion.Group) { // Because in release 1.1, /apis/extensions returns response with empty @@ -58,7 +104,7 @@ func NewAPIVersionHandler(serializer runtime.NegotiatedSerializer, groupVersion return &APIVersionHandler{ serializer: serializer, groupVersion: groupVersion, - apiResourceLister: apiResourceLister, + apiResourceLister: withContributedResources(groupVersion, apiResourceLister), } } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go index c295d0aa65933..ae96403648b3f 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go @@ -48,6 +48,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/util/dryrun" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kube-openapi/pkg/util/proto" utiltrace "k8s.io/utils/trace" "sigs.k8s.io/yaml" ) @@ -383,6 +384,7 @@ type smpPatcher struct { // Schema schemaReferenceObj runtime.Object fieldManager *fieldmanager.FieldManager + openapiModel proto.Schema } func (p *smpPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (runtime.Object, error) { @@ -396,7 +398,7 @@ func (p *smpPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (ru if err != nil { return nil, err } - if err := strategicPatchObject(p.defaulter, currentVersionedObject, p.patchBytes, versionedObjToUpdate, p.schemaReferenceObj); err != nil { + if err := strategicPatchObject(p.defaulter, currentVersionedObject, p.patchBytes, versionedObjToUpdate, p.schemaReferenceObj, p.openapiModel); err != nil { return nil, err } // Convert the object back to the hub version @@ -459,6 +461,7 @@ func strategicPatchObject( patchBytes []byte, objToUpdate runtime.Object, schemaReferenceObj runtime.Object, + openapiModel proto.Schema, ) error { originalObjMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(originalObject) if err != nil { @@ -470,7 +473,7 @@ func strategicPatchObject( return errors.NewBadRequest(err.Error()) } - if err := applyPatchToObject(defaulter, originalObjMap, patchMap, objToUpdate, schemaReferenceObj); err != nil { + if err := applyPatchToObject(defaulter, originalObjMap, patchMap, objToUpdate, schemaReferenceObj, openapiModel); err != nil { return err } return nil @@ -556,10 +559,17 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti if err != nil { return nil, false, err } + + var schema proto.Schema + modelsByGKV := scope.OpenapiModels + if modelsByGKV != nil { + schema = modelsByGKV[p.kind] + } p.mechanism = &smpPatcher{ patcher: p, schemaReferenceObj: schemaReferenceObj, fieldManager: scope.FieldManager, + openapiModel: schema, } // this case is unreachable if ServerSideApply is not enabled because we will have already rejected the content type case types.ApplyPatchType: @@ -612,8 +622,12 @@ func applyPatchToObject( patchMap map[string]interface{}, objToUpdate runtime.Object, schemaReferenceObj runtime.Object, + openapiModel proto.Schema, ) error { patchedObjMap, err := strategicpatch.StrategicMergeMapPatch(originalMap, patchMap, schemaReferenceObj) + if err == mergepatch.ErrUnsupportedStrategicMergePatchFormat && openapiModel !=nil { + patchedObjMap, err = strategicpatch.StrategicMergeMapPatchUsingLookupPatchMeta(originalMap, patchMap, strategicpatch.NewPatchMetaFromOpenAPI(openapiModel)) + } if err != nil { return interpretStrategicMergePatchError(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go index 97471637e56c8..26399f626a846 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go @@ -42,6 +42,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/metrics" + "k8s.io/apiserver/pkg/endpoints/openapi" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/rest" @@ -83,6 +84,8 @@ type RequestScope struct { HubGroupVersion schema.GroupVersion MaxRequestBodyBytes int64 + + OpenapiModels openapi.ModelsByGKV } func (scope *RequestScope) err(err error, w http.ResponseWriter, req *http.Request) { diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/openapi/openapi.go b/staging/src/k8s.io/apiserver/pkg/endpoints/openapi/openapi.go index e3bd028bbf918..56d0d50519054 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/openapi/openapi.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/openapi/openapi.go @@ -18,6 +18,7 @@ package openapi import ( "bytes" + "errors" "fmt" "reflect" "sort" @@ -31,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kube-openapi/pkg/util" + "k8s.io/kube-openapi/pkg/util/proto" ) var verbs = util.NewTrie([]string{"get", "log", "read", "replace", "patch", "delete", "deletecollection", "watch", "connect", "proxy", "list", "create", "patch"}) @@ -189,3 +191,76 @@ func (d *DefinitionNamer) GetDefinitionName(name string) (string, spec.Extension } return friendlyName(name), nil } + +type ModelsByGKV map[schema.GroupVersionKind]proto.Schema + +// NewOpenAPIData creates a new `Resources` out of the openapi models +func GetModelsByGKV(models proto.Models) (ModelsByGKV, error) { + result := map[schema.GroupVersionKind]proto.Schema{} + for _, modelName := range models.ListModels() { + model := models.LookupModel(modelName) + if model == nil { + return map[schema.GroupVersionKind]proto.Schema{}, errors.New("ListModels returns a model that can't be looked-up.") + } + gvkList := parseGroupVersionKind(model) + for _, gvk := range gvkList { + if len(gvk.Kind) > 0 { + key := schema.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind} + if key.Group == "core" { + key.Group = "" + } + result[key] = model + } + } + } + + return result, nil +} + +// Get and parse GroupVersionKind from the extension. Returns empty if it doesn't have one. +func parseGroupVersionKind(s proto.Schema) []schema.GroupVersionKind { + extensions := s.GetExtensions() + + gvkListResult := []schema.GroupVersionKind{} + + // Get the extensions + gvkExtension, ok := extensions[extensionGVK] + if !ok { + return []schema.GroupVersionKind{} + } + + // gvk extension must be a list of at least 1 element. + gvkList, ok := gvkExtension.([]interface{}) + if !ok { + return []schema.GroupVersionKind{} + } + + for _, gvk := range gvkList { + // gvk extension list must be a map with group, version, and + // kind fields + gvkMap, ok := gvk.(map[interface{}]interface{}) + if !ok { + continue + } + group, ok := gvkMap["group"].(string) + if !ok { + continue + } + version, ok := gvkMap["version"].(string) + if !ok { + continue + } + kind, ok := gvkMap["kind"].(string) + if !ok { + continue + } + + gvkListResult = append(gvkListResult, schema.GroupVersionKind{ + Group: group, + Version: version, + Kind: kind, + }) + } + + return gvkListResult +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index a4ab24f8e0db9..b0c159590ad76 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -615,7 +615,7 @@ func (e *Store) Get(ctx context.Context, name string, options *metav1.GetOptions if err != nil { return nil, err } - klog.Infof("DEBUG: GET key func returned: %s", key) +// klog.Infof("DEBUG: GET key func returned: %s", key) if err := e.Storage.Get(ctx, key, options.ResourceVersion, obj, false); err != nil { return nil, storeerr.InterpretGetError(err, e.qualifiedResourceFromContext(ctx), name) }