From cba3b8af6f890ae00be8d3dff842975b6c68c1fd Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Tue, 27 Apr 2021 09:38:14 +0200 Subject: [PATCH 1/4] Use a beefier crc instance --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 9085d20f..29a42a48 100755 --- a/ci.sh +++ b/ci.sh @@ -47,7 +47,7 @@ sudo mv out/linux-amd64/crc /usr/local/bin/ popd crc setup -crc start -p "${HOME}"/pull-secret -b crc_libvirt_*.crcbundle +crc start --disk-size 80 -m 24000 -c 10 -p "${HOME}"/pull-secret -b crc_libvirt_*.crcbundle mkdir -p /tmp/artifacts export KUBECONFIG="${HOME}"/.crc/machines/crc/kubeconfig From f1f5a1290aa5ecb67a6baf05d015ced0083bb0d3 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Fri, 23 Apr 2021 13:28:22 +0200 Subject: [PATCH 2/4] Run OpenShift conformance tests Rather than kubernetes/conformance, this attempts to run openshift/conformance. This disables all the instrumentation, network, node, scheduling and storage tests as I did not have time to look in details at the several dozens of failures they had. The goal is to eventually empty the ignore list and fix/ignore the ignored tests. --- ci.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/ci.sh b/ci.sh index 29a42a48..58ef0195 100755 --- a/ci.sh +++ b/ci.sh @@ -5,14 +5,26 @@ set -exuo pipefail sudo yum install -y podman make golang rsync cat > /tmp/ignoretests.txt << EOF -[sig-apps] Daemon set [Serial] should rollback without unnecessary restarts [Conformance] [Suite:openshift/conformance/serial/minimal] [Suite:k8s] -[sig-cli] Kubectl client Kubectl cluster-info should check if Kubernetes control plane services is included in cluster-info [Conformance] [Suite:openshift/conformance/parallel/minimal] [Suite:k8s] -[sig-scheduling] SchedulerPreemption [Serial] validates basic preemption works [Conformance] [Suite:openshift/conformance/serial/minimal] [Suite:k8s] -[sig-scheduling] SchedulerPreemption [Serial] validates lower priority pod preemption by critical pod [Conformance] [Suite:openshift/conformance/serial/minimal] [Suite:k8s] -[k8s.io] [sig-node] NoExecuteTaintManager Multiple Pods [Serial] evicts pods with minTolerationSeconds [Disruptive] [Conformance] [Suite:k8s] -[k8s.io] [sig-node] NoExecuteTaintManager Single Pod [Serial] removing taint cancels eviction [Disruptive] [Conformance] [Suite:k8s] +^"\[sig-arch\] Managed cluster should ensure control plane pods do not run in best-effort QoS \[Suite:openshift/conformance/parallel\]" +^"\[Serial\] \[sig-auth\]\[Feature:OAuthServer\] \[RequestHeaders\] \[IdP\] test RequestHeaders IdP \[Suite:openshift/conformance/serial\]" +^"\[sig-auth\]\[Feature:SCC\]\[Early\] should not have pod creation failures during install \[Suite:openshift/conformance/parallel\]" +^"\[sig-auth\]\[Feature:OpenShiftAuthorization\]\[Serial\] authorization TestAuthorizationResourceAccessReview should succeed \[Suite:openshift/conformance/serial\]" +^"\[sig-cli\] oc adm must-gather runs successfully for audit logs \[Suite:openshift/conformance/parallel\]" +^"\[sig-cli\] oc adm must-gather runs successfully \[Suite:openshift/conformance/parallel\]" +^"\[sig-cli\] oc observe works as expected \[Suite:openshift/conformance/parallel\]" +^"\[sig-cluster-lifecycle\]\[Feature:Machines\]\[Serial\] Managed cluster should grow and decrease when scaling different machineSets simultaneously \[Suite:openshift/conformance/serial\]" +^"\[sig-imageregistry\]\[Feature:Image\] oc tag should change image reference for internal images \[Suite:openshift/conformance/parallel\]" +^"\[sig-arch\] \[Conformance\] FIPS TestFIPS \[Suite:openshift/conformance/parallel/minimal\]" +^"\[sig-builds\]\[Feature:Builds\] Multi-stage image builds should succeed \[Suite:openshift/conformance/parallel\]" +^"\[sig-apps\] Daemon set \[Serial\] should rollback without unnecessary restarts \[Conformance\] \[Suite:openshift/conformance/serial/minimal\] \[Suite:k8s\]" +^"\[sig-instrumentation\] +^"\[sig-network\] +^"\[sig-node\] +^"\[sig-scheduling\] +^"\[sig-storage\] EOF + ./shellcheck.sh ./snc.sh @@ -51,7 +63,7 @@ crc start --disk-size 80 -m 24000 -c 10 -p "${HOME}"/pull-secret -b crc_libvirt_ mkdir -p /tmp/artifacts export KUBECONFIG="${HOME}"/.crc/machines/crc/kubeconfig -openshift-tests run kubernetes/conformance --dry-run | grep -F -v -f /tmp/ignoretests.txt | openshift-tests run -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f - +openshift-tests run openshift/conformance --dry-run | grep -v -f /tmp/ignoretests.txt | openshift-tests run --timeout 15m -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f - rc=$? echo "${rc}" > /tmp/test-return set -e From 118630c204edfb1676a791421718185e3358f812 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Tue, 8 Jun 2021 15:40:34 +0200 Subject: [PATCH 3/4] fixup! Run OpenShift conformance tests --- ci.sh | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/ci.sh b/ci.sh index 58ef0195..bb4d4cf6 100755 --- a/ci.sh +++ b/ci.sh @@ -5,25 +5,42 @@ set -exuo pipefail sudo yum install -y podman make golang rsync cat > /tmp/ignoretests.txt << EOF -^"\[sig-arch\] Managed cluster should ensure control plane pods do not run in best-effort QoS \[Suite:openshift/conformance/parallel\]" -^"\[Serial\] \[sig-auth\]\[Feature:OAuthServer\] \[RequestHeaders\] \[IdP\] test RequestHeaders IdP \[Suite:openshift/conformance/serial\]" -^"\[sig-auth\]\[Feature:SCC\]\[Early\] should not have pod creation failures during install \[Suite:openshift/conformance/parallel\]" +^"\[sig-builds\]\[Feature:Builds\] imagechangetriggers imagechangetriggers should trigger builds of all types \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-apps\] Daemon set \[Serial\] should rollback without unnecessary restarts \[Conformance\] \[Suite:openshift/conformance/serial/minimal\] \[Suite:k8s\]" +^"\[sig-arch\] Managed cluster should set requests but not limits \[Suite:openshift/conformance/parallel\]" ^"\[sig-auth\]\[Feature:OpenShiftAuthorization\]\[Serial\] authorization TestAuthorizationResourceAccessReview should succeed \[Suite:openshift/conformance/serial\]" -^"\[sig-cli\] oc adm must-gather runs successfully for audit logs \[Suite:openshift/conformance/parallel\]" +^"\[Serial\] \[sig-auth\]\[Feature:OAuthServer\] \[RequestHeaders\] \[IdP\] test RequestHeaders IdP \[Suite:openshift/conformance/serial\]" ^"\[sig-cli\] oc adm must-gather runs successfully \[Suite:openshift/conformance/parallel\]" +^"\[sig-cli\] oc adm must-gather runs successfully for audit logs \[Suite:openshift/conformance/parallel\]" ^"\[sig-cli\] oc observe works as expected \[Suite:openshift/conformance/parallel\]" +^"\[sig-autoscaling\] \[Feature:HPA\] Horizontal pod autoscaling (scale resource: CPU) ReplicationController light Should scale from 1 pod to 2 pods \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-etcd\] etcd leader changes are not excessive \[Late\] \[Suite:openshift/conformance/parallel\]" ^"\[sig-cluster-lifecycle\]\[Feature:Machines\]\[Serial\] Managed cluster should grow and decrease when scaling different machineSets simultaneously \[Suite:openshift/conformance/serial\]" -^"\[sig-imageregistry\]\[Feature:Image\] oc tag should change image reference for internal images \[Suite:openshift/conformance/parallel\]" -^"\[sig-arch\] \[Conformance\] FIPS TestFIPS \[Suite:openshift/conformance/parallel/minimal\]" -^"\[sig-builds\]\[Feature:Builds\] Multi-stage image builds should succeed \[Suite:openshift/conformance/parallel\]" -^"\[sig-apps\] Daemon set \[Serial\] should rollback without unnecessary restarts \[Conformance\] \[Suite:openshift/conformance/serial/minimal\] \[Suite:k8s\]" +^"\[sig-node\] Managed cluster should report ready nodes the entire duration of the test run \[Late\] \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-imageregistry\]\[Serial\]\[Suite:openshift/registry/serial\] Image signature workflow can push a signed image to openshift registry and verify it \[Suite:openshift/conformance/serial\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should enable openshift-monitoring to pull metrics \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should expose a health check on the metrics port \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should expose prometheus metrics for a route \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should expose the profiling endpoints \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should respond with 503 to unrecognized hosts \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should serve routes that were created from an ingress \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[Feature:Router\] The HAProxy router should support reencrypt to services backed by a serving certificate automatically \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-network\]\[endpoints\] admission TestEndpointAdmission \[Suite:openshift/conformance/parallel\]" +^"\[sig-storage\] CSI Volumes \[Driver: csi-hostpath\] \[Testpattern: Dynamic PV (block volmode)\] provisioning should provision storage with snapshot data source \[Feature:VolumeSnapshotDataSource\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI Volumes \[Driver: csi-hostpath\] \[Testpattern: Dynamic PV (default fs)\] provisioning should provision storage with snapshot data source \[Feature:VolumeSnapshotDataSource\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI Volumes \[Driver: csi-hostpath\] \[Testpattern: Dynamic Snapshot (delete policy)\] snapshottable\[Feature:VolumeSnapshotDataSource\] volume snapshot controller should check snapshot fields, check restore correctly works after modifying source data, check deletion \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI Volumes \[Driver: csi-hostpath\] \[Testpattern: Dynamic Snapshot (retain policy)\] snapshottable\[Feature:VolumeSnapshotDataSource\] volume snapshot controller should check snapshot fields, check restore correctly works after modifying source data, check deletion \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI Volumes \[Driver: csi-hostpath\] \[Testpattern: Pre-provisioned Snapshot (delete policy)\] snapshottable\[Feature:VolumeSnapshotDataSource\] volume snapshot controller should check snapshot fields, check restore correctly works after modifying source data, check deletion \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI Volumes \[Driver: csi-hostpath\] \[Testpattern: Pre-provisioned Snapshot (retain policy)\] snapshottable\[Feature:VolumeSnapshotDataSource\] volume snapshot controller should check snapshot fields, check restore correctly works after modifying source data, check deletion \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI mock volume CSI Volume Snapshots \[Feature:VolumeSnapshotDataSource\] volumesnapshotcontent and pvc in Bound state with deletion timestamp set should not get deleted while snapshot finalizer exists \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\] CSI mock volume CSI Volume Snapshots secrets \[Feature:VolumeSnapshotDataSource\] volume snapshot create/delete with secrets \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-storage\]\[Late\] Metrics should report short attach times \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" +^"\[sig-storage\]\[Late\] Metrics should report short mount times \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" ^"\[sig-instrumentation\] -^"\[sig-network\] -^"\[sig-node\] -^"\[sig-scheduling\] -^"\[sig-storage\] EOF +#^"\[sig-storage\] + ./shellcheck.sh ./snc.sh From a88d4b8cce77e85d52fa05526adfe2f6f38f2bd4 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Fri, 9 Jul 2021 16:28:01 +0200 Subject: [PATCH 4/4] rinse and repeat --- ci.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index bb4d4cf6..ca057b08 100755 --- a/ci.sh +++ b/ci.sh @@ -36,7 +36,24 @@ cat > /tmp/ignoretests.txt << EOF ^"\[sig-storage\] CSI mock volume CSI Volume Snapshots secrets \[Feature:VolumeSnapshotDataSource\] volume snapshot create/delete with secrets \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" ^"\[sig-storage\]\[Late\] Metrics should report short attach times \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" ^"\[sig-storage\]\[Late\] Metrics should report short mount times \[Skipped:Disconnected\] \[Suite:openshift/conformance/parallel\]" -^"\[sig-instrumentation\] +^"\[sig-instrumentation\]" +^"\[Serial\]\[sig-node\]\[Feature:TopologyManager\] Configured cluster with gu workload saturating NUMA nodes should reject pod requesting more cores than a single NUMA node have \[Suite:openshift/conformance/serial\]" +^"\[Serial\]\[sig-node\]\[Feature:TopologyManager\] Configured cluster with gu workload should guarantee NUMA-aligned cpu cores in gu pods with multiple pods, each with multiple containers requesting 1 core, only one requesting 1 device \[Suite:openshift/conformance/serial\]" +^"\[sig-auth\]\[Feature:LDAP\]\[Serial\] ldap group sync can sync groups from ldap \[Suite:openshift/conformance/serial\]" +^"\[sig-auth\]\[Feature:UserAPI\] users can manipulate groups \[Suite:openshift/conformance/parallel\]" +^"\[sig-imageregistry\]\[Feature:ImageTriggers\]\[Serial\] ImageStream API TestImageStreamMappingCreate \[Suite:openshift/conformance/serial\]" +^"\[sig-network\] IngressClass \[Feature:Ingress\] should set default value on new IngressClass \[Serial\] \[Suite:openshift/conformance/serial\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should allow egress access on one named port \[Feature:NetworkPolicy\] \[Skipped:Network/OVNKubernetes\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should allow egress access to server in CIDR block \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should allow ingress access from namespace on one named port \[Feature:NetworkPolicy\] \[Skipped:Network/OVNKubernetes\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should allow ingress access on one named port \[Feature:NetworkPolicy\] \[Skipped:Network/OVNKubernetes\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should enforce egress policy allowing traffic to a server in a different namespace based on PodSelector and NamespaceSelector \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should enforce except clause while egress access to server in CIDR block \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should enforce multiple egress policies with egress allow-all policy taking precedence \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should enforce policy to allow traffic from pods within server namespace based on PodSelector \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should ensure an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should support a 'default-deny-all' policy \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" +^"\[sig-network\] NetworkPolicy \[LinuxOnly\] NetworkPolicy between server and client should work with Ingress,Egress specified together \[Feature:NetworkPolicy\] \[Skipped:Network/OpenShiftSDN/Multitenant\] \[Skipped:Network/OpenShiftSDN\] \[Suite:openshift/conformance/parallel\] \[Suite:k8s\]" EOF #^"\[sig-storage\]