Skip to content

Commit 8cef71d

Browse files
bfourniebfournie
committed
Add version-specific operator validation for ISO_NO_REGISTRY mode
This commit adds a new validation function to verify that expected operators are installed after cluster deployment in ISO_NO_REGISTRY mode. Key features: - Version-specific operator lists for OCP 4.20 and 4.21 - Automatic version detection using openshift_version function - Clear error reporting for missing operators - 4.20 validates 7 operators - 4.21 validates 12 operators The validation runs after the API server is available and helps ensure that the virtualization bundle and other operators were properly installed during cluster creation.
1 parent 5901c75 commit 8cef71d

3 files changed

Lines changed: 221 additions & 44 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ agent_remove_extraworker_nodes:
4444
./agent/agent_remove_all_extraworker_nodes.sh
4545

4646
agent_post_install_validation:
47-
./agent/agent_post_install_validation.sh
47+
./agent/08_agent_post_install_validation.sh
4848

4949
redeploy: ocp_cleanup ironic_cleanup build_installer ironic install_config ocp_run bell
5050

‎agent/08_agent_post_install_validation.sh‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
#!/usr/bin/env bash
2+
set -euxo pipefail
3+
4+
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
5+
6+
LOGDIR=${SCRIPTDIR}/logs
7+
source $SCRIPTDIR/logging.sh
8+
source $SCRIPTDIR/common.sh
9+
source $SCRIPTDIR/release_info.sh
10+
11+
function validate_installed_manifests() {
12+
echo "Validating installed operator manifests..."
13+
14+
# Determine OCP version - manifest validation only supported in 4.21+
15+
ocp_version=$(openshift_version ${OCP_DIR})
16+
echo "Detected OpenShift version: ${ocp_version}"
17+
18+
major_version=$(echo "$ocp_version" | cut -d. -f1)
19+
minor_version=$(echo "$ocp_version" | cut -d. -f2)
20+
21+
# Skip validation for versions older than 4.21
22+
if [ "$major_version" -lt 4 ] || [ "$major_version" -eq 4 -a "$minor_version" -le 20 ]; then
23+
echo "Manifest validation not supported for version ${ocp_version}, skipping."
24+
return 0
25+
fi
26+
27+
# Suppress trace output for data processing
28+
set +x
29+
30+
# Get the ConfigMap as JSON
31+
if ! configmap_json=$(oc get configmap/olm-operator-manifests -n assisted-installer -o json 2>&1); then
32+
set -x
33+
echo "ERROR: Could not find ConfigMap olm-operator-manifests in assisted-installer namespace"
34+
return 1
35+
fi
36+
37+
# Get all keys that end with .metadata.yaml
38+
metadata_keys=$(echo "$configmap_json" | jq -r '.data | keys[] | select(endswith(".metadata.yaml"))')
39+
40+
if [ -z "$metadata_keys" ]; then
41+
set -x
42+
echo "ERROR: No metadata files found in ConfigMap"
43+
return 1
44+
fi
45+
46+
missing_resources=()
47+
48+
for metadata_key in $metadata_keys; do
49+
# Get the metadata content (plain YAML)
50+
metadata_yaml=$(echo "$configmap_json" | jq -r ".data[\"$metadata_key\"]")
51+
52+
# Extract manifest filenames (lines starting with "- " in YAML list)
53+
manifest_files=$(echo "$metadata_yaml" | grep "^- " | sed 's/^- //')
54+
55+
for manifest_file in $manifest_files; do
56+
# Get the base64-encoded manifest content and decode it
57+
manifest_yaml=$(echo "$configmap_json" | jq -r ".data[\"$manifest_file\"]" | base64 -d)
58+
59+
# Extract kind and name from the manifest YAML
60+
kind=$(echo "$manifest_yaml" | sed -n 's/^kind: *//p' | head -1 | tr -d '\r')
61+
name=$(echo "$manifest_yaml" | sed -n 's/^ name: *//p' | head -1 | tr -d '\r')
62+
63+
if [ -z "$kind" ] || [ -z "$name" ]; then
64+
set -x
65+
echo " WARNING: Could not extract kind or name from $manifest_file"
66+
set +x
67+
continue
68+
fi
69+
70+
# Try to get resources of this kind
71+
get_output=$(oc get "$kind" -A --no-headers 2>&1)
72+
get_exit_code=$?
73+
74+
if [ $get_exit_code -ne 0 ]; then
75+
# Check if it's because the resource type doesn't exist
76+
if echo "$get_output" | grep -q "error: the server doesn't have a resource type"; then
77+
missing_resources+=("$kind/$name (resource type '$kind' not available)")
78+
else
79+
# Some other error
80+
missing_resources+=("$kind/$name (error: $get_output)")
81+
fi
82+
else
83+
# Resource type exists, check if our specific resource is in the list
84+
# Check both column 1 (cluster-scoped) and column 2 (namespaced with -A)
85+
if ! echo "$get_output" | awk '{print $1, $2}' | grep -qw "$name"; then
86+
missing_resources+=("$kind/$name")
87+
fi
88+
fi
89+
done
90+
done
91+
92+
# Re-enable trace for final output
93+
set -x
94+
95+
if [ ${#missing_resources[@]} -gt 0 ]; then
96+
echo "ERROR: The following expected resources are not applied:"
97+
for missing_res in "${missing_resources[@]}"; do
98+
echo " - $missing_res"
99+
done
100+
return 1
101+
else
102+
echo "SUCCESS: All expected operator manifests are applied."
103+
fi
104+
}
105+
106+
function validate_installed_operators() {
107+
echo "Validating installed operators..."
108+
109+
# Determine OCP version
110+
ocp_version=$(openshift_version ${OCP_DIR})
111+
112+
# Build path to appliance-config.yaml based on version
113+
APPLIANCE_CONFIG="${SCRIPTDIR}/${OCP_DIR}/iso_builder/src/config/${ocp_version}/appliance-config.yaml"
114+
115+
if [ ! -f "${APPLIANCE_CONFIG}" ]; then
116+
echo "ERROR: Could not find appliance-config.yaml at: ${APPLIANCE_CONFIG}"
117+
return 1
118+
fi
119+
120+
echo "Reading operators from: ${APPLIANCE_CONFIG}"
121+
122+
# Suppress trace output for data processing
123+
set +x
124+
125+
# Extract operator names from appliance-config.yaml
126+
# Look for lines matching " - name: <operator-name>" under the packages section
127+
# These are indented with 8 spaces (under operators.packages array)
128+
expected_operators=()
129+
while IFS= read -r line; do
130+
expected_operators+=("$line")
131+
done < <(grep -A 999 "^operators:" "${APPLIANCE_CONFIG}" | \
132+
grep "^ - name:" | \
133+
sed 's/^ - name: *//' | \
134+
sort)
135+
136+
if [ ${#expected_operators[@]} -eq 0 ]; then
137+
set -x
138+
echo "ERROR: No operators found in ${APPLIANCE_CONFIG}"
139+
return 1
140+
fi
141+
142+
set -x
143+
echo "Expected operators (${#expected_operators[@]} total):"
144+
set +x
145+
146+
for op in "${expected_operators[@]}"; do
147+
echo " - $op"
148+
done
149+
150+
# Get list of installed operators (just the names, first column)
151+
installed_operators=$(oc get operators -o custom-columns=NAME:.metadata.name --no-headers)
152+
153+
missing_operators=()
154+
for expected_op in "${expected_operators[@]}"; do
155+
if ! echo "$installed_operators" | grep -q "^${expected_op}\."; then
156+
missing_operators+=("$expected_op")
157+
fi
158+
done
159+
160+
# Re-enable trace for final output
161+
set -x
162+
163+
if [ ${#missing_operators[@]} -gt 0 ]; then
164+
echo "ERROR: The following expected operators are not installed:"
165+
for missing_op in "${missing_operators[@]}"; do
166+
echo " - $missing_op"
167+
done
168+
echo ""
169+
echo "Installed operators:"
170+
oc get operators
171+
return 1
172+
else
173+
echo "SUCCESS: All expected operators are installed."
174+
oc get operators
175+
fi
176+
}
177+
178+
if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
179+
MAX_ATTEMPTS=120
180+
SLEEP_SECONDS=30 # Wait time between connection attempts
181+
ATTEMPT_COUNT=0
182+
183+
echo "Starting oc wait retry loop for API connection up to 60 minutes..."
184+
set +x # Disable debug tracing for cleaner output
185+
while ! (oc wait clusterversion version --for=condition=Available=True --timeout=2s 2>/dev/null && \
186+
oc wait clusterversion version --for=condition=Progressing=False --timeout=2s 2>/dev/null); do
187+
ATTEMPT_COUNT=$((ATTEMPT_COUNT + 1))
188+
if [ $ATTEMPT_COUNT -ge $MAX_ATTEMPTS ]; then
189+
echo ""
190+
set -x # Re-enable debug tracing
191+
echo "ERROR: API server connection failed after $MAX_ATTEMPTS attempts over 60 minutes."
192+
exit 1
193+
fi
194+
195+
echo -n "."
196+
sleep $SLEEP_SECONDS
197+
done
198+
echo ""
199+
set -x # Re-enable debug tracing
200+
echo "SUCCESS: API server connection established and ClusterVersion is available."
201+
# Run subsequent commands after successful cluster setup
202+
oc get packagemanifests -n openshift-marketplace
203+
204+
# Validate expected operators are installed
205+
validate_installed_operators || exit 1
206+
207+
# Validate operator manifests are applied
208+
validate_installed_manifests || exit 1
209+
fi
210+
211+
installed_control_plane_nodes=$(oc get nodes --selector=node-role.kubernetes.io/master | grep -v AGE | wc -l)
212+
213+
oc get nodes
214+
215+
if (( $NUM_MASTERS != $installed_control_plane_nodes )); then
216+
echo "Post install validation failed. Expected $NUM_MASTERS control plane nodes but found $installed_control_plane_nodes."
217+
exit 1
218+
fi
219+
220+
oc get clusterversion

‎agent/agent_post_install_validation.sh‎

Lines changed: 0 additions & 43 deletions
This file was deleted.

0 commit comments

Comments
 (0)