Skip to content

Commit 917e64c

Browse files
andypitcherandypitcher
authored
Merge pull request #63 from andypitcher/e2e-improvements
E2E for Rancher Monitoring and Logging
2 parents 7ffc5b2 + 764efdf commit 917e64c

4 files changed

Lines changed: 123 additions & 10 deletions

File tree

‎hack/e2e/centos8.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,4 @@ provision:
3030
script: |
3131
#!/bin/sh
3232
33-
yum in -y git container-selinux
33+
yum in -y git jq container-selinux setools

‎hack/e2e/centos9.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@ provision:
2424
script: |
2525
#!/bin/sh
2626
27-
yum in -y git container-selinux
27+
yum in -y git jq container-selinux setools

‎hack/e2e/setup-vm.sh‎

Lines changed: 103 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ function enforceSELinux(){
66
echo "> Check SELinux status"
77
# Short circuit if SELinux is not being enforced.
88
getenforce | grep -q Enforcing
9-
9+
# Remove dontaudits from policy for debugging
10+
sudo semodule -DB
11+
# Install rancher-selinux policy
1012
sudo dnf install -y /tmp/rancher-selinux.rpm
1113
}
1214

@@ -20,9 +22,14 @@ function installDependencies(){
2022

2123
local KUBECTL_VERSION
2224
KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
23-
24-
echo "> Installing kubectl ${KUBECTL_VERSION}"
25-
curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
25+
ARCH=$(uname -p)
26+
[[ "${ARCH}" == "aarch64" ]] && ARCH="arm64"
27+
[[ "${ARCH}" == "x86_64" ]] && ARCH="amd64"
28+
29+
echo "> Installing kubectl ${KUBECTL_VERSION} for ${ARCH}"
30+
curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl"
31+
curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl.sha256"
32+
echo "$(<kubectl.sha256) kubectl" | sha256sum -c -
2633
install -o root -g root -m 0755 kubectl /usr/bin/kubectl
2734
kubectl version --client=true
2835
}
@@ -93,8 +100,94 @@ function installRancherMonitoring(){
93100
kubectl wait --for=condition=ready -n cattle-monitoring-system pod -l app.kubernetes.io/name=prometheus-node-exporter --timeout=60s
94101
}
95102

96-
function E2E(){
97-
echo "<!-- Execute some RM op here -->"
103+
function installRancherLogging(){
104+
helm repo add rancher-charts https://charts.rancher.io/
105+
106+
helm upgrade --install=true \
107+
--labels=catalog.cattle.io/cluster-repo-name=rancher-charts \
108+
--namespace=cattle-logging-system --timeout=10m0s --wait=true \
109+
--create-namespace \
110+
rancher-logging-crd rancher-charts/rancher-logging-crd
111+
112+
# Install the chart with selinux enabled to true
113+
helm upgrade --install=true \
114+
--labels=catalog.cattle.io/cluster-repo-name=rancher-charts \
115+
--namespace=cattle-logging-system --timeout=10m0s --wait=true \
116+
--create-namespace \
117+
rancher-logging rancher-charts/rancher-logging \
118+
--set global.seLinux.enabled=true
119+
120+
# Ensure fluentbit daemonset is created
121+
kubectl wait --for=create -n cattle-logging-system daemonset/rancher-logging-root-fluentbit --timeout=60s
122+
# Wait for fluentbit pod to be ready
123+
kubectl wait --for=condition=ready -n cattle-logging-system pod -l app.kubernetes.io/name=fluentbit --timeout=60s
124+
}
125+
126+
function e2eRancherMonitoring(){
127+
CHART_CONTAINER_EXPECTED_SLTYPE="prom_node_exporter_t"
128+
CHART_CONTAINER_RUNNING_SLTYPE=""
129+
CHART_CONTAINER="node-exporter"
130+
CHART_POD_NAMESPACE="cattle-monitoring-system"
131+
CHART_POD=$(kubectl get pods -n ${CHART_POD_NAMESPACE} -o custom-columns=NAME:.metadata.name | grep ${CHART_CONTAINER})
132+
133+
echo "> Verify the presence of ${CHART_CONTAINER_EXPECTED_SLTYPE}"
134+
if [[ "$(seinfo -t ${CHART_CONTAINER_EXPECTED_SLTYPE} | grep -o ${CHART_CONTAINER_EXPECTED_SLTYPE})" == "${CHART_CONTAINER_EXPECTED_SLTYPE}" ]]; then
135+
echo "SELinux type is present: ${CHART_CONTAINER_EXPECTED_SLTYPE}"
136+
else
137+
echo "SELinux type is not present: ${CHART_CONTAINER_EXPECTED_SLTYPE}"
138+
exit 1
139+
fi
140+
141+
echo "> Verify expected SELinux context type ${CHART_CONTAINER_EXPECTED_SLTYPE} for container ${CHART_CONTAINER}"
142+
CHART_CONTAINER_RUNNING_SLTYPE=$(kubectl get pod ${CHART_POD} -n ${CHART_POD_NAMESPACE} -o json | jq -r '.spec.securityContext.seLinuxOptions.type')
143+
if [[ "${CHART_CONTAINER_RUNNING_SLTYPE}" == "${CHART_CONTAINER_EXPECTED_SLTYPE}" ]]; then
144+
echo "SELinux type is correct: ${CHART_CONTAINER_RUNNING_SLTYPE}"
145+
else
146+
echo "SELinux type is incorrect or not set: ${CHART_CONTAINER_RUNNING_SLTYPE}"
147+
exit 1
148+
fi
149+
150+
echo ">Look for any AVCs related to ${CHART_CONTAINER_RUNNING_SLTYPE}"
151+
if ausearch -m AVC,USER_AVC | grep ${CHART_CONTAINER_RUNNING_SLTYPE} > /dev/null; then
152+
echo "AVCs found for ${CHART_CONTAINER_RUNNING_SLTYPE}"
153+
ausearch -m AVC,USER_AVC | grep ${CHART_CONTAINER_RUNNING_SLTYPE}
154+
exit 1
155+
else
156+
echo "No AVCs found for ${CHART_CONTAINER_RUNNING_SLTYPE}"
157+
fi
158+
}
159+
160+
function e2eRancherLogging(){
161+
CHART_CONTAINER_EXPECTED_SLTYPE="rke_logreader_t"
162+
CHART_CONTAINER_RUNNING_SLTYPE=""
163+
CHART_CONTAINER="fluentbit"
164+
CHART_POD_NAMESPACE="cattle-logging-system"
165+
CHART_POD=$(kubectl get pods -n ${CHART_POD_NAMESPACE} -o custom-columns=NAME:.metadata.name | grep "${CHART_CONTAINER}")
166+
167+
echo "> Verify the presence of ${CHART_CONTAINER_EXPECTED_SLTYPE}"
168+
if [[ "$(seinfo -t ${CHART_CONTAINER_EXPECTED_SLTYPE} | grep -o ${CHART_CONTAINER_EXPECTED_SLTYPE})" == "${CHART_CONTAINER_EXPECTED_SLTYPE}" ]]; then
169+
echo "SELinux type is present: ${CHART_CONTAINER_EXPECTED_SLTYPE}"
170+
else
171+
echo "SELinux type is not present: ${CHART_CONTAINER_EXPECTED_SLTYPE}"
172+
fi
173+
174+
echo "> Verify expected SELinux context type ${CHART_CONTAINER_EXPECTED_SLTYPE} for container ${CHART_CONTAINER}"
175+
CHART_CONTAINER_RUNNING_SLTYPE=$(kubectl get pod ${CHART_POD} -n ${CHART_POD_NAMESPACE} -o json | jq -r '.spec.containers[0].securityContext.seLinuxOptions.type')
176+
if [[ "${CHART_CONTAINER_RUNNING_SLTYPE}" == "${CHART_CONTAINER_EXPECTED_SLTYPE}" ]]; then
177+
echo "SELinux type is correct: ${CHART_CONTAINER_RUNNING_SLTYPE}"
178+
else
179+
echo "SELinux type is incorrect or not set: ${CHART_CONTAINER_RUNNING_SLTYPE}"
180+
exit 1
181+
fi
182+
183+
echo ">Look for any AVCs related to ${CHART_CONTAINER_RUNNING_SLTYPE}"
184+
if ausearch -m AVC,USER_AVC | grep ${CHART_CONTAINER_RUNNING_SLTYPE} > /dev/null; then
185+
echo "AVCs found for ${CHART_CONTAINER_RUNNING_SLTYPE}"
186+
ausearch -m AVC,USER_AVC | grep ${CHART_CONTAINER_RUNNING_SLTYPE}
187+
exit 1
188+
else
189+
echo "No AVCs found for ${CHART_CONTAINER_RUNNING_SLTYPE}"
190+
fi
98191
}
99192

100193
function main(){
@@ -103,12 +196,14 @@ function main(){
103196
installRKE2
104197
installRancher
105198
installRancherMonitoring
106-
107-
E2E
199+
installRancherLogging
200+
e2eRancherMonitoring
201+
e2eRancherLogging
108202
}
109203

110204
# This is needed as Rocky does not include it in the PATH,
111205
# which is required for the Helm install.
112206
export PATH=$PATH:/usr/local/bin
113207

114208
main
209+

‎policy/centos8/rancher.te‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ allow rke_logreader_t syslogd_var_run_t:dir read;
4747
allow rke_logreader_t syslogd_var_run_t:file { getattr map open read };
4848
allow rke_logreader_t var_log_t:dir read;
4949
allow rke_logreader_t var_log_t:file { getattr map open read watch };
50+
allow rke_logreader_t self:tcp_socket listen;
5051

5152
########################
5253
# type rke_container_t #
@@ -103,3 +104,20 @@ manage_files_pattern(rke_network_t, var_run_t, var_run_t)
103104
allow rke_network_t kernel_t:system module_request;
104105
allow rke_network_t kernel_t:unix_dgram_socket sendto;
105106
allow rke_network_t self:netlink_route_socket nlmsg_write;
107+
108+
############################################################################
109+
# type prom_node_exporter_t #
110+
# target: prometheus-node-exporter container for Rancher monitoring chart #
111+
############################################################################
112+
gen_require(`
113+
type container_runtime_t;
114+
type security_t;
115+
class tcp_socket listen;
116+
')
117+
container_domain_template(prom_node_exporter, container)
118+
virt_sandbox_domain(prom_node_exporter_t)
119+
corenet_tcp_bind_generic_node(prom_node_exporter_t)
120+
corenet_tcp_bind_generic_port(prom_node_exporter_t)
121+
init_read_state(prom_node_exporter_t)
122+
allow prom_node_exporter_t self:tcp_socket listen;
123+
allow prom_node_exporter_t security_t:file { getattr open read };

0 commit comments

Comments
 (0)