From a70f1cb08ab1e352a829c79d34fd7c864f9fe3f3 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 30 Jul 2025 08:37:32 -0400 Subject: [PATCH 01/51] Adds RHEL9 Dockerfile and related files --- docker/rhel9/Dockerfile | 63 +++++++++++ docker/rhel9/README-ANSIBLE.md | 49 +++++++++ docker/rhel9/README.md | 163 ++++++++++++++++++++++++++++ docker/rhel9/check-lme-setup.ps1 | 45 ++++++++ docker/rhel9/check-lme-setup.sh | 47 ++++++++ docker/rhel9/docker-compose.yml | 38 +++++++ docker/rhel9/environment_example.sh | 1 + docker/rhel9/lme-init.sh | 34 ++++++ install.sh | 17 ++- 9 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 docker/rhel9/Dockerfile create mode 100644 docker/rhel9/README-ANSIBLE.md create mode 100644 docker/rhel9/README.md create mode 100644 docker/rhel9/check-lme-setup.ps1 create mode 100755 docker/rhel9/check-lme-setup.sh create mode 100644 docker/rhel9/docker-compose.yml create mode 100644 docker/rhel9/environment_example.sh create mode 100755 docker/rhel9/lme-init.sh diff --git a/docker/rhel9/Dockerfile b/docker/rhel9/Dockerfile new file mode 100644 index 00000000..b8c1d554 --- /dev/null +++ b/docker/rhel9/Dockerfile @@ -0,0 +1,63 @@ +# Base stage with common dependencies +FROM registry.access.redhat.com/ubi9/ubi:9.6 AS base + +ARG USER_ID=1002 +ARG GROUP_ID=1002 + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +RUN dnf update -y && dnf install -y \ + glibc-langpack-en sudo openssh-clients \ + && dnf clean all \ + && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ + && while getent passwd $USER_ID > /dev/null 2>&1; do USER_ID=$((USER_ID + 1)); done \ + && groupadd -g $GROUP_ID lme-user \ + && useradd -m -u $USER_ID -g lme-user lme-user \ + && usermod -aG wheel lme-user \ + && echo "lme-user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ + && echo "Defaults:lme-user !requiretty" >> /etc/sudoers \ + && echo "#%PAM-1.0" > /etc/pam.d/sudo \ + && echo "auth include system-auth" >> /etc/pam.d/sudo \ + && echo "account sufficient pam_permit.so" >> /etc/pam.d/sudo \ + && echo "password include system-auth" >> /etc/pam.d/sudo \ + && echo "session include system-auth" >> /etc/pam.d/sudo + +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 + +ENV BASE_DIR=/home/lme-user +WORKDIR $BASE_DIR + +# Lme stage with full dependencies +FROM base AS lme + +RUN dnf install -y \ + systemd systemd-sysv \ + && dnf clean all + +# Install EPEL repository +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ + && /usr/bin/crb enable \ + && dnf clean all + +RUN cd /lib/systemd/system/sysinit.target.wants/ && \ + ls | grep -v systemd-tmpfiles-setup | xargs rm -f $1 && \ + rm -f /lib/systemd/system/multi-user.target.wants/* && \ + rm -f /etc/systemd/system/*.wants/* && \ + rm -f /lib/systemd/system/local-fs.target.wants/* && \ + rm -f /lib/systemd/system/sockets.target.wants/*udev* && \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl* && \ + rm -f /lib/systemd/system/basic.target.wants/* && \ + rm -f /lib/systemd/system/anaconda.target.wants/* && \ + mkdir -p /etc/systemd/system/systemd-logind.service.d && \ + echo -e "[Service]\nProtectHostname=no" > /etc/systemd/system/systemd-logind.service.d/override.conf + +#COPY docker/rhel9/lme-setup.service /etc/systemd/system/ +# +#RUN chmod 644 /etc/systemd/system/lme-setup.service +# +## Enable the service +#RUN systemctl enable lme-setup.service + +CMD ["/lib/systemd/systemd"] \ No newline at end of file diff --git a/docker/rhel9/README-ANSIBLE.md b/docker/rhel9/README-ANSIBLE.md new file mode 100644 index 00000000..bafa8a95 --- /dev/null +++ b/docker/rhel9/README-ANSIBLE.md @@ -0,0 +1,49 @@ +# Ansible Installation on RHEL9/UBI9 Containers + +## Problem + +The EPEL `ansible` package for RHEL9/UBI9 requires `python3.9dist(ansible-core) >= 2.14.7`, but the `ansible-core` RPM is **not available** in any public repository for UBI9 or non-subscribed RHEL9. This results in a broken dependency and prevents installation via `dnf`: + +``` +dnf install ansible +# Error: nothing provides python3.9dist(ansible-core) >= 2.14.7 needed by ansible-1:7.7.0-1.el9.noarch from epel +``` + +## Why does this happen? +- Red Hat only provides the `ansible-core` RPM in the main RHEL subscription repositories, **not** in UBI or EPEL. +- UBI images are designed to be redistributable and do not include all RHEL content. +- EPEL expects the RHEL-provided `ansible-core` RPM, but it is not present in UBI or EPEL. + +## Solutions + +### 1. Use pip (Recommended for UBI9/Non-subscribed RHEL9) +Install Ansible using pip: + +``` +dnf install -y python3-pip +pip3 install ansible +``` + +This is the only way to get a working Ansible install on UBI9/RHEL9 containers without a RHEL subscription. This is also the method recommended by the [official Ansible documentation](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#pip-install). + +### 2. If you have a RHEL subscription +Register the container and enable the official RHEL repositories: + +``` +subscription-manager register +subscription-manager attach --auto +subscription-manager repos --enable ansible-2.9-for-rhel-9-x86_64-rpms +``` + +Then you can install Ansible via dnf: + +``` +dnf install ansible-core +``` + +### 3. Use a different base image +If you require a pure package-manager install, consider using Rocky Linux, CentOS Stream, or Fedora as your base image, where all dependencies are available via the package manager. + +## Summary +- **UBI9 and non-subscribed RHEL9 cannot install Ansible via dnf due to missing dependencies.** +- **Use pip to install Ansible, or use a different base image if you require dnf installation.** \ No newline at end of file diff --git a/docker/rhel9/README.md b/docker/rhel9/README.md new file mode 100644 index 00000000..30d7b232 --- /dev/null +++ b/docker/rhel9/README.md @@ -0,0 +1,163 @@ +# LME RHEL9 Container + +This directory contains the Docker configuration for running LME (Logging Made Easy) on RHEL9/UBI9. + +## Prerequisites + +### System Requirements +- **Docker**: Docker Engine 20.10+ or Docker Desktop +- **Docker Compose**: Version 2.0+ +- **Host System**: Linux, macOS, or Windows with Docker support +- **Memory**: Minimum 4GB RAM (8GB+ recommended) +- **Storage**: At least 10GB free space +- **Network**: Internet access for package downloads + +### Host System Prerequisites +- **cgroup v2 support**: Required for systemd in containers +- **SYS_ADMIN capability**: Required for privileged container operations +- **Port availability**: Ensure ports 5601, 443, 8220, and 9200 are not in use + +### Optional: RHEL Subscription (for package manager installation) +If you have a Red Hat Enterprise Linux subscription and want to use package manager installation instead of pip: + +1. **Register the container** with your RHEL subscription: + ```bash + docker exec -it lme subscription-manager register --username --password + ``` + +2. **Attach to a subscription**: + ```bash + docker exec -it lme subscription-manager attach --auto + ``` + +3. **Enable Ansible repositories**: + ```bash + docker exec -it lme subscription-manager repos --enable ansible-2.9-for-rhel-9-x86_64-rpms + ``` + +4. **Install Ansible via package manager**: + ```bash + docker exec -it lme dnf install -y ansible-core + ``` + +**Note**: Without a RHEL subscription, the install script will automatically fall back to pip installation. + +## Quick Start + +1. **Build and start the container**: + ```bash + docker compose up -d --build + ``` + +2. **Run the LME installation**: + ```bash + docker exec -it lme bash -c "cd /root/LME && sudo ./install.sh" + ``` + +3. **Access the services**: + - Kibana: http://localhost:5601 + - Elasticsearch: http://localhost:9200 + - Fleet Server: http://localhost:8220 + - HTTPS: https://localhost:443 + +## Container Features + +### Pre-installed Components +- **Base System**: RHEL9/UBI9 with systemd support +- **User Management**: `lme-user` with sudo privileges +- **Package Management**: EPEL repository enabled +- **System Services**: systemd with proper configuration +- **Network Tools**: openssh-clients for remote access + +### Security Features +- **Sudo Configuration**: Passwordless sudo for lme-user +- **PAM Configuration**: Custom sudo PAM setup for container environment +- **Privileged Mode**: Required for systemd and cgroup access +- **Security Options**: seccomp unconfined for compatibility + +### Volume Mounts +- **LME Source**: `/root/LME` - Mounts the LME source code +- **cgroup**: `/sys/fs/cgroup/systemd` - Required for systemd +- **Temporary Filesystems**: `/tmp`, `/run`, `/run/lock` + +## Environment Variables + +### Required +- `HOST_IP`: IP address for the container (set in environment.sh) + +### Optional +- `HOST_UID`: User ID for lme-user (default: 1001) +- `HOST_GID`: Group ID for lme-user (default: 1001) + +## Troubleshooting + +### Common Issues + +#### Ansible Installation Problems +- **Problem**: EPEL Ansible package has missing dependencies +- **Solution**: The install script automatically falls back to pip installation +- **Details**: See [README-ANSIBLE.md](README-ANSIBLE.md) for more information + +#### Systemd Issues +- **Problem**: Container fails to start with systemd +- **Solution**: Ensure cgroup v2 is enabled on the host +- **Check**: `docker exec -it lme systemctl status` + +#### Port Conflicts +- **Problem**: Port already in use error +- **Solution**: Change ports in docker-compose.yml or stop conflicting services +- **Alternative**: Use different port mappings + +#### Permission Issues +- **Problem**: Permission denied errors +- **Solution**: Ensure the container is running with proper privileges +- **Check**: `docker inspect lme | grep -i privileged` + +### Debugging Commands + +```bash +# Check container status +docker ps + +# View container logs +docker logs lme + +# Access container shell +docker exec -it lme bash + +# Check systemd status +docker exec -it lme systemctl status + +# Verify Ansible installation +docker exec -it lme ansible --version + +# Check available repositories +docker exec -it lme dnf repolist +``` + +## Development + +### Building from Source +```bash +# Build the container +docker compose build + +# Build with specific arguments +docker compose build --build-arg USER_ID=1000 --build-arg GROUP_ID=1000 +``` + +### Customizing the Container +- Modify `Dockerfile` to add additional packages +- Update `docker-compose.yml` for different port mappings +- Edit `environment.sh` to set custom environment variables + +## Support + +For issues related to: +- **Ansible installation**: See [README-ANSIBLE.md](README-ANSIBLE.md) +- **Container setup**: Check the troubleshooting section above +- **LME installation**: Refer to the main LME documentation + +## License + +This container configuration is part of the LME project. See the main LICENSE file for details. \ No newline at end of file diff --git a/docker/rhel9/check-lme-setup.ps1 b/docker/rhel9/check-lme-setup.ps1 new file mode 100644 index 00000000..d522368e --- /dev/null +++ b/docker/rhel9/check-lme-setup.ps1 @@ -0,0 +1,45 @@ +# Default timeout in minutes (30 minutes) +$timeoutMinutes = 30 +$startTime = Get-Date + +# Function to check if timeout has been reached +function Test-Timeout { + $currentTime = Get-Date + $elapsedTime = ($currentTime - $startTime).TotalMinutes + if ($elapsedTime -gt $timeoutMinutes) { + Write-Host "ERROR: Setup timed out after $timeoutMinutes minutes" + exit 1 + } +} + +Write-Host "Starting LME setup check..." + +# Main loop +while ($true) { + # Check if the timeout has been reached + Test-Timeout + + # Get the logs and check for completion + $logs = docker compose exec lme journalctl -u lme-setup -o cat --no-hostname + + # Check for successful completion + if ($logs -match "First-time initialization complete") { + Write-Host "SUCCESS: LME setup completed successfully" + exit 0 + } + + # Check for failure indicators + if ($logs -match "failed=1") { + Write-Host "ERROR: Ansible playbook reported failures" + exit 1 + } + + # Track progress through the playbooks + $recapCount = ($logs | Select-String "PLAY RECAP" -AllMatches).Matches.Count + if ($recapCount -gt 0) { + Write-Host "INFO: Detected $recapCount of 2 playbook completions..." + } + + # Wait before next check (60 seconds) + Start-Sleep -Seconds 60 +} \ No newline at end of file diff --git a/docker/rhel9/check-lme-setup.sh b/docker/rhel9/check-lme-setup.sh new file mode 100755 index 00000000..8807f664 --- /dev/null +++ b/docker/rhel9/check-lme-setup.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Default timeout in seconds (30 minutes) +TIMEOUT=1800 +START_TIME=$(date +%s) + +# Function to check if timeout has been reached +check_timeout() { + current_time=$(date +%s) + elapsed_time=$((current_time - START_TIME)) + if [ $elapsed_time -gt $TIMEOUT ]; then + echo "ERROR: Setup timed out after ${TIMEOUT} seconds" + exit 1 + fi +} + +echo "Starting LME setup check..." + +# Main loop +while true; do + # Check if the timeout has been reached + check_timeout + + # Get the logs and check for completion + logs=$(docker compose exec lme journalctl -u lme-setup -o cat --no-hostname) + + # Check for successful completion + if echo "$logs" | grep -q "First-time initialization complete"; then + echo "SUCCESS: LME setup completed successfully" + exit 0 + fi + + # Check for failure indicators + if echo "$logs" | grep -q "failed=1"; then + echo "ERROR: Ansible playbook reported failures" + exit 1 + fi + + # Track progress through the playbooks + recap_count=$(echo "$logs" | grep -c "PLAY RECAP") + if [ "$recap_count" -gt 0 ]; then + echo "INFO: Detected ${recap_count} of 2 playbook completions..." + fi + + # Wait before next check (60 seconds) + sleep 60 +done \ No newline at end of file diff --git a/docker/rhel9/docker-compose.yml b/docker/rhel9/docker-compose.yml new file mode 100644 index 00000000..20bbb710 --- /dev/null +++ b/docker/rhel9/docker-compose.yml @@ -0,0 +1,38 @@ +services: + lme: + build: + context: ../../ + dockerfile: docker/rhel9/Dockerfile + target: lme + args: + USER_ID: "${HOST_UID:-1001}" + GROUP_ID: "${HOST_GID:-1001}" + container_name: lme + working_dir: /root + volumes: + - ../../../LME:/root/LME + #- /sys/fs/cgroup:/sys/fs/cgroup:rslave + - /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd:rw + cap_add: + - SYS_ADMIN + security_opt: + - seccomp:unconfined + privileged: true + user: root + tmpfs: + - /tmp + - /run + - /run/lock + environment: + - PODMAN_IGNORE_CGROUPSV1_WARNING=1 + - LANG=en_US.UTF-8 + - LANGUAGE=en_US:en + - LC_ALL=en_US.UTF-8 + - container=docker + - HOST_IP=${HOST_IP} + command: ["/lib/systemd/systemd", "--system"] + ports: + - "5601:5601" + - "443:443" + - "8220:8220" + - "9200:9200" \ No newline at end of file diff --git a/docker/rhel9/environment_example.sh b/docker/rhel9/environment_example.sh new file mode 100644 index 00000000..8c3ddc1a --- /dev/null +++ b/docker/rhel9/environment_example.sh @@ -0,0 +1 @@ +#export HOST_IP=192.168.1.194 \ No newline at end of file diff --git a/docker/rhel9/lme-init.sh b/docker/rhel9/lme-init.sh new file mode 100755 index 00000000..0cc0940e --- /dev/null +++ b/docker/rhel9/lme-init.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +INIT_FLAG="/opt/.lme_initialized" + +if [ ! -f "$INIT_FLAG" ]; then + echo "Running first-time LME initialization..." + rm -rf /opt/lme/lme-environment.env + + # Copy environment file if it doesn't exist + + . /root/LME/docker/rhel9/environment.sh + + # Update IPVAR in the environment file with the passed HOST_IP + if [ ! -z "$HOST_IP" ]; then + echo "Using HOST_IP: $HOST_IP" + export IPVAR=$HOST_IP + else + echo "Warning: HOST_IP not set, using default IPVAR value" + fi + + cd /root/LME/ + export NON_INTERACTIVE=true + export AUTO_CREATE_ENV=true + export AUTO_IP=${IPVAR:-127.0.0.1} + ./install.sh --debug + + # Create flag file to indicate initialization is complete + touch "$INIT_FLAG" + echo "First-time initialization complete." +else + echo "LME already initialized, skipping first-time setup." + systemctl disable lme-setup.service + systemctl daemon-reload +fi \ No newline at end of file diff --git a/install.sh b/install.sh index ab21f22f..8c652164 100755 --- a/install.sh +++ b/install.sh @@ -66,6 +66,9 @@ done # Function to check if ansible is installed check_ansible() { + # Add /usr/local/bin to PATH for pip-installed packages + export PATH="/usr/local/bin:$PATH" + if command -v ansible &> /dev/null; then echo -e "${GREEN}✓ Ansible is already installed!${NC}" ansible --version | head -n 1 @@ -189,7 +192,19 @@ install_ansible() { ;; centos|rhel|rocky|almalinux) sudo dnf install -y epel-release - sudo dnf install -y ansible + # Try to install ansible via dnf first + if sudo dnf install -y ansible; then + echo -e "${GREEN}✓ Ansible installed via dnf${NC}" + else + echo -e "${YELLOW}⚠ dnf installation failed, trying pip installation...${NC}" + sudo dnf install -y python3-pip + sudo pip3 install ansible + # Create symlink to make pip-installed ansible available in PATH + if [ -f /usr/local/bin/ansible ] && [ ! -f /usr/bin/ansible ]; then + sudo ln -sf /usr/local/bin/ansible /usr/bin/ansible + echo -e "${GREEN}✓ Created symlink for ansible in /usr/bin${NC}" + fi + fi ;; arch|manjaro) sudo pacman -Sy --noconfirm ansible From 17999cef974fe8fcb143ecc035a76f5d2f35af94 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 1 Aug 2025 07:54:19 -0400 Subject: [PATCH 02/51] Adds RHEL9 to the ansible roles --- .github/workflows/cluster.yml | 9 ++ ansible/roles/base/defaults/main.yml | 25 +++- ansible/roles/base/tasks/main.yml | 10 +- ansible/roles/base/tasks/redhat.yml | 76 ++++++++++ ansible/roles/base/tasks/redhat_9.yml | 30 ++++ ansible/roles/base/tasks/setup_passwords.yml | 2 + ansible/roles/base/vars/default.yml | 2 + ansible/roles/base/vars/redhat.yml | 2 + ansible/roles/base/vars/redhat_9.yml | 2 + ansible/roles/base/vars/redhat_9_6.yml | 4 + ansible/roles/elasticsearch/tasks/main.yml | 8 + ansible/roles/nix/tasks/redhat.yml | 83 +++++++++++ ansible/roles/nix/vars/default.yml | 2 + ansible/roles/nix/vars/redhat.yml | 3 + ansible/roles/podman/tasks/redhat.yml | 15 ++ ansible/roles/podman/tasks/setup_secrets.yml | 12 +- ansible/roles/podman/vars/default.yml | 2 + ansible/roles/podman/vars/redhat.yml | 5 + docker/rhel9/README-CA-CERTIFICATES.md | 138 ++++++++++++++++++ docker/rhel9/README.md | 1 + install.sh | 3 +- testing/v2/development/Dockerfile | 99 +++++++++++++ testing/v2/installers/README.md | 37 ++++- .../azure/build_azure_linux_network.py | 20 +++ 24 files changed, 578 insertions(+), 12 deletions(-) create mode 100644 ansible/roles/base/tasks/redhat.yml create mode 100644 ansible/roles/base/tasks/redhat_9.yml create mode 100644 ansible/roles/base/vars/default.yml create mode 100644 ansible/roles/base/vars/redhat.yml create mode 100644 ansible/roles/base/vars/redhat_9.yml create mode 100644 ansible/roles/base/vars/redhat_9_6.yml create mode 100644 ansible/roles/nix/tasks/redhat.yml create mode 100644 ansible/roles/nix/vars/default.yml create mode 100644 ansible/roles/nix/vars/redhat.yml create mode 100644 ansible/roles/podman/tasks/redhat.yml create mode 100644 ansible/roles/podman/vars/default.yml create mode 100644 ansible/roles/podman/vars/redhat.yml create mode 100644 docker/rhel9/README-CA-CERTIFICATES.md diff --git a/.github/workflows/cluster.yml b/.github/workflows/cluster.yml index 088a3d75..292f6390 100644 --- a/.github/workflows/cluster.yml +++ b/.github/workflows/cluster.yml @@ -26,6 +26,14 @@ on: - ukwest - northeurope - westeurope + operating_system: + description: 'Operating system for Azure VM' + required: true + default: 'ubuntu' + type: choice + options: + - ubuntu + - rhel jobs: build-and-test-cluster: @@ -116,6 +124,7 @@ jobs: -vs Standard_E16d_v4 \ -l ${{ inputs.azure_region || 'centralus' }} \ -ast 23:00 \ + ${{ inputs.operating_system == 'rhel' && '--use-rhel' || '' }} \ -y " #-s ${{ env.IP_ADDRESS }}/32 \ diff --git a/ansible/roles/base/defaults/main.yml b/ansible/roles/base/defaults/main.yml index 1694f839..8e115c9f 100644 --- a/ansible/roles/base/defaults/main.yml +++ b/ansible/roles/base/defaults/main.yml @@ -4,22 +4,22 @@ # Default packages that apply to all distributions if not overridden common_packages: - - curl - wget - gnupg2 - sudo - git - - openssh-client - expect # Debian-specific packages debian_packages: + - curl - apt-transport-https - ca-certificates - gnupg - lsb-release - software-properties-common + - openssh-client - fuse-overlayfs - build-essential - python3-pip @@ -28,11 +28,13 @@ debian_packages: # Ubuntu-specific packages ubuntu_packages: + - curl - apt-transport-https - ca-certificates - gnupg - lsb-release - software-properties-common + - openssh-client - fuse-overlayfs - build-essential - python3-pip @@ -48,4 +50,21 @@ ubuntu_24_04_packages: ubuntu_22_04_packages: - python3.10 - python3.10-venv - - python3.10-dev \ No newline at end of file + - python3.10-dev + +# Red Hat-specific packages +redhat_packages: + - ca-certificates + - openssh-clients + - dnf-plugins-core + - fuse-overlayfs + - python3-pip + - python3-pexpect + - glibc-langpack-en + - xz + +# Red Hat 9 specific packages +redhat_9_packages: + - python3.11 + - python3.11-pip + - python3.11-devel \ No newline at end of file diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml index e804fe29..7e427ec5 100644 --- a/ansible/roles/base/tasks/main.yml +++ b/ansible/roles/base/tasks/main.yml @@ -14,10 +14,14 @@ include_tasks: "{{ ansible_distribution | lower }}.yml" when: ansible_distribution is defined -# Include version-specific tasks +# Include version-specific tasks with fallback - name: Include version-specific tasks - include_tasks: "{{ ansible_distribution | lower }}_{{ ansible_distribution_version | replace('.', '_') }}.yml" - when: ansible_distribution is defined and ansible_distribution_version is defined + include_tasks: "{{ item }}" + with_first_found: + - "{{ ansible_distribution | lower }}_{{ ansible_distribution_version | replace('.', '_') }}.yml" + - "{{ ansible_distribution | lower }}_{{ ansible_distribution_major_version }}.yml" + - "common.yml" + when: ansible_distribution is defined # Include common setup tasks that apply to all distributions - name: Include common directory setup tasks diff --git a/ansible/roles/base/tasks/redhat.yml b/ansible/roles/base/tasks/redhat.yml new file mode 100644 index 00000000..e33962e4 --- /dev/null +++ b/ansible/roles/base/tasks/redhat.yml @@ -0,0 +1,76 @@ +--- +# Red Hat-specific tasks for base role + +- name: Update dnf cache + dnf: + update_cache: yes + become: yes + register: dnf_update + retries: 60 + delay: 10 + until: dnf_update is success + ignore_errors: "{{ ansible_check_mode }}" + +- name: Check if curl-minimal is installed + command: rpm -q curl-minimal + register: curl_minimal_check + failed_when: false + changed_when: false + +- name: Debug - curl-minimal status + debug: + msg: "curl-minimal is {{ 'installed' if curl_minimal_check.rc == 0 else 'not installed' }}. Using curl-minimal instead of full curl to avoid conflicts." + +- name: Debug - Show common packages to be installed + debug: + msg: "Installing common packages: {{ common_packages | join(', ') }}" + +- name: Install common packages + dnf: + name: "{{ common_packages }}" + state: present + become: yes + register: dnf_install + retries: 60 + delay: 10 + until: dnf_install is success + ignore_errors: "{{ ansible_check_mode }}" + +- name: Debug - Show common packages install result + debug: + var: dnf_install + when: debug_mode | default(false) + +- name: Debug - Show Red Hat packages to be installed + debug: + msg: "Installing Red Hat packages: {{ redhat_packages | join(', ') }}" + +- name: Install required Red Hat packages + dnf: + name: "{{ redhat_packages }}" + state: present + become: yes + register: dnf_install_redhat + retries: 60 + delay: 10 + until: dnf_install_redhat is success + ignore_errors: "{{ ansible_check_mode }}" + +- name: Debug - Show Red Hat packages install result + debug: + var: dnf_install_redhat + when: debug_mode | default(false) + +- name: Create CA certificates symlink for compatibility (Red Hat systems) + file: + src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + dest: /etc/ssl/certs/ca-certificates.crt + state: link + force: yes + become: yes + when: ansible_os_family == 'RedHat' + +- name: Set timezone + timezone: + name: "{{ timezone_area | default('Etc') }}/{{ timezone_zone | default('UTC') }}" + become: yes \ No newline at end of file diff --git a/ansible/roles/base/tasks/redhat_9.yml b/ansible/roles/base/tasks/redhat_9.yml new file mode 100644 index 00000000..c7d75fe4 --- /dev/null +++ b/ansible/roles/base/tasks/redhat_9.yml @@ -0,0 +1,30 @@ +--- +# Red Hat 9-specific tasks for base role + +- name: Install EPEL repository + dnf: + name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + state: present + become: yes + register: epel_install + retries: 3 + delay: 5 + until: epel_install is success + ignore_errors: "{{ ansible_check_mode }}" + +- name: Enable PowerTools/CRB repository + command: dnf config-manager --set-enabled crb + become: yes + changed_when: true + ignore_errors: true + +- name: Install Red Hat 9-specific packages + dnf: + name: "{{ redhat_9_packages }}" + state: present + become: yes + register: dnf_install_rh9 + retries: 60 + delay: 10 + until: dnf_install_rh9 is success + ignore_errors: "{{ ansible_check_mode }}" \ No newline at end of file diff --git a/ansible/roles/base/tasks/setup_passwords.yml b/ansible/roles/base/tasks/setup_passwords.yml index ea9cfb62..84b54fa4 100644 --- a/ansible/roles/base/tasks/setup_passwords.yml +++ b/ansible/roles/base/tasks/setup_passwords.yml @@ -67,6 +67,7 @@ path: /root/.profile line: "export ANSIBLE_VAULT_PASSWORD_FILE=\"{{ password_file }}\"" state: present + create: yes become: yes - name: Ensure ANSIBLE_VAULT_PASSWORD_FILE is set in .bashrc @@ -74,6 +75,7 @@ path: /root/.bashrc line: "export ANSIBLE_VAULT_PASSWORD_FILE=\"{{ password_file }}\"" state: present + create: yes become: yes - name: Setup Podman secrets configuration diff --git a/ansible/roles/base/vars/default.yml b/ansible/roles/base/vars/default.yml new file mode 100644 index 00000000..2ef57ca4 --- /dev/null +++ b/ansible/roles/base/vars/default.yml @@ -0,0 +1,2 @@ +--- +# Default variables when no distribution-specific vars are found \ No newline at end of file diff --git a/ansible/roles/base/vars/redhat.yml b/ansible/roles/base/vars/redhat.yml new file mode 100644 index 00000000..6b82b355 --- /dev/null +++ b/ansible/roles/base/vars/redhat.yml @@ -0,0 +1,2 @@ +--- +# Red Hat-specific variables \ No newline at end of file diff --git a/ansible/roles/base/vars/redhat_9.yml b/ansible/roles/base/vars/redhat_9.yml new file mode 100644 index 00000000..0fb72f7b --- /dev/null +++ b/ansible/roles/base/vars/redhat_9.yml @@ -0,0 +1,2 @@ +--- +# Red Hat 9-specific variables \ No newline at end of file diff --git a/ansible/roles/base/vars/redhat_9_6.yml b/ansible/roles/base/vars/redhat_9_6.yml new file mode 100644 index 00000000..706744e9 --- /dev/null +++ b/ansible/roles/base/vars/redhat_9_6.yml @@ -0,0 +1,4 @@ +--- +# Red Hat Enterprise Linux 9.6-specific variables +# This file ensures that OS-specific variable loading works properly +# Package definitions are inherited from defaults/main.yml \ No newline at end of file diff --git a/ansible/roles/elasticsearch/tasks/main.yml b/ansible/roles/elasticsearch/tasks/main.yml index e291a481..d2899a04 100644 --- a/ansible/roles/elasticsearch/tasks/main.yml +++ b/ansible/roles/elasticsearch/tasks/main.yml @@ -1,12 +1,20 @@ --- # Elasticsearch setup tasks +- name: Debug - Show what global_secrets contains + debug: + msg: "global_secrets variable: {{ global_secrets | default('UNDEFINED') }}" + - name: Set playbook variables ansible.builtin.set_fact: local_es_url: "{{ env_dict.LOCAL_ES_URL | default('') }}" elastic_username: "{{ env_dict.ELASTIC_USERNAME | default('') }}" elastic_password: "{{ global_secrets.elastic | default('') }}" +- name: Debug - Show extracted values + debug: + msg: "Elasticsearch config - URL: {{ local_es_url }}, Username: {{ elastic_username }}, Password length: {{ elastic_password | length }}" + # Create Read-Only User - name: Wait for Elasticsearch to be ready uri: diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml new file mode 100644 index 00000000..5b8f5a6c --- /dev/null +++ b/ansible/roles/nix/tasks/redhat.yml @@ -0,0 +1,83 @@ +--- +# Red Hat-specific Nix setup + +- name: Check if Nix is already installed + stat: + path: /nix/var/nix/profiles/default/bin/nix + register: nix_installed + +- name: Create nix installation directory + file: + path: /opt/nix-install + state: directory + mode: '0755' + become: yes + when: not nix_installed.stat.exists + +- name: Create temporary directory for Nix install + file: + path: /opt/nix-install/tmp + state: directory + mode: '0755' + become: yes + when: not nix_installed.stat.exists + +- name: Download Nix installer + get_url: + url: https://nixos.org/nix/install + dest: /opt/nix-install/install-nix.sh + mode: '0755' + become: yes + when: not nix_installed.stat.exists + +- name: Install Nix package manager + shell: sh /opt/nix-install/install-nix.sh --daemon --yes + become: yes + environment: + TMPDIR: /opt/nix-install/tmp + when: not nix_installed.stat.exists + register: nix_install_result + +- name: Debug Nix installation result + debug: + var: nix_install_result + when: debug_mode | default(false) and not nix_installed.stat.exists + +- name: Ensure nix-daemon service is started + systemd: + name: nix-daemon + state: started + enabled: yes + become: yes + when: not nix_installed.stat.exists + +- name: Add Nix channel + command: nix-channel --add https://nixos.org/channels/nixpkgs-unstable nixpkgs + become: yes + environment: + PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" + +- name: Update Nix channel + command: nix-channel --update + become: yes + environment: + PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" + +- name: Check if nix-users group exists + group: + name: nix-users + state: present + become: yes + +- name: Add user to nix-users group + user: + name: "{{ install_user }}" + groups: nix-users + append: yes + become: yes + +- name: Install required packages + command: nix-env -iA nixpkgs.podman nixpkgs.docker-compose + become: yes + environment: + PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" diff --git a/ansible/roles/nix/vars/default.yml b/ansible/roles/nix/vars/default.yml new file mode 100644 index 00000000..455510ab --- /dev/null +++ b/ansible/roles/nix/vars/default.yml @@ -0,0 +1,2 @@ +--- +# Default variables for nix when no distribution-specific vars are found \ No newline at end of file diff --git a/ansible/roles/nix/vars/redhat.yml b/ansible/roles/nix/vars/redhat.yml new file mode 100644 index 00000000..230ee022 --- /dev/null +++ b/ansible/roles/nix/vars/redhat.yml @@ -0,0 +1,3 @@ +--- +# Red Hat-specific variables for nix +install_user: "{{ ansible_user_id }}" \ No newline at end of file diff --git a/ansible/roles/podman/tasks/redhat.yml b/ansible/roles/podman/tasks/redhat.yml new file mode 100644 index 00000000..ec8277b8 --- /dev/null +++ b/ansible/roles/podman/tasks/redhat.yml @@ -0,0 +1,15 @@ +--- +# Red Hat-specific tasks for podman setup + +- name: Enable linger for user + command: "loginctl enable-linger {{ install_user }}" + become: yes + changed_when: true + ignore_errors: true + +- name: Ensure containers directory exists + file: + path: /etc/containers + state: directory + mode: '0755' + become: yes \ No newline at end of file diff --git a/ansible/roles/podman/tasks/setup_secrets.yml b/ansible/roles/podman/tasks/setup_secrets.yml index d71ba9d7..6c24ef37 100644 --- a/ansible/roles/podman/tasks/setup_secrets.yml +++ b/ansible/roles/podman/tasks/setup_secrets.yml @@ -1,9 +1,20 @@ --- # Extract and set global secrets +- name: Debug - Test podman secret ls directly + shell: podman secret ls + register: podman_secret_test + become: yes + ignore_errors: true + +- name: Debug - Show podman secret ls output + debug: + msg: "Podman secret ls result: {{ podman_secret_test.stdout_lines if podman_secret_test.rc == 0 else 'FAILED: ' + podman_secret_test.stderr }}" + - name: Source extract_secrets and capture output ansible.builtin.shell: | set -a + export PATH=$PATH:/nix/var/nix/profiles/default/bin source {{ playbook_dir }}/../scripts/extract_secrets.sh -q echo "elastic=$elastic" echo "wazuh=$wazuh" @@ -26,7 +37,6 @@ loop: "{{ extract_secrets_vars.stdout_lines }}" when: item != '' and '=' in item no_log: "{{ not debug_mode }}" - delegate_to: localhost - name: Verify global secrets were set debug: diff --git a/ansible/roles/podman/vars/default.yml b/ansible/roles/podman/vars/default.yml new file mode 100644 index 00000000..1528c0e4 --- /dev/null +++ b/ansible/roles/podman/vars/default.yml @@ -0,0 +1,2 @@ +--- +# Default variables for podman when no distribution-specific vars are found \ No newline at end of file diff --git a/ansible/roles/podman/vars/redhat.yml b/ansible/roles/podman/vars/redhat.yml new file mode 100644 index 00000000..1a79497e --- /dev/null +++ b/ansible/roles/podman/vars/redhat.yml @@ -0,0 +1,5 @@ +--- +# Red Hat-specific variables for podman + +# Red Hat uses different systemctl service names in some cases +nix_daemon_service: "nix-daemon" \ No newline at end of file diff --git a/docker/rhel9/README-CA-CERTIFICATES.md b/docker/rhel9/README-CA-CERTIFICATES.md new file mode 100644 index 00000000..ca497036 --- /dev/null +++ b/docker/rhel9/README-CA-CERTIFICATES.md @@ -0,0 +1,138 @@ +# CA Certificate Path Compatibility Issue + +## Overview + +Red Hat Enterprise Linux (RHEL) and Fedora-based systems use a different file system layout for CA certificates compared to Debian/Ubuntu systems. This creates a compatibility issue when running containers that expect Debian-style certificate paths. + +## The Problem + +### Certificate Path Differences + +**Red Hat/Fedora Systems:** +``` +/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem # Main CA certificate bundle +/etc/pki/ca-trust/extracted/pem/ca-bundle.crt # Alternative format +/etc/ssl/certs/ca-bundle.crt # Symlink to PKI location +``` + +**Debian/Ubuntu Systems:** +``` +/etc/ssl/certs/ca-certificates.crt # Main CA certificate bundle +/usr/share/ca-certificates/ # Individual certificates +``` + +### Container Impact + +LME containers, particularly Kibana, are configured to mount the system's CA certificate bundle for validating external SSL connections: + +```yaml +# From quadlet/lme-kibana.container +Volume=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro +``` + +**The Issue:** When running on RHEL/Fedora hosts, this file path doesn't exist by default, causing container startup failures with: +``` +Error: statfs /etc/ssl/certs/ca-certificates.crt: no such file or directory +``` + +## The Solution + +### Automatic Workaround (Ansible) + +The LME Ansible installation automatically creates the required compatibility symlink on Red Hat family systems: + +```yaml +# From ansible/roles/base/tasks/redhat.yml +- name: Create CA certificates symlink for compatibility (Red Hat systems) + file: + src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + dest: /etc/ssl/certs/ca-certificates.crt + state: link + force: yes + become: yes + when: ansible_os_family == 'RedHat' +``` + +### Manual Workaround + +If running containers manually or troubleshooting, create the symlink on the host system: + +```bash +# Create the target directory if it doesn't exist +sudo mkdir -p /etc/ssl/certs + +# Create the compatibility symlink +sudo ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/ssl/certs/ca-certificates.crt + +# Verify the symlink +ls -la /etc/ssl/certs/ca-certificates.crt +``` + +## Technical Details + +### Certificate Content + +Both files contain the same content - hundreds of trusted root CA certificates in PEM format: +``` +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw... +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/... +-----END CERTIFICATE----- +[hundreds more certificates...] +``` + +### LME Certificate Architecture + +LME uses a two-tier certificate system: + +1. **System CA Certificates** (this fix addresses): + - Purpose: Validate external SSL connections + - Location: `/etc/ssl/certs/ca-certificates.crt` (symlinked) + - Used by: Package downloads, external API calls, health checks + +2. **Internal LME Certificates**: + - Purpose: Secure inter-service communication + - Location: `lme_certs` volume + - Contains: Internal CA, service certificates for Elasticsearch, Kibana, etc. + +### Why This Matters + +Without the system CA bundle available at the expected path: +- Containers fail to start due to mount point errors +- SSL certificate validation fails for external connections +- Health checks and API calls cannot verify certificate authenticity +- Package downloads and updates may fail + +## Verification + +After applying the fix, verify the setup: + +```bash +# Check that the symlink exists +ls -la /etc/ssl/certs/ca-certificates.crt + +# Verify it points to the correct file +readlink /etc/ssl/certs/ca-certificates.crt + +# Test that containers can access it +docker run --rm -v /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro \ + registry.access.redhat.com/ubi9/ubi:latest \ + cat /etc/ssl/certs/ca-certificates.crt | head -5 +``` + +## Container Development Notes + +When developing containers for cross-platform compatibility: + +1. **Use flexible CA paths**: Check for both Debian and Red Hat certificate locations +2. **Document certificate requirements**: Clearly specify which certificate files containers need +3. **Test on multiple base images**: Verify compatibility with both UBI and Ubuntu base images +4. **Consider init containers**: Use setup containers to create necessary symlinks if needed + +## Related Files + +- `ansible/roles/base/tasks/redhat.yml` - Automatic symlink creation +- `quadlet/lme-kibana.container` - Container configuration requiring the certificate +- `quadlet/lme-elasticsearch.container` - Related certificate configuration \ No newline at end of file diff --git a/docker/rhel9/README.md b/docker/rhel9/README.md index 30d7b232..956aef7e 100644 --- a/docker/rhel9/README.md +++ b/docker/rhel9/README.md @@ -155,6 +155,7 @@ docker compose build --build-arg USER_ID=1000 --build-arg GROUP_ID=1000 For issues related to: - **Ansible installation**: See [README-ANSIBLE.md](README-ANSIBLE.md) +- **CA certificate compatibility**: See [README-CA-CERTIFICATES.md](README-CA-CERTIFICATES.md) - **Container setup**: Check the troubleshooting section above - **LME installation**: Refer to the main LME documentation diff --git a/install.sh b/install.sh index 8c652164..ed8ef3c8 100755 --- a/install.sh +++ b/install.sh @@ -202,6 +202,7 @@ install_ansible() { # Create symlink to make pip-installed ansible available in PATH if [ -f /usr/local/bin/ansible ] && [ ! -f /usr/bin/ansible ]; then sudo ln -sf /usr/local/bin/ansible /usr/bin/ansible + sudo ln -sf /usr/local/bin/ansible-vault /usr/bin/ansible-vault echo -e "${GREEN}✓ Created symlink for ansible in /usr/bin${NC}" fi fi @@ -415,7 +416,7 @@ run_playbook() { # Add debug mode if enabled if [ "$DEBUG_MODE" = "true" ]; then echo -e "${YELLOW}Debug mode enabled - verbose output will be shown${NC}" - ANSIBLE_OPTS="$ANSIBLE_OPTS -e debug_mode=true" + ANSIBLE_OPTS="$ANSIBLE_OPTS -e debug_mode=true -vvvv" fi # Run the main installation playbook diff --git a/testing/v2/development/Dockerfile b/testing/v2/development/Dockerfile index b69fd33c..9702c277 100644 --- a/testing/v2/development/Dockerfile +++ b/testing/v2/development/Dockerfile @@ -29,6 +29,90 @@ RUN echo "lme-user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers ENV BASE_DIR=/home/lme-user WORKDIR $BASE_DIR +# Red Hat stage with RHEL9/UBI9 base +FROM registry.access.redhat.com/ubi9/ubi:9.6 AS rhel-base + +ARG USER_ID=1001 +ARG GROUP_ID=1001 + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +RUN dnf update -y && dnf install -y \ + glibc-langpack-en sudo openssh-clients sshpass \ + && dnf clean all \ + && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ + && while getent passwd $USER_ID > /dev/null 2>&1; do USER_ID=$((USER_ID + 1)); done \ + && groupadd -g $GROUP_ID lme-user \ + && useradd -m -u $USER_ID -g lme-user lme-user \ + && usermod -aG wheel lme-user \ + && echo "lme-user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ + && echo "Defaults:lme-user !requiretty" >> /etc/sudoers + +ENV BASE_DIR=/home/lme-user +WORKDIR $BASE_DIR + +# Red Hat stage with full dependencies +FROM rhel-base AS rhel + +RUN dnf install -y \ + systemd systemd-sysv \ + && dnf clean all + +# Install EPEL repository +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ + && /usr/bin/crb enable \ + && dnf clean all + +# Install development tools and dependencies +RUN dnf install -y \ + python3 \ + python3-pip \ + zip \ + git \ + curl \ + wget \ + cron \ + vim \ + freerdp \ + pkg-config \ + cairo-devel \ + dbus-devel \ + && dnf clean all + +# Install PowerShell for RHEL +RUN curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/pki/rpm-gpg/microsoft.asc.gpg > /dev/null \ + && echo -e "[packages-microsoft-com-prod]\nname=packages-microsoft-com-prod\nbaseurl=https://packages.microsoft.com/rhel/9/prod\nenabled=1\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/microsoft.asc.gpg" > /etc/yum.repos.d/microsoft-prod.repo \ + && dnf install -y powershell \ + && dnf clean all + +# Install Azure CLI +RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc \ + && dnf install -y azure-cli \ + && dnf clean all + +# Install Chrome for testing +RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/pki/rpm-gpg/google-chrome.asc.gpg \ + && echo -e "[google-chrome]\nname=google-chrome\nbaseurl=https://dl.google.com/linux/chrome/rpm/stable/x86_64\nenabled=1\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/google-chrome.asc.gpg" > /etc/yum.repos.d/google-chrome.repo \ + && dnf install -y google-chrome-stable \ + && dnf clean all + +# Configure systemd for containers +RUN cd /lib/systemd/system/sysinit.target.wants/ && \ + ls | grep -v systemd-tmpfiles-setup | xargs rm -f $1 && \ + rm -f /lib/systemd/system/multi-user.target.wants/* && \ + rm -f /etc/systemd/system/*.wants/* && \ + rm -f /lib/systemd/system/local-fs.target.wants/* && \ + rm -f /lib/systemd/system/sockets.target.wants/*udev* && \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl* && \ + rm -f /lib/systemd/system/basic.target.wants/* && \ + rm -f /lib/systemd/system/anaconda.target.wants/* && \ + mkdir -p /etc/systemd/system/systemd-logind.service.d && \ + echo -e "[Service]\nProtectHostname=no" > /etc/systemd/system/systemd-logind.service.d/override.conf + +CMD ["/lib/systemd/systemd"] + # Ubuntu stage with full dependencies FROM base AS ubuntu @@ -94,3 +178,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ USER lme-user CMD ["sleep", "infinity"] + +# Pipeline-RHEL stage with minimal dependencies for Red Hat +FROM rhel-base AS pipeline-rhel + +RUN dnf install -y \ + python3 \ + python3-pip \ + openssh-clients \ + curl \ + && rpm --import https://packages.microsoft.com/keys/microsoft.asc \ + && dnf install -y azure-cli \ + && dnf clean all + +USER lme-user +CMD ["sleep", "infinity"] diff --git a/testing/v2/installers/README.md b/testing/v2/installers/README.md index e60abe58..5e664fd8 100644 --- a/testing/v2/installers/README.md +++ b/testing/v2/installers/README.md @@ -1,6 +1,8 @@ # Installation Guide #### Attention: Run these commands in the order presented in this document. Some commands depend on variables set in previous commands. Not all commands need to be run. There are some optional commands depending on the testing scenario. +**Note:** This guide supports both **Ubuntu 22.04** (default) and **Red Hat Enterprise Linux 9** as base operating systems. Use the `--use-rhel` flag to deploy RHEL instead of Ubuntu. + ## Initial Setup Variables First, set these variables in your terminal: @@ -26,21 +28,34 @@ cd testing/v2/installers ### Creating Azure Machine(s) -Linux only: +#### Ubuntu Linux (default): ```bash ./azure/build_azure_linux_network.py -g $RESOURCE_GROUP -s $PUBLIC_IP -vs $VM_SIZE -l $LOCATION -ast $AUTO_SHUTDOWN_TIME ``` -Linux and Windows (just add the -w flag): +#### Red Hat Enterprise Linux 9: +```bash +./azure/build_azure_linux_network.py -g $RESOURCE_GROUP -s $PUBLIC_IP -vs $VM_SIZE -l $LOCATION -ast $AUTO_SHUTDOWN_TIME --use-rhel +``` + +#### Linux and Windows (add the -w flag to either Ubuntu or RHEL): +Ubuntu + Windows: ```bash ./azure/build_azure_linux_network.py -g $RESOURCE_GROUP -s $PUBLIC_IP -vs $VM_SIZE -l $LOCATION -ast $AUTO_SHUTDOWN_TIME -w ``` +RHEL + Windows: +```bash +./azure/build_azure_linux_network.py -g $RESOURCE_GROUP -s $PUBLIC_IP -vs $VM_SIZE -l $LOCATION -ast $AUTO_SHUTDOWN_TIME --use-rhel -w +``` + After VM creation, set these additional variables: ```bash # These are generated during VM creation export VM_IP=$(cat $RESOURCE_GROUP.ip.txt) export VM_PASSWORD=$(cat $RESOURCE_GROUP.password.txt) +echo $VM_IP +echo $VM_PASSWORD ``` ### Installing lme-v2 @@ -98,13 +113,14 @@ sudo ./install_local.sh # Press enter for subscription and tenant prompts ``` -## Optional: Ubuntu 24.04 Setup +## Optional: Alternative Linux Distributions + Remember to activate venv first: ```bash source ~/LME/venv/bin/activate ``` -Create the network: +### Ubuntu 24.04 Setup ```bash ./azure/build_azure_linux_network.py \ -g $RESOURCE_GROUP \ @@ -117,6 +133,19 @@ Create the network: -is 24_04-daily-lts-gen2 ``` +### RHEL 8 Setup (alternative to RHEL 9) +```bash +./azure/build_azure_linux_network.py \ + -g $RESOURCE_GROUP \ + -s "0.0.0.0" \ + -vs $VM_SIZE \ + -l $LOCATION \ + -ast $AUTO_SHUTDOWN_TIME \ + -pub RedHat \ + -io RHEL \ + -is 8-lvm-gen2 +``` + ## Creating Additional VMs (Non-Network Attack Scenarios) ### Windows VM diff --git a/testing/v2/installers/azure/build_azure_linux_network.py b/testing/v2/installers/azure/build_azure_linux_network.py index 3f868652..39d9ae85 100755 --- a/testing/v2/installers/azure/build_azure_linux_network.py +++ b/testing/v2/installers/azure/build_azure_linux_network.py @@ -715,8 +715,28 @@ def main( action="store_true", help="Add a Windows server with default settings", ) + parser.add_argument( + "--use-rhel", + action="store_true", + help="Use Red Hat Enterprise Linux 9 instead of Ubuntu 22.04", + ) args = parser.parse_args() + + # Override image parameters if RHEL is requested + if args.use_rhel: + # Only override if user didn't specify custom values + if args.image_publisher == "Canonical": + args.image_publisher = "RedHat" + if args.image_offer == "0001-com-ubuntu-server-jammy": + args.image_offer = "RHEL" + if args.image_sku == "22_04-lts-gen2": + args.image_sku = "9-lvm-gen2" + args.machine_name = "rhel" if args.machine_name == "ubuntu" else args.machine_name + print(f"Using Red Hat Enterprise Linux image: {args.image_publisher}:{args.image_offer}:{args.image_sku}") + else: + print("Using Ubuntu 22.04 image") + check_ports_protocals_and_priorities( args.ports, args.priorities, args.protocols ) From dd816d8053ff542327f9e67b05cefaa22a0f8439 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 1 Aug 2025 07:57:08 -0400 Subject: [PATCH 03/51] Import the redhat 9 gpg key --- ansible/roles/base/tasks/redhat_9.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ansible/roles/base/tasks/redhat_9.yml b/ansible/roles/base/tasks/redhat_9.yml index c7d75fe4..6acf1b9c 100644 --- a/ansible/roles/base/tasks/redhat_9.yml +++ b/ansible/roles/base/tasks/redhat_9.yml @@ -1,6 +1,17 @@ --- # Red Hat 9-specific tasks for base role +- name: Import EPEL GPG key + rpm_key: + key: https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9 + state: present + become: yes + register: epel_key_import + retries: 3 + delay: 5 + until: epel_key_import is success + ignore_errors: "{{ ansible_check_mode }}" + - name: Install EPEL repository dnf: name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm From 018416432e916287e7288c5381cecb5733375790 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Mon, 11 Aug 2025 13:49:19 +0000 Subject: [PATCH 04/51] RHEL 9: SELinux-friendly Nix install flow and container-wide policy; compile/load via nix role; guarded SELinux mode changes --- .../selinux/container_wide_filtered_dedup.te | 25 ++ ansible/roles/nix/tasks/redhat.yml | 90 ++++++++ scripts/expand_rhel_disk.sh | 218 ++++++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te create mode 100755 scripts/expand_rhel_disk.sh diff --git a/ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te b/ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te new file mode 100644 index 00000000..3410a8b3 --- /dev/null +++ b/ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te @@ -0,0 +1,25 @@ +allow container_t cert_t:file getattr; +allow container_t cgroup_t:file { open read }; +allow container_t ephemeral_port_t:tcp_socket name_connect; +allow container_t fusefs_t:dir { add_name create remove_name write }; +allow container_t fusefs_t:file { append create execute execute_no_trans lock map open read rename setattr unlink write }; +allow container_t fusefs_t:lnk_file read; +allow container_t fusefs_t:sock_file write; +allow container_t http_port_t:tcp_socket name_connect; +allow container_t init_t:fifo_file { getattr ioctl read write }; +allow container_t proc_net_t:lnk_file read; +allow container_t random_device_t:chr_file getattr; +allow container_t self:cap_userns dac_override; +allow container_t self:netlink_route_socket { bind create getattr nlmsg_read }; +allow container_t self:process execmem; +allow container_t self:tcp_socket { accept connect create getattr getopt setopt shutdown }; +allow container_t self:udp_socket { connect create getattr setopt }; +allow container_t sysfs_t:file { open read }; +allow container_t sysfs_t:lnk_file read; +allow container_t unreserved_port_t:tcp_socket name_connect; +allow container_t wap_wsp_port_t:tcp_socket name_connect; +allow init_t container_ro_file_t:blk_file { create rename unlink }; +allow init_t container_t:process siginh; +allow init_t container_var_lib_t:fifo_file { create open read unlink write }; +allow init_t container_var_lib_t:sock_file { create setattr unlink write }; +allow init_t container_var_run_t:file write; diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index 5b8f5a6c..977a74c2 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -1,6 +1,34 @@ --- # Red Hat-specific Nix setup +- name: Detect if SELinux tooling is available + command: which getenforce + register: selinux_tooling + changed_when: false + failed_when: false + become: yes + +- name: Set SELinux availability fact + set_fact: + selinux_available: "{{ selinux_tooling.rc == 0 }}" + +- name: Get current SELinux mode + command: getenforce + register: getenforce_out + changed_when: false + failed_when: false + become: yes + when: selinux_available | default(false) + +- name: Remember if SELinux was enforcing + set_fact: + selinux_was_enforcing: "{{ selinux_available | default(false) and (getenforce_out.stdout | default('') | trim) == 'Enforcing' }}" + +- name: Set SELinux to permissive for Nix install + command: setenforce 0 + when: selinux_was_enforcing + become: yes + - name: Check if Nix is already installed stat: path: /nix/var/nix/profiles/default/bin/nix @@ -81,3 +109,65 @@ become: yes environment: PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" + +- name: Ensure SELinux policy tools are present + package: + name: + - policycoreutils + - policycoreutils-python-utils + - checkpolicy + - libsemanage-devel + state: present + become: yes + +- name: Deploy container-wide SELinux TE (filtered dedup) + copy: + src: selinux/container_wide_filtered_dedup.te + dest: /etc/selinux/lme/container_wide_filtered_dedup.te + owner: root + group: root + mode: '0644' + become: yes + +- name: Compile SELinux module (container_wide_filtered) + shell: | + set -e + mkdir -p /etc/selinux/lme + checkmodule -M -m -o /etc/selinux/lme/container_wide_filtered.mod /etc/selinux/lme/container_wide_filtered_dedup.te + semodule_package -o /etc/selinux/lme/container_wide_filtered.pp -m /etc/selinux/lme/container_wide_filtered.mod + args: + executable: /bin/bash + become: yes + +- name: Load SELinux module (container_wide_filtered) + command: semodule -i /etc/selinux/lme/container_wide_filtered.pp + become: yes + register: semodule_load + changed_when: semodule_load.rc == 0 + failed_when: false + +- name: Verify SELinux module loaded + command: semodule -l + register: semodule_list + changed_when: false + become: yes + +- name: Assert container_wide_filtered module present + assert: + that: + - semodule_list.stdout is search("^container_wide_filtered(\\s|$)") + fail_msg: "container_wide_filtered module not loaded" + +- name: Restore SELinux enforcing mode + command: setenforce 1 + when: selinux_was_enforcing + become: yes + +- name: Persist SELinux enforcement across reboots (optional) + lineinfile: + path: /etc/selinux/config + regexp: '^SELINUX=' + line: 'SELINUX=enforcing' + create: no + when: selinux_was_enforcing + become: yes diff --git a/scripts/expand_rhel_disk.sh b/scripts/expand_rhel_disk.sh new file mode 100755 index 00000000..bc8f6f42 --- /dev/null +++ b/scripts/expand_rhel_disk.sh @@ -0,0 +1,218 @@ +#!/bin/bash + +# LME Disk Expansion Script for RHEL Systems +# This script fixes the common issue where RHEL auto-partitioning doesn't use the full disk +# Expands the main LVM partition and /var filesystem to use all available space + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging function +log() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" +} + +success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +error() { + echo -e "${RED}[ERROR]${NC} $1" + exit 1 +} + +# Check if running as root +if [[ $EUID -ne 0 ]]; then + error "This script must be run as root (use sudo)" +fi + +# Function to check if disk expansion is needed +check_disk_space() { + local disk="/dev/sda" + local var_usage=$(df /var | awk 'NR==2 {print $5}' | sed 's/%//') + local var_size=$(df -h /var | awk 'NR==2 {print $2}') + + log "Current /var filesystem: ${var_size} (${var_usage}% used)" + + # Check if /var is less than 50GB (indicating it needs expansion) + local var_size_gb=$(df /var | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') + if [[ $var_size_gb -lt 50 ]]; then + warning "/var is only ${var_size_gb}GB - expansion recommended for LME deployment" + return 0 + else + success "/var is ${var_size_gb}GB - sufficient space available" + return 1 + fi +} + +# Function to backup partition table +backup_partition_table() { + local backup_file="/root/partition_backup_$(date +%Y%m%d_%H%M%S).dump" + log "Creating partition table backup: $backup_file" + sfdisk -d /dev/sda > "$backup_file" + success "Partition table backed up to $backup_file" +} + +# Function to fix GPT and expand partition +expand_disk() { + local disk="/dev/sda" + local lvm_partition="/dev/sda4" + + log "Starting disk expansion process..." + + # Fix GPT table and get partition info + log "Fixing GPT partition table..." + parted $disk print 2>&1 | grep -q "fix the GPT" && { + echo "Fix" | parted $disk print > /dev/null 2>&1 + success "GPT table fixed" + } || { + log "GPT table already correct" + } + + # Get current disk size + local disk_size=$(parted $disk print | grep "Disk.*:" | awk '{print $3}') + log "Total disk size: $disk_size" + + # Expand partition 4 to use full disk + log "Expanding LVM partition to use full disk..." + parted $disk resizepart 4 100% 2>/dev/null || { + # Alternative approach if 100% doesn't work + parted $disk resizepart 4 $disk_size + } + success "LVM partition expanded" + + # Resize physical volume + log "Resizing physical volume..." + pvresize $lvm_partition + success "Physical volume resized" + + # Get volume group name (usually rootvg for RHEL) + local vg_name=$(pvdisplay $lvm_partition | grep "VG Name" | awk '{print $3}') + log "Volume group: $vg_name" + + # Show available space + local free_space=$(vgdisplay $vg_name | grep "Free.*Size" | awk '{print $6 $7}') + log "Available free space: $free_space" + + if [[ "$free_space" == "0" ]]; then + warning "No free space available in volume group" + return 1 + fi + + # Extend /var logical volume + log "Extending /var logical volume..." + lvextend -l +100%FREE /dev/$vg_name/varlv + success "/var logical volume extended" + + # Grow XFS filesystem + log "Growing XFS filesystem..." + xfs_growfs /var + success "XFS filesystem grown" + + return 0 +} + +# Function to verify results +verify_expansion() { + log "Verifying disk expansion results..." + + # Show new disk layout + echo + log "=== Final Disk Layout ===" + lsblk | grep -E "(sda|rootvg)" + + echo + log "=== /var Filesystem Status ===" + df -h /var + + echo + log "=== Volume Group Status ===" + vgdisplay rootvg | grep -E "(VG Size|Free.*Size)" + + # Check if /var is now larger than 50GB + local var_size_gb=$(df /var | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') + if [[ $var_size_gb -gt 50 ]]; then + success "Disk expansion successful! /var is now ${var_size_gb}GB" + else + error "Disk expansion may have failed - /var is still only ${var_size_gb}GB" + fi +} + +# Main execution +main() { + log "LME RHEL Disk Expansion Script Starting..." + log "This script will expand your disk partitions to use full available space" + + # Check if expansion is needed + if ! check_disk_space; then + log "Disk expansion not needed - exiting" + exit 0 + fi + + # Confirm with user unless --yes flag is provided + if [[ "${1:-}" != "--yes" ]]; then + echo + warning "This script will modify your disk partitions." + warning "While these operations are generally safe, always ensure you have backups." + echo + read -p "Do you want to continue? [y/N]: " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log "Operation cancelled by user" + exit 0 + fi + fi + + # Create backup + backup_partition_table + + # Perform expansion + if expand_disk; then + verify_expansion + echo + success "=== DISK EXPANSION COMPLETED SUCCESSFULLY ===" + success "Your system now has significantly more space for LME containers and data" + echo + log "You can now proceed with LME installation" + else + error "Disk expansion failed - check the logs above" + fi +} + +# Script usage +show_usage() { + echo "Usage: $0 [--yes]" + echo " --yes Skip confirmation prompts (for automation)" + echo + echo "This script expands RHEL disk partitions to use all available space." + echo "Specifically designed for Azure VMs where auto-partitioning is conservative." +} + +# Handle command line arguments +case "${1:-}" in + -h|--help) + show_usage + exit 0 + ;; + --yes) + main --yes + ;; + "") + main + ;; + *) + error "Unknown option: $1" + show_usage + exit 1 + ;; +esac From 01e536ce7e05205902e6f63edecc848bdde749af Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Mon, 11 Aug 2025 15:18:42 +0000 Subject: [PATCH 05/51] Modified SELinux policy to use container_policy module --- .../nix/files/selinux/container_policy.te | 58 +++++++++++++++++++ ansible/roles/nix/tasks/redhat.yml | 22 +++---- 2 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 ansible/roles/nix/files/selinux/container_policy.te diff --git a/ansible/roles/nix/files/selinux/container_policy.te b/ansible/roles/nix/files/selinux/container_policy.te new file mode 100644 index 00000000..0df6c9b2 --- /dev/null +++ b/ansible/roles/nix/files/selinux/container_policy.te @@ -0,0 +1,58 @@ +module container_policy 1.0; + +require { + type container_t; + type cert_t; + type cgroup_t; + type ephemeral_port_t; + type fusefs_t; + type http_port_t; + type init_t; + type proc_net_t; + type random_device_t; + type sysfs_t; + type unreserved_port_t; + type wap_wsp_port_t; + type container_ro_file_t; + type container_var_lib_t; + type container_var_run_t; + + class file { getattr open read append create execute execute_no_trans lock map rename setattr unlink write }; + class dir { add_name create remove_name write }; + class lnk_file read; + class sock_file write; + class tcp_socket { name_connect accept connect create getattr getopt setopt shutdown }; + class udp_socket { connect create getattr setopt }; + class netlink_route_socket { bind create getattr nlmsg_read }; + class process { execmem siginh }; + class fifo_file { getattr ioctl read write create open unlink }; + class chr_file getattr; + class blk_file { create rename unlink }; + class cap_userns dac_override; +} + +allow container_t cert_t:file getattr; +allow container_t cgroup_t:file { open read }; +allow container_t ephemeral_port_t:tcp_socket name_connect; +allow container_t fusefs_t:dir { add_name create remove_name write }; +allow container_t fusefs_t:file { append create execute execute_no_trans lock map open read rename setattr unlink write }; +allow container_t fusefs_t:lnk_file read; +allow container_t fusefs_t:sock_file write; +allow container_t http_port_t:tcp_socket name_connect; +allow container_t init_t:fifo_file { getattr ioctl read write }; +allow container_t proc_net_t:lnk_file read; +allow container_t random_device_t:chr_file getattr; +allow container_t self:cap_userns dac_override; +allow container_t self:netlink_route_socket { bind create getattr nlmsg_read }; +allow container_t self:process execmem; +allow container_t self:tcp_socket { accept connect create getattr getopt setopt shutdown }; +allow container_t self:udp_socket { connect create getattr setopt }; +allow container_t sysfs_t:file { open read }; +allow container_t sysfs_t:lnk_file read; +allow container_t unreserved_port_t:tcp_socket name_connect; +allow container_t wap_wsp_port_t:tcp_socket name_connect; +allow init_t container_ro_file_t:blk_file { create rename unlink }; +allow init_t container_t:process siginh; +allow init_t container_var_lib_t:fifo_file { create open read unlink write }; + allow init_t container_var_lib_t:sock_file write; +allow init_t container_var_run_t:file write; diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index 977a74c2..42a8aa5b 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -120,27 +120,27 @@ state: present become: yes -- name: Deploy container-wide SELinux TE (filtered dedup) +- name: Deploy container-wide SELinux TE (container_policy) copy: - src: selinux/container_wide_filtered_dedup.te - dest: /etc/selinux/lme/container_wide_filtered_dedup.te + src: selinux/container_policy.te + dest: /etc/selinux/lme/container_policy.te owner: root group: root mode: '0644' become: yes -- name: Compile SELinux module (container_wide_filtered) +- name: Compile SELinux module (container_policy) shell: | set -e mkdir -p /etc/selinux/lme - checkmodule -M -m -o /etc/selinux/lme/container_wide_filtered.mod /etc/selinux/lme/container_wide_filtered_dedup.te - semodule_package -o /etc/selinux/lme/container_wide_filtered.pp -m /etc/selinux/lme/container_wide_filtered.mod + checkmodule -M -m -o /etc/selinux/lme/container_policy.mod /etc/selinux/lme/container_policy.te + semodule_package -o /etc/selinux/lme/container_policy.pp -m /etc/selinux/lme/container_policy.mod args: executable: /bin/bash become: yes -- name: Load SELinux module (container_wide_filtered) - command: semodule -i /etc/selinux/lme/container_wide_filtered.pp +- name: Load SELinux module (container_policy) + command: semodule -i /etc/selinux/lme/container_policy.pp become: yes register: semodule_load changed_when: semodule_load.rc == 0 @@ -152,11 +152,11 @@ changed_when: false become: yes -- name: Assert container_wide_filtered module present +- name: Assert container_policy module present assert: that: - - semodule_list.stdout is search("^container_wide_filtered(\\s|$)") - fail_msg: "container_wide_filtered module not loaded" + - semodule_list.stdout is search("^container_policy(\\s|$)") + fail_msg: "container_policy module not loaded" - name: Restore SELinux enforcing mode command: setenforce 1 From 6844a7eae255561d36e55b158f035143269a45eb Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 13 Aug 2025 09:43:43 +0000 Subject: [PATCH 06/51] Adds a script to fix SELinux contexts for quadlet units on RHEL 9 --- scripts/selinux_quadlet_fix.sh | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 scripts/selinux_quadlet_fix.sh diff --git a/scripts/selinux_quadlet_fix.sh b/scripts/selinux_quadlet_fix.sh new file mode 100644 index 00000000..3c6edcce --- /dev/null +++ b/scripts/selinux_quadlet_fix.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Purpose: Make Podman Quadlet work on RHEL with SELinux Enforcing when Podman is from Nix +# Notes: +# - Run as root. This script sets persistent SELinux file contexts and regenerates quadlet units. +# - It assumes a Nix-installed Podman with generator and quadlet under /nix/store. + +if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then + echo "[ERROR] Run this script as root (sudo)." >&2 + exit 1 +fi + +echo "[INFO] SELinux mode: $(getenforce || true)" + +echo "[INFO] (Optional) Ensure SELinux tooling is present" +if command -v dnf >/dev/null 2>&1; then + dnf -y install selinux-policy selinux-policy-targeted policycoreutils policycoreutils-python-utils libselinux-utils >/dev/null 2>&1 || true +fi + +echo "[INFO] Add persistent SELinux file contexts for Nix-installed Podman/Quadlet" +# Generator symlink must be readable by init_t (non-exec type for symlink) +semanage fcontext -a -t lib_t '/nix/store/.*/lib/systemd/system-generators/podman-system-generator' || true + +# Quadlet helper (not a container runtime) - readable/executable by init +semanage fcontext -a -t bin_t '/nix/store/.*/libexec/podman/quadlet' || true + +# Container runtime stack must be container_runtime_exec_t so it transitions to container_runtime_t +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/bin/podman' || true +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/bin/\.podman-wrapped' || true +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/bin/conmon' || true +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/bin/crun' || true +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/bin/runc' || true +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/bin/netavark' || true + +# Nix-provided helper wrapper paths for conmon/crun +semanage fcontext -a -t container_runtime_exec_t '/nix/store/.*/podman-helper-binary-wrapper/bin/.*' || true + +# Bash used by wrapper +semanage fcontext -a -t shell_exec_t '/nix/store/.*/bin/bash' || true + +# Dynamic loader and shared libraries used by quadlet/podman +semanage fcontext -a -t ld_so_t '/nix/store/.*/lib/ld-linux-x86-64\.so\.2' || true +semanage fcontext -a -t lib_t '/nix/store/.*/lib/.*\.so(\..*)?' || true +semanage fcontext -a -t lib_t '/nix/store/.*/lib64/.*\.so(\..*)?' || true + +# Nix profile symlink (systemd may read through this path) +semanage fcontext -a -t bin_t '/nix/var/nix/profiles/default(/.*)?' || true + +# Quadlet directory should be etc_t; ensure permissions 0755 +semanage fcontext -a -t etc_t '/etc/containers/systemd(/.*)?' || true +chmod 0755 /etc/containers/systemd || true + +echo "[INFO] Apply SELinux contexts (this may take a moment)" +restorecon -Rv /nix/store \ + /nix/var/nix/profiles/default \ + /usr/lib/systemd/system-generators/podman-system-generator \ + /etc/containers/systemd | tail -n 50 || true + +echo "[INFO] Enable common container SELinux booleans" +setsebool -P container_manage_cgroup on || true +setsebool -P container_use_devices on || true +setsebool -P container_read_certs on || true + +echo "[INFO] Run Podman quadlet generator and reload systemd" +/usr/lib/systemd/system-generators/podman-system-generator \ + /run/systemd/generator \ + /run/systemd/generator.early \ + /run/systemd/generator.late || true + +systemctl daemon-reload || true + +echo "[INFO] Generated LME units under /run/systemd/generator:" +ls -1 /run/systemd/generator/*lme*.service 2>/dev/null || true + +echo "[INFO] (Optional) Start orchestrator" +systemctl start lme.service || true + +echo "[DONE] SELinux contexts applied and quadlet units generated." + From 46eb6030a800a7c77b17d80c3b60dae10c107f72 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 14 Aug 2025 13:23:59 +0000 Subject: [PATCH 07/51] Adds the quadlet fix script --- ansible/roles/nix/tasks/redhat.yml | 49 +++++++++++++++++++++++++++--- scripts/selinux_quadlet_fix.sh | 0 2 files changed, 44 insertions(+), 5 deletions(-) mode change 100644 => 100755 scripts/selinux_quadlet_fix.sh diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index 42a8aa5b..6ab7c10a 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -120,6 +120,15 @@ state: present become: yes +- name: Ensure SELinux policy directory exists + file: + path: /etc/selinux/lme + state: directory + owner: root + group: root + mode: '0755' + become: yes + - name: Deploy container-wide SELinux TE (container_policy) copy: src: selinux/container_policy.te @@ -132,31 +141,61 @@ - name: Compile SELinux module (container_policy) shell: | set -e - mkdir -p /etc/selinux/lme checkmodule -M -m -o /etc/selinux/lme/container_policy.mod /etc/selinux/lme/container_policy.te semodule_package -o /etc/selinux/lme/container_policy.pp -m /etc/selinux/lme/container_policy.mod args: executable: /bin/bash become: yes + register: selinux_compile + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Debug SELinux compile result + debug: + var: selinux_compile + when: debug_mode | default(false) - name: Load SELinux module (container_policy) command: semodule -i /etc/selinux/lme/container_policy.pp become: yes register: semodule_load changed_when: semodule_load.rc == 0 - failed_when: false + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Ensure SELinux module enabled (container_policy) + command: semodule -e container_policy + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Debug SELinux module load result + debug: + var: semodule_load + when: debug_mode | default(false) - name: Verify SELinux module loaded - command: semodule -l - register: semodule_list + shell: semodule -l | grep -E "^container_policy(\\s|$)" || true + args: + executable: /bin/bash + register: container_policy_present changed_when: false become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' - name: Assert container_policy module present assert: that: - - semodule_list.stdout is search("^container_policy(\\s|$)") + - container_policy_present.rc == 0 fail_msg: "container_policy module not loaded" + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' - name: Restore SELinux enforcing mode command: setenforce 1 diff --git a/scripts/selinux_quadlet_fix.sh b/scripts/selinux_quadlet_fix.sh old mode 100644 new mode 100755 From 3bcf34c3b94737f42f8a8e812b9ab7d5b6de95b8 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 14 Aug 2025 10:14:01 -0400 Subject: [PATCH 08/51] Fixes the SELinux install to only install when selinux_available is true --- ansible/roles/nix/tasks/redhat.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index 6ab7c10a..95ca6ab4 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -119,6 +119,7 @@ - libsemanage-devel state: present become: yes + when: selinux_available | default(false) - name: Ensure SELinux policy directory exists file: @@ -128,6 +129,7 @@ group: root mode: '0755' become: yes + when: selinux_available | default(false) - name: Deploy container-wide SELinux TE (container_policy) copy: @@ -137,6 +139,7 @@ group: root mode: '0644' become: yes + when: selinux_available | default(false) - name: Compile SELinux module (container_policy) shell: | From 06375018572444165979a029add1ad3c9e364c4a Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 14 Aug 2025 17:22:19 +0000 Subject: [PATCH 09/51] Collects the SELinux setup tasks into a single role --- .../roles/base/files/selinux/lme_policy.fc | 36 +++++ .../files/selinux/lme_policy.te} | 16 +- ansible/roles/base/tasks/redhat.yml | 27 ++++ ansible/roles/base/tasks/selinux_setup.yml | 149 ++++++++++++++++++ .../selinux/container_wide_filtered_dedup.te | 25 --- ansible/roles/nix/tasks/redhat.yml | 99 ++---------- 6 files changed, 237 insertions(+), 115 deletions(-) create mode 100644 ansible/roles/base/files/selinux/lme_policy.fc rename ansible/roles/{nix/files/selinux/container_policy.te => base/files/selinux/lme_policy.te} (82%) create mode 100644 ansible/roles/base/tasks/selinux_setup.yml delete mode 100644 ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te diff --git a/ansible/roles/base/files/selinux/lme_policy.fc b/ansible/roles/base/files/selinux/lme_policy.fc new file mode 100644 index 00000000..2b0a017c --- /dev/null +++ b/ansible/roles/base/files/selinux/lme_policy.fc @@ -0,0 +1,36 @@ +#============================================================================ +# LME SELinux File Contexts for Nix-installed Podman/Quadlet +#============================================================================ + +# Podman system generator (symlinked to /usr/lib/systemd/system-generators/) +/nix/store/.*/lib/systemd/system-generators/podman-system-generator -- gen_context(system_u:object_r:lib_t:s0) + +# Quadlet helper binary (not container runtime) +/nix/store/.*/libexec/podman/quadlet -- gen_context(system_u:object_r:bin_t:s0) + +# Container runtime binaries - must be container_runtime_exec_t for proper domain transitions +/nix/store/.*/bin/podman -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/bin/\.podman-wrapped -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/bin/conmon -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/bin/crun -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/bin/runc -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/bin/netavark -- gen_context(system_u:object_r:container_runtime_exec_t:s0) + +# Nix-provided helper wrapper paths +/nix/store/.*/podman-helper-binary-wrapper/bin/.* -- gen_context(system_u:object_r:container_runtime_exec_t:s0) + +# Shell used by wrappers +/nix/store/.*/bin/bash -- gen_context(system_u:object_r:shell_exec_t:s0) + +# Dynamic loader and shared libraries +/nix/store/.*/lib/ld-linux-x86-64\.so\.2 -- gen_context(system_u:object_r:ld_so_t:s0) +/nix/store/.*/lib/.*\.so(\..*)?\ -- gen_context(system_u:object_r:lib_t:s0) +/nix/store/.*/lib64/.*\.so(\..*)?\ -- gen_context(system_u:object_r:lib_t:s0) + +# Nix profile symlinks (systemd may read through these paths) +/nix/var/nix/profiles/default(/.*)? gen_context(system_u:object_r:bin_t:s0) + +# Quadlet configuration directory +/etc/containers/systemd(/.*)? gen_context(system_u:object_r:etc_t:s0) + + diff --git a/ansible/roles/nix/files/selinux/container_policy.te b/ansible/roles/base/files/selinux/lme_policy.te similarity index 82% rename from ansible/roles/nix/files/selinux/container_policy.te rename to ansible/roles/base/files/selinux/lme_policy.te index 0df6c9b2..c7b67958 100644 --- a/ansible/roles/nix/files/selinux/container_policy.te +++ b/ansible/roles/base/files/selinux/lme_policy.te @@ -1,4 +1,4 @@ -module container_policy 1.0; +module lme_policy 1.0; require { type container_t; @@ -16,11 +16,17 @@ require { type container_ro_file_t; type container_var_lib_t; type container_var_run_t; + type container_runtime_exec_t; + type bin_t; + type lib_t; + type etc_t; + type ld_so_t; + type shell_exec_t; class file { getattr open read append create execute execute_no_trans lock map rename setattr unlink write }; class dir { add_name create remove_name write }; class lnk_file read; - class sock_file write; + class sock_file { write create setattr unlink }; class tcp_socket { name_connect accept connect create getattr getopt setopt shutdown }; class udp_socket { connect create getattr setopt }; class netlink_route_socket { bind create getattr nlmsg_read }; @@ -31,6 +37,10 @@ require { class cap_userns dac_override; } +#============================================================================ +# Container Policy Rules (merged from container_policy.te) +#============================================================================ + allow container_t cert_t:file getattr; allow container_t cgroup_t:file { open read }; allow container_t ephemeral_port_t:tcp_socket name_connect; @@ -54,5 +64,5 @@ allow container_t wap_wsp_port_t:tcp_socket name_connect; allow init_t container_ro_file_t:blk_file { create rename unlink }; allow init_t container_t:process siginh; allow init_t container_var_lib_t:fifo_file { create open read unlink write }; - allow init_t container_var_lib_t:sock_file write; +allow init_t container_var_lib_t:sock_file { create setattr unlink write }; allow init_t container_var_run_t:file write; diff --git a/ansible/roles/base/tasks/redhat.yml b/ansible/roles/base/tasks/redhat.yml index e33962e4..4fded320 100644 --- a/ansible/roles/base/tasks/redhat.yml +++ b/ansible/roles/base/tasks/redhat.yml @@ -61,6 +61,33 @@ var: dnf_install_redhat when: debug_mode | default(false) +# SELinux setup - run early before any Nix/Podman installation +- name: Detect if SELinux tooling is available (base role) + command: which getenforce + register: base_selinux_tooling + changed_when: false + failed_when: false + become: yes + +- name: Set SELinux availability fact (base role) + set_fact: + base_selinux_available: "{{ base_selinux_tooling.rc == 0 }}" + +- name: Get current SELinux mode (base role) + command: getenforce + register: base_getenforce_out + changed_when: false + failed_when: false + become: yes + when: base_selinux_available | default(false) + +- name: Setup SELinux policies for LME + include_tasks: selinux_setup.yml + when: + - ansible_os_family == 'RedHat' + - base_selinux_available | default(false) + - (base_getenforce_out.stdout | default('') | trim) != 'Disabled' + - name: Create CA certificates symlink for compatibility (Red Hat systems) file: src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem diff --git a/ansible/roles/base/tasks/selinux_setup.yml b/ansible/roles/base/tasks/selinux_setup.yml new file mode 100644 index 00000000..0d55333e --- /dev/null +++ b/ansible/roles/base/tasks/selinux_setup.yml @@ -0,0 +1,149 @@ +--- +# SELinux setup tasks for LME - run early to ensure proper file labeling + +- name: Detect if SELinux tooling is available + command: which getenforce + register: selinux_tooling + changed_when: false + failed_when: false + become: yes + +- name: Set SELinux availability fact + set_fact: + selinux_available: "{{ selinux_tooling.rc == 0 }}" + +- name: Get current SELinux mode + command: getenforce + register: getenforce_out + changed_when: false + failed_when: false + become: yes + when: selinux_available | default(false) + +- name: Remember if SELinux was enforcing + set_fact: + selinux_was_enforcing: "{{ selinux_available | default(false) and (getenforce_out.stdout | default('') | trim) == 'Enforcing' }}" + +- name: Display SELinux status + debug: + msg: | + SELinux Status: + - Available: {{ selinux_available | default(false) }} + - Current mode: {{ getenforce_out.stdout | default('N/A') }} + - Was enforcing: {{ selinux_was_enforcing | default(false) }} + when: debug_mode | default(false) + +# Install SELinux policy tools early +- name: Ensure SELinux policy tools are present + package: + name: + - policycoreutils + - policycoreutils-python-utils + - checkpolicy + - libsemanage-devel + - selinux-policy + - selinux-policy-targeted + - libselinux-utils + state: present + become: yes + when: selinux_available | default(false) + +- name: Ensure SELinux policy directory exists + file: + path: /etc/selinux/lme + state: directory + owner: root + group: root + mode: '0755' + become: yes + when: selinux_available | default(false) + +# Deploy unified LME SELinux policy (container + nix/podman contexts) +- name: Deploy LME unified SELinux policy + copy: + src: selinux/lme_policy.te + dest: /etc/selinux/lme/lme_policy.te + owner: root + group: root + mode: '0644' + become: yes + when: selinux_available | default(false) + register: selinux_policy_deployed + +- name: Deploy LME SELinux file contexts + copy: + src: selinux/lme_policy.fc + dest: /etc/selinux/lme/lme_policy.fc + owner: root + group: root + mode: '0644' + become: yes + when: selinux_available | default(false) + register: selinux_fc_deployed + +- name: Compile SELinux module (lme_policy) + shell: | + set -e + cd /etc/selinux/lme + checkmodule -M -m -o lme_policy.mod lme_policy.te + semodule_package -o lme_policy.pp -m lme_policy.mod -f lme_policy.fc + args: + executable: /bin/bash + become: yes + register: selinux_compile + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_policy_deployed.changed or selinux_fc_deployed.changed + +- name: Debug SELinux compile result + debug: + var: selinux_compile + when: debug_mode | default(false) and selinux_compile is defined + +- name: Load SELinux module (lme_policy) + command: semodule -i /etc/selinux/lme/lme_policy.pp + become: yes + register: semodule_load + changed_when: semodule_load.rc == 0 + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_compile.changed | default(false) + +- name: Ensure SELinux module enabled (lme_policy) + command: semodule -e lme_policy + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Verify SELinux module loaded + shell: semodule -l | grep -E "^lme_policy(\\s|$)" || true + args: + executable: /bin/bash + register: lme_policy_present + changed_when: false + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Assert LME policy module present + assert: + that: + - lme_policy_present.rc == 0 + fail_msg: "lme_policy module not loaded" + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Display SELinux module status + debug: + msg: | + SELinux Module Status: + - LME policy loaded: {{ 'Yes' if lme_policy_present.rc == 0 else 'No' }} + - Ready for Nix/Podman installation with proper contexts + when: + - debug_mode | default(false) + - selinux_available | default(false) diff --git a/ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te b/ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te deleted file mode 100644 index 3410a8b3..00000000 --- a/ansible/roles/nix/files/selinux/container_wide_filtered_dedup.te +++ /dev/null @@ -1,25 +0,0 @@ -allow container_t cert_t:file getattr; -allow container_t cgroup_t:file { open read }; -allow container_t ephemeral_port_t:tcp_socket name_connect; -allow container_t fusefs_t:dir { add_name create remove_name write }; -allow container_t fusefs_t:file { append create execute execute_no_trans lock map open read rename setattr unlink write }; -allow container_t fusefs_t:lnk_file read; -allow container_t fusefs_t:sock_file write; -allow container_t http_port_t:tcp_socket name_connect; -allow container_t init_t:fifo_file { getattr ioctl read write }; -allow container_t proc_net_t:lnk_file read; -allow container_t random_device_t:chr_file getattr; -allow container_t self:cap_userns dac_override; -allow container_t self:netlink_route_socket { bind create getattr nlmsg_read }; -allow container_t self:process execmem; -allow container_t self:tcp_socket { accept connect create getattr getopt setopt shutdown }; -allow container_t self:udp_socket { connect create getattr setopt }; -allow container_t sysfs_t:file { open read }; -allow container_t sysfs_t:lnk_file read; -allow container_t unreserved_port_t:tcp_socket name_connect; -allow container_t wap_wsp_port_t:tcp_socket name_connect; -allow init_t container_ro_file_t:blk_file { create rename unlink }; -allow init_t container_t:process siginh; -allow init_t container_var_lib_t:fifo_file { create open read unlink write }; -allow init_t container_var_lib_t:sock_file { create setattr unlink write }; -allow init_t container_var_run_t:file write; diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index 95ca6ab4..a76bc031 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -110,102 +110,27 @@ environment: PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" -- name: Ensure SELinux policy tools are present - package: - name: - - policycoreutils - - policycoreutils-python-utils - - checkpolicy - - libsemanage-devel - state: present - become: yes - when: selinux_available | default(false) - -- name: Ensure SELinux policy directory exists - file: - path: /etc/selinux/lme - state: directory - owner: root - group: root - mode: '0755' - become: yes - when: selinux_available | default(false) - -- name: Deploy container-wide SELinux TE (container_policy) - copy: - src: selinux/container_policy.te - dest: /etc/selinux/lme/container_policy.te - owner: root - group: root - mode: '0644' - become: yes - when: selinux_available | default(false) - -- name: Compile SELinux module (container_policy) - shell: | - set -e - checkmodule -M -m -o /etc/selinux/lme/container_policy.mod /etc/selinux/lme/container_policy.te - semodule_package -o /etc/selinux/lme/container_policy.pp -m /etc/selinux/lme/container_policy.mod - args: - executable: /bin/bash - become: yes - register: selinux_compile - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Debug SELinux compile result - debug: - var: selinux_compile - when: debug_mode | default(false) - -- name: Load SELinux module (container_policy) - command: semodule -i /etc/selinux/lme/container_policy.pp - become: yes - register: semodule_load - changed_when: semodule_load.rc == 0 - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Ensure SELinux module enabled (container_policy) - command: semodule -e container_policy - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Debug SELinux module load result - debug: - var: semodule_load - when: debug_mode | default(false) - -- name: Verify SELinux module loaded - shell: semodule -l | grep -E "^container_policy(\\s|$)" || true - args: - executable: /bin/bash - register: container_policy_present - changed_when: false - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Assert container_policy module present - assert: - that: - - container_policy_present.rc == 0 - fail_msg: "container_policy module not loaded" +- name: After Podman installation, apply SELinux booleans + seboolean: + name: "{{ item }}" + state: yes + persistent: yes + become: yes + loop: + - container_manage_cgroup + - container_use_devices + - container_read_certs when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' + ignore_errors: yes # Some booleans might not exist on all systems - name: Restore SELinux enforcing mode command: setenforce 1 when: selinux_was_enforcing become: yes -- name: Persist SELinux enforcement across reboots (optional) +- name: Persist SELinux enforcement across reboots lineinfile: path: /etc/selinux/config regexp: '^SELINUX=' From bf4c4a5f64d50f5a83508bd2fce572ba9d415637 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 14 Aug 2025 19:33:49 +0000 Subject: [PATCH 10/51] Adds container-selinux package to base role if selinux is available --- ansible/roles/base/tasks/selinux_setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/base/tasks/selinux_setup.yml b/ansible/roles/base/tasks/selinux_setup.yml index 0d55333e..ea7e182d 100644 --- a/ansible/roles/base/tasks/selinux_setup.yml +++ b/ansible/roles/base/tasks/selinux_setup.yml @@ -44,6 +44,7 @@ - selinux-policy - selinux-policy-targeted - libselinux-utils + - container-selinux state: present become: yes when: selinux_available | default(false) From 9603043ae40892f3832461bc9a0d25dafd313afe Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Mon, 18 Aug 2025 11:57:26 +0000 Subject: [PATCH 11/51] Fixes errors in SELinux policy and setup --- .../roles/base/files/selinux/lme_policy.fc | 44 ++++++---- ansible/roles/base/tasks/main.yml | 12 +-- ansible/roles/base/tasks/selinux_setup.yml | 85 ++++++++++++++++++- ansible/roles/nix/tasks/main.yml | 22 +++++ ansible/roles/nix/tasks/redhat.yml | 16 ++++ ansible/roles/podman/tasks/main.yml | 57 +++++++++++-- ansible/roles/podman/tasks/quadlet_setup.yml | 32 +++++++ ansible/site.yml | 3 +- 8 files changed, 241 insertions(+), 30 deletions(-) diff --git a/ansible/roles/base/files/selinux/lme_policy.fc b/ansible/roles/base/files/selinux/lme_policy.fc index 2b0a017c..1e8ba1e3 100644 --- a/ansible/roles/base/files/selinux/lme_policy.fc +++ b/ansible/roles/base/files/selinux/lme_policy.fc @@ -3,34 +3,48 @@ #============================================================================ # Podman system generator (symlinked to /usr/lib/systemd/system-generators/) -/nix/store/.*/lib/systemd/system-generators/podman-system-generator -- gen_context(system_u:object_r:lib_t:s0) +/nix/store/.*/lib/systemd/system-generators/podman-system-generator -- system_u:object_r:bin_t:s0 # Quadlet helper binary (not container runtime) -/nix/store/.*/libexec/podman/quadlet -- gen_context(system_u:object_r:bin_t:s0) +/nix/store/.*/libexec/podman/quadlet -- system_u:object_r:bin_t:s0 # Container runtime binaries - must be container_runtime_exec_t for proper domain transitions -/nix/store/.*/bin/podman -- gen_context(system_u:object_r:container_runtime_exec_t:s0) -/nix/store/.*/bin/\.podman-wrapped -- gen_context(system_u:object_r:container_runtime_exec_t:s0) -/nix/store/.*/bin/conmon -- gen_context(system_u:object_r:container_runtime_exec_t:s0) -/nix/store/.*/bin/crun -- gen_context(system_u:object_r:container_runtime_exec_t:s0) -/nix/store/.*/bin/runc -- gen_context(system_u:object_r:container_runtime_exec_t:s0) -/nix/store/.*/bin/netavark -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/bin/podman -- system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/\.podman-wrapped -- system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/conmon -- system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/crun -- system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/runc -- system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/netavark -- system_u:object_r:container_runtime_exec_t:s0 # Nix-provided helper wrapper paths -/nix/store/.*/podman-helper-binary-wrapper/bin/.* -- gen_context(system_u:object_r:container_runtime_exec_t:s0) +/nix/store/.*/podman-helper-binary-wrapper/bin/.* -- system_u:object_r:container_runtime_exec_t:s0 # Shell used by wrappers -/nix/store/.*/bin/bash -- gen_context(system_u:object_r:shell_exec_t:s0) +/nix/store/.*/bin/bash -- system_u:object_r:shell_exec_t:s0 # Dynamic loader and shared libraries -/nix/store/.*/lib/ld-linux-x86-64\.so\.2 -- gen_context(system_u:object_r:ld_so_t:s0) -/nix/store/.*/lib/.*\.so(\..*)?\ -- gen_context(system_u:object_r:lib_t:s0) -/nix/store/.*/lib64/.*\.so(\..*)?\ -- gen_context(system_u:object_r:lib_t:s0) +/nix/store/.*/lib/ld-linux-x86-64\.so\.2 -- system_u:object_r:ld_so_t:s0 +/nix/store/.*/lib/.*\.so(\..*)? -- system_u:object_r:lib_t:s0 +/nix/store/.*/lib64/.*\.so(\..*)? -- system_u:object_r:lib_t:s0 # Nix profile symlinks (systemd may read through these paths) -/nix/var/nix/profiles/default(/.*)? gen_context(system_u:object_r:bin_t:s0) +/nix/var/nix/profiles/default(/.*)? system_u:object_r:bin_t:s0 # Quadlet configuration directory -/etc/containers/systemd(/.*)? gen_context(system_u:object_r:etc_t:s0) +/etc/containers/systemd(/.*)? system_u:object_r:etc_t:s0 + +# Nix systemd unit files +/nix/var/nix/profiles(/.*)? system_u:object_r:bin_t:s0 +/nix/var/nix/profiles/default/lib/systemd/system(/.*)? system_u:object_r:systemd_unit_file_t:s0 +/nix/store/.*/lib/systemd/system/.*\.service system_u:object_r:systemd_unit_file_t:s0 +/nix/store/.*/lib/systemd/system/.*\.socket system_u:object_r:systemd_unit_file_t:s0 + +# Nix executables needed by systemd +/nix/store/.*/bin/nix-daemon system_u:object_r:bin_t:s0 +/nix/store/.*/bin/nix system_u:object_r:bin_t:s0 +/nix/store/.*/lib/systemd/system-generators/podman-system-generator -- system_u:object_r:bin_t:s0 + +# Allow systemd to execute our custom generator +/etc/systemd/system-generators(/.*)? gen_context(system_u:object_r:systemd_generator_exec_t,s0) diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml index 7e427ec5..ce723aa4 100644 --- a/ansible/roles/base/tasks/main.yml +++ b/ansible/roles/base/tasks/main.yml @@ -1,13 +1,15 @@ --- # Include OS-specific variables -- name: Include OS-specific variables +- name: Set OS-specific variables include_vars: "{{ item }}" with_first_found: - - "{{ ansible_distribution | lower }}_{{ ansible_distribution_version | replace('.', '_') }}.yml" - - "{{ ansible_distribution | lower }}.yml" + - "redhat_{{ ansible_distribution_major_version }}_{{ ansible_distribution_minor_version }}.yml" - "{{ ansible_os_family | lower }}.yml" - - "default.yml" - tags: always + - main.yml + +- name: Include SELinux setup tasks + include_tasks: selinux_setup.yml + when: lme_install_selinux | default(true) # Include common OS tasks first - name: Include common OS tasks diff --git a/ansible/roles/base/tasks/selinux_setup.yml b/ansible/roles/base/tasks/selinux_setup.yml index ea7e182d..3e58926c 100644 --- a/ansible/roles/base/tasks/selinux_setup.yml +++ b/ansible/roles/base/tasks/selinux_setup.yml @@ -87,7 +87,11 @@ set -e cd /etc/selinux/lme checkmodule -M -m -o lme_policy.mod lme_policy.te - semodule_package -o lme_policy.pp -m lme_policy.mod -f lme_policy.fc + # Try to include file contexts; if that fails, build a package without them + if ! semodule_package -o lme_policy.pp -m lme_policy.mod -f lme_policy.fc; then + echo "Warning: building SELinux module without file contexts" >&2 + semodule_package -o lme_policy_no_fc.pp -m lme_policy.mod + fi args: executable: /bin/bash become: yes @@ -102,15 +106,90 @@ var: selinux_compile when: debug_mode | default(false) and selinux_compile is defined +- name: Check if SELinux module already present (pre-load) + shell: semodule -l | grep -E "^lme_policy(\\s|$)" || true + args: + executable: /bin/bash + register: lme_policy_present_pre + changed_when: false + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +# --------------------------------------------------------------------------- +# Ensure Nix-provided systemd unit files have correct SELinux labels +# --------------------------------------------------------------------------- +- name: Define SELinux fcontext for Nix systemd unit files + shell: | + set -e + semanage fcontext -a -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.service' || \ + semanage fcontext -m -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.service' + semanage fcontext -a -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.socket' || \ + semanage fcontext -m -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.socket' + semanage fcontext -a -t systemd_unit_file_t '/nix/var/nix/profiles/default/lib/systemd/system(/.*)?' || \ + semanage fcontext -m -t systemd_unit_file_t '/nix/var/nix/profiles/default/lib/systemd/system(/.*)?' + args: + executable: /bin/bash + become: yes + changed_when: false + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Apply persistent fcontext for nix profile symlink + command: semanage fcontext -a -t systemd_unit_file_t "{{ nix_profile_symlink_path }}(/.*)?" + become: yes + changed_when: false + failed_when: false + +- name: Apply persistent fcontext for systemd generators + command: semanage fcontext -a -t systemd_generator_exec_t "/etc/systemd/system-generators(/.*)?" + become: yes + changed_when: false + failed_when: false + +- name: Restore context for nix profile + command: restorecon -Rv "{{ nix_profile_symlink_path }}" + become: yes + changed_when: false + +- name: Restore context for systemd generators + command: restorecon -Rv /etc/systemd/system-generators + become: yes + changed_when: false + +- name: Reload systemd daemon to apply changes + systemd: + daemon_reload: yes + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' - name: Load SELinux module (lme_policy) - command: semodule -i /etc/selinux/lme/lme_policy.pp + shell: | + set -e + cd /etc/selinux/lme + if [ -f lme_policy.pp ]; then + if semodule -i lme_policy.pp; then + exit 0 + fi + fi + if [ -f lme_policy_no_fc.pp ]; then + semodule -i lme_policy_no_fc.pp + else + echo "No module package found to install" >&2 + exit 1 + fi + args: + executable: /bin/bash become: yes register: semodule_load changed_when: semodule_load.rc == 0 when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' - - selinux_compile.changed | default(false) + - selinux_compile.changed | default(false) or (lme_policy_present_pre.rc | default(1)) != 0 - name: Ensure SELinux module enabled (lme_policy) command: semodule -e lme_policy diff --git a/ansible/roles/nix/tasks/main.yml b/ansible/roles/nix/tasks/main.yml index 37228479..e7b6ba3e 100644 --- a/ansible/roles/nix/tasks/main.yml +++ b/ansible/roles/nix/tasks/main.yml @@ -55,6 +55,28 @@ become: yes ignore_errors: yes +- name: Create podman symlink in /usr/bin for systemd generators + file: + src: "{{ nix_profile_symlink_path }}/bin/podman" + dest: /usr/bin/podman + state: link + force: yes + become: yes + +- name: Ensure correct podman systemd generator is linked in the nix profile + file: + src: "{{ nix_profile_symlink_path }}/libexec/podman/quadlet" + dest: "{{ nix_profile_symlink_path }}/lib/systemd/system-generators/podman-system-generator" + state: link + force: yes + become: yes + +- name: Restore contexts on podman symlinks (best-effort) + command: restorecon -v /usr/local/bin/podman /usr/bin/podman + changed_when: false + failed_when: false + become: yes + - name: Source updated PATH shell: source ~/.profile args: diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index a76bc031..68792e52 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -71,11 +71,27 @@ var: nix_install_result when: debug_mode | default(false) and not nix_installed.stat.exists +- name: Reload systemd units after Nix installation + systemd: + daemon_reload: yes + become: yes + when: not nix_installed.stat.exists + - name: Ensure nix-daemon service is started systemd: name: nix-daemon state: started enabled: yes + daemon_reload: yes + become: yes + when: not nix_installed.stat.exists + +- name: Ensure nix-daemon socket is enabled and started (for socket activation) + systemd: + name: nix-daemon.socket + state: started + enabled: yes + daemon_reload: yes become: yes when: not nix_installed.stat.exists diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index e4bc0a48..468d247a 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -16,13 +16,58 @@ - "common.yml" # These tasks are common for all distributions -- name: Ensure Nix daemon is running - systemd: - name: "{{ nix_daemon_service }}" - state: started - enabled: yes +- name: Check if Nix daemon unit exists + shell: systemctl cat "{{ nix_daemon_service }}" >/dev/null 2>&1 || systemctl list-unit-files | grep -E "^{{ nix_daemon_service }}(\\.service)?\\s" >/dev/null 2>&1 + args: + executable: /bin/bash + register: nix_daemon_unit_check + changed_when: false + failed_when: false become: yes - notify: restart nix-daemon + +- name: Check if Nix appears installed (nix binary) + stat: + path: /nix/var/nix/profiles/default/bin/nix + register: nix_binary_stat + +- name: Ensure Nix daemon is running + block: + - name: Start/enable nix-daemon via systemd module + systemd: + name: "{{ nix_daemon_service }}" + state: started + enabled: yes + daemon_reload: yes + become: yes + notify: restart nix-daemon + - name: Ensure nix-daemon.socket is enabled (socket activation) + systemd: + name: "{{ nix_daemon_service }}.socket" + state: started + enabled: yes + daemon_reload: yes + become: yes + rescue: + - name: Fallback start of nix-daemon via systemctl + shell: | + set -e + systemctl daemon-reload + systemctl enable --now {{ nix_daemon_service }}.socket || true + systemctl enable --now {{ nix_daemon_service }} + args: + executable: /bin/bash + become: yes + when: nix_daemon_unit_check.rc == 0 and nix_binary_stat.stat.exists + +- name: Skip Nix daemon management (unit not present) + debug: + msg: "Skipping nix-daemon management because the unit is not present. Run with tags 'nix' first or full play to install Nix." + when: nix_daemon_unit_check.rc != 0 + +- name: Skip Nix daemon management (Nix not installed yet) + debug: + msg: "Skipping nix-daemon management because Nix is not installed. Run with tags 'nix' first or full play to install Nix." + when: nix_daemon_unit_check.rc == 0 and not nix_binary_stat.stat.exists - name: Wait for Nix daemon to be ready wait_for: diff --git a/ansible/roles/podman/tasks/quadlet_setup.yml b/ansible/roles/podman/tasks/quadlet_setup.yml index 41d67b3f..2fa3dbc7 100644 --- a/ansible/roles/podman/tasks/quadlet_setup.yml +++ b/ansible/roles/podman/tasks/quadlet_setup.yml @@ -37,6 +37,38 @@ mode: '0644' become: yes +# Ensure quadlet helper and systemd generator are available from standard paths +- name: Ensure /usr/libexec/podman exists + file: + path: /usr/libexec/podman + state: directory + owner: root + group: root + mode: '0755' + become: yes + +- name: Link quadlet helper into /usr/libexec/podman + file: + src: /nix/var/nix/profiles/default/libexec/podman/quadlet + dest: /usr/libexec/podman/quadlet + state: link + force: true + become: yes + +- name: Ensure podman systemd generator is linked + file: + src: /nix/var/nix/profiles/default/libexec/podman/quadlet + dest: /usr/lib/systemd/system-generators/podman-system-generator + state: link + force: true + become: yes + +- name: Restore contexts on quadlet/generator paths (best-effort) + command: restorecon -v /usr/libexec/podman/quadlet /usr/lib/systemd/system-generators/podman-system-generator + changed_when: false + failed_when: false + become: yes + - name: Reload systemd daemon systemd: daemon_reload: yes diff --git a/ansible/site.yml b/ansible/site.yml index 20d01fed..0e670b48 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -36,11 +36,12 @@ # Default timezone settings timezone_area: "Etc" # Change to your area: America, Europe, Asia, etc. timezone_zone: "UTC" # Change to your timezone: New_York, London, Tokyo, etc. + nix_profile_symlink_path: /nix/var/nix/profiles/default roles: - role: base tags: ['base', 'all'] - role: nix - tags: ['base', 'all'] + tags: ['base', 'system', 'all'] - role: podman tags: ['system', 'all'] - role: elasticsearch From d416ad8301d570b28d22ceeacee76bd5eb024647 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Mon, 18 Aug 2025 13:00:36 +0000 Subject: [PATCH 12/51] Break out the selinux policies for nix and podman --- .../roles/base/files/selinux/lme_policy.fc | 50 +------------- .../roles/base/files/selinux/nix_policy.fc | 26 +++++++ .../roles/base/files/selinux/nix_policy.te | 16 +++++ .../roles/base/files/selinux/podman_policy.fc | 27 ++++++++ .../roles/base/files/selinux/podman_policy.te | 14 ++++ ansible/roles/base/tasks/selinux_setup.yml | 49 ------------- ansible/roles/nix/tasks/main.yml | 68 ++++++++++++++++++- ansible/roles/podman/tasks/quadlet_setup.yml | 62 ++++++++++++++++- 8 files changed, 214 insertions(+), 98 deletions(-) create mode 100644 ansible/roles/base/files/selinux/nix_policy.fc create mode 100644 ansible/roles/base/files/selinux/nix_policy.te create mode 100644 ansible/roles/base/files/selinux/podman_policy.fc create mode 100644 ansible/roles/base/files/selinux/podman_policy.te diff --git a/ansible/roles/base/files/selinux/lme_policy.fc b/ansible/roles/base/files/selinux/lme_policy.fc index 1e8ba1e3..a74062a2 100644 --- a/ansible/roles/base/files/selinux/lme_policy.fc +++ b/ansible/roles/base/files/selinux/lme_policy.fc @@ -1,50 +1,6 @@ #============================================================================ -# LME SELinux File Contexts for Nix-installed Podman/Quadlet +# LME SELinux File Contexts (generic) #============================================================================ -# Podman system generator (symlinked to /usr/lib/systemd/system-generators/) -/nix/store/.*/lib/systemd/system-generators/podman-system-generator -- system_u:object_r:bin_t:s0 - -# Quadlet helper binary (not container runtime) -/nix/store/.*/libexec/podman/quadlet -- system_u:object_r:bin_t:s0 - -# Container runtime binaries - must be container_runtime_exec_t for proper domain transitions -/nix/store/.*/bin/podman -- system_u:object_r:container_runtime_exec_t:s0 -/nix/store/.*/bin/\.podman-wrapped -- system_u:object_r:container_runtime_exec_t:s0 -/nix/store/.*/bin/conmon -- system_u:object_r:container_runtime_exec_t:s0 -/nix/store/.*/bin/crun -- system_u:object_r:container_runtime_exec_t:s0 -/nix/store/.*/bin/runc -- system_u:object_r:container_runtime_exec_t:s0 -/nix/store/.*/bin/netavark -- system_u:object_r:container_runtime_exec_t:s0 - -# Nix-provided helper wrapper paths -/nix/store/.*/podman-helper-binary-wrapper/bin/.* -- system_u:object_r:container_runtime_exec_t:s0 - -# Shell used by wrappers -/nix/store/.*/bin/bash -- system_u:object_r:shell_exec_t:s0 - -# Dynamic loader and shared libraries -/nix/store/.*/lib/ld-linux-x86-64\.so\.2 -- system_u:object_r:ld_so_t:s0 -/nix/store/.*/lib/.*\.so(\..*)? -- system_u:object_r:lib_t:s0 -/nix/store/.*/lib64/.*\.so(\..*)? -- system_u:object_r:lib_t:s0 - -# Nix profile symlinks (systemd may read through these paths) -/nix/var/nix/profiles/default(/.*)? system_u:object_r:bin_t:s0 - -# Quadlet configuration directory -/etc/containers/systemd(/.*)? system_u:object_r:etc_t:s0 - -# Nix systemd unit files -/nix/var/nix/profiles(/.*)? system_u:object_r:bin_t:s0 -/nix/var/nix/profiles/default/lib/systemd/system(/.*)? system_u:object_r:systemd_unit_file_t:s0 -/nix/store/.*/lib/systemd/system/.*\.service system_u:object_r:systemd_unit_file_t:s0 -/nix/store/.*/lib/systemd/system/.*\.socket system_u:object_r:systemd_unit_file_t:s0 - -# Nix executables needed by systemd -/nix/store/.*/bin/nix-daemon system_u:object_r:bin_t:s0 -/nix/store/.*/bin/nix system_u:object_r:bin_t:s0 -/nix/store/.*/lib/systemd/system-generators/podman-system-generator -- system_u:object_r:bin_t:s0 - -# Allow systemd to execute our custom generator -/etc/systemd/system-generators(/.*)? gen_context(system_u:object_r:systemd_generator_exec_t,s0) - - +# Intentionally left minimal: Nix/Podman contexts have been split into +# nix_policy.fc and podman_policy.fc and are loaded by their respective roles. diff --git a/ansible/roles/base/files/selinux/nix_policy.fc b/ansible/roles/base/files/selinux/nix_policy.fc new file mode 100644 index 00000000..d34e6e67 --- /dev/null +++ b/ansible/roles/base/files/selinux/nix_policy.fc @@ -0,0 +1,26 @@ +#============================================================================ +# Nix SELinux File Contexts (Nix store and profiles) +#============================================================================ + +# Nix profile symlinks (systemd may read through these paths) +/nix/var/nix/profiles/default(/.*)? system_u:object_r:bin_t:s0 +/nix/var/nix/profiles(/.*)? system_u:object_r:bin_t:s0 + +# Nix systemd unit files +/nix/var/nix/profiles/default/lib/systemd/system(/.*)? system_u:object_r:systemd_unit_file_t:s0 +/nix/store/.*/lib/systemd/system/.*\.service system_u:object_r:systemd_unit_file_t:s0 +/nix/store/.*/lib/systemd/system/.*\.socket system_u:object_r:systemd_unit_file_t:s0 + +# Nix executables needed by system components +/nix/store/.*/bin/nix-daemon system_u:object_r:bin_t:s0 +/nix/store/.*/bin/nix system_u:object_r:bin_t:s0 + +# Shell used by wrappers +/nix/store/.*/bin/bash system_u:object_r:shell_exec_t:s0 + +# Dynamic loader and shared libraries +/nix/store/.*/lib/ld-linux-x86-64\.so\.2 system_u:object_r:ld_so_t:s0 +/nix/store/.*/lib/.*\.so(\..*)? system_u:object_r:lib_t:s0 +/nix/store/.*/lib64/.*\.so(\..*)? system_u:object_r:lib_t:s0 + + diff --git a/ansible/roles/base/files/selinux/nix_policy.te b/ansible/roles/base/files/selinux/nix_policy.te new file mode 100644 index 00000000..dfda07c7 --- /dev/null +++ b/ansible/roles/base/files/selinux/nix_policy.te @@ -0,0 +1,16 @@ +module nix_policy 1.0; + +require { + type bin_t; + type ld_so_t; + type lib_t; + type shell_exec_t; + type systemd_unit_file_t; + class file { getattr open read execute execute_no_trans map }; + class lnk_file read; +} + +# Minimal Nix policy: primarily file contexts; no allows beyond default +# Kept small because main behavior relies on contexts provided in nix_policy.fc + + diff --git a/ansible/roles/base/files/selinux/podman_policy.fc b/ansible/roles/base/files/selinux/podman_policy.fc new file mode 100644 index 00000000..9a463e06 --- /dev/null +++ b/ansible/roles/base/files/selinux/podman_policy.fc @@ -0,0 +1,27 @@ +#============================================================================ +# Podman/Quadlet SELinux File Contexts (when installed via Nix) +#============================================================================ + +# Container runtime binaries - must be container_runtime_exec_t for proper domain transitions +/nix/store/.*/bin/podman system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/\.podman-wrapped system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/conmon system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/crun system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/runc system_u:object_r:container_runtime_exec_t:s0 +/nix/store/.*/bin/netavark system_u:object_r:container_runtime_exec_t:s0 + +# Quadlet helper binary and systemd generator +/nix/store/.*/libexec/podman/quadlet system_u:object_r:bin_t:s0 +/nix/store/.*/lib/systemd/system-generators/podman-system-generator system_u:object_r:bin_t:s0 + +# Wrapper helper paths installed by Nix builds +/nix/store/.*/podman-helper-binary-wrapper/bin/.* system_u:object_r:container_runtime_exec_t:s0 + +# Quadlet configuration directory +/etc/containers/systemd(/.*)? system_u:object_r:etc_t:s0 + +# Systemd generator target paths (use init_exec_t per RHEL defaults) +/etc/systemd/system-generators(/.*)? system_u:object_r:init_exec_t:s0 +/usr/lib/systemd/system-generators(/.*)? system_u:object_r:init_exec_t:s0 + + diff --git a/ansible/roles/base/files/selinux/podman_policy.te b/ansible/roles/base/files/selinux/podman_policy.te new file mode 100644 index 00000000..beffe3d6 --- /dev/null +++ b/ansible/roles/base/files/selinux/podman_policy.te @@ -0,0 +1,14 @@ +module podman_policy 1.0; + +require { + type container_runtime_exec_t; + type etc_t; + type bin_t; + type init_exec_t; + class file { getattr open read execute execute_no_trans map }; + class lnk_file read; +} + +# Minimal Podman policy: contexts are defined in podman_policy.fc. No custom allows here. + + diff --git a/ansible/roles/base/tasks/selinux_setup.yml b/ansible/roles/base/tasks/selinux_setup.yml index 3e58926c..2962ecc3 100644 --- a/ansible/roles/base/tasks/selinux_setup.yml +++ b/ansible/roles/base/tasks/selinux_setup.yml @@ -117,55 +117,6 @@ - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' -# --------------------------------------------------------------------------- -# Ensure Nix-provided systemd unit files have correct SELinux labels -# --------------------------------------------------------------------------- -- name: Define SELinux fcontext for Nix systemd unit files - shell: | - set -e - semanage fcontext -a -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.service' || \ - semanage fcontext -m -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.service' - semanage fcontext -a -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.socket' || \ - semanage fcontext -m -t systemd_unit_file_t '/nix/store/.*/lib/systemd/system/.*\.socket' - semanage fcontext -a -t systemd_unit_file_t '/nix/var/nix/profiles/default/lib/systemd/system(/.*)?' || \ - semanage fcontext -m -t systemd_unit_file_t '/nix/var/nix/profiles/default/lib/systemd/system(/.*)?' - args: - executable: /bin/bash - become: yes - changed_when: false - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Apply persistent fcontext for nix profile symlink - command: semanage fcontext -a -t systemd_unit_file_t "{{ nix_profile_symlink_path }}(/.*)?" - become: yes - changed_when: false - failed_when: false - -- name: Apply persistent fcontext for systemd generators - command: semanage fcontext -a -t systemd_generator_exec_t "/etc/systemd/system-generators(/.*)?" - become: yes - changed_when: false - failed_when: false - -- name: Restore context for nix profile - command: restorecon -Rv "{{ nix_profile_symlink_path }}" - become: yes - changed_when: false - -- name: Restore context for systemd generators - command: restorecon -Rv /etc/systemd/system-generators - become: yes - changed_when: false - -- name: Reload systemd daemon to apply changes - systemd: - daemon_reload: yes - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - name: Load SELinux module (lme_policy) shell: | set -e diff --git a/ansible/roles/nix/tasks/main.yml b/ansible/roles/nix/tasks/main.yml index e7b6ba3e..df02e152 100644 --- a/ansible/roles/nix/tasks/main.yml +++ b/ansible/roles/nix/tasks/main.yml @@ -80,4 +80,70 @@ - name: Source updated PATH shell: source ~/.profile args: - executable: /bin/bash \ No newline at end of file + executable: /bin/bash + +# --------------------------------------------------------------------------- +# Nix SELinux policy: deploy, compile, load, and label after Nix is installed +# --------------------------------------------------------------------------- +- name: Deploy Nix SELinux policy files + copy: + src: "{{ clone_directory }}/ansible/roles/base/files/selinux/nix_policy.te" + dest: /etc/selinux/lme/nix_policy.te + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Deploy Nix SELinux file contexts + copy: + src: "{{ clone_directory }}/ansible/roles/base/files/selinux/nix_policy.fc" + dest: /etc/selinux/lme/nix_policy.fc + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Compile Nix SELinux module + shell: | + set -e + cd /etc/selinux/lme + checkmodule -M -m -o nix_policy.mod nix_policy.te + semodule_package -o nix_policy.pp -m nix_policy.mod -f nix_policy.fc + args: + executable: /bin/bash + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Load Nix SELinux module + command: semodule -i /etc/selinux/lme/nix_policy.pp + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Ensure Nix SELinux module enabled + command: semodule -e nix_policy + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Check if nix profile exists + stat: + path: "{{ nix_profile_symlink_path }}" + register: nix_profile_exists + +- name: Restore context for nix profile (post-install) + command: restorecon -Rv "{{ nix_profile_symlink_path }}" + become: yes + changed_when: false + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - nix_profile_exists.stat.exists \ No newline at end of file diff --git a/ansible/roles/podman/tasks/quadlet_setup.yml b/ansible/roles/podman/tasks/quadlet_setup.yml index 2fa3dbc7..31189c30 100644 --- a/ansible/roles/podman/tasks/quadlet_setup.yml +++ b/ansible/roles/podman/tasks/quadlet_setup.yml @@ -79,4 +79,64 @@ name: lme.service state: started enabled: yes - become: yes \ No newline at end of file + become: yes + +# --------------------------------------------------------------------------- +# Podman SELinux policy: deploy, compile, load, and label after Podman is set up +# --------------------------------------------------------------------------- +- name: Deploy Podman SELinux policy files + copy: + src: "{{ clone_directory }}/ansible/roles/base/files/selinux/podman_policy.te" + dest: /etc/selinux/lme/podman_policy.te + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Deploy Podman SELinux file contexts + copy: + src: "{{ clone_directory }}/ansible/roles/base/files/selinux/podman_policy.fc" + dest: /etc/selinux/lme/podman_policy.fc + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Compile Podman SELinux module + shell: | + set -e + cd /etc/selinux/lme + checkmodule -M -m -o podman_policy.mod podman_policy.te + semodule_package -o podman_policy.pp -m podman_policy.mod -f podman_policy.fc + args: + executable: /bin/bash + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Load Podman SELinux module + command: semodule -i /etc/selinux/lme/podman_policy.pp + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Ensure Podman SELinux module enabled + command: semodule -e podman_policy + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Restore contexts on quadlet/generator paths + command: restorecon -Rv /usr/libexec/podman/quadlet /usr/lib/systemd/system-generators/podman-system-generator + changed_when: false + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' \ No newline at end of file From ee521f422f78117c60a10a22e44e2a81674bac39 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 19 Aug 2025 13:06:15 +0000 Subject: [PATCH 13/51] Setting further selinux policies for redhat 9.1 --- .gitignore | 4 ++++ ansible/roles/base/files/selinux/lme_policy.te | 17 ++++++++++++++++- ansible/roles/base/files/selinux/nix_policy.fc | 2 ++ .../roles/base/files/selinux/podman_policy.fc | 3 +++ ansible/roles/nix/tasks/main.yml | 10 +++++++++- ansible/roles/nix/tasks/redhat.yml | 18 ++++++++++-------- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 97c3cd88..09547cd7 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ testing/upgrade_testing/ .cache/ .venv/ *.env + +# SELinux build artifacts +ansible/roles/base/files/selinux/*.mod +ansible/roles/base/files/selinux/*.pp diff --git a/ansible/roles/base/files/selinux/lme_policy.te b/ansible/roles/base/files/selinux/lme_policy.te index c7b67958..35740110 100644 --- a/ansible/roles/base/files/selinux/lme_policy.te +++ b/ansible/roles/base/files/selinux/lme_policy.te @@ -22,6 +22,10 @@ require { type etc_t; type ld_so_t; type shell_exec_t; + type init_exec_t; + type kvm_device_t; + type fs_t; + type user_home_t; class file { getattr open read append create execute execute_no_trans lock map rename setattr unlink write }; class dir { add_name create remove_name write }; @@ -32,7 +36,8 @@ require { class netlink_route_socket { bind create getattr nlmsg_read }; class process { execmem siginh }; class fifo_file { getattr ioctl read write create open unlink }; - class chr_file getattr; + class chr_file { getattr read write }; + class filesystem quotamod; class blk_file { create rename unlink }; class cap_userns dac_override; } @@ -66,3 +71,13 @@ allow init_t container_t:process siginh; allow init_t container_var_lib_t:fifo_file { create open read unlink write }; allow init_t container_var_lib_t:sock_file { create setattr unlink write }; allow init_t container_var_run_t:file write; + +# Additional allows derived from nix_lme.te (focused, minimal set) +allow init_t container_var_run_t:file { append create rename setattr }; +allow init_t kvm_device_t:chr_file { read write }; +allow init_t fs_t:filesystem quotamod; +allow init_t user_home_t:file { open read }; + +# Allow systemd (init_t) to read generator symlink labeled init_exec_t +allow init_t init_exec_t:lnk_file read; +allow init_t bin_t:lnk_file read; diff --git a/ansible/roles/base/files/selinux/nix_policy.fc b/ansible/roles/base/files/selinux/nix_policy.fc index d34e6e67..43f68299 100644 --- a/ansible/roles/base/files/selinux/nix_policy.fc +++ b/ansible/roles/base/files/selinux/nix_policy.fc @@ -23,4 +23,6 @@ /nix/store/.*/lib/.*\.so(\..*)? system_u:object_r:lib_t:s0 /nix/store/.*/lib64/.*\.so(\..*)? system_u:object_r:lib_t:s0 +/nix/var/nix/daemon-socket(/.*)? system_u:object_r:container_var_run_t:s0 + diff --git a/ansible/roles/base/files/selinux/podman_policy.fc b/ansible/roles/base/files/selinux/podman_policy.fc index 9a463e06..4705db62 100644 --- a/ansible/roles/base/files/selinux/podman_policy.fc +++ b/ansible/roles/base/files/selinux/podman_policy.fc @@ -13,6 +13,9 @@ # Quadlet helper binary and systemd generator /nix/store/.*/libexec/podman/quadlet system_u:object_r:bin_t:s0 /nix/store/.*/lib/systemd/system-generators/podman-system-generator system_u:object_r:bin_t:s0 +/usr/lib/systemd/system-generators/podman-system-generator system_u:object_r:init_exec_t:s0 + +/nix/store/.*/libexec/podman(/.*)? system_u:object_r:bin_t:s0 # Wrapper helper paths installed by Nix builds /nix/store/.*/podman-helper-binary-wrapper/bin/.* system_u:object_r:container_runtime_exec_t:s0 diff --git a/ansible/roles/nix/tasks/main.yml b/ansible/roles/nix/tasks/main.yml index df02e152..df4c395a 100644 --- a/ansible/roles/nix/tasks/main.yml +++ b/ansible/roles/nix/tasks/main.yml @@ -146,4 +146,12 @@ when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' - - nix_profile_exists.stat.exists \ No newline at end of file + - nix_profile_exists.stat.exists + +- name: Restore context for Nix daemon socket directory + command: restorecon -Rv /nix/var/nix/daemon-socket + become: yes + changed_when: false + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' \ No newline at end of file diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index 68792e52..a533c116 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -136,6 +136,7 @@ - container_manage_cgroup - container_use_devices - container_read_certs + - domain_can_mmap_files when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' @@ -146,11 +147,12 @@ when: selinux_was_enforcing become: yes -- name: Persist SELinux enforcement across reboots - lineinfile: - path: /etc/selinux/config - regexp: '^SELINUX=' - line: 'SELINUX=enforcing' - create: no - when: selinux_was_enforcing - become: yes +# It may not be a good thing to persist SELinux enforcement across reboots +# - name: Persist SELinux enforcement across reboots +# lineinfile: +# path: /etc/selinux/config +# regexp: '^SELINUX=' +# line: 'SELINUX=enforcing' +# create: no +# when: selinux_was_enforcing +# become: yes From 73612206993070dc020d6677b6f394868285b2e4 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 20 Aug 2025 10:06:14 +0000 Subject: [PATCH 14/51] Fixes the expand_rhel_disk.sh script --- scripts/expand_rhel_disk.sh | 340 +++++++++++++++++++++++++++++------- 1 file changed, 274 insertions(+), 66 deletions(-) diff --git a/scripts/expand_rhel_disk.sh b/scripts/expand_rhel_disk.sh index bc8f6f42..b33a0a5e 100755 --- a/scripts/expand_rhel_disk.sh +++ b/scripts/expand_rhel_disk.sh @@ -2,7 +2,7 @@ # LME Disk Expansion Script for RHEL Systems # This script fixes the common issue where RHEL auto-partitioning doesn't use the full disk -# Expands the main LVM partition and /var filesystem to use all available space +# Doubles the root partition size and allocates remaining space to /var set -euo pipefail @@ -39,18 +39,24 @@ fi # Function to check if disk expansion is needed check_disk_space() { local disk="/dev/sda" + local root_usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') + local root_size=$(df -h / | awk 'NR==2 {print $2}') local var_usage=$(df /var | awk 'NR==2 {print $5}' | sed 's/%//') local var_size=$(df -h /var | awk 'NR==2 {print $2}') + log "Current / filesystem: ${root_size} (${root_usage}% used)" log "Current /var filesystem: ${var_size} (${var_usage}% used)" - # Check if /var is less than 50GB (indicating it needs expansion) - local var_size_gb=$(df /var | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') - if [[ $var_size_gb -lt 50 ]]; then - warning "/var is only ${var_size_gb}GB - expansion recommended for LME deployment" + # Check disk space usage vs total disk size + local disk_size_bytes=$(lsblk -b -n -o SIZE /dev/sda | head -1) + local used_space_bytes=$(df --output=used / /var | tail -n +2 | awk '{sum += $1} END {print sum * 1024}') + local usage_percent=$((used_space_bytes * 100 / disk_size_bytes)) + + if [[ $usage_percent -lt 80 ]]; then + warning "Disk usage is only ${usage_percent}% - expansion recommended" return 0 else - success "/var is ${var_size_gb}GB - sufficient space available" + success "Disk usage is ${usage_percent}% - expansion may not be needed" return 1 fi } @@ -63,63 +69,249 @@ backup_partition_table() { success "Partition table backed up to $backup_file" } -# Function to fix GPT and expand partition +# Function to check and fix GPT table non-interactively +check_and_fix_gpt() { + local disk="/dev/sda" + log "Checking GPT partition table..." + + # Use parted in script mode with --fix to handle GPT issues non-interactively + if ! parted --script --fix "$disk" print > /dev/null 2>&1; then + error "Failed to read or fix partition table" + fi + + success "GPT table checked and fixed if needed" +} + +# Function to get partition information +get_partition_info() { + local disk="/dev/sda" + + # Get current partition layout + log "Analyzing current partition layout..." + parted --script "$disk" print free + + # Find the root partition (usually mounted at /) + local root_partition=$(df / | awk 'NR==2 {print $1}' | grep -o 'sda[0-9]*') + local root_part_num=${root_partition#sda} + + # Get current root partition size and end + local root_info=$(parted --script "$disk" print | grep "^ *$root_part_num ") + local root_start=$(echo "$root_info" | awk '{print $2}') + local root_end=$(echo "$root_info" | awk '{print $3}') + local root_size=$(echo "$root_info" | awk '{print $4}') + + log "Root partition ($root_partition): Start=$root_start, End=$root_end, Size=$root_size" + + # Get total disk size + local disk_end=$(parted --script "$disk" print | grep "^Disk /dev/sda:" | awk '{print $3}') + log "Total disk size: $disk_end" + + echo "$root_part_num|$root_start|$root_end|$root_size|$disk_end" +} + +# Function to calculate new partition sizes +calculate_new_sizes() { + local partition_info="$1" + IFS='|' read -r root_part_num root_start root_end root_size disk_end <<< "$partition_info" + + # Convert sizes to MB for calculation + local root_size_mb=$(echo "$root_size" | sed 's/GB/000/' | sed 's/MB//' | sed 's/\..*//') + local disk_end_mb=$(echo "$disk_end" | sed 's/GB/000/' | sed 's/MB//' | sed 's/\..*//') + local root_start_mb=$(echo "$root_start" | sed 's/GB/000/' | sed 's/MB//' | sed 's/\..*//') + + # Double the root partition size + local new_root_size_mb=$((root_size_mb * 2)) + local new_root_end_mb=$((root_start_mb + new_root_size_mb)) + + # Calculate /var partition start and end + local var_start_mb=$((new_root_end_mb + 1)) + local var_end_mb=$disk_end_mb + + log "Calculated sizes:" + log " Root partition: ${root_start_mb}MB - ${new_root_end_mb}MB (${new_root_size_mb}MB)" + log " Var partition: ${var_start_mb}MB - ${var_end_mb}MB ($((var_end_mb - var_start_mb))MB)" + + echo "${root_part_num}|${new_root_end_mb}MB|${var_start_mb}MB|${var_end_mb}MB" +} + +# Function to expand and repartition disk expand_disk() { local disk="/dev/sda" - local lvm_partition="/dev/sda4" log "Starting disk expansion process..." - # Fix GPT table and get partition info - log "Fixing GPT partition table..." - parted $disk print 2>&1 | grep -q "fix the GPT" && { - echo "Fix" | parted $disk print > /dev/null 2>&1 - success "GPT table fixed" - } || { - log "GPT table already correct" - } - - # Get current disk size - local disk_size=$(parted $disk print | grep "Disk.*:" | awk '{print $3}') - log "Total disk size: $disk_size" - - # Expand partition 4 to use full disk - log "Expanding LVM partition to use full disk..." - parted $disk resizepart 4 100% 2>/dev/null || { - # Alternative approach if 100% doesn't work - parted $disk resizepart 4 $disk_size - } + # Check and fix GPT table + check_and_fix_gpt + + # Get partition information + local partition_info=$(get_partition_info) + local new_sizes=$(calculate_new_sizes "$partition_info") + IFS='|' read -r root_part_num new_root_end var_start var_end <<< "$new_sizes" + + # Check if we have LVM setup + local vg_name="" + if command -v vgdisplay >/dev/null 2>&1; then + vg_name=$(vgdisplay 2>/dev/null | grep "VG Name" | head -1 | awk '{print $3}' || echo "") + fi + + if [[ -n "$vg_name" ]]; then + log "LVM detected with volume group: $vg_name" + expand_with_lvm "$disk" "$root_part_num" "$new_root_end" "$var_start" "$var_end" "$vg_name" + else + log "No LVM detected - using direct partition expansion" + expand_without_lvm "$disk" "$root_part_num" "$new_root_end" "$var_start" "$var_end" + fi + + return 0 +} + +# Function to expand with LVM +expand_with_lvm() { + local disk="$1" + local root_part_num="$2" + local new_root_end="$3" + local var_start="$4" + local var_end="$5" + local vg_name="$6" + + # Find the LVM partition (usually the last one) + local lvm_part_num=$(parted --script "$disk" print | grep "lvm" | tail -1 | awk '{print $1}') + local lvm_partition="/dev/sda${lvm_part_num}" + + log "Expanding LVM partition ${lvm_partition} to use full disk..." + + # Expand the LVM partition to use all available space + parted --script "$disk" resizepart "$lvm_part_num" 100% success "LVM partition expanded" # Resize physical volume log "Resizing physical volume..." - pvresize $lvm_partition + pvresize "$lvm_partition" success "Physical volume resized" - # Get volume group name (usually rootvg for RHEL) - local vg_name=$(pvdisplay $lvm_partition | grep "VG Name" | awk '{print $3}') - log "Volume group: $vg_name" + # Get available free space (in MB) robustly + local total_free_mb=$(vgs "$vg_name" --noheadings --units m --nosuffix -o vg_free | awk '{print int($1)}') + log "Available free space in VG ${vg_name}: ${total_free_mb} MB" - # Show available space - local free_space=$(vgdisplay $vg_name | grep "Free.*Size" | awk '{print $6 $7}') - log "Available free space: $free_space" - - if [[ "$free_space" == "0" ]]; then + if [[ -z "${total_free_mb}" ]] || [[ "${total_free_mb}" -le 0 ]]; then warning "No free space available in volume group" return 1 fi - # Extend /var logical volume - log "Extending /var logical volume..." - lvextend -l +100%FREE /dev/$vg_name/varlv - success "/var logical volume extended" + # Determine logical volumes for / and /var + local root_lv=$(findmnt -n -o SOURCE / 2>/dev/null || true) + if [[ -z "$root_lv" ]]; then + root_lv="/dev/${vg_name}/rootlv" + fi + local var_lv=$(findmnt -n -o SOURCE /var 2>/dev/null || true) + if [[ -n "$var_lv" ]] && [[ "$var_lv" == "$root_lv" ]]; then + var_lv="" + fi - # Grow XFS filesystem - log "Growing XFS filesystem..." - xfs_growfs /var - success "XFS filesystem grown" + # Get current root LV size (in MB) + local current_root_mb=$(lvs --noheadings --units m --nosuffix -o LV_SIZE "$root_lv" 2>/dev/null | awk '{print int($1)}') + if [[ -z "$current_root_mb" ]]; then + error "Unable to determine current root LV size for $root_lv" + fi - return 0 + # Target: double root size, remainder to /var + local desired_root_mb=$((current_root_mb * 2)) + local required_increase_mb=$((desired_root_mb - current_root_mb)) + if [[ "$required_increase_mb" -le 0 ]]; then + required_increase_mb=0 + fi + + local root_expand_mb=$required_increase_mb + if [[ "$root_expand_mb" -gt "$total_free_mb" ]]; then + root_expand_mb=$total_free_mb + fi + local var_expand_mb=$((total_free_mb - root_expand_mb)) + + # Extend root logical volume + log "Extending root logical volume ($root_lv) by ${root_expand_mb}MB..." + if [[ $root_expand_mb -gt 0 ]]; then + lvextend -L "+${root_expand_mb}m" "$root_lv" + else + log "Root LV already at or above desired size; skipping root extension" + fi + if [[ -e "$root_lv" ]] && [[ $root_expand_mb -gt 0 ]]; then + # Grow root filesystem + log "Growing root filesystem..." + if [[ "$(findmnt -n -o FSTYPE /)" == "xfs" ]]; then + xfs_growfs / + else + resize2fs "$root_lv" + fi + success "Root filesystem extended" + fi + + # Extend /var logical volume if present + if [[ -n "$var_lv" ]] && [[ -e "$var_lv" ]] && [[ $var_expand_mb -gt 0 ]]; then + log "Extending /var logical volume ($var_lv) by ${var_expand_mb}MB..." + lvextend -L "+${var_expand_mb}m" "$var_lv" + + # Grow /var filesystem + log "Growing /var filesystem..." + if [[ "$(findmnt -n -o FSTYPE /var)" == "xfs" ]]; then + xfs_growfs /var + else + resize2fs "$var_lv" + fi + success "/var filesystem extended" + else + # If no separate /var LV, allocate any remaining space to root + if [[ $var_expand_mb -gt 0 ]]; then + log "No separate /var LV found; allocating remaining ${var_expand_mb}MB to root..." + lvextend -L "+${var_expand_mb}m" "$root_lv" + if [[ "$(findmnt -n -o FSTYPE /)" == "xfs" ]]; then + xfs_growfs / + else + resize2fs "$root_lv" + fi + fi + fi +} + +# Function to expand without LVM (direct partitions) +expand_without_lvm() { + local disk="$1" + local root_part_num="$2" + local new_root_end="$3" + local var_start="$4" + local var_end="$5" + + log "Expanding root partition to ${new_root_end}..." + parted --script "$disk" resizepart "$root_part_num" "$new_root_end" + + # Grow root filesystem + log "Growing root filesystem..." + local root_partition="/dev/sda${root_part_num}" + if mount | grep -q "xfs.*on / "; then + xfs_growfs / + else + resize2fs "$root_partition" + fi + success "Root partition and filesystem expanded" + + # Create new partition for /var if there's remaining space + local remaining_space_mb=$(($(echo "$var_end" | sed 's/MB//') - $(echo "$var_start" | sed 's/MB//'))) + if [[ $remaining_space_mb -gt 1000 ]]; then # Only if more than 1GB + log "Creating new partition for /var expansion..." + local new_var_part_num=$((root_part_num + 1)) + parted --script "$disk" mkpart primary ext4 "$var_start" "$var_end" + + # Format the new partition + local new_var_partition="/dev/sda${new_var_part_num}" + log "Formatting new partition ${new_var_partition}..." + mkfs.ext4 -F "$new_var_partition" + + success "New partition created for /var expansion" + warning "Manual steps required:" + warning "1. Mount new partition: mount ${new_var_partition} /mnt" + warning "2. Copy /var data: cp -a /var/* /mnt/" + warning "3. Update /etc/fstab to mount ${new_var_partition} at /var" + warning "4. Reboot to activate new /var mount" + fi } # Function to verify results @@ -129,40 +321,53 @@ verify_expansion() { # Show new disk layout echo log "=== Final Disk Layout ===" - lsblk | grep -E "(sda|rootvg)" + lsblk | grep -E "(sda|rootvg|centos|rhel)" + + echo + log "=== Root Filesystem Status ===" + df -h / echo log "=== /var Filesystem Status ===" df -h /var - echo - log "=== Volume Group Status ===" - vgdisplay rootvg | grep -E "(VG Size|Free.*Size)" + # Check for LVM + if command -v vgdisplay >/dev/null 2>&1; then + local vg_name=$(vgdisplay 2>/dev/null | grep "VG Name" | head -1 | awk '{print $3}' || echo "") + if [[ -n "$vg_name" ]]; then + echo + log "=== Volume Group Status ===" + vgdisplay "$vg_name" | grep -E "(VG Size|Free.*Size)" + fi + fi - # Check if /var is now larger than 50GB + # Calculate expansion success + local root_size_gb=$(df / | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') local var_size_gb=$(df /var | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') - if [[ $var_size_gb -gt 50 ]]; then - success "Disk expansion successful! /var is now ${var_size_gb}GB" - else - error "Disk expansion may have failed - /var is still only ${var_size_gb}GB" - fi + + success "Disk expansion completed!" + success "Root (/) filesystem: ${root_size_gb}GB" + success "/var filesystem: ${var_size_gb}GB" } # Main execution main() { log "LME RHEL Disk Expansion Script Starting..." - log "This script will expand your disk partitions to use full available space" + log "This script will double the root partition and allocate remaining space to /var" # Check if expansion is needed if ! check_disk_space; then - log "Disk expansion not needed - exiting" - exit 0 + if [[ "${1:-}" != "--force" ]]; then + log "Disk expansion may not be needed - use --force to proceed anyway" + exit 0 + fi fi # Confirm with user unless --yes flag is provided - if [[ "${1:-}" != "--yes" ]]; then + if [[ "${1:-}" != "--yes" ]] && [[ "${1:-}" != "--force" ]]; then echo warning "This script will modify your disk partitions." + warning "It will double the root partition size and allocate remaining space to /var." warning "While these operations are generally safe, always ensure you have backups." echo read -p "Do you want to continue? [y/N]: " -n 1 -r @@ -181,7 +386,7 @@ main() { verify_expansion echo success "=== DISK EXPANSION COMPLETED SUCCESSFULLY ===" - success "Your system now has significantly more space for LME containers and data" + success "Root partition has been doubled and /var has been allocated remaining space" echo log "You can now proceed with LME installation" else @@ -191,11 +396,14 @@ main() { # Script usage show_usage() { - echo "Usage: $0 [--yes]" - echo " --yes Skip confirmation prompts (for automation)" + echo "Usage: $0 [--yes|--force]" + echo " --yes Skip confirmation prompts (for automation)" + echo " --force Force expansion even if not deemed necessary" echo - echo "This script expands RHEL disk partitions to use all available space." - echo "Specifically designed for Azure VMs where auto-partitioning is conservative." + echo "This script expands RHEL disk partitions:" + echo "- Doubles the root (/) partition size" + echo "- Allocates remaining space to /var" + echo "- Works with both LVM and direct partitions" } # Handle command line arguments @@ -204,8 +412,8 @@ case "${1:-}" in show_usage exit 0 ;; - --yes) - main --yes + --yes|--force) + main "$1" ;; "") main From e3b942da7c08dc1ef826f7c0cdad5e8ab74dbf3c Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 20 Aug 2025 11:32:21 +0000 Subject: [PATCH 15/51] Adds SELinux context fixes for Nix store packages --- ansible/roles/nix/tasks/redhat.yml | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index a533c116..b0ac0ca0 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -77,6 +77,30 @@ become: yes when: not nix_installed.stat.exists +- name: Fix SELinux contexts on core Nix paths (after installation) + command: restorecon -Rv "{{ item }}" + become: yes + changed_when: false + failed_when: false + loop: + - "/nix/var/nix/profiles/" + - "/nix/var/nix/daemon-socket/" + when: + - not nix_installed.stat.exists + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +# TODO: We may or may not need this. +# - name: Fix nix-daemon service file documentation reference (prevents 'bad' status) +# lineinfile: +# path: "/nix/var/nix/profiles/default/lib/systemd/system/nix-daemon.service" +# regexp: '^Documentation=man:nix-daemon' +# line: 'Documentation=https://nixos.org/manual' +# backup: no +# become: yes +# when: not nix_installed.stat.exists +# failed_when: false + - name: Ensure nix-daemon service is started systemd: name: nix-daemon @@ -126,6 +150,18 @@ environment: PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" +- name: Fix SELinux contexts after package installation (covers newly installed packages) + command: restorecon -Rv "{{ item }}" + become: yes + changed_when: false + failed_when: false + loop: + - "/nix/var/nix/profiles/" + - "/nix/store/" + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - name: After Podman installation, apply SELinux booleans seboolean: name: "{{ item }}" From b823d7ab1a443af5fe4b28bb805c9921005b5748 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 20 Aug 2025 12:29:49 +0000 Subject: [PATCH 16/51] Load the SELinux policy for Podman Quadlet and restorecon the files --- ansible/roles/podman/tasks/quadlet_setup.yml | 35 ++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/ansible/roles/podman/tasks/quadlet_setup.yml b/ansible/roles/podman/tasks/quadlet_setup.yml index 31189c30..da748d66 100644 --- a/ansible/roles/podman/tasks/quadlet_setup.yml +++ b/ansible/roles/podman/tasks/quadlet_setup.yml @@ -69,17 +69,7 @@ failed_when: false become: yes -- name: Reload systemd daemon - systemd: - daemon_reload: yes - become: yes - -- name: Start LME service - systemd: - name: lme.service - state: started - enabled: yes - become: yes +# (moved) systemd reload/start to after SELinux module load and restorecon # --------------------------------------------------------------------------- # Podman SELinux policy: deploy, compile, load, and label after Podman is set up @@ -134,9 +124,28 @@ - (getenforce_out.stdout | default('') | trim) != 'Disabled' - name: Restore contexts on quadlet/generator paths - command: restorecon -Rv /usr/libexec/podman/quadlet /usr/lib/systemd/system-generators/podman-system-generator + command: restorecon -Rv \ + /usr/libexec/podman/quadlet \ + /usr/lib/systemd/system-generators/podman-system-generator \ + /usr/lib/systemd/system-generators \ + /etc/systemd/system-generators \ + /etc/containers/systemd \ + /nix/var/nix/profiles/default \ + /nix/store changed_when: false become: yes when: - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' \ No newline at end of file + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Reload systemd daemon + systemd: + daemon_reload: yes + become: yes + +- name: Start LME service + systemd: + name: lme.service + state: started + enabled: yes + become: yes \ No newline at end of file From f6521f73500ca19beb9b0cf6e98f2e376a42127b Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 20 Aug 2025 13:40:16 +0000 Subject: [PATCH 17/51] Reorder SELinux tasks to ensure proper context restoration --- ansible/roles/nix/tasks/main.yml | 64 +++------------ ansible/roles/nix/tasks/redhat.yml | 83 ++++++++++++++++++-- ansible/roles/podman/tasks/main.yml | 72 +++++++++++++++++ ansible/roles/podman/tasks/quadlet_setup.yml | 73 +++++------------ 4 files changed, 177 insertions(+), 115 deletions(-) diff --git a/ansible/roles/nix/tasks/main.yml b/ansible/roles/nix/tasks/main.yml index df4c395a..9c5bf696 100644 --- a/ansible/roles/nix/tasks/main.yml +++ b/ansible/roles/nix/tasks/main.yml @@ -71,11 +71,15 @@ force: yes become: yes -- name: Restore contexts on podman symlinks (best-effort) +# Apply contexts to podman symlinks after they're created +- name: Apply SELinux contexts to podman symlinks (post-symlink) command: restorecon -v /usr/local/bin/podman /usr/bin/podman changed_when: false failed_when: false become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' - name: Source updated PATH shell: source ~/.profile @@ -83,75 +87,29 @@ executable: /bin/bash # --------------------------------------------------------------------------- -# Nix SELinux policy: deploy, compile, load, and label after Nix is installed +# Final SELinux context restoration (post-install cleanup) # --------------------------------------------------------------------------- -- name: Deploy Nix SELinux policy files - copy: - src: "{{ clone_directory }}/ansible/roles/base/files/selinux/nix_policy.te" - dest: /etc/selinux/lme/nix_policy.te - owner: root - group: root - mode: '0644' - become: yes - when: - - selinux_available | default(false) - -- name: Deploy Nix SELinux file contexts - copy: - src: "{{ clone_directory }}/ansible/roles/base/files/selinux/nix_policy.fc" - dest: /etc/selinux/lme/nix_policy.fc - owner: root - group: root - mode: '0644' - become: yes - when: - - selinux_available | default(false) - -- name: Compile Nix SELinux module - shell: | - set -e - cd /etc/selinux/lme - checkmodule -M -m -o nix_policy.mod nix_policy.te - semodule_package -o nix_policy.pp -m nix_policy.mod -f nix_policy.fc - args: - executable: /bin/bash - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Load Nix SELinux module - command: semodule -i /etc/selinux/lme/nix_policy.pp - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Ensure Nix SELinux module enabled - command: semodule -e nix_policy - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - - name: Check if nix profile exists stat: path: "{{ nix_profile_symlink_path }}" register: nix_profile_exists -- name: Restore context for nix profile (post-install) +# Final restorecon: Ensure all contexts are correct after all operations +- name: Final SELinux context restoration for nix profile (post-install #3) command: restorecon -Rv "{{ nix_profile_symlink_path }}" become: yes changed_when: false + failed_when: false when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' - nix_profile_exists.stat.exists -- name: Restore context for Nix daemon socket directory +- name: Final SELinux context restoration for daemon socket (post-install #3) command: restorecon -Rv /nix/var/nix/daemon-socket become: yes changed_when: false + failed_when: false when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' \ No newline at end of file diff --git a/ansible/roles/nix/tasks/redhat.yml b/ansible/roles/nix/tasks/redhat.yml index b0ac0ca0..b0146b15 100644 --- a/ansible/roles/nix/tasks/redhat.yml +++ b/ansible/roles/nix/tasks/redhat.yml @@ -29,6 +29,58 @@ when: selinux_was_enforcing become: yes +# --------------------------------------------------------------------------- +# Deploy and load Nix SELinux policy BEFORE installation +# --------------------------------------------------------------------------- +- name: Deploy Nix SELinux policy files (pre-install) + copy: + src: "{{ clone_directory | default(playbook_dir + '/../..') }}/ansible/roles/base/files/selinux/nix_policy.te" + dest: /etc/selinux/lme/nix_policy.te + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Deploy Nix SELinux file contexts (pre-install) + copy: + src: "{{ clone_directory | default(playbook_dir + '/../..') }}/ansible/roles/base/files/selinux/nix_policy.fc" + dest: /etc/selinux/lme/nix_policy.fc + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Compile Nix SELinux module (pre-install) + shell: | + set -e + cd /etc/selinux/lme + checkmodule -M -m -o nix_policy.mod nix_policy.te + semodule_package -o nix_policy.pp -m nix_policy.mod -f nix_policy.fc + args: + executable: /bin/bash + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Load Nix SELinux module (pre-install) + command: semodule -i /etc/selinux/lme/nix_policy.pp + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Ensure Nix SELinux module enabled (pre-install) + command: semodule -e nix_policy + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - name: Check if Nix is already installed stat: path: /nix/var/nix/profiles/default/bin/nix @@ -71,13 +123,9 @@ var: nix_install_result when: debug_mode | default(false) and not nix_installed.stat.exists -- name: Reload systemd units after Nix installation - systemd: - daemon_reload: yes - become: yes - when: not nix_installed.stat.exists -- name: Fix SELinux contexts on core Nix paths (after installation) +# First restorecon: Apply contexts to basic Nix structure immediately after install +- name: Apply SELinux contexts to core Nix paths (post-install #1) command: restorecon -Rv "{{ item }}" become: yes changed_when: false @@ -85,6 +133,20 @@ loop: - "/nix/var/nix/profiles/" - "/nix/var/nix/daemon-socket/" + - "/nix/store/" + when: + - not nix_installed.stat.exists + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +# Apply contexts to systemd service files before starting services +- name: Apply SELinux contexts to systemd service files + command: restorecon -Rv "{{ item }}" + become: yes + changed_when: false + failed_when: false + loop: + - "/nix/var/nix/profiles/default/lib/systemd/system/" when: - not nix_installed.stat.exists - selinux_available | default(false) @@ -101,6 +163,12 @@ # when: not nix_installed.stat.exists # failed_when: false +- name: Reload systemd units after Nix installation + systemd: + daemon_reload: yes + become: yes + when: not nix_installed.stat.exists + - name: Ensure nix-daemon service is started systemd: name: nix-daemon @@ -150,7 +218,8 @@ environment: PATH: "{{ ansible_env.PATH }}:/nix/var/nix/profiles/default/bin" -- name: Fix SELinux contexts after package installation (covers newly installed packages) +# Second restorecon: Apply contexts to newly installed packages +- name: Apply SELinux contexts after package installation (post-install #2) command: restorecon -Rv "{{ item }}" become: yes changed_when: false diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml index 468d247a..a9fb8079 100644 --- a/ansible/roles/podman/tasks/main.yml +++ b/ansible/roles/podman/tasks/main.yml @@ -15,6 +15,26 @@ - "{{ ansible_os_family | lower }}.yml" - "common.yml" +# Set up SELinux detection variables (needed for pre-install module loading) +- name: Detect if SELinux tooling is available + command: which getenforce + register: selinux_tooling + changed_when: false + failed_when: false + become: yes + +- name: Set SELinux availability fact + set_fact: + selinux_available: "{{ selinux_tooling.rc == 0 }}" + +- name: Get current SELinux mode + command: getenforce + register: getenforce_out + changed_when: false + failed_when: false + become: yes + when: selinux_available | default(false) + # These tasks are common for all distributions - name: Check if Nix daemon unit exists shell: systemctl cat "{{ nix_daemon_service }}" >/dev/null 2>&1 || systemctl list-unit-files | grep -E "^{{ nix_daemon_service }}(\\.service)?\\s" >/dev/null 2>&1 @@ -74,6 +94,58 @@ timeout: 10 when: ansible_play_hosts_all.index(inventory_hostname) == 0 +# --------------------------------------------------------------------------- +# Deploy and load Podman SELinux policy BEFORE installation +# --------------------------------------------------------------------------- +- name: Deploy Podman SELinux policy files (pre-install) + copy: + src: "{{ clone_directory | default(playbook_dir + '/../..') }}/ansible/roles/base/files/selinux/podman_policy.te" + dest: /etc/selinux/lme/podman_policy.te + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Deploy Podman SELinux file contexts (pre-install) + copy: + src: "{{ clone_directory | default(playbook_dir + '/../..') }}/ansible/roles/base/files/selinux/podman_policy.fc" + dest: /etc/selinux/lme/podman_policy.fc + owner: root + group: root + mode: '0644' + become: yes + when: + - selinux_available | default(false) + +- name: Compile Podman SELinux module (pre-install) + shell: | + set -e + cd /etc/selinux/lme + checkmodule -M -m -o podman_policy.mod podman_policy.te + semodule_package -o podman_policy.pp -m podman_policy.mod -f podman_policy.fc + args: + executable: /bin/bash + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Load Podman SELinux module (pre-install) + command: semodule -i /etc/selinux/lme/podman_policy.pp + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + +- name: Ensure Podman SELinux module enabled (pre-install) + command: semodule -e podman_policy + become: yes + when: + - selinux_available | default(false) + - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - name: Install Podman using Nix command: nix-env -iA nixpkgs.podman become: yes diff --git a/ansible/roles/podman/tasks/quadlet_setup.yml b/ansible/roles/podman/tasks/quadlet_setup.yml index da748d66..e0c46e00 100644 --- a/ansible/roles/podman/tasks/quadlet_setup.yml +++ b/ansible/roles/podman/tasks/quadlet_setup.yml @@ -69,71 +69,34 @@ failed_when: false become: yes -# (moved) systemd reload/start to after SELinux module load and restorecon - # --------------------------------------------------------------------------- -# Podman SELinux policy: deploy, compile, load, and label after Podman is set up +# Apply SELinux contexts after podman installation and symlink creation # --------------------------------------------------------------------------- -- name: Deploy Podman SELinux policy files - copy: - src: "{{ clone_directory }}/ansible/roles/base/files/selinux/podman_policy.te" - dest: /etc/selinux/lme/podman_policy.te - owner: root - group: root - mode: '0644' - become: yes - when: - - selinux_available | default(false) -- name: Deploy Podman SELinux file contexts - copy: - src: "{{ clone_directory }}/ansible/roles/base/files/selinux/podman_policy.fc" - dest: /etc/selinux/lme/podman_policy.fc - owner: root - group: root - mode: '0644' - become: yes - when: - - selinux_available | default(false) - -- name: Compile Podman SELinux module - shell: | - set -e - cd /etc/selinux/lme - checkmodule -M -m -o podman_policy.mod podman_policy.te - semodule_package -o podman_policy.pp -m podman_policy.mod -f podman_policy.fc - args: - executable: /bin/bash - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Load Podman SELinux module - command: semodule -i /etc/selinux/lme/podman_policy.pp - become: yes - when: - - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' - -- name: Ensure Podman SELinux module enabled - command: semodule -e podman_policy +# Apply SELinux contexts to paths (now that podman_policy module defines the contexts) +- name: Apply SELinux contexts to system generator paths (if they exist) + command: restorecon -Rv "{{ item }}" + changed_when: false + failed_when: false become: yes + loop: + - "/usr/libexec/podman/quadlet" + - "/usr/lib/systemd/system-generators/podman-system-generator" + - "/usr/lib/systemd/system-generators" + - "/etc/systemd/system-generators" # May be created by systemd after module load + - "/etc/containers/systemd" when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' -- name: Restore contexts on quadlet/generator paths - command: restorecon -Rv \ - /usr/libexec/podman/quadlet \ - /usr/lib/systemd/system-generators/podman-system-generator \ - /usr/lib/systemd/system-generators \ - /etc/systemd/system-generators \ - /etc/containers/systemd \ - /nix/var/nix/profiles/default \ - /nix/store +- name: Apply SELinux contexts to Nix paths (final podman restorecon) + command: restorecon -Rv "{{ item }}" changed_when: false + failed_when: false become: yes + loop: + - "/nix/var/nix/profiles/default" + - "/nix/store" when: - selinux_available | default(false) - (getenforce_out.stdout | default('') | trim) != 'Disabled' From 62bbe474dab99d81572574a95c469aba7385b683 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 21 Aug 2025 09:35:55 +0000 Subject: [PATCH 18/51] Adds RHEL 9.1 to the docker workflow --- .github/workflows/docker.yml | 249 +++++++++++++++++++++++++++++++++++ docker/rhel9/Dockerfile | 4 +- 2 files changed, 251 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f8ce6aa9..4a11fa25 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,8 @@ name: Docker Pipeline # sudo act --bind --workflows .github/workflows/docker.yml --job build-24-04 --secret-file .env # or # sudo act --bind --workflows .github/workflows/docker.yml --job build-d12-10 --secret-file .env +# or +# sudo act --bind --workflows .github/workflows/docker.yml --job build-rhel9 --secret-file .env on: workflow_dispatch: inputs: @@ -769,6 +771,253 @@ jobs: az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait " + - name: Stop and remove containers + if: always() + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} down + docker system prune -af + + build-rhel9: + runs-on: self-hosted + + env: + UNIQUE_ID: ${{ github.run_number }}_rhel9_${{ github.run_id }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + CONTAINER_TYPE: "rhel9" + AZURE_IP: "" + IP_ADDRESS: "" + + steps: + - name: Generate random number + shell: bash + run: | + RANDOM_NUM=$(shuf -i 1000000000-9999999999 -n 1) + echo "UNIQUE_ID=${RANDOM_NUM}_rhel9_${{ github.run_number }}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4.1.1 + + - name: Get branch name + shell: bash + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV + else + echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV + fi + + - name: Set the environment for docker compose + run: | + cd testing/v2/development + echo "HOST_UID=$(id -u)" > .env + echo "HOST_GID=$(id -g)" >> .env + echo "HOST_IP=10.1.0.5" >> .env + PUBLIC_IP=$(curl -s https://api.ipify.org) + echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV + + + - name: Start pipeline container + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline + + - name: Install Python requirements + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers/azure && \ + pip install -r requirements.txt + " + + - name: Build an Azure instance + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T \ + -e AZURE_CLIENT_ID \ + -e AZURE_CLIENT_SECRET \ + -e AZURE_TENANT_ID \ + -e AZURE_SUBSCRIPTION_ID \ + pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + python3 ./azure/build_azure_linux_network.py \ + -g pipe-${{ env.UNIQUE_ID }} \ + -s 0.0.0.0/0 \ + -vs Standard_B4s_v2 \ + -l ${{ inputs.azure_region || 'centralus' }} \ + -ast 23:00 \ + -y + " + #-s ${{ env.IP_ADDRESS }}/32 \ + + - name: Retrieve Azure IP + run: | + cd testing/v2/development + AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt") + echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV + echo "Azure IP: $AZURE_IP" + echo "Azure IP retrieved successfully" + + - name: Retrieve Azure Password + run: | + cd testing/v2/development + AZURE_PASS=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.password.txt") + echo "AZURE_PASS=$AZURE_PASS" >> $GITHUB_ENV + echo "Azure Password retrieved successfully" + + # wait for the azure instance to be ready + - name: Wait for Azure instance to be ready + run: | + sleep 30 + + - name: Copy SSH Key to Azure instance + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + ./lib/copy_ssh_key.sh lme-user ${{ env.AZURE_IP }} /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.password.txt + " + + - name: Clone repository on Azure instance + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + if [ ! -d LME ]; then + git clone https://github.com/cisagov/LME.git; + fi + cd LME + if [ \"${{ env.BRANCH_NAME }}\" != \"main\" ]; then + git fetch + git checkout ${{ env.BRANCH_NAME }} + fi + ' + " + + - name: Install Docker on Azure instance + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS 'chmod +x ~/LME/docker/install_latest_docker_in_ubuntu.sh && \ + sudo ~/LME/docker/install_latest_docker_in_ubuntu.sh && \ + sudo usermod -aG docker \$USER && \ + sudo systemctl enable docker && \ + sudo systemctl start docker' + " + + - name: Install test prerequisites on Azure instance + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd LME/testing/tests && \ + sudo apt-get update && \ + sudo apt-get install -y python3.10-venv && \ + wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + sudo apt install -y ./google-chrome-stable_current_amd64.deb && \ + python3 -m venv venv && \ + source venv/bin/activate && \ + pip install -r requirements.txt + ' + " + + - name: Test Docker container + run: | + cd testing/v2/development + + # Set environment + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd ~/LME/docker/${{ env.CONTAINER_TYPE }} + echo \"HOST_IP=10.1.0.5\" > .env + ' + " + + # Build container + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd ~/LME/docker/${{ env.CONTAINER_TYPE }} + sudo docker compose up -d + ' + " + + # Deploy LME + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd ~/LME/docker/${{ env.CONTAINER_TYPE }} + sudo docker compose exec -T lme bash -c \"NON_INTERACTIVE=true AUTO_CREATE_ENV=true /root/LME/install.sh -i 10.1.0.5 -d\" + ' + " + + # Extract passwords + ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd ~/LME/docker/${{ env.CONTAINER_TYPE }} + SECRETS=\$(sudo docker compose exec -T lme bash -c \". ~/LME/scripts/extract_secrets.sh -p\") + echo \"\$SECRETS\" | grep \"^elastic=\" | cut -d= -f2- + ' + ") + + KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd ~/LME/docker/${{ env.CONTAINER_TYPE }} + SECRETS=\$(sudo docker compose exec -T lme bash -c \". ~/LME/scripts/extract_secrets.sh -p\") + echo \"\$SECRETS\" | grep \"^kibana_system=\" | cut -d= -f2- + ' + ") + + # Run tests + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + sleep 360 + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS ' + cd ~/LME/testing/tests + echo \"Container: ${{ env.CONTAINER_TYPE }}\" > .env + echo \"ELASTIC_PASSWORD=$ES_PASSWORD\" >> .env + echo \"KIBANA_PASSWORD=$KIBANA_PASSWORD\" >> .env + echo \"elastic=$ES_PASSWORD\" >> .env + source venv/bin/activate + echo \"Running tests for container ${{ env.CONTAINER_TYPE }}\" + pytest -v api_tests/linux_only/ selenium_tests/linux_only/ + ' + " + + - name: Cleanup Azure resources + if: always() + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_SECRET: ${{ secrets.AZURE_SECRET }} + AZURE_TENANT: ${{ secrets.AZURE_TENANT }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_SECRET --tenant $AZURE_TENANT + az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait + " + - name: Stop and remove containers if: always() run: | diff --git a/docker/rhel9/Dockerfile b/docker/rhel9/Dockerfile index b8c1d554..a421e96a 100644 --- a/docker/rhel9/Dockerfile +++ b/docker/rhel9/Dockerfile @@ -9,7 +9,7 @@ ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 RUN dnf update -y && dnf install -y \ - glibc-langpack-en sudo openssh-clients \ + glibc-langpack-en sudo openssh-clients git \ && dnf clean all \ && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ && while getent passwd $USER_ID > /dev/null 2>&1; do USER_ID=$((USER_ID + 1)); done \ @@ -60,4 +60,4 @@ RUN cd /lib/systemd/system/sysinit.target.wants/ && \ ## Enable the service #RUN systemctl enable lme-setup.service -CMD ["/lib/systemd/systemd"] \ No newline at end of file +CMD ["/lib/systemd/systemd"] From abcd1dc79a502ec47425dfcb1cfe6c8bfefae52c Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 09:52:54 +0000 Subject: [PATCH 19/51] Updates the expand_rhel_disk.sh script to double the root partition size and double the /home partition size. --- scripts/expand_rhel_disk.sh | 183 ++++++++++++++++++++++++++++++------ 1 file changed, 154 insertions(+), 29 deletions(-) diff --git a/scripts/expand_rhel_disk.sh b/scripts/expand_rhel_disk.sh index b33a0a5e..abd1f71f 100755 --- a/scripts/expand_rhel_disk.sh +++ b/scripts/expand_rhel_disk.sh @@ -2,7 +2,7 @@ # LME Disk Expansion Script for RHEL Systems # This script fixes the common issue where RHEL auto-partitioning doesn't use the full disk -# Doubles the root partition size and allocates remaining space to /var +# Doubles the root partition size, doubles the /home partition size, and allocates remaining space to /var set -euo pipefail @@ -41,15 +41,18 @@ check_disk_space() { local disk="/dev/sda" local root_usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//') local root_size=$(df -h / | awk 'NR==2 {print $2}') + local home_usage=$(df /home 2>/dev/null | awk 'NR==2 {print $5}' | sed 's/%//' || echo "N/A") + local home_size=$(df -h /home 2>/dev/null | awk 'NR==2 {print $2}' || echo "N/A") local var_usage=$(df /var | awk 'NR==2 {print $5}' | sed 's/%//') local var_size=$(df -h /var | awk 'NR==2 {print $2}') log "Current / filesystem: ${root_size} (${root_usage}% used)" + log "Current /home filesystem: ${home_size} (${home_usage}% used)" log "Current /var filesystem: ${var_size} (${var_usage}% used)" # Check disk space usage vs total disk size local disk_size_bytes=$(lsblk -b -n -o SIZE /dev/sda | head -1) - local used_space_bytes=$(df --output=used / /var | tail -n +2 | awk '{sum += $1} END {print sum * 1024}') + local used_space_bytes=$(df --output=used / /home /var 2>/dev/null | tail -n +2 | awk '{sum += $1} END {print sum * 1024}') local usage_percent=$((used_space_bytes * 100 / disk_size_bytes)) if [[ $usage_percent -lt 80 ]]; then @@ -198,11 +201,15 @@ expand_with_lvm() { return 1 fi - # Determine logical volumes for / and /var + # Determine logical volumes for /, /home, and /var local root_lv=$(findmnt -n -o SOURCE / 2>/dev/null || true) if [[ -z "$root_lv" ]]; then root_lv="/dev/${vg_name}/rootlv" fi + local home_lv=$(findmnt -n -o SOURCE /home 2>/dev/null || true) + if [[ -n "$home_lv" ]] && [[ "$home_lv" == "$root_lv" ]]; then + home_lv="" + fi local var_lv=$(findmnt -n -o SOURCE /var 2>/dev/null || true) if [[ -n "$var_lv" ]] && [[ "$var_lv" == "$root_lv" ]]; then var_lv="" @@ -214,18 +221,41 @@ expand_with_lvm() { error "Unable to determine current root LV size for $root_lv" fi - # Target: double root size, remainder to /var + # Get current /home LV size (in MB) if it exists + local current_home_mb=0 + if [[ -n "$home_lv" ]] && [[ -e "$home_lv" ]]; then + current_home_mb=$(lvs --noheadings --units m --nosuffix -o LV_SIZE "$home_lv" 2>/dev/null | awk '{print int($1)}') + if [[ -z "$current_home_mb" ]]; then + current_home_mb=0 + fi + fi + + # Target: double root size, double /home size (if exists), remainder to /var local desired_root_mb=$((current_root_mb * 2)) - local required_increase_mb=$((desired_root_mb - current_root_mb)) - if [[ "$required_increase_mb" -le 0 ]]; then - required_increase_mb=0 + local required_root_increase_mb=$((desired_root_mb - current_root_mb)) + if [[ "$required_root_increase_mb" -le 0 ]]; then + required_root_increase_mb=0 + fi + + local desired_home_mb=$((current_home_mb * 2)) + local required_home_increase_mb=$((desired_home_mb - current_home_mb)) + if [[ "$required_home_increase_mb" -le 0 ]]; then + required_home_increase_mb=0 fi - local root_expand_mb=$required_increase_mb - if [[ "$root_expand_mb" -gt "$total_free_mb" ]]; then - root_expand_mb=$total_free_mb + # Calculate actual allocations based on available space + local root_expand_mb=$required_root_increase_mb + local home_expand_mb=$required_home_increase_mb + local total_required_mb=$((root_expand_mb + home_expand_mb)) + + if [[ "$total_required_mb" -gt "$total_free_mb" ]]; then + # Scale down proportionally if not enough space + local scale_factor=$(echo "scale=4; $total_free_mb / $total_required_mb" | bc -l) + root_expand_mb=$(echo "scale=0; $root_expand_mb * $scale_factor / 1" | bc) + home_expand_mb=$(echo "scale=0; $home_expand_mb * $scale_factor / 1" | bc) fi - local var_expand_mb=$((total_free_mb - root_expand_mb)) + + local var_expand_mb=$((total_free_mb - root_expand_mb - home_expand_mb)) # Extend root logical volume log "Extending root logical volume ($root_lv) by ${root_expand_mb}MB..." @@ -245,6 +275,38 @@ expand_with_lvm() { success "Root filesystem extended" fi + # Extend /home logical volume if present + if [[ -n "$home_lv" ]] && [[ -e "$home_lv" ]] && [[ $home_expand_mb -gt 0 ]]; then + log "Extending /home logical volume ($home_lv) by ${home_expand_mb}MB..." + lvextend -L "+${home_expand_mb}m" "$home_lv" + + # Grow /home filesystem + log "Growing /home filesystem..." + if [[ "$(findmnt -n -o FSTYPE /home)" == "xfs" ]]; then + xfs_growfs /home + else + resize2fs "$home_lv" + fi + success "/home filesystem extended" + elif [[ $current_home_mb -eq 0 ]] && [[ $home_expand_mb -gt 0 ]]; then + # Create new /home LV if it doesn't exist and we have space allocated for it + log "Creating new /home logical volume (${home_expand_mb}MB)..." + lvcreate -L "${home_expand_mb}m" -n homelv "$vg_name" + local new_home_lv="/dev/${vg_name}/homelv" + + # Format the new /home LV + log "Formatting new /home logical volume..." + mkfs.ext4 -F "$new_home_lv" + + success "New /home logical volume created" + warning "Manual steps required for /home:" + warning "1. Backup current /home: cp -a /home /home.backup" + warning "2. Mount new LV: mount ${new_home_lv} /mnt" + warning "3. Copy /home data: cp -a /home.backup/* /mnt/" + warning "4. Update /etc/fstab to mount ${new_home_lv} at /home" + warning "5. Reboot to activate new /home mount" + fi + # Extend /var logical volume if present if [[ -n "$var_lv" ]] && [[ -e "$var_lv" ]] && [[ $var_expand_mb -gt 0 ]]; then log "Extending /var logical volume ($var_lv) by ${var_expand_mb}MB..." @@ -293,24 +355,80 @@ expand_without_lvm() { fi success "Root partition and filesystem expanded" - # Create new partition for /var if there's remaining space + # Calculate space allocation: half of remaining space to /home, half to /var local remaining_space_mb=$(($(echo "$var_end" | sed 's/MB//') - $(echo "$var_start" | sed 's/MB//'))) - if [[ $remaining_space_mb -gt 1000 ]]; then # Only if more than 1GB - log "Creating new partition for /var expansion..." - local new_var_part_num=$((root_part_num + 1)) - parted --script "$disk" mkpart primary ext4 "$var_start" "$var_end" + if [[ $remaining_space_mb -gt 2000 ]]; then # Only if more than 2GB total + local home_space_mb=$((remaining_space_mb / 2)) + local var_space_mb=$((remaining_space_mb - home_space_mb)) + + local home_start="$var_start" + local home_end_mb=$(($(echo "$var_start" | sed 's/MB//') + home_space_mb)) + local home_end="${home_end_mb}MB" + local new_var_start="${home_end_mb}MB" + + # Create new partition for /home + log "Creating new partition for /home expansion (${home_space_mb}MB)..." + local new_home_part_num=$((root_part_num + 1)) + parted --script "$disk" mkpart primary ext4 "$home_start" "$home_end" + + # Format the new /home partition + local new_home_partition="/dev/sda${new_home_part_num}" + log "Formatting new /home partition ${new_home_partition}..." + mkfs.ext4 -F "$new_home_partition" - # Format the new partition - local new_var_partition="/dev/sda${new_var_part_num}" - log "Formatting new partition ${new_var_partition}..." - mkfs.ext4 -F "$new_var_partition" + success "New partition created for /home expansion" - success "New partition created for /var expansion" - warning "Manual steps required:" - warning "1. Mount new partition: mount ${new_var_partition} /mnt" - warning "2. Copy /var data: cp -a /var/* /mnt/" - warning "3. Update /etc/fstab to mount ${new_var_partition} at /var" - warning "4. Reboot to activate new /var mount" + # Create new partition for /var with remaining space + if [[ $var_space_mb -gt 1000 ]]; then # Only if more than 1GB + log "Creating new partition for /var expansion (${var_space_mb}MB)..." + local new_var_part_num=$((new_home_part_num + 1)) + parted --script "$disk" mkpart primary ext4 "$new_var_start" "$var_end" + + # Format the new /var partition + local new_var_partition="/dev/sda${new_var_part_num}" + log "Formatting new /var partition ${new_var_partition}..." + mkfs.ext4 -F "$new_var_partition" + + success "New partition created for /var expansion" + + warning "Manual steps required:" + warning "For /home:" + warning "1. Backup current /home: cp -a /home /home.backup" + warning "2. Mount new /home partition: mount ${new_home_partition} /mnt" + warning "3. Copy /home data: cp -a /home.backup/* /mnt/" + warning "4. Update /etc/fstab to mount ${new_home_partition} at /home" + warning "For /var:" + warning "5. Mount new /var partition: mount ${new_var_partition} /mnt2" + warning "6. Copy /var data: cp -a /var/* /mnt2/" + warning "7. Update /etc/fstab to mount ${new_var_partition} at /var" + warning "8. Reboot to activate new mounts" + else + warning "Manual steps required for /home:" + warning "1. Backup current /home: cp -a /home /home.backup" + warning "2. Mount new partition: mount ${new_home_partition} /mnt" + warning "3. Copy /home data: cp -a /home.backup/* /mnt/" + warning "4. Update /etc/fstab to mount ${new_home_partition} at /home" + warning "5. Reboot to activate new /home mount" + fi + else + # If not enough space for both, just create /var partition as before + if [[ $remaining_space_mb -gt 1000 ]]; then # Only if more than 1GB + log "Creating new partition for /var expansion..." + local new_var_part_num=$((root_part_num + 1)) + parted --script "$disk" mkpart primary ext4 "$var_start" "$var_end" + + # Format the new partition + local new_var_partition="/dev/sda${new_var_part_num}" + log "Formatting new partition ${new_var_partition}..." + mkfs.ext4 -F "$new_var_partition" + + success "New partition created for /var expansion" + warning "Manual steps required:" + warning "1. Mount new partition: mount ${new_var_partition} /mnt" + warning "2. Copy /var data: cp -a /var/* /mnt/" + warning "3. Update /etc/fstab to mount ${new_var_partition} at /var" + warning "4. Reboot to activate new /var mount" + fi fi } @@ -327,6 +445,10 @@ verify_expansion() { log "=== Root Filesystem Status ===" df -h / + echo + log "=== /home Filesystem Status ===" + df -h /home + echo log "=== /var Filesystem Status ===" df -h /var @@ -343,17 +465,19 @@ verify_expansion() { # Calculate expansion success local root_size_gb=$(df / | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') + local home_size_gb=$(df /home 2>/dev/null | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}' || echo "N/A") local var_size_gb=$(df /var | awk 'NR==2 {print $2}' | awk '{print int($1/1024/1024)}') success "Disk expansion completed!" success "Root (/) filesystem: ${root_size_gb}GB" + success "/home filesystem: ${home_size_gb}GB" success "/var filesystem: ${var_size_gb}GB" } # Main execution main() { log "LME RHEL Disk Expansion Script Starting..." - log "This script will double the root partition and allocate remaining space to /var" + log "This script will double the root partition, double the /home partition, and allocate remaining space to /var" # Check if expansion is needed if ! check_disk_space; then @@ -367,7 +491,7 @@ main() { if [[ "${1:-}" != "--yes" ]] && [[ "${1:-}" != "--force" ]]; then echo warning "This script will modify your disk partitions." - warning "It will double the root partition size and allocate remaining space to /var." + warning "It will double the root partition size, double the /home partition size, and allocate remaining space to /var." warning "While these operations are generally safe, always ensure you have backups." echo read -p "Do you want to continue? [y/N]: " -n 1 -r @@ -386,7 +510,7 @@ main() { verify_expansion echo success "=== DISK EXPANSION COMPLETED SUCCESSFULLY ===" - success "Root partition has been doubled and /var has been allocated remaining space" + success "Root partition has been doubled, /home partition has been doubled, and /var has been allocated remaining space" echo log "You can now proceed with LME installation" else @@ -402,6 +526,7 @@ show_usage() { echo echo "This script expands RHEL disk partitions:" echo "- Doubles the root (/) partition size" + echo "- Doubles the /home partition size (if it exists)" echo "- Allocates remaining space to /var" echo "- Works with both LVM and direct partitions" } From 6e596a8f07a94cae4e2a5b1478ba112f7385d709 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 06:51:32 -0400 Subject: [PATCH 20/51] Rename Wazuh dashboard files: remove 'dumped' from filenames --- ...dent_response.dumped.ndjson => wazuh_incident_response.ndjson} | 0 ...are_detection.dumped.ndjson => wazuh_malware_detection.ndjson} | 0 ...security_events.dumped.ndjson => wazuh_security_events.ndjson} | 0 ...vulnerabilities.dumped.ndjson => wazuh_vulnerabilities.ndjson} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename dashboards/wazuh/{wazuh_incident_response.dumped.ndjson => wazuh_incident_response.ndjson} (100%) rename dashboards/wazuh/{wazuh_malware_detection.dumped.ndjson => wazuh_malware_detection.ndjson} (100%) rename dashboards/wazuh/{wazuh_security_events.dumped.ndjson => wazuh_security_events.ndjson} (100%) rename dashboards/wazuh/{wazuh_vulnerabilities.dumped.ndjson => wazuh_vulnerabilities.ndjson} (100%) diff --git a/dashboards/wazuh/wazuh_incident_response.dumped.ndjson b/dashboards/wazuh/wazuh_incident_response.ndjson similarity index 100% rename from dashboards/wazuh/wazuh_incident_response.dumped.ndjson rename to dashboards/wazuh/wazuh_incident_response.ndjson diff --git a/dashboards/wazuh/wazuh_malware_detection.dumped.ndjson b/dashboards/wazuh/wazuh_malware_detection.ndjson similarity index 100% rename from dashboards/wazuh/wazuh_malware_detection.dumped.ndjson rename to dashboards/wazuh/wazuh_malware_detection.ndjson diff --git a/dashboards/wazuh/wazuh_security_events.dumped.ndjson b/dashboards/wazuh/wazuh_security_events.ndjson similarity index 100% rename from dashboards/wazuh/wazuh_security_events.dumped.ndjson rename to dashboards/wazuh/wazuh_security_events.ndjson diff --git a/dashboards/wazuh/wazuh_vulnerabilities.dumped.ndjson b/dashboards/wazuh/wazuh_vulnerabilities.ndjson similarity index 100% rename from dashboards/wazuh/wazuh_vulnerabilities.dumped.ndjson rename to dashboards/wazuh/wazuh_vulnerabilities.ndjson From a3b7be2b73af16c469a5cc43dd93191ad51fa216 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 11:14:12 +0000 Subject: [PATCH 21/51] Adds a new installer for RHEL 9 --- .../v2/installers/install_v2/install_rhel.sh | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 testing/v2/installers/install_v2/install_rhel.sh diff --git a/testing/v2/installers/install_v2/install_rhel.sh b/testing/v2/installers/install_v2/install_rhel.sh new file mode 100755 index 00000000..68a9b9e9 --- /dev/null +++ b/testing/v2/installers/install_v2/install_rhel.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -e + +# Check if the required arguments are provided +if [ $# -lt 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Set the remote server details from the command-line arguments +user=$1 +hostname=$2 +password_file=$3 +branch=$4 + +# Store the original working directory +ORIGINAL_DIR="$(pwd)" + +# Get the directory of the script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Change to the parent directory of the script +cd "$SCRIPT_DIR/.." + +# Copy the SSH key to the remote machine +./lib/copy_ssh_key.sh $user $hostname $password_file + +echo "Checking OS version" +ssh -o StrictHostKeyChecking=no $user@$hostname 'cat /etc/os-release' + +echo "Updating apt" +ssh -o StrictHostKeyChecking=no $user@$hostname 'sudo dnf -y install git' + +echo "Checking out code" +ssh -o StrictHostKeyChecking=no $user@$hostname "cd ~ && rm -rf LME && git clone https://github.com/cisagov/LME.git" +if [ "${branch}" != "main" ]; then + ssh -o StrictHostKeyChecking=no $user@$hostname " + cd ~/LME && + git fetch --all --tags && + if git show-ref --tags --verify --quiet \"refs/tags/${branch}\"; then + echo \"Checking out tag: ${branch}\" + git checkout ${branch} + else + echo \"Checking out branch: ${branch}\" + git checkout -t origin/${branch} + fi + " +fi +echo "Code cloned to $HOME/LME" + +echo "Expanding disks" +ssh -o StrictHostKeyChecking=no $user@$hostname "cd ~/LME && sudo ./scripts/expand_rhel_disk.sh" + +echo "Running LME installer" +ssh -o StrictHostKeyChecking=no $user@$hostname "export NON_INTERACTIVE=true && export AUTO_CREATE_ENV=true && export AUTO_IP=10.1.0.5 && cd ~/LME && ./install.sh --debug" + +echo "Installation and configuration completed successfully." + +# Change back to the original directory +cd "$ORIGINAL_DIR" \ No newline at end of file From 88174872439d0eb6bb28be118b21ecdcbd0b1547 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 11:20:42 +0000 Subject: [PATCH 22/51] Updates the RHEL 9 pipeline to use the new installer --- .github/workflows/linux_only_redhat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_only_redhat.yml b/.github/workflows/linux_only_redhat.yml index 456dbdc6..ec5bd0ff 100644 --- a/.github/workflows/linux_only_redhat.yml +++ b/.github/workflows/linux_only_redhat.yml @@ -101,7 +101,7 @@ jobs: ls -la && \ cd /home/lme-user/LME/testing/v2/installers && \ IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ - ./install_v2/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" ${{ env.BRANCH_NAME }} + ./install_v2/install_rhel.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" ${{ env.BRANCH_NAME }} " - name: Retrieve Elastic password From 8184a87b41fccc46c5688114830b79e9d5f2b187 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 11:37:23 +0000 Subject: [PATCH 23/51] Updates the RHEL 9 installer to use the --yes flag for the expand_rhel_disk.sh script --- testing/v2/installers/install_v2/install_rhel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/v2/installers/install_v2/install_rhel.sh b/testing/v2/installers/install_v2/install_rhel.sh index 68a9b9e9..c0246e00 100755 --- a/testing/v2/installers/install_v2/install_rhel.sh +++ b/testing/v2/installers/install_v2/install_rhel.sh @@ -50,7 +50,7 @@ fi echo "Code cloned to $HOME/LME" echo "Expanding disks" -ssh -o StrictHostKeyChecking=no $user@$hostname "cd ~/LME && sudo ./scripts/expand_rhel_disk.sh" +ssh -o StrictHostKeyChecking=no $user@$hostname "cd ~/LME && sudo ./scripts/expand_rhel_disk.sh --yes" echo "Running LME installer" ssh -o StrictHostKeyChecking=no $user@$hostname "export NON_INTERACTIVE=true && export AUTO_CREATE_ENV=true && export AUTO_IP=10.1.0.5 && cd ~/LME && ./install.sh --debug" From bfe32e5cacf87b50c5645bd30eb43d2ee4c285ec Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 12:22:54 +0000 Subject: [PATCH 24/51] Update to run all tests on RHEL 9 --- testing/v2/installers/install_v2/install_rhel.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/v2/installers/install_v2/install_rhel.sh b/testing/v2/installers/install_v2/install_rhel.sh index c0246e00..b94f1817 100755 --- a/testing/v2/installers/install_v2/install_rhel.sh +++ b/testing/v2/installers/install_v2/install_rhel.sh @@ -1,5 +1,4 @@ #!/bin/bash - set -e # Check if the required arguments are provided From 0216c4125128ee343504b4cb5274f253e92ebe2d Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 22 Aug 2025 13:02:29 +0000 Subject: [PATCH 25/51] Remove caddy from the certificate setup --- config/setup/instances.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/config/setup/instances.yml b/config/setup/instances.yml index fb45133c..3fe8c098 100644 --- a/config/setup/instances.yml +++ b/config/setup/instances.yml @@ -42,10 +42,3 @@ instances: - "localhost" ip: - "127.0.0.1" - - - name: "caddy" - dns: - - "lme-caddy" - - "localhost" - ip: - - "127.0.0.1" From e1801551db3465025b0a83a2cbc94e4543e4013f Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 26 Aug 2025 09:54:52 +0000 Subject: [PATCH 26/51] Adds RHEL 9 cluster workflow --- .github/workflows/cluster_redhat.yml | 302 ++++++++++++++++++ .../lib/install_agent_windows_azure.sh | 264 +++++++++++++++ 2 files changed, 566 insertions(+) create mode 100644 .github/workflows/cluster_redhat.yml create mode 100644 testing/v2/installers/lib/install_agent_windows_azure.sh diff --git a/.github/workflows/cluster_redhat.yml b/.github/workflows/cluster_redhat.yml new file mode 100644 index 00000000..fbe97d78 --- /dev/null +++ b/.github/workflows/cluster_redhat.yml @@ -0,0 +1,302 @@ +name: Cluster RedHat + +on: + workflow_dispatch: + inputs: + azure_region: + description: 'Azure region to deploy resources' + required: true + default: 'centralus' + type: choice + options: + - centralus + - eastus + - eastus2 + - westus + - westus2 + - westus3 + - northcentralus + - southcentralus + - canadacentral + - canadaeast + - uksouth + - ukwest + - northeurope + - westeurope + +jobs: + build-and-test-cluster-redhat: + runs-on: self-hosted + env: + UNIQUE_ID: ${{ github.run_id }}-${{ github.run_number }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + IP_ADDRESS: "" + AZURE_IP: "" + WINDOWS_IP: "" + ENROLLMENT_TOKEN: "" + ES_PASSWORD: "" + KIBANA_PASSWORD: "" + ELASTIC_AGENT_VERSION: "8.18.3" + + steps: + - name: Generate random number + shell: bash + run: | + RANDOM_NUM=$(shuf -i 1000000000-9999999999 -n 1) + echo "UNIQUE_ID=${RANDOM_NUM}_cluster_redhat_${{ github.run_number }}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4.1.1 + + - name: Setup environment variables + run: | + PUBLIC_IP=$(curl -s https://api.ipify.org) + echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV + + - name: Get branch name + shell: bash + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV + else + echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV + fi + + - name: Set the environment for docker-compose + run: | + cd testing/v2/development + # Get the UID and GID of the current user + echo "HOST_UID=$(id -u)" > .env + echo "HOST_GID=$(id -g)" >> .env + cat .env + + - name: Build pipeline container + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} build pipeline --no-cache + + - name: Start pipeline container + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline + + - name: Install Python requirements + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers/azure && \ + pip install -r requirements.txt + " + + - name: Build Azure RHEL instance with Windows VM + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T \ + -e AZURE_CLIENT_ID \ + -e AZURE_CLIENT_SECRET \ + -e AZURE_TENANT_ID \ + -e AZURE_SUBSCRIPTION_ID \ + pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + python3 ./azure/build_azure_linux_network.py \ + -g pipe-${{ env.UNIQUE_ID }} \ + -s ${{ env.IP_ADDRESS }}/32 \ + -vs Standard_E16d_v4 \ + -l ${{ inputs.azure_region || 'centralus' }} \ + -ast 23:00 \ + --use-rhel \ + -w \ + -y + " + + - name: Install LME on Azure RHEL instance + run: | + cd testing/v2/development + sleep 60 + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ./install_v2/install_rhel.sh lme-user \$IP_ADDRESS \"pipe-${{ env.UNIQUE_ID }}.password.txt\" ${{ env.BRANCH_NAME }} + " + + - name: Wait for LME installation to complete + run: | + echo "Waiting for LME installation to fully complete..." + sleep 300 + + - name: Get Azure and Windows IP addresses + run: | + cd testing/v2/development + AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt") + echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV + echo "Azure IP: $AZURE_IP" + + # Get Windows VM IP from Azure resource group - try multiple methods + WINDOWS_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + # Try getting public IP directly + az network public-ip show --resource-group pipe-${{ env.UNIQUE_ID }} --name ws1-public-ip --query ipAddress --output tsv 2>/dev/null || \ + # Try getting from VM network interfaces + az vm show --resource-group pipe-${{ env.UNIQUE_ID }} --name ws1 --show-details --query publicIps --output tsv 2>/dev/null || \ + # Try listing all IPs in resource group and find ws1 + az vm list-ip-addresses --resource-group pipe-${{ env.UNIQUE_ID }} --query '[?virtualMachine.name==\`ws1\`].virtualMachine.network.publicIpAddresses[0].ipAddress' --output tsv 2>/dev/null || \ + echo '' + ") + + if [ -z "$WINDOWS_IP" ]; then + echo "Warning: Could not retrieve Windows VM public IP, using private IP instead" + WINDOWS_IP="10.1.0.4" + fi + echo "WINDOWS_IP=$WINDOWS_IP" >> $GITHUB_ENV + echo "Windows IP: $WINDOWS_IP" + + echo "Azure IP: $AZURE_IP, Windows IP: $WINDOWS_IP" + + - name: Retrieve Elastic password + env: + AZURE_IP: ${{ env.AZURE_IP }} + run: | + cd testing/v2/development + ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$elastic'" | tail -n 1 | tr -d '\n') + echo "::add-mask::$ES_PASSWORD" + echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV + echo "Elastic password retrieved successfully" + KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$kibana_system'" | tail -n 1 | tr -d '\n') + echo "::add-mask::$KIBANA_PASSWORD" + echo "KIBANA_PASSWORD=$KIBANA_PASSWORD" >> $GITHUB_ENV + echo "Kibana password retrieved successfully" + + - name: Install test requirements on Azure RHEL instance + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers && \ + IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \ + ssh lme-user@\$IP_ADDRESS 'whoami && hostname && \ + wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm && \ + sudo yum install -y ./google-chrome-stable_current_x86_64.rpm && \ + cd /home/lme-user/LME/testing/tests && \ + python3 -m venv venv && \ + source venv/bin/activate && \ + pip install -r requirements.txt ' + " + + - name: Retrieve Elastic policy ID and enrollment token + env: + KIBANA_URL: "https://localhost:5601" + ES_USERNAME: "elastic" + ES_PASSWORD: ${{ env.ES_PASSWORD }} + run: | + cd testing/v2/development + + # Retrieve policy ID + POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + ssh lme-user@${{ env.AZURE_IP }} ' + curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \ + -H \"kbn-xsrf: true\" \ + -H \"Content-Type: application/json\" | + jq -r '.items[0].id' + ' + ") + echo "Retrieved Policy ID: $POLICY_ID" + + # Retrieve enrollment token using the policy ID + ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + ssh lme-user@${{ env.AZURE_IP }} ' + curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \ + -H \"kbn-xsrf: true\" \ + -H \"Content-Type: application/json\" \ + -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" | + jq -r .item.api_key + ' + ") + echo "Retrieved enrollment token: $ENROLLMENT_TOKEN" + + # Mask the enrollment token in logs and set it as an environment variable + echo "::add-mask::$ENROLLMENT_TOKEN" + echo "ENROLLMENT_TOKEN=$ENROLLMENT_TOKEN" >> $GITHUB_ENV + echo "Policy ID and Enrollment Token retrieved successfully" + + - name: Install the Elastic Agent on Windows Azure VM + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + cd /home/lme-user/LME/testing/v2/installers/lib/ && \ + chmod +x install_agent_windows_azure.sh && \ + ./install_agent_windows_azure.sh \ + --resource-group pipe-${{ env.UNIQUE_ID }} \ + --vm-name ws1 \ + --hostip ${{ env.AZURE_IP }} \ + --token ${{ env.ENROLLMENT_TOKEN }} \ + --version ${{ env.ELASTIC_AGENT_VERSION }} + " + + - name: Check if the Windows Elastic agent is reporting + env: + ES_PASSWORD: ${{ env.ES_PASSWORD }} + run: | + sleep 360 + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \ + 'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh windows' + " + + - name: Wait 5 minutes after Windows agent check + run: | + sleep 300 + + - name: Run api tests on Azure RHEL instance + env: + ES_PASSWORD: ${{ env.ES_PASSWORD }} + KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \ + echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \ + echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \ + echo elastic=\"$ES_PASSWORD\" >> .env && \ + source venv/bin/activate && \ + pytest -v api_tests/' + " + + - name: Run selenium tests on Azure RHEL instance + env: + ES_PASSWORD: ${{ env.ES_PASSWORD }} + KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }} + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \ + echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \ + echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \ + echo elastic=\"$ES_PASSWORD\" >> .env && \ + source venv/bin/activate && \ + pytest -v selenium_tests/' + " + + - name: Cleanup Azure resources + if: always() + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_SECRET: ${{ secrets.AZURE_SECRET }} + AZURE_TENANT: ${{ secrets.AZURE_TENANT }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " + az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_SECRET --tenant $AZURE_TENANT + az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait + " + + - name: Stop and remove containers + if: always() + run: | + cd testing/v2/development + docker compose -p ${{ env.UNIQUE_ID }} down + docker system prune -af diff --git a/testing/v2/installers/lib/install_agent_windows_azure.sh b/testing/v2/installers/lib/install_agent_windows_azure.sh new file mode 100644 index 00000000..1ea151f3 --- /dev/null +++ b/testing/v2/installers/lib/install_agent_windows_azure.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash + +# Azure-native Windows Elastic Agent installer +# Uses Azure CLI 'az vm run-command' instead of SSH/minimega for remote Windows management + +# Default values +VERSION="8.18.3" +ARCHITECTURE="windows-x86_64" +HOST_IP="10.1.0.5" +CLIENT_IP="" +RESOURCE_GROUP="" +VM_NAME="ws1" +PORT="8220" +ENROLLMENT_TOKEN="" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --version) + VERSION="$2" + shift 2 + ;; + --arch) + ARCHITECTURE="$2" + shift 2 + ;; + --hostip) + HOST_IP="$2" + shift 2 + ;; + --clientip) + CLIENT_IP="$2" + shift 2 + ;; + --resource-group) + RESOURCE_GROUP="$2" + shift 2 + ;; + --vm-name) + VM_NAME="$2" + shift 2 + ;; + --port) + PORT="$2" + shift 2 + ;; + --token) + ENROLLMENT_TOKEN="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 --resource-group --vm-name --hostip --token [--version ] [--port ]" + exit 1 + ;; + esac +done + +# Validate required parameters +if [[ -z "$RESOURCE_GROUP" ]]; then + echo "Error: --resource-group is required" + exit 1 +fi + +if [[ -z "$ENROLLMENT_TOKEN" ]]; then + echo "Error: --token (enrollment token) is required" + exit 1 +fi + +if [[ -z "$HOST_IP" ]]; then + echo "Error: --hostip is required" + exit 1 +fi + +echo "Installing Elastic Agent on Azure Windows VM..." +echo "Resource Group: $RESOURCE_GROUP" +echo "VM Name: $VM_NAME" +echo "Host IP: $HOST_IP" +echo "Version: $VERSION" + +# Function to run PowerShell commands on Azure Windows VM +run_azure_powershell() { + local command="$1" + local description="$2" + + echo "Running: $description" + + # Use az vm run-command to execute PowerShell on the Windows VM + local result=$(az vm run-command invoke \ + --command-id RunPowerShellScript \ + --resource-group "$RESOURCE_GROUP" \ + --name "$VM_NAME" \ + --scripts "$command" \ + --output json 2>/dev/null) + + if [[ $? -ne 0 ]]; then + echo "Error: Failed to run command on Windows VM: $description" + return 1 + fi + + # Extract and display the output + local stdout=$(echo "$result" | jq -r '.value[0].message' 2>/dev/null || echo "No output") + echo "Output: $stdout" + + return 0 +} + +# Step 1: Download Elastic Agent to Windows VM +echo "Step 1: Downloading Elastic Agent to Windows VM..." +download_command=' +$url = "https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-'"$VERSION"'-'"$ARCHITECTURE"'.zip" +$output = "C:\elastic-agent-'"$VERSION"'-'"$ARCHITECTURE"'.zip" +Write-Host "Downloading from: $url" +Write-Host "Saving to: $output" +try { + Invoke-WebRequest -Uri $url -OutFile $output -UseBasicParsing + Write-Host "Download completed successfully" + if (Test-Path $output) { + $fileSize = (Get-Item $output).length + Write-Host "File size: $fileSize bytes" + } +} catch { + Write-Host "Download failed: $($_.Exception.Message)" + exit 1 +} +' + +if ! run_azure_powershell "$download_command" "Download Elastic Agent"; then + echo "Failed to download Elastic Agent" + exit 1 +fi + +# Step 2: Extract the archive +echo "Step 2: Extracting Elastic Agent archive..." +extract_command=' +$zipPath = "C:\elastic-agent-'"$VERSION"'-'"$ARCHITECTURE"'.zip" +$extractPath = "C:\elastic-agent-extract" +Write-Host "Extracting $zipPath to $extractPath" +try { + if (Test-Path $extractPath) { + Remove-Item -Path $extractPath -Recurse -Force + } + Expand-Archive -Path $zipPath -DestinationPath $extractPath -Force + Write-Host "Extraction completed" + + # List contents to verify extraction + $contents = Get-ChildItem -Path $extractPath -Recurse -Name + Write-Host "Extracted contents:" + $contents | ForEach-Object { Write-Host " $_" } +} catch { + Write-Host "Extraction failed: $($_.Exception.Message)" + exit 1 +} +' + +if ! run_azure_powershell "$extract_command" "Extract Elastic Agent"; then + echo "Failed to extract Elastic Agent" + exit 1 +fi + +# Step 3: Install Elastic Agent +echo "Step 3: Installing Elastic Agent..." +install_command=' +$extractPath = "C:\elastic-agent-extract" +$agentPath = Get-ChildItem -Path $extractPath -Directory | Select-Object -First 1 +$agentExePath = Join-Path -Path $agentPath.FullName -ChildPath "elastic-agent.exe" + +Write-Host "Agent executable path: $agentExePath" + +if (-not (Test-Path $agentExePath)) { + Write-Host "Error: elastic-agent.exe not found at $agentExePath" + exit 1 +} + +# Install the agent +$installArgs = @( + "install" + "--non-interactive" + "--force" + "--url=https://'"$HOST_IP"':'"$PORT"'" + "--insecure" + "--enrollment-token='"$ENROLLMENT_TOKEN"'" +) + +Write-Host "Installing Elastic Agent with arguments: $($installArgs -join \" \")" + +try { + $process = Start-Process -FilePath $agentExePath -ArgumentList $installArgs -Wait -PassThru -NoNewWindow + Write-Host "Installation process exit code: $($process.ExitCode)" + + if ($process.ExitCode -eq 0) { + Write-Host "Elastic Agent installation completed successfully" + } else { + Write-Host "Elastic Agent installation failed with exit code: $($process.ExitCode)" + exit 1 + } +} catch { + Write-Host "Installation failed: $($_.Exception.Message)" + exit 1 +} +' + +if ! run_azure_powershell "$install_command" "Install Elastic Agent"; then + echo "Failed to install Elastic Agent" + exit 1 +fi + +# Step 4: Wait for service to start +echo "Step 4: Waiting for Elastic Agent service to start..." +sleep 60 + +# Step 5: Check agent service status +echo "Step 5: Checking Elastic Agent service status..." +status_command=' +try { + $service = Get-Service -Name "Elastic Agent" -ErrorAction SilentlyContinue + if ($service) { + Write-Host "Elastic Agent service status: $($service.Status)" + if ($service.Status -eq "Running") { + Write-Host "Elastic Agent service is running successfully" + } else { + Write-Host "Warning: Elastic Agent service is not running" + } + } else { + Write-Host "Error: Elastic Agent service not found" + exit 1 + } +} catch { + Write-Host "Error checking service status: $($_.Exception.Message)" + exit 1 +} +' + +if ! run_azure_powershell "$status_command" "Check Elastic Agent service"; then + echo "Failed to check Elastic Agent service status" + exit 1 +fi + +# Step 6: Cleanup downloaded files +echo "Step 6: Cleaning up temporary files..." +cleanup_command=' +try { + $zipPath = "C:\elastic-agent-'"$VERSION"'-'"$ARCHITECTURE"'.zip" + $extractPath = "C:\elastic-agent-extract" + + if (Test-Path $zipPath) { + Remove-Item -Path $zipPath -Force + Write-Host "Removed: $zipPath" + } + + if (Test-Path $extractPath) { + Remove-Item -Path $extractPath -Recurse -Force + Write-Host "Removed: $extractPath" + } + + Write-Host "Cleanup completed" +} catch { + Write-Host "Cleanup failed: $($_.Exception.Message)" +} +' + +run_azure_powershell "$cleanup_command" "Cleanup temporary files" + +echo "Elastic Agent installation on Azure Windows VM completed successfully!" From ebfaa4f13f61f61d38b3c95a4f77b13a677bbc84 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 26 Aug 2025 06:38:43 -0400 Subject: [PATCH 27/51] Adds RHEL remote installation instructions --- testing/v2/installers/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testing/v2/installers/README.md b/testing/v2/installers/README.md index 5e664fd8..e4eb4f72 100644 --- a/testing/v2/installers/README.md +++ b/testing/v2/installers/README.md @@ -59,10 +59,17 @@ echo $VM_PASSWORD ``` ### Installing lme-v2 + +#### Ubuntu Linux (default): ```bash ./install_v2/install.sh $LME_USER $VM_IP $RESOURCE_GROUP.password.txt your-branch-name ``` +#### Red Hat Enterprise Linux: +```bash +./install_v2/install_rhel.sh $LME_USER $VM_IP $RESOURCE_GROUP.password.txt your-branch-name +``` + ## Setting Up Minimega Clients ### Connecting to VMs From 1de310c61bfee4ed2ce7496ba2dfa3af4146b648 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 26 Aug 2025 07:07:33 -0400 Subject: [PATCH 28/51] Adds RHEL cluster workflow to trigger on PRs --- .github/workflows/cluster.yml | 2 -- .github/workflows/cluster_redhat.yml | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cluster.yml b/.github/workflows/cluster.yml index 60ee438b..5fa6a1d0 100644 --- a/.github/workflows/cluster.yml +++ b/.github/workflows/cluster.yml @@ -33,7 +33,6 @@ on: type: choice options: - ubuntu - - rhel jobs: build-and-test-cluster: @@ -124,7 +123,6 @@ jobs: -vs Standard_E16d_v4 \ -l ${{ inputs.azure_region || 'centralus' }} \ -ast 23:00 \ - ${{ inputs.operating_system == 'rhel' && '--use-rhel' || '' }} \ -y " #-s ${{ env.IP_ADDRESS }}/32 \ diff --git a/.github/workflows/cluster_redhat.yml b/.github/workflows/cluster_redhat.yml index fbe97d78..9f25c7b8 100644 --- a/.github/workflows/cluster_redhat.yml +++ b/.github/workflows/cluster_redhat.yml @@ -1,6 +1,9 @@ name: Cluster RedHat on: + pull_request: + branches: + - 'cbaxley-563-redhat-9-1' workflow_dispatch: inputs: azure_region: From 47cf62256804777ba37d36a5a731bf44a5b55564 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 26 Aug 2025 07:14:42 -0400 Subject: [PATCH 29/51] Adds RHEL cluster workflow to trigger on PRs --- .github/workflows/cluster_redhat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster_redhat.yml b/.github/workflows/cluster_redhat.yml index 9f25c7b8..4419d2e3 100644 --- a/.github/workflows/cluster_redhat.yml +++ b/.github/workflows/cluster_redhat.yml @@ -3,7 +3,7 @@ name: Cluster RedHat on: pull_request: branches: - - 'cbaxley-563-redhat-9-1' + - '*' workflow_dispatch: inputs: azure_region: From 25e3fd65aa673fed84530d1d40b6b9de284fa69d Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 27 Aug 2025 08:40:52 -0400 Subject: [PATCH 30/51] Fixes Azure Windows agent installation output parsing --- .github/workflows/cluster_redhat.yml | 3 +- .../lib/install_agent_windows_azure.sh | 75 +++++++- .../v2/installers/lib/test_azure_output.sh | 168 ++++++++++++++++++ 3 files changed, 240 insertions(+), 6 deletions(-) mode change 100644 => 100755 testing/v2/installers/lib/install_agent_windows_azure.sh create mode 100755 testing/v2/installers/lib/test_azure_output.sh diff --git a/.github/workflows/cluster_redhat.yml b/.github/workflows/cluster_redhat.yml index 4419d2e3..a7e625a9 100644 --- a/.github/workflows/cluster_redhat.yml +++ b/.github/workflows/cluster_redhat.yml @@ -235,7 +235,8 @@ jobs: --vm-name ws1 \ --hostip ${{ env.AZURE_IP }} \ --token ${{ env.ENROLLMENT_TOKEN }} \ - --version ${{ env.ELASTIC_AGENT_VERSION }} + --version ${{ env.ELASTIC_AGENT_VERSION }} \ + --debug " - name: Check if the Windows Elastic agent is reporting diff --git a/testing/v2/installers/lib/install_agent_windows_azure.sh b/testing/v2/installers/lib/install_agent_windows_azure.sh old mode 100644 new mode 100755 index 1ea151f3..80cc3435 --- a/testing/v2/installers/lib/install_agent_windows_azure.sh +++ b/testing/v2/installers/lib/install_agent_windows_azure.sh @@ -12,6 +12,7 @@ RESOURCE_GROUP="" VM_NAME="ws1" PORT="8220" ENROLLMENT_TOKEN="" +DEBUG_MODE=false # Parse command line arguments while [[ $# -gt 0 ]]; do @@ -48,9 +49,13 @@ while [[ $# -gt 0 ]]; do ENROLLMENT_TOKEN="$2" shift 2 ;; + --debug) + DEBUG_MODE=true + shift + ;; *) echo "Unknown option: $1" - echo "Usage: $0 --resource-group --vm-name --hostip --token [--version ] [--port ]" + echo "Usage: $0 --resource-group --vm-name --hostip --token [--version ] [--port ] [--debug]" exit 1 ;; esac @@ -98,9 +103,69 @@ run_azure_powershell() { return 1 fi - # Extract and display the output - local stdout=$(echo "$result" | jq -r '.value[0].message' 2>/dev/null || echo "No output") - echo "Output: $stdout" + # Debug: Show raw JSON response if debug mode is enabled + if [[ "$DEBUG_MODE" == "true" ]]; then + echo "DEBUG: Raw JSON response:" + echo "$result" | jq '.' 2>/dev/null || echo "$result" + echo "DEBUG: End raw JSON response" + fi + + # Extract and display the output - handle both stdout and stderr properly + local stdout="" + local stderr="" + + # Parse all messages from the response + if command -v jq >/dev/null 2>&1; then + # Get all messages and separate stdout/stderr (match prefix explicitly) + local messages=$(echo "$result" | jq -r '.value[] | select(.code | startswith("ComponentStatus/StdOut/")) | .message' 2>/dev/null) + local errors=$(echo "$result" | jq -r '.value[] | select(.code | startswith("ComponentStatus/StdErr/")) | .message' 2>/dev/null) + + # If no specific stdout/stderr found, try the legacy format + if [[ -z "$messages" && -z "$errors" ]]; then + messages=$(echo "$result" | jq -r '.value[0].message' 2>/dev/null) + fi + + # If still no output, try to get any message from the response + if [[ -z "$messages" && -z "$errors" ]]; then + messages=$(echo "$result" | jq -r '.value[] | .message' 2>/dev/null | tr '\n' ' ') + fi + + stdout="$messages" + stderr="$errors" + else + # Fallback if jq is not available + stdout=$(echo "$result" | grep -o '"message":"[^"]*"' | sed 's/"message":"//g' | sed 's/"$//g' | tr '\n' ' ') + fi + + # Detect errors and print output + local has_error=false + if [[ -n "$stderr" ]]; then + echo "Error: $stderr" + has_error=true + fi + + if [[ -n "$stdout" ]]; then + if [[ "$stdout" == *"Error:"* ]] || [[ "$stdout" == *"ParserError"* ]] || [[ "$stdout" == *"Exception"* ]]; then + echo "Error detected in output: $stdout" + has_error=true + else + echo "Output: $stdout" + fi + fi + + if [[ -z "$stdout" && -z "$stderr" ]]; then + echo "Output: No output" + if [[ "$DEBUG_MODE" == "true" ]]; then + echo "DEBUG: No output found in JSON response. This might indicate:" + echo "DEBUG: 1. The command completed successfully but produced no output" + echo "DEBUG: 2. The JSON structure is different than expected" + echo "DEBUG: 3. The command failed silently" + fi + fi + + if [[ "$has_error" == "true" ]]; then + return 1 + fi return 0 } @@ -182,7 +247,7 @@ $installArgs = @( "--enrollment-token='"$ENROLLMENT_TOKEN"'" ) -Write-Host "Installing Elastic Agent with arguments: $($installArgs -join \" \")" +Write-Host "Installing Elastic Agent with arguments: $($installArgs -join ([char]32))" try { $process = Start-Process -FilePath $agentExePath -ArgumentList $installArgs -Wait -PassThru -NoNewWindow diff --git a/testing/v2/installers/lib/test_azure_output.sh b/testing/v2/installers/lib/test_azure_output.sh new file mode 100755 index 00000000..8ff2ba5a --- /dev/null +++ b/testing/v2/installers/lib/test_azure_output.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +# Test script to debug Azure CLI output parsing +# This script helps understand what the Azure CLI returns and how to parse it + +# Default values +RESOURCE_GROUP="" +VM_NAME="" +DEBUG_MODE=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --resource-group) + RESOURCE_GROUP="$2" + shift 2 + ;; + --vm-name) + VM_NAME="$2" + shift 2 + ;; + --debug) + DEBUG_MODE=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 --resource-group --vm-name [--debug]" + exit 1 + ;; + esac +done + +# Validate required parameters +if [[ -z "$RESOURCE_GROUP" ]]; then + echo "Error: --resource-group is required" + exit 1 +fi + +if [[ -z "$VM_NAME" ]]; then + echo "Error: --vm-name is required" + exit 1 +fi + +echo "Testing Azure CLI output parsing..." +echo "Resource Group: $RESOURCE_GROUP" +echo "VM Name: $VM_NAME" +echo "Debug Mode: $DEBUG_MODE" +echo "" + +# Test 1: Simple command that should produce output +echo "Test 1: Simple Write-Host command" +test_command='Write-Host "Hello from Azure Windows VM"' + +echo "Running: $test_command" +result=$(az vm run-command invoke \ + --command-id RunPowerShellScript \ + --resource-group "$RESOURCE_GROUP" \ + --name "$VM_NAME" \ + --scripts "$test_command" \ + --output json 2>/dev/null) + +if [[ $? -ne 0 ]]; then + echo "Error: Failed to run test command" + exit 1 +fi + +echo "Raw JSON response:" +echo "$result" | jq '.' 2>/dev/null || echo "$result" +echo "" + +# Test 2: Command that should produce error output +echo "Test 2: Command that should produce error output" +test_command='Write-Error "This is a test error message"' + +echo "Running: $test_command" +result=$(az vm run-command invoke \ + --command-id RunPowerShellScript \ + --resource-group "$RESOURCE_GROUP" \ + --name "$VM_NAME" \ + --scripts "$test_command" \ + --output json 2>/dev/null) + +if [[ $? -ne 0 ]]; then + echo "Error: Failed to run test command" + exit 1 +fi + +echo "Raw JSON response:" +echo "$result" | jq '.' 2>/dev/null || echo "$result" +echo "" + +# Test 3: Command that should produce both stdout and stderr +echo "Test 3: Command that should produce both stdout and stderr" +test_command='Write-Host "This is stdout"; Write-Error "This is stderr"' + +echo "Running: $test_command" +result=$(az vm run-command invoke \ + --command-id RunPowerShellScript \ + --resource-group "$RESOURCE_GROUP" \ + --name "$VM_NAME" \ + --scripts "$test_command" \ + --output json 2>/dev/null) + +if [[ $? -ne 0 ]]; then + echo "Error: Failed to run test command" + exit 1 +fi + +echo "Raw JSON response:" +echo "$result" | jq '.' 2>/dev/null || echo "$result" +echo "" + +echo "Test 4: Parsing output using improved logic" +test_command='Write-Host "Test output for parsing"; Write-Error "Test error for parsing"' + +echo "Running: $test_command" +result=$(az vm run-command invoke \ + --command-id RunPowerShellScript \ + --resource-group "$RESOURCE_GROUP" \ + --name "$VM_NAME" \ + --scripts "$test_command" \ + --output json 2>/dev/null) + +if [[ $? -ne 0 ]]; then + echo "Error: Failed to run test command" + exit 1 +fi + +if command -v jq >/dev/null 2>&1; then + echo "Parsing with jq..." + + # List codes present + echo "Codes present:" + echo "$result" | jq -r '.value[] | .code' + echo "" + + # Concise, readable parsed output (preferred filters) + stdout_parsed=$(echo "$result" | jq -r '.value[] | select(.code | startswith("ComponentStatus/StdOut/")) | .message') + stderr_parsed=$(echo "$result" | jq -r '.value[] | select(.code | startswith("ComponentStatus/StdErr/")) | .message') + + echo "Readable parsed output:" + echo "StdOut:" + if [[ -z "$stdout_parsed" ]]; then + echo "(none)" + else + echo "$stdout_parsed" + fi + echo "" + echo "StdErr:" + if [[ -z "$stderr_parsed" ]]; then + echo "(none)" + else + echo "$stderr_parsed" + fi + echo "" + + # Minimal structure check + echo "Structure summary:" + echo "$result" | jq '.value[] | {code: .code, messageLen: (.message|tostring|length)}' || echo "Failed to analyze JSON structure" +else + echo "jq not available, using fallback parsing..." + stdout=$(echo "$result" | grep -o '"message":"[^"]*"' | sed 's/"message":"//g' | sed 's/"$//g' | tr '\n' ' ') + echo "Fallback parsing result: '$stdout'" +fi + +echo "" +echo "Test completed!" From 10a741cd2126eac352c8e6337649ad922946ee12 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 27 Aug 2025 08:48:50 -0400 Subject: [PATCH 31/51] Checks Azure Windows agent installation output for errors --- .../lib/install_agent_windows_azure.sh | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/testing/v2/installers/lib/install_agent_windows_azure.sh b/testing/v2/installers/lib/install_agent_windows_azure.sh index 80cc3435..32ecc0a8 100755 --- a/testing/v2/installers/lib/install_agent_windows_azure.sh +++ b/testing/v2/installers/lib/install_agent_windows_azure.sh @@ -137,21 +137,29 @@ run_azure_powershell() { stdout=$(echo "$result" | grep -o '"message":"[^"]*"' | sed 's/"message":"//g' | sed 's/"$//g' | tr '\n' ' ') fi - # Detect errors and print output + # Detect errors and print output (treat non-fatal stderr as note) local has_error=false - if [[ -n "$stderr" ]]; then - echo "Error: $stderr" - has_error=true - fi - + local error_patterns='error|exception|failed|parsererror|write-error|writeerrorexception|categoryinfo|fullyqualifiederrorid' + + # Print stdout and check for error-like content if [[ -n "$stdout" ]]; then - if [[ "$stdout" == *"Error:"* ]] || [[ "$stdout" == *"ParserError"* ]] || [[ "$stdout" == *"Exception"* ]]; then + if echo "$stdout" | grep -Eqi "$error_patterns"; then echo "Error detected in output: $stdout" has_error=true else echo "Output: $stdout" fi fi + + # Print stderr as note unless it contains error-like content + if [[ -n "$stderr" ]]; then + if echo "$stderr" | grep -Eqi "$error_patterns"; then + echo "Error: $stderr" + has_error=true + else + echo "Note (stderr): $stderr" + fi + fi if [[ -z "$stdout" && -z "$stderr" ]]; then echo "Output: No output" From 1bf4faffb1529986ef7362d2440529a5ef756742 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 27 Aug 2025 09:23:19 -0400 Subject: [PATCH 32/51] Passes the azure environment variables to the Azure Windows agent installation script --- .github/workflows/cluster_redhat.yml | 33 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cluster_redhat.yml b/.github/workflows/cluster_redhat.yml index a7e625a9..5d728c74 100644 --- a/.github/workflows/cluster_redhat.yml +++ b/.github/workflows/cluster_redhat.yml @@ -225,19 +225,30 @@ jobs: echo "Policy ID and Enrollment Token retrieved successfully" - name: Install the Elastic Agent on Windows Azure VM + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} run: | cd testing/v2/development - docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c " - cd /home/lme-user/LME/testing/v2/installers/lib/ && \ - chmod +x install_agent_windows_azure.sh && \ - ./install_agent_windows_azure.sh \ - --resource-group pipe-${{ env.UNIQUE_ID }} \ - --vm-name ws1 \ - --hostip ${{ env.AZURE_IP }} \ - --token ${{ env.ENROLLMENT_TOKEN }} \ - --version ${{ env.ELASTIC_AGENT_VERSION }} \ - --debug - " + docker compose -p ${{ env.UNIQUE_ID }} exec -T \ + -e AZURE_CLIENT_ID \ + -e AZURE_CLIENT_SECRET \ + -e AZURE_TENANT_ID \ + -e AZURE_SUBSCRIPTION_ID \ + pipeline bash -c " + az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID && \ + cd /home/lme-user/LME/testing/v2/installers/lib/ && \ + chmod +x install_agent_windows_azure.sh && \ + ./install_agent_windows_azure.sh \ + --resource-group pipe-${{ env.UNIQUE_ID }} \ + --vm-name ws1 \ + --hostip ${{ env.AZURE_IP }} \ + --token ${{ env.ENROLLMENT_TOKEN }} \ + --version ${{ env.ELASTIC_AGENT_VERSION }} \ + --debug + " - name: Check if the Windows Elastic agent is reporting env: From ada98b8a81959ba6eaf8feb31e6035a47531f737 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 27 Aug 2025 11:40:05 -0400 Subject: [PATCH 33/51] Uses the default ip instead of the AZURE_IP environment variable --- .github/workflows/cluster_redhat.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cluster_redhat.yml b/.github/workflows/cluster_redhat.yml index 5d728c74..611071ff 100644 --- a/.github/workflows/cluster_redhat.yml +++ b/.github/workflows/cluster_redhat.yml @@ -244,7 +244,6 @@ jobs: ./install_agent_windows_azure.sh \ --resource-group pipe-${{ env.UNIQUE_ID }} \ --vm-name ws1 \ - --hostip ${{ env.AZURE_IP }} \ --token ${{ env.ENROLLMENT_TOKEN }} \ --version ${{ env.ELASTIC_AGENT_VERSION }} \ --debug From ce00735214a1d90b9c32766eb9464e1495b9b446 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 28 Aug 2025 07:43:10 -0400 Subject: [PATCH 34/51] Updates some of the tools and documentation for testing --- docker/README.md | 2 +- testing/InstallTestbed.ps1 | 402 -------------------------- testing/Readme.md | 93 +----- testing/SetupTestbed.ps1 | 494 -------------------------------- testing/internet_toggle.ps1 | 47 --- testing/tests/Dockerfile | 5 +- testing/v2/installers/README.md | 12 - 7 files changed, 5 insertions(+), 1050 deletions(-) delete mode 100644 testing/InstallTestbed.ps1 delete mode 100644 testing/SetupTestbed.ps1 delete mode 100644 testing/internet_toggle.ps1 diff --git a/docker/README.md b/docker/README.md index a5ce6433..218570dd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,7 +17,7 @@ You can choose either the 22.04 or 24.04 directories to build the container. Note: We have installed Docker desktop on Windows and Linux and have been able to build and run the container. ### Special instructions for Windows running Linux -If running Linux on a hypervisor or virtual machine, you may need to modify the GRUB configuration in your VM: +If running Linux on a hypervisor or virtual machine, you may need to modify the GRUB configuration in your VM (only if you have problems): 1. Add the following to the `GRUB_CMDLINE_LINUX` line in `/etc/default/grub`: ```bash diff --git a/testing/InstallTestbed.ps1 b/testing/InstallTestbed.ps1 deleted file mode 100644 index 6e7b8be0..00000000 --- a/testing/InstallTestbed.ps1 +++ /dev/null @@ -1,402 +0,0 @@ -param ( - [Alias("g")] - [Parameter(Mandatory = $true)] - [string]$ResourceGroup, - - [Alias("w")] - [string]$DomainController = "DC1", - - [Alias("l")] - [string]$LinuxVM = "LS1", - - [Alias("n")] - [int]$NumClients = 2, - - [Alias("m")] - [Parameter( - HelpMessage = "(minimal) Only install the linux server. Useful for testing the linux server without the windows clients" - )] - [switch]$LinuxOnly, - - [Alias("v")] - [string]$Version = $false, - - [Alias("b")] - [string]$Branch = $false -) - -# If you were to need the password from the SetupTestbed.ps1 script, you could use this: -# $Password = Get-Content "${ResourceGroup}.password.txt" - - -$ProcessSeparator = "`n----------------------------------------`n" - -# Define our library path -$LibraryPath = Join-Path -Path $PSScriptRoot -ChildPath "configure\azure_scripts\lib\utilityFunctions.ps1" - -# Check if the library file exists -if (Test-Path -Path $LibraryPath) { - # Dot-source the library script - . $LibraryPath -} -else { - Write-Error "Library script not found at path: $LibraryPath" -} - -if ($Version -ne $false -and -not ($Version -match '^[0-9]+\.[0-9]+\.[0-9]+$')) { - Write-Host "Invalid version format: $Version. Expected format: X.Y.Z (e.g., 1.3.0)" - exit 1 -} - -# Create a container to keep files for the VM -Write-Output "Creating a container to keep files for the VM..." -$createBlobResponse = ./configure/azure_scripts/create_blob_container.ps1 ` - -ResourceGroup $ResourceGroup -Write-Output $createBlobResponse -Write-Output $ProcessSeparator - -# Source the variables from the file -Write-Output "`nSourcing the variables from the file..." -. ./configure/azure_scripts/config.ps1 - -# Remove old code if it exists -if (Test-Path ./configure.zip) { - Remove-Item ./configure.zip -Force -Confirm:$false -ErrorAction SilentlyContinue -} - -Write-Output $ProcessSeparator - -# Zip up the installer scripts for the VM -Write-Output "`nZipping up the installer scripts for the VMs..." -./configure/azure_scripts/zip_my_parents_parent.ps1 -Write-Output $ProcessSeparator - -# Upload the zip file to the container and get a key to download it -Write-Output "`nUploading the zip file to the container and getting a key to download it..." -$FileDownloadUrl = ./configure/azure_scripts/copy_file_to_container.ps1 ` - -LocalFilePath "configure.zip" ` - -ContainerName $ContainerName ` - -StorageAccountName $StorageAccountName ` - -StorageAccountKey $StorageAccountKey - -Write-Output "File download URL: $FileDownloadUrl" -Write-Output $ProcessSeparator - -Write-Output "`nChanging directory to the azure scripts..." -Set-Location configure/azure_scripts -Write-Output $ProcessSeparator - -if (-Not $LinuxOnly) { - Write-Output "`nInstalling on the windows clients..." - # Make our directory on the VM - Write-Output "`nMaking our directory on the VM..." - $createDirResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name $DomainController ` - --resource-group $ResourceGroup ` - --scripts "if (-not (Test-Path -Path 'C:\lme')) { New-Item -Path 'C:\lme' -ItemType Directory }" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$createDirResponse") - Write-Output $ProcessSeparator - - # Download the zip file to the VM - Write-Output "`nDownloading the zip file to the VM..." - $downloadZipFileResponse = .\download_in_container.ps1 ` - -VMName $DomainController ` - -ResourceGroup $ResourceGroup ` - -FileDownloadUrl "$FileDownloadUrl" ` - -DestinationFilePath "configure.zip" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$downloadZipFileResponse") - Write-Output $ProcessSeparator - - # Extract the zip file - Write-Output "`nExtracting the zip file..." - $extractArchiveResponse = .\extract_archive.ps1 ` - -VMName $DomainController ` - -ResourceGroup $ResourceGroup ` - -FileName "configure.zip" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$extractArchiveResponse") - Write-Output $ProcessSeparator - - # Run the install script for chapter 1 - Write-Output "`nRunning the install script for chapter 1..." - $installChapter1Response = .\run_script_in_container.ps1 ` - -ResourceGroup $ResourceGroup ` - -VMName $DomainController ` - -ScriptPathOnVM "C:\lme\configure\install_chapter_1.ps1" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installChapter1Response") - Write-Output $ProcessSeparator - - # Update the group policy on the remote machines - Write-Output "`nUpdating the group policy on the remote machines..." - Invoke-GPUpdateOnVMs -ResourceGroup $ResourceGroup -numberOfClients $NumClients - Write-Output $ProcessSeparator - - # Wait for the services to start - Write-Output "`nWaiting for the services to start..." - Start-Sleep 10 - - # See if we can see the forwarding computers in the DC - write-host "`nChecking if we can see the forwarding computers in the DC..." - $listForwardingComputersResponse = .\run_script_in_container.ps1 ` - -ResourceGroup $ResourceGroup ` - -VMName $DomainController ` - -ScriptPathOnVM "C:\lme\configure\list_computers_forwarding_events.ps1" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$listForwardingComputersResponse") - Write-Output $ProcessSeparator - - # Install the sysmon service on DC1 from chapter 2 - Write-Output "`nInstalling the sysmon service on DC1 from chapter 2..." - $installChapter2Response = .\run_script_in_container.ps1 ` - -ResourceGroup $ResourceGroup ` - -VMName $DomainController ` - -ScriptPathOnVM "C:\lme\configure\install_chapter_2.ps1" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installChapter2Response") - Write-Output $ProcessSeparator - - # Update the group policy on the remote machines - Write-Output "`nUpdating the group policy on the remote machines..." - Invoke-GPUpdateOnVMs -ResourceGroup $ResourceGroup -numberOfClients $NumClients - Write-Output $ProcessSeparator - - # Wait for the services to start - Write-Output "`nWaiting for the services to start. Generally they don't show..." - Start-Sleep 10 - - # See if you can see sysmon running on the machine - Write-Output "`nSeeing if you can see sysmon running on a machine..." - $showSysmonResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name "C1" ` - --resource-group $ResourceGroup ` - --scripts 'Get-Service | Where-Object { $_.DisplayName -like "*Sysmon*" }' - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$showSysmonResponse") - Write-Output $ProcessSeparator -} - -Write-Output "`nInstalling on the linux server..." -# Download the installers on LS1 -Write-Output "`nDownloading the installers on LS1..." -$downloadLinuxZipFileResponse = .\download_in_container.ps1 ` - -VMName $LinuxVM ` - -ResourceGroup $ResourceGroup ` - -FileDownloadUrl "$FileDownloadUrl" ` - -DestinationFilePath "configure.zip" ` - -os "linux" -Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$downloadLinuxZipFileResponse") -Write-Output $ProcessSeparator - -# Install unzip on LS1 -Write-Output "`nInstalling unzip on LS1..." -$installUnzipResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts 'apt-get install unzip -y' -Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installUnzipResponse") -Write-Output $ProcessSeparator - -# Unzip the file on LS1 -Write-Output "`nUnzipping the file on LS1..." -$extractLinuxArchiveResponse = .\extract_archive.ps1 ` - -VMName $LinuxVM ` - -ResourceGroup $ResourceGroup ` - -FileName "configure.zip" ` - -Os "Linux" -Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$extractLinuxArchiveResponse") -Write-Output $ProcessSeparator - -Write-Output "`nMaking the installer files executable and updating the system packages on LS1..." -$updateLinuxResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts 'chmod +x /home/admin.ackbar/lme/configure/* && /home/admin.ackbar/lme/configure/linux_update_system.sh' -Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$updateLinuxResponse") -Write-Output $ProcessSeparator - -$versionArgument = "" -if ($Branch -ne $false) { - $versionArgument = " -b '$($Branch)'" -} elseif ($Version -ne $false) { - $versionArgument = " -v $Version" -} -Write-Output "`nRunning the lme installer on LS1..." -$installLmeResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts "/home/admin.ackbar/lme/configure/linux_install_lme.sh $versionArgument" -Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installLmeResponse") -Write-Output $ProcessSeparator - -# Check if the response contains the need to reboot -$rebootCheckstring = $installLmeResponse | Out-String -if ($rebootCheckstring -match "reboot is required in order to proceed with the install") { - # Have to check for the reboot thing here - Write-Output "`nRebooting ${LinuxVM}..." - az vm restart ` - --resource-group $ResourceGroup ` - --name $LinuxVM - Write-Output $ProcessSeparator - - Write-Output "`nRunning the lme installer on LS1..." - $installLmeResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts "/home/admin.ackbar/lme/configure/linux_install_lme.sh $versionArgument" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installLmeResponse") - Write-Output $ProcessSeparator -} - -# Capture the output of the install script -Write-Output "`nCapturing the output of the install script for ES passwords..." -$getElasticsearchPasswordsResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts 'sed -n "/^## elastic/,/^####################/p" "/opt/lme/Chapter 3 Files/output.log"' - -Write-Output $ProcessSeparator - -if (-Not $LinuxOnly){ - # Generate key using expect on linux - Write-Output "`nGenerating key using expect on linux..." - $generateKeyResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts '/home/admin.ackbar/lme/configure/linux_make_private_key.exp' - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$generateKeyResponse") - Write-Output $ProcessSeparator - - # Add the public key to the authorized_keys file on LS1 - Write-Output "`nAdding the public key to the authorized_keys file on LS1..." - $authorizePrivateKeyResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts '/home/admin.ackbar/lme/configure/linux_authorize_private_key.sh' - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$authorizePrivateKeyResponse") - Write-Output $ProcessSeparator - - # Cat the private key and capture that to the azure shell - Write-Output "`nCat the private key and capture that to the azure shell..." - $jsonResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts 'cat /home/admin.ackbar/.ssh/id_rsa' - $privateKey = Get-PrivateKeyFromJson -jsonResponse "$jsonResponse" - - # Save the private key to a file - Write-Output "`nSaving the private key to a file..." - $privateKeyPath = ".\id_rsa" - Set-Content -Path $privateKeyPath -Value $privateKey - Write-Output $ProcessSeparator - - # Upload the private key to the container and get a key to download it - Write-Output "`nUploading the private key to the container and getting a key to download it..." - $KeyDownloadUrl = ./copy_file_to_container.ps1 ` - -LocalFilePath "id_rsa" ` - -ContainerName $ContainerName ` - -StorageAccountName $StorageAccountName ` - -StorageAccountKey $StorageAccountKey - - # Download the private key to DC1 - Write-Output "`nDownloading the private key to DC1..." - $downloadPrivateKeyResponse = .\download_in_container.ps1 ` - -VMName $DomainController ` - -ResourceGroup $ResourceGroup ` - -FileDownloadUrl "$KeyDownloadUrl" ` - -DestinationFilePath "id_rsa" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$downloadPrivateKeyResponse") - Write-Output $ProcessSeparator - - # Change the ownership of the private key file on DC1 - Write-Output "`nChanging the ownership of the private key file on DC1..." - $chownPrivateKeyResponse = .\run_script_in_container.ps1 ` - -ResourceGroup $ResourceGroup ` - -VMName $DomainController ` - -ScriptPathOnVM "C:\lme\configure\chown_dc1_private_key.ps1" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$chownPrivateKeyResponse") - Write-Output $ProcessSeparator - - # Remove the private key from the local machine - Remove-Item -Path $privateKeyPath - - # Use the azure shell to run scp on DC1 to copy the files from LS1 to DC1 - Write-Output "`nUsing the azure shell to run scp on DC1 to copy the files from LS1 to DC1..." - $scpResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name $DomainController ` - --resource-group $ResourceGroup ` - --scripts 'scp -o StrictHostKeyChecking=no -i "C:\lme\id_rsa" admin.ackbar@ls1:/home/admin.ackbar/files_for_windows.zip "C:\lme\"' - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$scpResponse") - Write-Output $ProcessSeparator - - # Extract the files on DC1 - Write-Output "`nExtracting the files on DC1..." - $extractFilesForWindowsResponse = .\extract_archive.ps1 ` - -VMName $DomainController ` - -ResourceGroup $ResourceGroup ` - -FileName "files_for_windows.zip" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$extractFilesForWindowsResponse") - Write-Output $ProcessSeparator - - # Install winlogbeat on DC1 - Write-Output "`nInstalling winlogbeat on DC1..." - $installWinlogbeatResponse = .\run_script_in_container.ps1 ` - -ResourceGroup $ResourceGroup ` - -VMName $DomainController ` - -ScriptPathOnVM "C:\lme\configure\winlogbeat_install.ps1" - - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installWinlogbeatResponse") - Write-Output $ProcessSeparator -} - - -Write-Output "`nRunning the tests for lme on LS1..." -$runTestResponse = az vm run-command invoke ` - --command-id RunShellScript ` - --name $LinuxVM ` - --resource-group $ResourceGroup ` - --scripts '/home/admin.ackbar/lme/configure/linux_test_install.sh' | ConvertFrom-Json - -$message = $runTestResponse.value[0].message -Write-Host "$message`n" -Write-Host "--------------------------------------------" - -# Check if there is stderr content in the message field -if ($message -match '\[stderr\]\n(.+)$') { - Write-Host "Tests failed" - exit 1 -} else { - Write-Host "Tests succeeded" -} - -Write-Output "`nInstall completed." - -$EsPasswords = (Format-AzVmRunCommandOutput -JsonResponse "$getElasticsearchPasswordsResponse")[0].StdOut -# Output the passwords -$EsPasswords - -# Write the passwords to a file -$PasswordPath = "..\..\${ResourceGroup}.password.txt" -$EsPasswords | Out-File -Append -FilePath $PasswordPath - -# Constructing a string that will hold all the command-line parameters to be written to the file -$paramsToWrite = @" -ResourceGroup: $ResourceGroup -DomainController: $DomainController -LinuxVM: $LinuxVM -NumClients: $NumClients -LinuxOnly: $($LinuxOnly.IsPresent) -Version: $Version -Branch: $Branch -"@ - -# Output the parameters to the end of the password file -$paramsToWrite | Out-File -Append -FilePath $PasswordPath - -Get-Content -Path $PasswordPath \ No newline at end of file diff --git a/testing/Readme.md b/testing/Readme.md index 8577bf09..8bdf8159 100644 --- a/testing/Readme.md +++ b/testing/Readme.md @@ -1,92 +1 @@ -# SetupTestbed.ps1 -This script creates a "blank slate" for testing/configuring LME. - -Using the Azure CLI, it creates the following: -- A resource group -- A virtual network, subnet, and network security group -- 2 VMs: "DC1," a Windows server, and "LS1," a Linux server -- Client VMs: Windows clients "C1", "C2", etc. up to 16 based on user input -- Promotes DC1 to a domain controller -- Adds C1 to the managed domain -- Adds a DNS entry pointing to LS1 - -This script does not install LME; it simply creates a fresh environment that's ready to have LME installed. - -## Usage -| **Parameter** | **Alias** | **Description** | **Required** | -|--------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| -| $ResourceGroup | -g | The name of the resource group that will be created for storing all testbed resources. | Yes | -| $NumClients | -n | The number of Windows clients to create; maximum 16; defaults to 2 | No | -| $AutoShutdownTime | | The auto-shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900); auto-shutdown not configured if not provided | No | -| $AutoShutdownEmail | | An email to be notified if a VM is auto-shutdown. | No | -| $AllowedSources | -s | Comma-Separated list of CIDR prefixes or IP ranges, e.g. XX.XX.XX.XX/YY,XX.XX.XX.XX/YY,etc..., that are allowed to connect to the VMs via RDP and ssh. | Yes | -| $Location | -l | The region you would like to build the assets in. Defaults to westus | No | -| $NoPrompt | -y | Switch, run the script with no prompt (useful for automated runs). By default, the script will prompt the user to review paramters and confirm before continuing. | No | -| $LinuxOnly | -m | Run a minimal install of only the linux server | No | - -Example: -``` -./SetupTestbed.ps1 -ResourceGroup Example1 -NumClients 2 -AutoShutdownTime 0000 -AllowedSources "1.2.3.4,1.2.3.5" -y -``` - -## Running Using Azure Shell -| **#** | **Step** | **Screenshot** | -|-------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| -| 1 | Open a cloud shell by navigating to portal.azure.com and clicking the shell icon. | ![image](/docs/imgs/testing-screenshots/shell.png) | -| 2 | Select PowerShell. | ![image](/docs/imgs/testing-secreenshots/shell2.png) | -| 3 | Clone the repo `git clone https://github.com/cisagov/LME.git` and then `cd LME\testing` | | -| 4 | Run the script, providing values for the parameters when promoted (see [Usage](#usage)). The script will take ~20 minutes to run to completion. | ![image](/docs/imgs/testing-screenshots/shell4.png) | -| 5 | Save the login credentials printed to the terminal at the end (They will also be in a file called `<$ResourceGroup>.password.txt`). At this point you can login to each VM using RDP (for the Windows servers) or SSH (for the Linux server). | ![image](/docs/imgs/testing-screenshots/shell5.png) | -| 6 | When you're done testing, simply delete the resource group to clean up all resources created. | ![image](/docs/imgs/testing-screenshots/delete.png) | - -# Extra Functionality: - -## Clean Up ResourceGroup: - -1. open a shell like before -2. run command: `az group delete --name [NAME_YOUP_ROVIDED_ABOVE]` - -## Disable Internet: -Run the following commands in the azure shell. - -```powershell -./internet_toggle.ps1 -RG [NAME_YOU_PROVIDED_ABOVE] [-NSG OPTIONAL_NSG_GROUP] [-enable] -``` - -Flags: - - enable: deletes the DENYINTERNET/DENYLOADBALANCER rules - - NSG: sets NSG to a custom NSG if desired [NSG1 default] - -## Install LME on the cluster: -### InstallTestbed.ps1 -## Usage -| **Parameter** | **Alias** | **Description** | **Required** | -|-------------------|-----------|----------------------------------------------------------------------------------------|--------------| -| $ResourceGroup | -g | The name of the resource group that will be created for storing all testbed resources. | Yes | -| $NumClients | -n | The number of Windows clients you have created; defaults to 2 | No | -| $DomainController | -w | The name of the domain controller in the cluster; defaults to "DC1" | No | -| $LinuxVm | -l | The name of the linux server in the cluster; defaults to "LS1" | No | -| $LinuxOnly | -m | Run a minimal install of only the linux server | No | -| $Version | -v | Optionally provide a version to install if you want a specific one. `-v 1.3.2` | No | -| $Branch | -b | Optionally provide a branch to install if you want a specific one `-b your_branch` | No | - -Example: -``` -./InstallTestbed.ps1 -ResourceGroup YourResourceGroup -# Or if you want to save the output to a file -./InstallTestbed.ps1 -ResourceGroup YourResourceGroup | Tee-Object -FilePath "./YourResourceGroup.output.log" -``` -| **#** | **Step** | **Screenshot** | -|-------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------| -| 1 | Open a cloud shell by navigating to portal.azure.com and clicking the shell icon. | ![image](/docs/imgs/testing-screenshots/shell.png) | -| 2 | Select PowerShell. | ![image](/docs/imgs/testing-secreenshots/shell2.png) | -| 3.a | If you have already cloned the LME repo then make sure you are in the `LME\testing` directory and run git pull before changing to the testing directory. | | -| 3.b | If you haven't cloned it, clone the github repo in the home directory. `git clone https://github.com/cisagov/LME.git` and then `cd LME\testing`. | | -| 4 | Now you can run one of the commands from the Examples above. | | -| 5 | Save the login credentials printed to the terminal at the end. *See note* | | -| 6 | When you're done testing, simply delete the resource group to clean up all resources created. | | - -Note: When the script finishes you will be in the azure_scripts directory, and you should see the elasticsearch credentials printed to the terminal. -You will need to `cd ../../` to get back to the LME directory. All the passwords should also be in the `<$ResourceGroup>.password.txt` file. - - +See the Readme for the installers in `v2/installers` and for the tests in the `tests` directory \ No newline at end of file diff --git a/testing/SetupTestbed.ps1 b/testing/SetupTestbed.ps1 deleted file mode 100644 index 59dc856b..00000000 --- a/testing/SetupTestbed.ps1 +++ /dev/null @@ -1,494 +0,0 @@ -<# - Creates a "blank slate" for testing/configuring LME. - - Creates the following: - - A resource group - - A virtual network, subnet, and network security group - - 2 VMs: "DC1," a Windows server, and "LS1," a Linux server. You can use -m for only the linux server - - Client VMs: Windows clients "C1", "C2", etc. up to 16 based on user input - - Promotes DC1 to a domain controller - - Adds "C" clients to the managed domain - - Adds a DNS entry pointing to LS1 - - This script should do all the work for you, simply specify a new resource group, - the number of desired clients, and optionally Auto-shutdown configuration - each time you run it. Be sure to copy the username/password it outputs at the end. - After completion, login to the VMs using RDP (for the Windows machines) or ssh (for the - linux server) to configure/test LME. -#> - -param ( - [Parameter( - HelpMessage = "Auto-Shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900). Convert timezone as necesary: (e.g. 05:30 pm ET -> 9:30 pm UTC -> 21:30 -> 2130)" - )] - $AutoShutdownTime = $null, - - [Parameter( - HelpMessage = "Auto-shutdown notification email" - )] - $AutoShutdownEmail = $null, - - [Alias("l")] - [Parameter( - HelpMessage = "Location where the cluster will be built. Default westus" - )] - [string]$Location = "westus", - - [Alias("g")] - [Parameter(Mandatory = $true)] - [string]$ResourceGroup, - - [Alias("n")] - [Parameter( - HelpMessage = "Number of clients to create (Max: 16)" - )] - [int]$NumClients = 2, - - [Alias("s")] - [Parameter(Mandatory = $true, - HelpMessage = "XX.XX.XX.XX/YY,XX.XX.XX.XX/YY,etc... Comma-Separated list of CIDR prefixes or IP ranges" - )] - [string]$AllowedSources, - - [Alias("y")] - [Parameter( - HelpMessage = "Run the script with no prompt (useful for automated runs)" - )] - [switch]$NoPrompt, - - [Alias("m")] - [Parameter( - HelpMessage = "(minimal) Only install the linux server. Useful for testing the linux server without the windows clients" - )] - [switch]$LinuxOnly -) - -$ProcessSeparator = "`n----------------------------------------`n" - -# Define our library path -$libraryPath = Join-Path -Path $PSScriptRoot -ChildPath "configure\azure_scripts\lib\utilityFunctions.ps1" - -# Check if the library file exists -if (Test-Path -Path $libraryPath) { - # Dot-source the library script - . $libraryPath -} -else { - Write-Error "Library script not found at path: $libraryPathCreating Network Port 22 rule..." -} - - -#DEFAULTS: -#Desired Netowrk Mapping: -$VNetPrefix = "10.1.0.0/16" -$SubnetPrefix = "10.1.0.0/24" -$DcIP = "10.1.0.4" -$LsIP = "10.1.0.5" - -#Default Azure Region: -# $Location = "westus" - -#Domain information: -$VMAdmin = "admin.ackbar" -$DomainName = "lme.local" - -#Port options: https://learn.microsoft.com/en-us/cli/azure/network/nsg/rule?view=azure-cli-latest#az-network-nsg-rule-create -$Ports = 22, 3389, 443, 9200, 5044 -$Priorities = 1001, 1002, 1003, 1004, 1005 -$Protocols = "Tcp", "Tcp", "Tcp", "Tcp", "Tcp" - -# Variables used for Azure tags -$CurrentUser = $(az account show | ConvertFrom-Json).user.name -$Today = $(Get-Date).ToString("yyyy-MM-dd") -$Project = "LME" - -function Get-RandomPassword { - param ( - [Parameter(Mandatory)] - [int]$Length - ) - $TokenSet = @{ - L = [Char[]]'abcdefghijkmnopqrstuvwxyz' - U = [Char[]]'ABCDEFGHIJKMNPQRSTUVWXYZ' - N = [Char[]]'23456789' - } - - $Lower = Get-Random -Count 5 -InputObject $TokenSet.L - $Upper = Get-Random -Count 5 -InputObject $TokenSet.U - $Number = Get-Random -Count 5 -InputObject $TokenSet.N - - $StringSet = $Lower + $Number + $Upper - - (Get-Random -Count $Length -InputObject $StringSet) -join '' -} - -function Set-AutoShutdown { - param ( - [Parameter(Mandatory)] - [string]$VMName - ) - - Write-Output "`nCreating Auto-Shutdown Rule for $VMName at time $AutoShutdownTime..." - if ($null -ne $AutoShutdownEmail) { - $autoShutdownResponse = az vm auto-shutdown ` - -g $ResourceGroup ` - -n $VMName ` - --time $AutoShutdownTime ` - --email $AutoShutdownEmail - Write-Output $autoShutdownResponse - } - else { - $autoShutdownResponse = az vm auto-shutdown ` - -g $ResourceGroup ` - -n $VMName ` - --time $AutoShutdownTime - Write-Output $autoShutdownResponse - } -} - -function Set-NetworkRules { - param ( - [Parameter(Mandatory)] - $AllowedSourcesList - ) - - if ($Ports.length -ne $Priorities.length) { - Write-Output "Priorities and Ports length should be equal!" - Exit 1 - } - if ($Ports.length -ne $Protocols.length) { - Write-Output "Protocols and Ports length should be equal!" - Exit 1 - } - - for ($i = 0; $i -le $Ports.length - 1; $i++) { - $port = $Ports[$i] - $priority = $Priorities[$i] - $protocol = $Protocols[$i] - Write-Output "`nCreating Network Port $port rule..." - $command = "az network nsg rule create --name Network_Port_Rule_$port " + - "--resource-group $ResourceGroup " + - "--nsg-name NSG1 " + - "--priority $priority " + - "--direction Inbound " + - "--access Allow " + - "--protocol $protocol " + - "--source-address-prefixes $AllowedSourcesList " + - "--destination-address-prefixes '*' " + - "--destination-port-ranges $port " + - "--description 'Allow inbound from $sources on $port via $protocol connections.' " - - Write-Output "Running command: $command" - - $networkRuleResponse = Invoke-Expression $command - Write-Output $networkRuleResponse - - } -} - - -######################## -# Validation of Globals # -######################## -$AllowedSourcesList = $AllowedSources -Split "," -if ($AllowedSourcesList.length -lt 1) { - Write-Output "**ERROR**: Variable AllowedSources must be set (set with -AllowedSources or -s)" - Exit 1 -} - -if ($null -ne $AutoShutdownTime) { - if (-not ( $AutoShutdownTime -match '^([01][0-9]|2[0-3])[0-5][0-9]$')) { - Write-Output "**ERROR** Invalid time" - Write-Output "Enter the Auto-Shutdown time in UTC (HHMM, e.g. 2230, 0000, 1900), `n`tConvert timezone as necesary: (e.g. 05:30 pm ET -> 9:30 pm UTC -> 21:30 -> 2130)" - Exit 1 - } -} - -if (($NumClients -lt 1 -or $NumClients -gt 16) -and -Not $LinuxOnly) { - Write-Output "The number of clients must be at least 1 and no more than 16." - $NumClients = $NumClients -as [int] - Exit 1 -} - -################ -# Confirmation # -################ -Write-Output "Supplied configuration:`n" - -Write-Output "Location: $Location" -Write-Output "Resource group: $ResourceGroup" -Write-Output "Number of clients: $NumClients" -Write-Output "Allowed sources (IP's): $AllowedSourcesList" -Write-Output "Auto-shutdown time: $AutoShutdownTime" -Write-Output "Auto-shutdown e-mail: $AutoShutdownEmail" -if ($LinuxOnly) { - Write-Output "Creating a linux server only" -} - -if (-Not $NoPrompt) { - do { - $Proceed = Read-Host "`nProceed? (Y/n)" - } until ($Proceed -eq "y" -or $Proceed -eq "Y" -or $Proceed -eq "n" -or $Proceed -eq "N") - - if ($Proceed -eq "n" -or $Proceed -eq "N") { - Write-Output "Setup canceled" - Exit - } -} - -######################## -# Setup resource group # -######################## -Write-Output "`nCreating resource group..." -$createResourceGroupResponse = az group create --name $ResourceGroup ` - --location $Location ` - --tags project=$Project created=$Today createdBy=$CurrentUser -Write-Output $createResourceGroupResponse - -################# -# Setup network # -################# - -Write-Output "`nCreating virtual network..." -$createVirtualNetworkResponse = az network vnet create --resource-group $ResourceGroup ` - --name VNet1 ` - --address-prefix $VNetPrefix ` - --subnet-name SNet1 ` - --subnet-prefix $SubnetPrefix ` - --tags project=$Project created=$Today createdBy=$CurrentUser -Write-Output $createVirtualNetworkResponse - -Write-Output "`nCreating nsg..." -$createNsgResponse = az network nsg create --name NSG1 ` - --resource-group $ResourceGroup ` - --location $Location ` - --tags project=$Project created=$Today createdBy=$CurrentUser -Write-Output $createNsgResponse - -Set-NetworkRules -AllowedSourcesList $AllowedSourcesList - -################## -# Create the VMs # -################## -$VMPassword = Get-RandomPassword 12 -Write-Output "`nWriting $VMAdmin password to ${ResourceGroup}.password.txt" -$VMPassword | Out-File -FilePath "${ResourceGroup}.password.txt" -Encoding UTF8 - - -Write-Output "`nCreating LS1..." -$createLs1Response = az vm create ` - --name LS1 ` - --resource-group $ResourceGroup ` - --nsg NSG1 ` - --image Ubuntu2204 ` - --admin-username $VMAdmin ` - --admin-password $VMPassword ` - --vnet-name VNet1 ` - --subnet SNet1 ` - --public-ip-sku Standard ` - --size Standard_E2d_v4 ` - --os-disk-size-gb 128 ` - --private-ip-address $LsIP ` - --tags project=$Project created=$Today createdBy=$CurrentUser -Write-Output $createLs1Response - -if (-Not $LinuxOnly){ - Write-Output "`nCreating DC1..." - $createDc1Response = az vm create ` - --name DC1 ` - --resource-group $ResourceGroup ` - --nsg NSG1 ` - --image Win2019Datacenter ` - --admin-username $VMAdmin ` - --admin-password $VMPassword ` - --vnet-name VNet1 ` - --subnet SNet1 ` - --public-ip-sku Standard ` - --private-ip-address $DcIP ` - --tags project=$Project created=$Today createdBy=$CurrentUser - Write-Output $createDc1Response - for ($i = 1; $i -le $NumClients; $i++) { - Write-Output "`nCreating C$i..." - $createClientResponse = az vm create ` - --name C$i ` - --resource-group $ResourceGroup ` - --nsg NSG1 ` - --image Win2019Datacenter ` - --admin-username $VMAdmin ` - --admin-password $VMPassword ` - --vnet-name VNet1 ` - --subnet SNet1 ` - --public-ip-sku Standard ` - --tags project=$Project created=$Today createdBy=$CurrentUser - Write-Output $createClientResponse - } -} - -########################### -# Configure Auto-Shutdown # -########################### - -if ($null -ne $AutoShutdownTime) { - Set-AutoShutdown "LS1" - if (-Not $LinuxOnly){ - Set-AutoShutdown "DC1" - for ($i = 1; $i -le $NumClients; $i++) { - Set-AutoShutdown "C$i" - } - } -} - -#################### -# Setup the domain # -#################### -if (-Not $LinuxOnly){ - Write-Output "`nInstalling AD Domain services on DC1..." - $addDomainServicesResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name DC1 ` - --scripts "Add-WindowsFeature AD-Domain-Services -IncludeManagementTools" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addDomainServicesResponse") - -# Write-Output "`nRestarting DC1..." -# az vm restart ` -# --resource-group $ResourceGroup ` -# --name DC1 ` - - Write-Output "`nCreating the ADDS forest..." - $installAddsForestResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name DC1 ` - --scripts "`$Password = ConvertTo-SecureString `"$VMPassword`" -AsPlainText -Force; ` - Install-ADDSForest -DomainName $DomainName -Force -SafeModeAdministratorPassword `$Password" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$installAddsForestResponse") - - Write-Output "`nRestarting DC1..." - az vm restart ` - --resource-group $ResourceGroup ` - --name DC1 ` - - for ($i = 1; $i -le $NumClients; $i++) { - Write-Output "`nAdding DC IP address to C$i host file..." - $addIpResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name C$i ` - --scripts "Add-Content -Path `$env:windir\System32\drivers\etc\hosts -Value `"`n$DcIP`t$DomainName`" -Force" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addIpResponse") - - Write-Output "`nSetting C$i DNS server to DC1..." - $setDnsResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name C$i ` - --scripts "Get-Netadapter | Set-DnsClientServerAddress -ServerAddresses $DcIP" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$setDnsResponse") - - Write-Output "`nRestarting C$i..." - az vm restart ` - --resource-group $ResourceGroup ` - --name C$i ` - - Write-Output "`nAdding C$i to the domain..." - $addToDomainResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name C$i ` - --scripts "`$Password = ConvertTo-SecureString `"$VMPassword`" -AsPlainText -Force; ` - `$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $DomainName\$VMAdmin, `$Password; ` - Add-Computer -DomainName $DomainName -Credential `$Credential -Restart" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addToDomainResponse") - - # The following command fixes this issue: - # https://serverfault.com/questions/754012/windows-10-unable-to-access-sysvol-and-netlogon - Write-Output "`nModifying C$i register to allow access to sysvol..." - $addToSysvolResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name C$i ` - --scripts "cmd.exe /c `"%COMSPEC% /C reg add HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\NetworkProvider\HardenedPaths /v \\*\SYSVOL /d RequireMutualAuthentication=0 /t REG_SZ`"" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addToSysvolResponse") - } -} - -Write-Output $ProcessSeparator -Write-Output "`nVM login info:" -Write-Output "ResourceGroup: $( $ResourceGroup )" -Write-Output "Username: $( $VMAdmin )" -Write-Output "Password: $( $VMPassword )" -Write-Output "SAVE THE ABOVE INFO`n" -Write-Output $ProcessSeparator - -if (-Not $LinuxOnly){ - Write-Output "`nAdding DNS entry for Linux server..." - Write-Warning "NOTE: To verify, log on to DC1 and run 'Resolve-DnsName ls1' in PowerShell. - If it returns NXDOMAIN, you'll need to add it manually." - Write-Output "The time is $( Get-Date )." - # Define the PowerShell script with the DomainName variable interpolated - $scriptContent = @" -`$scriptBlock = { - Add-DnsServerResourceRecordA -Name LS1 -ZoneName $DomainName. -AllowUpdateAny -IPv4Address $LsIP -TimeToLive 01:00:00 -AsJob -} -`$job = Start-Job -ScriptBlock `$scriptBlock -`$timeout = 120 -if (Wait-Job -Job `$job -Timeout `$timeout) { - Receive-Job -Job `$job - Write-Host 'The script completed within the timeout period.' -} else { - Stop-Job -Job `$job - Remove-Job -Job `$job - Write-Host 'The script timed out after `$timeout seconds.' -} -"@ - - # Convert the script to a Base64-encoded string - $bytes = [System.Text.Encoding]::Unicode.GetBytes($scriptContent) - $encodedScript = [Convert]::ToBase64String($bytes) - - - # Run the encoded script on the Azure VM - Write-Output "`nAdding script to add DNS entry for Linux server. No output expected..." - $createDnsScriptResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name DC1 ` - --resource-group $ResourceGroup ` - --scripts "Set-Content -Path 'C:\AddDnsRecord.ps1' -Value ([System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String('$encodedScript')))" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$createDnsScriptResponse") - - Write-Output "`nRunning script to add DNS entry for Linux server. It could time out or not. Check output of the next command..." - $addDnsRecordResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name DC1 ` - --resource-group $ResourceGroup ` - --scripts "C:\AddDnsRecord.ps1" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$addDnsRecordResponse") - - Write-Output "`nAdding ls1 to hosts file..." - $writeToHostsFileResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name DC1 ` - --resource-group $ResourceGroup ` - --scripts "Add-Content -Path 'C:\windows\system32\drivers\etc\hosts' -Value '$LsIP ls1.$DomainName ls1'" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$writeToHostsFileResponse") - - Write-Host "Checking if ls1 resolves. This should resolve to ls1.lme.local->${LsIP}, not another domain..." - $resolveLs1Response = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --resource-group $ResourceGroup ` - --name DC1 ` - --scripts "Resolve-DnsName ls1" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$resolveLs1Response") - - Write-Host "Removing the Dns script. No output expected..." - $removeDnsRecordScriptResponse = az vm run-command invoke ` - --command-id RunPowerShellScript ` - --name DC1 ` - --resource-group $ResourceGroup ` - --scripts "Remove-Item -Path 'C:\AddDnsRecord.ps1' -Force" - Show-FormattedOutput -FormattedOutput (Format-AzVmRunCommandOutput -JsonResponse "$removeDnsRecordScriptResponse") - -} - -Write-Output "Done." diff --git a/testing/internet_toggle.ps1 b/testing/internet_toggle.ps1 deleted file mode 100644 index b4567d0d..00000000 --- a/testing/internet_toggle.ps1 +++ /dev/null @@ -1,47 +0,0 @@ - - -param ( - [Parameter(Mandatory)] - [Alias("RG")] - [string]$ResourceGroup, - [string]$NSG = "NSG1", - [switch]$enable = $false -) - -function enable { - $list=az network nsg rule list -g $ResourceGroup --nsg-name $NSG | jq -r 'map(.name) | .[]' - - if ($list.contains("DENYINTERNET")){ - az network nsg rule delete --name DENYINTERNET -g $ResourceGroup --nsg-name $NSG - } - if ($list.contains("DENYLOAD")){ - az network nsg rule delete --name DENYLOAD -g $ResourceGroup --nsg-name $NSG - } -} - -function disable { - az network nsg rule create --name DENYINTERNET ` - --resource-group $ResourceGroup ` - --nsg-name $NSG ` - --priority 4096 ` - --direction OutBound ` - --access Deny ` - --destination-address-prefixes Internet ` - --destination-port-ranges '*' - - az network nsg rule create --name DENYLOAD ` - --resource-group $ResourceGroup ` - --nsg-name $NSG ` - --priority 4095 ` - --direction OutBound ` - --access Deny ` - --destination-address-prefixes AzureLoadBalancer ` - --destination-port-ranges '*' -} - -if ($enable) { - enable -} -else { - disable -} diff --git a/testing/tests/Dockerfile b/testing/tests/Dockerfile index 6f261c0c..fd4f9df3 100644 --- a/testing/tests/Dockerfile +++ b/testing/tests/Dockerfile @@ -1,5 +1,5 @@ -# Use Ubuntu 22.04 as base image -FROM ubuntu:22.04 +# Use Ubuntu 24.04 as base image +FROM ubuntu:24.04 # Set environment variable to avoid interactive dialogues during build ENV DEBIAN_FRONTEND=noninteractive @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y \ python3-venv \ python3-pip \ zip \ + git \ && rm -rf /var/lib/apt/lists/* # Set work directory diff --git a/testing/v2/installers/README.md b/testing/v2/installers/README.md index e4eb4f72..853f5abc 100644 --- a/testing/v2/installers/README.md +++ b/testing/v2/installers/README.md @@ -140,18 +140,6 @@ source ~/LME/venv/bin/activate -is 24_04-daily-lts-gen2 ``` -### RHEL 8 Setup (alternative to RHEL 9) -```bash -./azure/build_azure_linux_network.py \ - -g $RESOURCE_GROUP \ - -s "0.0.0.0" \ - -vs $VM_SIZE \ - -l $LOCATION \ - -ast $AUTO_SHUTDOWN_TIME \ - -pub RedHat \ - -io RHEL \ - -is 8-lvm-gen2 -``` ## Creating Additional VMs (Non-Network Attack Scenarios) From 05f75d9f0e95454895427f8571fc99a5b725bd48 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 29 Aug 2025 12:40:04 +0000 Subject: [PATCH 35/51] Clean up SELinux setup and vars --- ansible/roles/backup_lme/tasks/main.yml | 14 +++++--- ansible/roles/base/tasks/selinux_setup.yml | 13 ++++--- ansible/roles/base/tasks/selinux_vars.yml | 42 ++++++++++++++++++++++ ansible/rollback_lme.yml | 35 +++++++++++++++--- 4 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 ansible/roles/base/tasks/selinux_vars.yml diff --git a/ansible/roles/backup_lme/tasks/main.yml b/ansible/roles/backup_lme/tasks/main.yml index 7d6b3b3e..696e5da3 100644 --- a/ansible/roles/backup_lme/tasks/main.yml +++ b/ansible/roles/backup_lme/tasks/main.yml @@ -32,6 +32,12 @@ debug: msg: "Backup base directory: {{ backup_base_dir }}" +- name: Include SELinux variable setup + include_role: + name: base + tasks_from: selinux_vars.yml + tags: selinux + - name: Display backup location debug: msg: "Backups will be stored in: {{ backup_base_dir }}/backups" @@ -120,12 +126,12 @@ mkdir -p "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/etc_containers_systemd" # Copy the LME installation - cp -a "{{ lme_install_dir }}/." "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/lme/" + {{ cp_preserve_cmd }} "{{ lme_install_dir }}/." "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/lme/" LME_COPY_STATUS=$? # Copy the vault and password files from /etc/lme if [ -d "/etc/lme" ]; then - cp -a "/etc/lme/." "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/etc_lme/" + {{ cp_preserve_cmd }} "/etc/lme/." "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/etc_lme/" ETC_LME_COPY_STATUS=$? else echo "Warning: /etc/lme directory not found" @@ -134,7 +140,7 @@ # Copy the systemd container files from /etc/containers/systemd if [ -d "/etc/containers/systemd" ]; then - cp -a "/etc/containers/systemd/." "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/etc_containers_systemd/" + {{ cp_preserve_cmd }} "/etc/containers/systemd/." "{{ backup_base_dir }}/backups/{{ backup_timestamp }}/etc_containers_systemd/" SYSTEMD_COPY_STATUS=$? echo "Systemd container files backed up successfully" else @@ -266,7 +272,7 @@ # Copy the volume contents if [ -d "$VOLUME_PATH" ] && [ "$(ls -A $VOLUME_PATH)" ]; then - cp -a "$VOLUME_PATH/." "$VOLUME_DIR/data/" + {{ cp_preserve_cmd }} "$VOLUME_PATH/." "$VOLUME_DIR/data/" if [ $? -eq 0 ]; then echo "SUCCESS" > "$VOLUME_DIR/backup_status.txt" echo "Volume {{ item }} backed up successfully" diff --git a/ansible/roles/base/tasks/selinux_setup.yml b/ansible/roles/base/tasks/selinux_setup.yml index 2962ecc3..6ccd9a8e 100644 --- a/ansible/roles/base/tasks/selinux_setup.yml +++ b/ansible/roles/base/tasks/selinux_setup.yml @@ -1,6 +1,9 @@ --- # SELinux setup tasks for LME - run early to ensure proper file labeling +- name: Load SELinux facts + import_tasks: selinux_vars.yml + - name: Detect if SELinux tooling is available command: which getenforce register: selinux_tooling @@ -98,7 +101,7 @@ register: selinux_compile when: - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_active | default(false) - selinux_policy_deployed.changed or selinux_fc_deployed.changed - name: Debug SELinux compile result @@ -115,7 +118,7 @@ become: yes when: - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_active | default(false) - name: Load SELinux module (lme_policy) shell: | @@ -139,7 +142,7 @@ changed_when: semodule_load.rc == 0 when: - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_active | default(false) - selinux_compile.changed | default(false) or (lme_policy_present_pre.rc | default(1)) != 0 - name: Ensure SELinux module enabled (lme_policy) @@ -147,7 +150,7 @@ become: yes when: - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_active | default(false) - name: Verify SELinux module loaded shell: semodule -l | grep -E "^lme_policy(\\s|$)" || true @@ -158,7 +161,7 @@ become: yes when: - selinux_available | default(false) - - (getenforce_out.stdout | default('') | trim) != 'Disabled' + - selinux_active | default(false) - name: Assert LME policy module present assert: diff --git a/ansible/roles/base/tasks/selinux_vars.yml b/ansible/roles/base/tasks/selinux_vars.yml new file mode 100644 index 00000000..ff864ea5 --- /dev/null +++ b/ansible/roles/base/tasks/selinux_vars.yml @@ -0,0 +1,42 @@ +--- +# Minimal SELinux variable detection - sets facts only, no configuration changes + +- name: Ensure SELinux facts are present (subset) + setup: + gather_subset: + - '!all' + - '!min' + - selinux + +- name: Set SELinux facts using built-in Ansible facts + set_fact: + selinux_enabled: "{{ ansible_selinux.status is defined and ansible_selinux.status == 'enabled' }}" + selinux_enforcing: "{{ ansible_selinux.mode is defined and ansible_selinux.mode == 'enforcing' }}" + selinux_permissive: "{{ ansible_selinux.mode is defined and ansible_selinux.mode == 'permissive' }}" + selinux_disabled: "{{ ansible_selinux.status is not defined or ansible_selinux.status == 'disabled' }}" + + # Convenience variables + selinux_context_supported: "{{ ansible_selinux.status is defined and ansible_selinux.status == 'enabled' }}" + selinux_active: "{{ ansible_selinux.status is defined and ansible_selinux.status == 'enabled' and ansible_selinux.mode != 'disabled' }}" + + # For backup/rollback - whether cp can preserve contexts + selinux_preserve_contexts: "{{ ansible_selinux.status is defined and ansible_selinux.status == 'enabled' }}" + + # Precomputed command for cp + cp_preserve_cmd: "{{ 'cp -a --preserve=context' if (ansible_selinux.status is defined and ansible_selinux.status == 'enabled') else 'cp -a' }}" + +- name: Display SELinux status (list format) + debug: + msg: + - "SELinux status: {{ ansible_selinux.status | default('unknown') }}" + - "SELinux mode: {{ ansible_selinux.mode | default('unknown') }}" + - "Enabled: {{ selinux_enabled }}" + - "Enforcing: {{ selinux_enforcing }}" + - "Permissive: {{ selinux_permissive }}" + - "Disabled: {{ selinux_disabled }}" + - "Active (not disabled): {{ selinux_active }}" + - "Context supported: {{ selinux_context_supported }}" + - "cp preserve cmd: {{ cp_preserve_cmd }}" + when: debug_mode | default(false) + + diff --git a/ansible/rollback_lme.yml b/ansible/rollback_lme.yml index 031e0271..b9f6fb7f 100644 --- a/ansible/rollback_lme.yml +++ b/ansible/rollback_lme.yml @@ -64,6 +64,12 @@ debug: msg: "Using podman from: {{ podman_cmd }}" + - name: Include SELinux variable setup + include_role: + name: base + tasks_from: selinux_vars.yml + tags: selinux + - name: Get Podman graphroot location shell: | # Get the full path to the storage directory @@ -280,13 +286,13 @@ shell: | # Restore LME installation mkdir -p {{ lme_install_dir }} - cp -a {{ selected_backup_dir }}/lme/. {{ lme_install_dir }}/ + {{ cp_preserve_cmd }} {{ selected_backup_dir }}/lme/. {{ lme_install_dir }}/ LME_RESTORE_STATUS=$? # Restore vault files if they exist in backup if [ -d "{{ selected_backup_dir }}/etc_lme" ]; then mkdir -p /etc/lme - cp -af {{ selected_backup_dir }}/etc_lme/. /etc/lme/ + {{ cp_preserve_cmd }} -f {{ selected_backup_dir }}/etc_lme/. /etc/lme/ VAULT_RESTORE_STATUS=$? # Ensure proper permissions on restored vault files @@ -305,7 +311,7 @@ # Restore systemd container files if they exist in backup if [ -d "{{ selected_backup_dir }}/etc_containers_systemd" ]; then mkdir -p /etc/containers/systemd - cp -af {{ selected_backup_dir }}/etc_containers_systemd/. /etc/containers/systemd/ + {{ cp_preserve_cmd }} -f {{ selected_backup_dir }}/etc_containers_systemd/. /etc/containers/systemd/ SYSTEMD_RESTORE_STATUS=$? # Ensure proper permissions on restored systemd files @@ -331,6 +337,16 @@ executable: /bin/bash when: backup_stat.stat.exists register: restore_result + + - name: Restore SELinux contexts for restored paths + command: restorecon -Rv "{{ item }}" + loop: + - "{{ lme_install_dir }}" + - "/etc/lme" + - "/etc/containers/systemd" + when: + - selinux_active | default(false) + changed_when: false - name: Reload systemd daemon after restoring systemd files systemd: @@ -579,7 +595,7 @@ if [ -d "$BACKUP_DIR" ]; then # Copy the backup data to the volume - cp -a "$BACKUP_DIR/." "$VOLUME_PATH/" + {{ cp_preserve_cmd }} "$BACKUP_DIR/." "$VOLUME_PATH/" if [ $? -eq 0 ]; then echo "Restored {{ item.item }} to ${VOLUME_PATH}" exit 0 @@ -596,6 +612,17 @@ loop: "{{ volume_paths.results }}" register: volume_restore_result + - name: Restore SELinux contexts for volumes + shell: | + VOLUME_PATH=$({{ podman_cmd }} volume inspect "{{ item }}" --format "{{ '{{' }}.Mountpoint{{ '}}' }}") + restorecon -Rv "$VOLUME_PATH" + args: + executable: /bin/bash + loop: "{{ volume_names }}" + when: + - selinux_active | default(false) + changed_when: false + - name: Set volume list for verification set_fact: volume_list: "{{ volume_names | join(' ') }}" From fd6d745abadb8578967188f37d6ad99c7b9b4b16 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 29 Aug 2025 10:53:27 -0400 Subject: [PATCH 36/51] Updates the example args for the azure linux network script --- testing/v2/installers/README.md | 7 ++++--- testing/v2/installers/azure/build_azure_linux_network.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/testing/v2/installers/README.md b/testing/v2/installers/README.md index 853f5abc..b2194b9a 100644 --- a/testing/v2/installers/README.md +++ b/testing/v2/installers/README.md @@ -131,13 +131,14 @@ source ~/LME/venv/bin/activate ```bash ./azure/build_azure_linux_network.py \ -g $RESOURCE_GROUP \ - -s "0.0.0.0" \ + -s "0.0.0.0/0" \ -vs $VM_SIZE \ -l $LOCATION \ -ast $AUTO_SHUTDOWN_TIME \ -pub Canonical \ - -io 0001-com-ubuntu-server-noble-daily \ - -is 24_04-daily-lts-gen2 + -io ubuntu-24_04-lts \ + -is server \ + --no-prompt ``` diff --git a/testing/v2/installers/azure/build_azure_linux_network.py b/testing/v2/installers/azure/build_azure_linux_network.py index 39d9ae85..089fd1a5 100755 --- a/testing/v2/installers/azure/build_azure_linux_network.py +++ b/testing/v2/installers/azure/build_azure_linux_network.py @@ -735,7 +735,13 @@ def main( args.machine_name = "rhel" if args.machine_name == "ubuntu" else args.machine_name print(f"Using Red Hat Enterprise Linux image: {args.image_publisher}:{args.image_offer}:{args.image_sku}") else: - print("Using Ubuntu 22.04 image") + # Detect Ubuntu version based on image parameters + if args.image_offer == "ubuntu-24_04-lts" or "24" in args.image_sku: + print(f"Using Ubuntu 24.04 image: {args.image_publisher}:{args.image_offer}:{args.image_sku}") + elif args.image_offer == "0001-com-ubuntu-server-jammy" or "22" in args.image_sku: + print(f"Using Ubuntu 22.04 image: {args.image_publisher}:{args.image_offer}:{args.image_sku}") + else: + print(f"Using Ubuntu image: {args.image_publisher}:{args.image_offer}:{args.image_sku}") check_ports_protocals_and_priorities( args.ports, args.priorities, args.protocols From c9a0dea38f19b6281ffa715da451b074d9eba6f6 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 2 Sep 2025 05:13:22 -0400 Subject: [PATCH 37/51] Change install.sh to install EPEL repository on RHEL9 --- install.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index ed8ef3c8..a5ec62d4 100755 --- a/install.sh +++ b/install.sh @@ -178,7 +178,7 @@ install_ansible() { # Set noninteractive mode for apt-based installations export DEBIAN_FRONTEND=noninteractive - + case $DISTRO in ubuntu|debian|linuxmint|pop) sudo ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime @@ -191,8 +191,20 @@ install_ansible() { sudo dnf install -y ansible ;; centos|rhel|rocky|almalinux) - sudo dnf install -y epel-release - # Try to install ansible via dnf first + # Install EPEL repository first + echo "Installing EPEL repository..." + if ! sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm -E %rhel).noarch.rpm; then + echo -e "${RED}✗ Failed to install EPEL repository${NC}" + exit 1 + fi + echo -e "${GREEN}✓ EPEL repository installed${NC}" + + # For RHEL, you might also need to enable CodeReady Builder repository + if [[ "$DISTRO" == "rhel" ]]; then + sudo subscription-manager repos --enable codeready-builder-for-rhel-$(rpm -E %rhel)-$(arch)-rpms 2>/dev/null || true + fi + + # Now try to install ansible via dnf if sudo dnf install -y ansible; then echo -e "${GREEN}✓ Ansible installed via dnf${NC}" else From 3764b7aac7a4907ecf51bc61b5fbf873e6dc78ef Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 2 Sep 2025 06:41:34 -0400 Subject: [PATCH 38/51] Adds CodeReady Builder for RHEL 9 --- ansible/roles/base/tasks/redhat_9.yml | 31 ++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/ansible/roles/base/tasks/redhat_9.yml b/ansible/roles/base/tasks/redhat_9.yml index 6acf1b9c..e19bc52e 100644 --- a/ansible/roles/base/tasks/redhat_9.yml +++ b/ansible/roles/base/tasks/redhat_9.yml @@ -23,10 +23,39 @@ until: epel_install is success ignore_errors: "{{ ansible_check_mode }}" -- name: Enable PowerTools/CRB repository +- name: Install dnf-plugins-core + dnf: + name: dnf-plugins-core + state: present + become: yes + +- name: Check available repositories + command: dnf repolist --all + register: available_repos + changed_when: false + become: yes + +- name: Enable CRB repository (Rocky/AlmaLinux) command: dnf config-manager --set-enabled crb become: yes changed_when: true + when: "'crb' in available_repos.stdout" + ignore_errors: true + +- name: Enable PowerTools repository (CentOS) + command: dnf config-manager --set-enabled powertools + become: yes + changed_when: true + when: "'powertools' in available_repos.stdout" + ignore_errors: true + +- name: Enable CodeReady Builder for RHEL (if registered) + command: subscription-manager repos --enable codeready-builder-for-rhel-9-x86_64-rpms + become: yes + changed_when: true + when: + - "'codeready-builder' in available_repos.stdout" + - ansible_distribution == "RedHat" ignore_errors: true - name: Install Red Hat 9-specific packages From 2303d77b921294f3986032cd9d410ab8b02a2a25 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 4 Sep 2025 07:46:30 -0400 Subject: [PATCH 39/51] Sbom generator fixes for Red Hat 9 --- docker/rhel9/Dockerfile | 8 +- docker/rhel9/README.md | 84 ++++++++++++++++--- scripts/sbom/generate-ansible-sbom.py | 116 ++++++++++++++++++++------ 3 files changed, 167 insertions(+), 41 deletions(-) diff --git a/docker/rhel9/Dockerfile b/docker/rhel9/Dockerfile index a421e96a..21058c8e 100644 --- a/docker/rhel9/Dockerfile +++ b/docker/rhel9/Dockerfile @@ -9,7 +9,7 @@ ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 RUN dnf update -y && dnf install -y \ - glibc-langpack-en sudo openssh-clients git \ + glibc-langpack-en sudo openssh-clients git vim \ && dnf clean all \ && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ && while getent passwd $USER_ID > /dev/null 2>&1; do USER_ID=$((USER_ID + 1)); done \ @@ -37,9 +37,9 @@ RUN dnf install -y \ && dnf clean all # Install EPEL repository -RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ - && /usr/bin/crb enable \ - && dnf clean all +# RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ +# && /usr/bin/crb enable \ +# && dnf clean all RUN cd /lib/systemd/system/sysinit.target.wants/ && \ ls | grep -v systemd-tmpfiles-setup | xargs rm -f $1 && \ diff --git a/docker/rhel9/README.md b/docker/rhel9/README.md index 956aef7e..1317ef3d 100644 --- a/docker/rhel9/README.md +++ b/docker/rhel9/README.md @@ -23,27 +23,35 @@ If you have a Red Hat Enterprise Linux subscription and want to use package mana 1. **Register the container** with your RHEL subscription: ```bash docker exec -it lme subscription-manager register --username --password + # or + docker exec -it lme subscription-manager register --activationkey=YOUR_ACTIVATION_KEY --org=YOUR_ORG_ID ``` -2. **Attach to a subscription**: +2. **Attach to a subscription (if needed)**: ```bash docker exec -it lme subscription-manager attach --auto ``` -3. **Enable Ansible repositories**: +3. **Enable Ansible repositories (should be installed by install.sh)**: ```bash - docker exec -it lme subscription-manager repos --enable ansible-2.9-for-rhel-9-x86_64-rpms + dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm ``` -4. **Install Ansible via package manager**: - ```bash - docker exec -it lme dnf install -y ansible-core - ``` **Note**: Without a RHEL subscription, the install script will automatically fall back to pip installation. ## Quick Start +### Prerequisites Setup +1. **Set the HOST_IP environment variable**: + ```bash + # Copy and edit the environment file + cp environment_example.sh environment.sh + # Edit environment.sh to set your HOST_IP + nano environment.sh + ``` + +### Option 1: Manual Installation (Current Default) 1. **Build and start the container**: ```bash docker compose up -d --build @@ -51,14 +59,26 @@ If you have a Red Hat Enterprise Linux subscription and want to use package mana 2. **Run the LME installation**: ```bash - docker exec -it lme bash -c "cd /root/LME && sudo ./install.sh" + docker exec -it lme bash -c "cd /root/LME && sudo ./install.sh -d" + ``` + +### Option 2: Monitor Setup Progress +If you want to monitor the installation process: +1. **Run the setup checker** (in a separate terminal): + ```bash + # For Linux/macOS + ./check-lme-setup.sh + + # For Windows PowerShell + .\check-lme-setup.ps1 ``` -3. **Access the services**: - - Kibana: http://localhost:5601 - - Elasticsearch: http://localhost:9200 - - Fleet Server: http://localhost:8220 - - HTTPS: https://localhost:443 +### Access the Services +Once installation is complete: +- Kibana: http://localhost:5601 +- Elasticsearch: http://localhost:9200 +- Fleet Server: http://localhost:8220 +- HTTPS: https://localhost:443 ## Container Features @@ -89,6 +109,12 @@ If you have a Red Hat Enterprise Linux subscription and want to use package mana - `HOST_UID`: User ID for lme-user (default: 1001) - `HOST_GID`: Group ID for lme-user (default: 1001) +### Container Environment Variables (Auto-configured) +The following environment variables are automatically set by docker-compose: +- `PODMAN_IGNORE_CGROUPSV1_WARNING`: Suppresses podman cgroup warnings +- `LANG`, `LANGUAGE`, `LC_ALL`: Locale settings (en_US.UTF-8) +- `container`: Set to "docker" for container detection + ## Troubleshooting ### Common Issues @@ -113,6 +139,12 @@ If you have a Red Hat Enterprise Linux subscription and want to use package mana - **Solution**: Ensure the container is running with proper privileges - **Check**: `docker inspect lme | grep -i privileged` +#### Volume Mount Issues +- **Problem**: "No such file or directory" when accessing /root/LME +- **Solution**: Ensure you're running docker-compose from the correct directory +- **Details**: The volume mount `../../../LME:/root/LME` expects the LME directory to be 3 levels up from your current location +- **Fix**: Run docker-compose from the correct path or adjust the volume mount in docker-compose.yml + ### Debugging Commands ```bash @@ -133,8 +165,26 @@ docker exec -it lme ansible --version # Check available repositories docker exec -it lme dnf repolist + +# Monitor setup progress (if using automated setup) +./check-lme-setup.sh # Linux/macOS +.\check-lme-setup.ps1 # Windows PowerShell ``` +### Setup Monitoring +The directory includes setup monitoring scripts that can track installation progress: +- `check-lme-setup.sh`: Linux/macOS script to monitor setup +- `check-lme-setup.ps1`: Windows PowerShell script to monitor setup + +These scripts: +- Monitor for 30 minutes by default +- Check for successful completion messages +- Report Ansible playbook failures +- Track progress through multiple playbook executions +- Exit with appropriate status codes for automation + +**Note**: These scripts expect an `lme-setup` systemd service to be running. Currently, the automated setup service is disabled in the RHEL9 container configuration, so these scripts are primarily useful for development or if you enable the automated setup service. + ## Development ### Building from Source @@ -151,6 +201,14 @@ docker compose build --build-arg USER_ID=1000 --build-arg GROUP_ID=1000 - Update `docker-compose.yml` for different port mappings - Edit `environment.sh` to set custom environment variables +### Differences from Other Docker Setups +The RHEL9 container setup differs from other LME Docker configurations (22.04, 24.04, d12.10): +- **Manual Installation**: Currently requires manual execution of install.sh +- **No Automated Service**: The lme-setup.service is commented out in the Dockerfile +- **UBI9 Base**: Uses Red Hat Universal Base Image instead of Ubuntu/Debian +- **EPEL Dependencies**: Relies on EPEL repository for additional packages +- **Ansible Installation**: Automatically falls back to pip installation due to missing ansible-core in EPEL + ## Support For issues related to: diff --git a/scripts/sbom/generate-ansible-sbom.py b/scripts/sbom/generate-ansible-sbom.py index f526dff7..cb171d8d 100644 --- a/scripts/sbom/generate-ansible-sbom.py +++ b/scripts/sbom/generate-ansible-sbom.py @@ -24,8 +24,10 @@ def get_os_version() -> Dict[str, str]: data = {} with open(release_path, 'r') as fp: for line in fp.readlines(): - key, value = line.strip().split("=", 1) - data[key] = value + line = line.strip() + if line and "=" in line: + key, value = line.split("=", 1) + data[key] = value info["name"] = data.get("NAME", "").strip("\"") @@ -33,26 +35,44 @@ def get_os_version() -> Dict[str, str]: return info -def get_package_version(package: str) -> str: +def get_package_version(package: str, os_info: Dict[str, str]) -> str: version = 'unknown' - + + os_name = os_info.get("name", "").lower() + try: - result = subprocess.run(['apt-cache', 'show', package], capture_output=True, text=True, timeout=10) - if result.returncode == 0: - version_match = re.search(r'Version:\s*([^\n]+)', result.stdout) - if version_match: - version = version_match.group(1).strip() + # Determine which package manager to use based on OS + if 'red hat' in os_name or 'rhel' in os_name or 'centos' in os_name or 'fedora' in os_name: + # Use dnf/yum for Red Hat-based systems + for cmd in ['dnf', 'yum']: + try: + result = subprocess.run([cmd, 'info', package], capture_output=True, text=True, timeout=10) + if result.returncode == 0: + version_match = re.search(r'Version\s*:\s*([^\n]+)', result.stdout) + if version_match: + version = version_match.group(1).strip() + break + except FileNotFoundError: + continue + else: + # Use apt for Debian/Ubuntu systems + result = subprocess.run(['apt-cache', 'show', package], capture_output=True, text=True, timeout=10) + if result.returncode == 0: + version_match = re.search(r'Version:\s*([^\n]+)', result.stdout) + if version_match: + version = version_match.group(1).strip() except(subprocess.TimeoutExpired, subprocess.SubprocessError, FileNotFoundError): version = 'unknown' return version class Package(): - def __init__(self, name: str, version: str, file: Path, pkg_type: str): + def __init__(self, name: str, version: str, file: Path, pkg_type: str, os_info: Dict[str, str] = None): self.name = name self.version = version self.file = file self.package_type = pkg_type + self.os_info = os_info or {} self.hash_string = self.make_hash_string() self.reference_locator = self.get_reference_locator() @@ -60,8 +80,27 @@ def __init__(self, name: str, version: str, file: Path, pkg_type: str): def get_reference_locator(self) -> str: reference_locator = "NOASSERTION" - if self.package_type == "apt": - reference_locator = f"pkg:deb/debian/{self.name}" + + if self.package_type == "rpm": + # For Red Hat-based systems + os_name = self.os_info.get("name", "").lower() + if 'red hat' in os_name or 'rhel' in os_name: + reference_locator = f"pkg:rpm/redhat/{self.name}" + elif 'centos' in os_name: + reference_locator = f"pkg:rpm/centos/{self.name}" + elif 'fedora' in os_name: + reference_locator = f"pkg:rpm/fedora/{self.name}" + else: + reference_locator = f"pkg:rpm/{self.name}" + elif self.package_type == "deb": + # For Debian-based systems + os_name = self.os_info.get("name", "").lower() + if 'ubuntu' in os_name: + reference_locator = f"pkg:deb/ubuntu/{self.name}" + elif 'debian' in os_name: + reference_locator = f"pkg:deb/debian/{self.name}" + else: + reference_locator = f"pkg:deb/{self.name}" elif self.package_type == "nix": reference_locator = f"pkg:nix/{self.name}" @@ -227,7 +266,7 @@ def make_sbom_data(self, root_package_id: str) -> SbomPart: sbom_part = SbomPart(root_package_id) spdx_file_id = sbom_part.add_file(self.file_path, self.baseDir) for pkg in self.nix_packages: - p = Package(pkg, "unknown", self.file_path, "nix") + p = Package(pkg, "unknown", self.file_path, "nix", {}) sbom_part.add_package(p, spdx_file_id) return sbom_part @@ -244,20 +283,35 @@ def is_valid_release_type(self, release: str) -> bool: os_release = self.os_info.get("version", "").lower() os_release = os_release.replace(".", "_") - # print(os_name, os_release, release) - - if " " in os_name: - os_name = os_name.split(" ")[0] #debian + # print(f"Checking release '{release}' against OS '{os_name}' version '{os_release}'") - if 'common' in release: + # Always include common packages + if 'common' in release.lower(): return True + + # Handle Red Hat Enterprise Linux + if 'red hat' in os_name or 'rhel' in os_name: + if 'redhat' in release.lower(): + # If there are no numbers in the release name, assume it applies to all versions + if not re.search(r'\d', release): + return True + # Check for version match (e.g., redhat_9 matches version 9.x) + major_version = os_release.split('_')[0] if '_' in os_release else os_release.split('.')[0] + if major_version in release.lower(): + return True + return False + + # Handle other distributions + if " " in os_name: + os_name = os_name.split(" ")[0] # Take first word (e.g., "red" from "red hat") + if os_name in release.lower(): - #if there are no numbers, assume no version name + # If there are no numbers, assume no version name if not re.search(r'\d', release): return True - if os_release in release.lower(): return True + return False def get_apt_packages_list(self): @@ -270,8 +324,15 @@ def get_apt_packages_list(self): if self.is_valid_release_type(release_type): packages.update(package_list) + # Determine package type based on OS + os_name = self.os_info.get("name", "").lower() + if 'red hat' in os_name or 'rhel' in os_name or 'centos' in os_name or 'fedora' in os_name: + pkg_type = "rpm" + else: + pkg_type = "deb" + for package in packages: - pkg = Package(package, get_package_version(package), self.filepath, pkg_type="apt") + pkg = Package(package, get_package_version(package, self.os_info), self.filepath, pkg_type, self.os_info) self.package_details.append(pkg) def make_sbom_data(self, root_package_id: str) -> SbomPart: @@ -294,11 +355,18 @@ def main(): os_info = get_os_version() nix_dir = base_dir / "roles" / "nix" / "tasks" - nix_file_path = nix_dir / "ubuntu.yml" #default to ubuntu - if 'ubuntu' in os_info.get("name", "").lower(): + os_name = os_info.get("name", "").lower() + + # Select the appropriate nix file based on OS + if 'red hat' in os_name or 'rhel' in os_name: + nix_file_path = nix_dir / "redhat.yml" + elif 'ubuntu' in os_name: nix_file_path = nix_dir / "ubuntu.yml" - elif 'debian' in os_info.get("name", "").lower(): + elif 'debian' in os_name: nix_file_path = nix_dir / "debian.yml" + else: + # Default to common if no specific match + nix_file_path = nix_dir / "common.yml" nixParser = NixPackageParser(nix_file_path, base_dir) From fac65cb697bc761f40df2853fb7592fa60971fe4 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 11 Sep 2025 10:21:04 +0000 Subject: [PATCH 40/51] Adds script to configure Red Hat firewall --- ansible/roles/base/tasks/redhat.yml | 15 ++ scripts/configure_rhel_firewall.sh | 270 ++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100755 scripts/configure_rhel_firewall.sh diff --git a/ansible/roles/base/tasks/redhat.yml b/ansible/roles/base/tasks/redhat.yml index 4fded320..5a92c0bc 100644 --- a/ansible/roles/base/tasks/redhat.yml +++ b/ansible/roles/base/tasks/redhat.yml @@ -97,6 +97,21 @@ become: yes when: ansible_os_family == 'RedHat' +- name: Disable and stop firewalld service + systemd: + name: firewalld + enabled: no + state: stopped + become: yes + ignore_errors: yes + register: firewalld_disable + tags: ['firewall'] + +- name: Debug - Show firewalld disable result + debug: + msg: "Firewalld service {{ 'successfully disabled' if firewalld_disable.changed else 'was already disabled or not installed' }}" + when: debug_mode | default(false) + - name: Set timezone timezone: name: "{{ timezone_area | default('Etc') }}/{{ timezone_zone | default('UTC') }}" diff --git a/scripts/configure_rhel_firewall.sh b/scripts/configure_rhel_firewall.sh new file mode 100755 index 00000000..42d65947 --- /dev/null +++ b/scripts/configure_rhel_firewall.sh @@ -0,0 +1,270 @@ +#!/bin/bash +# LME Red Hat Firewall Configuration Script +# This script automatically detects and configures firewalld rules for LME +# Compatible with Red Hat Enterprise Linux, CentOS, and Fedora systems + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Check if running as root +if [[ $EUID -eq 0 ]]; then + SUDO="" +else + SUDO="sudo" +fi + +# Function to check if firewalld is installed and running +check_firewalld() { + log_info "Checking firewalld status..." + + # Check if firewalld is installed + if ! command -v firewall-cmd &> /dev/null; then + log_error "firewalld is not installed. Installing..." + $SUDO dnf install -y firewalld + fi + + # Check if firewalld service is running + if ! $SUDO systemctl is-active --quiet firewalld; then + log_warning "firewalld is not running. Starting..." + $SUDO systemctl enable --now firewalld + fi + + # Verify firewalld is responding + if ! $SUDO firewall-cmd --state &> /dev/null; then + log_error "firewalld is not responding properly" + exit 1 + fi + + log_success "firewalld is installed and running" +} + +# Function to detect LME container network +detect_lme_network() { + log_info "Detecting LME container network..." + + # Check if podman is installed + if ! command -v podman &> /dev/null; then + log_error "podman is not installed or not in PATH" + return 1 + fi + + # Try to get LME network information + local lme_subnet + if lme_subnet=$($SUDO -i podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then + if [[ -n "$lme_subnet" && "$lme_subnet" != "null" ]]; then + echo "$lme_subnet" + return 0 + fi + fi + + log_warning "Could not detect LME network. LME containers may not be running." + log_info "Default podman subnet 10.88.0.0/16 will be used as fallback" + echo "10.88.0.0/16" + return 0 +} + +# Function to detect podman interfaces +detect_podman_interfaces() { + log_info "Detecting podman network interfaces..." + + local interfaces=() + + # Try to get LME network interface (most important) + local lme_interface + if lme_interface=$($SUDO -i podman network inspect lme 2>/dev/null | jq -r '.[].network_interface' 2>/dev/null); then + if [[ -n "$lme_interface" && "$lme_interface" != "null" ]]; then + interfaces+=("$lme_interface") + log_info "Found LME network interface: $lme_interface" + fi + fi + + # Look for common podman interfaces + for iface in podman0 podman1 cni-podman0 cni-podman1; do + if ip link show "$iface" &> /dev/null; then + if [[ ! " ${interfaces[@]} " =~ " ${iface} " ]]; then + interfaces+=("$iface") + log_info "Found additional podman interface: $iface" + fi + fi + done + + if [[ ${#interfaces[@]} -eq 0 ]]; then + log_warning "No podman interfaces detected. Container networking may not be configured." + echo "" + else + printf '%s\n' "${interfaces[@]}" + fi +} + +# Function to configure firewall rules +configure_firewall() { + local lme_subnet="$1" + local interfaces=("${@:2}") + + log_info "Configuring firewall rules for LME..." + + # Add LME ports to public zone + local lme_ports=(1514 1515 8220 9200 5601 443) + + log_info "Adding LME ports to public zone..." + for port in "${lme_ports[@]}"; do + if $SUDO firewall-cmd --permanent --zone=public --add-port="${port}/tcp" &> /dev/null; then + log_success "Added port ${port}/tcp to public zone" + else + log_warning "Port ${port}/tcp may already be configured" + fi + done + + # Optional: Add Wazuh API port + read -p "Do you want to enable Wazuh API port 55000? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + if $SUDO firewall-cmd --permanent --zone=public --add-port=55000/tcp &> /dev/null; then + log_success "Added Wazuh API port 55000/tcp to public zone" + else + log_warning "Port 55000/tcp may already be configured" + fi + fi + + # Configure container networking + log_info "Configuring container networking..." + + # Add container subnet to trusted zone + if [[ -n "$lme_subnet" ]]; then + if $SUDO firewall-cmd --permanent --zone=trusted --add-source="$lme_subnet" &> /dev/null; then + log_success "Added container subnet $lme_subnet to trusted zone" + else + log_warning "Container subnet $lme_subnet may already be configured" + fi + fi + + # Add podman interfaces to trusted zone + for iface in "${interfaces[@]}"; do + if [[ -n "$iface" ]]; then + if $SUDO firewall-cmd --permanent --zone=trusted --add-interface="$iface" &> /dev/null; then + log_success "Added interface $iface to trusted zone" + else + log_warning "Interface $iface may already be configured" + fi + fi + done + + # Enable masquerading for container traffic + if $SUDO firewall-cmd --permanent --add-masquerade &> /dev/null; then + log_success "Enabled masquerading for container traffic" + else + log_warning "Masquerading may already be enabled" + fi + + # Reload firewall to apply changes + log_info "Reloading firewall configuration..." + $SUDO firewall-cmd --reload + log_success "Firewall configuration reloaded" +} + +# Function to verify configuration +verify_configuration() { + log_info "Verifying firewall configuration..." + + echo + echo "=== Current Firewall Configuration ===" + echo + + echo "Public Zone (External Access):" + $SUDO firewall-cmd --zone=public --list-ports + echo + + echo "Trusted Zone (Container Networks):" + $SUDO firewall-cmd --zone=trusted --list-all + echo + + echo "=== Active Zones ===" + $SUDO firewall-cmd --get-active-zones + echo +} + +# Function to provide troubleshooting information +provide_troubleshooting() { + log_info "Troubleshooting Information:" + echo + echo "If you experience connectivity issues:" + echo "1. Check if LME containers are running:" + echo " sudo -i podman ps" + echo + echo "2. Test container-to-container communication:" + echo " sudo -i podman exec lme-kibana curl -s http://lme-elasticsearch:9200/_cluster/health" + echo + echo "3. Test external access (replace with your server IP):" + echo " curl -v http://YOUR_SERVER_IP:5601" + echo + echo "4. Check firewall logs for blocked connections:" + echo " sudo journalctl -u firewalld | tail -20" + echo + echo "5. Temporarily disable firewall for testing:" + echo " sudo systemctl stop firewalld" + echo " # Test your connections" + echo " sudo systemctl start firewalld" + echo +} + +# Main execution +main() { + echo "========================================" + echo "LME Red Hat Firewall Configuration" + echo "========================================" + echo + + # Check prerequisites + check_firewalld + + # Detect network configuration + lme_subnet=$(detect_lme_network) + readarray -t interfaces <<< "$(detect_podman_interfaces)" + + # Display detected configuration + echo + log_info "Detected Configuration:" + echo " LME Container Subnet: $lme_subnet" + if [[ ${#interfaces[@]} -gt 0 && -n "${interfaces[0]}" ]]; then + echo " Podman Interfaces: ${interfaces[*]}" + else + echo " Podman Interfaces: None detected" + fi + echo + + # Confirm before proceeding + read -p "Do you want to configure firewalld with these settings? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "Configuration cancelled by user" + exit 0 + fi + + # Configure firewall + configure_firewall "$lme_subnet" "${interfaces[@]}" + + # Verify configuration + verify_configuration + + # Provide troubleshooting info + provide_troubleshooting + + log_success "LME firewall configuration completed!" +} + +# Run main function if script is executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi From 3886a3999f1fc22ac9db3f1a4a6c5adf29bf2ee8 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 11 Sep 2025 12:58:01 +0000 Subject: [PATCH 41/51] Adds nftables configuration script for Red Hat 9 --- scripts/configure_lme_nftables.sh | 401 ++++++++++++++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100755 scripts/configure_lme_nftables.sh diff --git a/scripts/configure_lme_nftables.sh b/scripts/configure_lme_nftables.sh new file mode 100755 index 00000000..b161d323 --- /dev/null +++ b/scripts/configure_lme_nftables.sh @@ -0,0 +1,401 @@ +#!/bin/bash +# LME nftables Configuration Script +# Direct nftables configuration equivalent to the firewalld LME setup +# Compatible with systems that prefer nftables over firewalld + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Check if running as root +if [[ $EUID -eq 0 ]]; then + SUDO="" +else + SUDO="sudo" +fi + +# Function to check if nftables is available +check_nftables() { + log_info "Checking nftables availability..." + + # Check if nft command exists + if ! command -v nft &> /dev/null; then + log_error "nftables is not installed. Installing..." + if command -v dnf &> /dev/null; then + $SUDO dnf install -y nftables + elif command -v apt &> /dev/null; then + $SUDO apt update && $SUDO apt install -y nftables + else + log_error "Could not determine package manager to install nftables" + exit 1 + fi + fi + + # Ensure nftables service is enabled + if ! $SUDO systemctl is-enabled --quiet nftables; then + log_info "Enabling nftables service..." + $SUDO systemctl enable nftables + fi + + log_success "nftables is available" +} + +# Function to detect LME container network +detect_lme_network() { + log_info "Detecting LME container network..." + + # Check if podman is installed + if ! command -v podman &> /dev/null; then + log_error "podman is not installed or not in PATH" + return 1 + fi + + # Try to get LME network information + local lme_subnet + if lme_subnet=$($SUDO -i podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then + if [[ -n "$lme_subnet" && "$lme_subnet" != "null" ]]; then + echo "$lme_subnet" + return 0 + fi + fi + + log_warning "Could not detect LME network. Using default podman subnet" + echo "10.88.0.0/16" + return 0 +} + +# Function to detect network interfaces +detect_interfaces() { + log_info "Detecting network interfaces..." + + # Get primary network interface (usually eth0, but could be others) + local primary_iface + primary_iface=$(ip route | grep default | head -n1 | awk '{print $5}') + + # Get podman interface + local podman_iface="" + for iface in podman0 podman1 cni-podman0 cni-podman1; do + if ip link show "$iface" &> /dev/null; then + podman_iface="$iface" + break + fi + done + + echo "$primary_iface $podman_iface" +} + +# Function to backup existing nftables rules +backup_nftables() { + log_info "Backing up existing nftables configuration..." + + local backup_file="/etc/nftables/lme_backup_$(date +%Y%m%d_%H%M%S).nft" + $SUDO mkdir -p /etc/nftables + + if $SUDO nft list ruleset > /dev/null 2>&1; then + $SUDO nft list ruleset > "$backup_file" || { + log_warning "Could not create backup, continuing anyway..." + } + log_success "Current ruleset backed up to $backup_file" + fi +} + +# Function to create LME nftables configuration +create_lme_nftables() { + local container_subnet="$1" + local primary_iface="$2" + local podman_iface="$3" + local enable_wazuh_api="$4" + + log_info "Creating LME nftables configuration..." + + # Create the nftables configuration file + local config_file="/etc/nftables/lme.nft" + $SUDO mkdir -p /etc/nftables + + cat > /tmp/lme_nftables.conf << EOF +#!/usr/sbin/nft -f +# LME nftables configuration +# Equivalent to firewalld LME setup + +# Flush existing rules (comment out if you want to preserve existing rules) +# flush ruleset + +table inet lme_filter { + # Input chain - handle incoming connections + chain input { + type filter hook input priority filter; policy drop; + + # Allow loopback traffic + iifname "lo" accept + + # Allow established and related connections + ct state established,related accept + + # Allow ICMP + ip protocol icmp accept + ip6 nexthdr ipv6-icmp accept + + # Allow SSH (modify port if needed) + tcp dport 22 accept + + # LME service ports on primary interface + iifname "$primary_iface" tcp dport { 1514, 1515, 8220, 9200, 5601, 443 } accept +EOF + + # Add Wazuh API port if requested + if [[ "$enable_wazuh_api" == "yes" ]]; then + cat >> /tmp/lme_nftables.conf << EOF + + # Wazuh API port + iifname "$primary_iface" tcp dport 55000 accept +EOF + fi + + # Continue with the rest of the configuration + cat >> /tmp/lme_nftables.conf << EOF + + # Allow all traffic from container network + ip saddr $container_subnet accept + + # Allow all traffic on podman interface +EOF + + if [[ -n "$podman_iface" ]]; then + cat >> /tmp/lme_nftables.conf << EOF + iifname "$podman_iface" accept +EOF + fi + + cat >> /tmp/lme_nftables.conf << EOF + + # Log and drop everything else + log prefix "LME_DROPPED: " drop + } + + # Forward chain - handle traffic forwarding + chain forward { + type filter hook forward priority filter; policy drop; + + # Allow established and related connections + ct state established,related accept + + # Allow forwarding from container network + ip saddr $container_subnet accept + + # Allow forwarding to container network + ip daddr $container_subnet accept +EOF + + if [[ -n "$podman_iface" ]]; then + cat >> /tmp/lme_nftables.conf << EOF + + # Allow forwarding on podman interface + iifname "$podman_iface" accept + oifname "$podman_iface" accept +EOF + fi + + cat >> /tmp/lme_nftables.conf << EOF + + # Log and drop everything else + log prefix "LME_FWD_DROPPED: " drop + } +} + +table ip lme_nat { + # NAT chain for masquerading container traffic + chain postrouting { + type nat hook postrouting priority srcnat; policy accept; + + # Masquerade traffic from container network going out primary interface + ip saddr $container_subnet oifname "$primary_iface" masquerade + + # General masquerading for container traffic (except loopback) + oifname != "lo" masquerade + } +} +EOF + + # Move the configuration file to its final location + $SUDO mv /tmp/lme_nftables.conf "$config_file" + $SUDO chmod 640 "$config_file" + + log_success "nftables configuration created at $config_file" +} + +# Function to apply nftables configuration +apply_nftables() { + local config_file="/etc/nftables/lme.nft" + + log_info "Applying nftables configuration..." + + # Test the configuration first + if ! $SUDO nft -c -f "$config_file"; then + log_error "nftables configuration has syntax errors" + return 1 + fi + + # Apply the configuration + $SUDO nft -f "$config_file" + log_success "nftables configuration applied" + + # Add to main nftables config to persist across reboots + local main_config="/etc/nftables.conf" + if [[ -f "$main_config" ]]; then + if ! grep -q "include.*lme.nft" "$main_config"; then + echo 'include "/etc/nftables/lme.nft"' | $SUDO tee -a "$main_config" > /dev/null + log_success "LME configuration added to main nftables config" + fi + else + # Create main config if it doesn't exist + echo 'include "/etc/nftables/lme.nft"' | $SUDO tee "$main_config" > /dev/null + log_success "Created main nftables config with LME rules" + fi + + # Ensure nftables service will start on boot + $SUDO systemctl enable nftables +} + +# Function to verify configuration +verify_configuration() { + log_info "Verifying nftables configuration..." + + echo + echo "=== Current nftables Configuration ===" + echo + + echo "Filter table (lme_filter):" + $SUDO nft list table inet lme_filter 2>/dev/null || log_warning "lme_filter table not found" + echo + + echo "NAT table (lme_nat):" + $SUDO nft list table ip lme_nat 2>/dev/null || log_warning "lme_nat table not found" + echo + + echo "=== All Active Rules ===" + $SUDO nft list ruleset | grep -A 10 -B 2 "lme" || log_info "No LME-specific rules found in output" + echo +} + +# Function to provide troubleshooting information +provide_troubleshooting() { + log_info "Troubleshooting Information:" + echo + echo "If you experience connectivity issues:" + echo "1. Check if LME containers are running:" + echo " sudo -i podman ps" + echo + echo "2. Test container-to-container communication:" + echo " sudo -i podman exec lme-kibana curl -s http://lme-elasticsearch:9200/_cluster/health" + echo + echo "3. Test external access (replace with your server IP):" + echo " curl -v http://YOUR_SERVER_IP:5601" + echo + echo "4. Check nftables rules:" + echo " sudo nft list ruleset" + echo + echo "5. Monitor dropped packets:" + echo " sudo journalctl -f | grep LME_DROPPED" + echo + echo "6. Temporarily flush rules for testing:" + echo " sudo nft delete table inet lme_filter" + echo " sudo nft delete table ip lme_nat" + echo " # Test your connections" + echo " sudo nft -f /etc/nftables/lme.nft" + echo + echo "7. Restore from backup if needed:" + echo " ls -la /etc/nftables/lme_backup_*" + echo " sudo nft -f /etc/nftables/lme_backup_YYYYMMDD_HHMMSS.nft" + echo +} + +# Function to disable firewalld if running +disable_firewalld() { + if systemctl is-active --quiet firewalld; then + log_warning "firewalld is currently running" + read -p "Do you want to stop and disable firewalld? (recommended for nftables) (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + $SUDO systemctl stop firewalld + $SUDO systemctl disable firewalld + log_success "firewalld stopped and disabled" + else + log_warning "firewalld is still running - this may conflict with nftables rules" + fi + fi +} + +# Main execution +main() { + echo "========================================" + echo "LME nftables Configuration" + echo "========================================" + echo + + # Check if firewalld conflicts + disable_firewalld + + # Check prerequisites + check_nftables + + # Detect network configuration + container_subnet=$(detect_lme_network) + read -r primary_iface podman_iface <<< "$(detect_interfaces)" + + # Display detected configuration + echo + log_info "Detected Configuration:" + echo " Container Subnet: $container_subnet" + echo " Primary Interface: $primary_iface" + echo " Podman Interface: ${podman_iface:-"None detected"}" + echo + + # Ask about Wazuh API + read -p "Do you want to enable Wazuh API port 55000? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + enable_wazuh_api="yes" + else + enable_wazuh_api="no" + fi + + # Confirm before proceeding + read -p "Do you want to configure nftables with these settings? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "Configuration cancelled by user" + exit 0 + fi + + # Backup existing configuration + backup_nftables + + # Create and apply nftables configuration + create_lme_nftables "$container_subnet" "$primary_iface" "$podman_iface" "$enable_wazuh_api" + apply_nftables + + # Verify configuration + verify_configuration + + # Provide troubleshooting info + provide_troubleshooting + + log_success "LME nftables configuration completed!" + log_info "Configuration will persist across reboots via /etc/nftables.conf" +} + +# Run main function if script is executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi From 817c5bb0569077b852068adc13d476c470d24fad Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Fri, 12 Sep 2025 11:09:24 +0000 Subject: [PATCH 42/51] Change sudo use in configure_lme_nftables.sh --- scripts/configure_lme_nftables.sh | 56 ++++++++++++++----------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/scripts/configure_lme_nftables.sh b/scripts/configure_lme_nftables.sh index b161d323..9e039505 100755 --- a/scripts/configure_lme_nftables.sh +++ b/scripts/configure_lme_nftables.sh @@ -18,12 +18,6 @@ log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } -# Check if running as root -if [[ $EUID -eq 0 ]]; then - SUDO="" -else - SUDO="sudo" -fi # Function to check if nftables is available check_nftables() { @@ -33,9 +27,9 @@ check_nftables() { if ! command -v nft &> /dev/null; then log_error "nftables is not installed. Installing..." if command -v dnf &> /dev/null; then - $SUDO dnf install -y nftables + sudo dnf install -y nftables elif command -v apt &> /dev/null; then - $SUDO apt update && $SUDO apt install -y nftables + sudo apt update && sudo apt install -y nftables else log_error "Could not determine package manager to install nftables" exit 1 @@ -43,9 +37,9 @@ check_nftables() { fi # Ensure nftables service is enabled - if ! $SUDO systemctl is-enabled --quiet nftables; then + if ! sudo systemctl is-enabled --quiet nftables; then log_info "Enabling nftables service..." - $SUDO systemctl enable nftables + sudo systemctl enable nftables fi log_success "nftables is available" @@ -53,31 +47,31 @@ check_nftables() { # Function to detect LME container network detect_lme_network() { - log_info "Detecting LME container network..." + log_info "Detecting LME container network..." >&2 # Check if podman is installed if ! command -v podman &> /dev/null; then - log_error "podman is not installed or not in PATH" + log_error "podman is not installed or not in PATH" >&2 return 1 fi # Try to get LME network information local lme_subnet - if lme_subnet=$($SUDO -i podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then + if lme_subnet=$(sudo -i podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then if [[ -n "$lme_subnet" && "$lme_subnet" != "null" ]]; then echo "$lme_subnet" return 0 fi fi - log_warning "Could not detect LME network. Using default podman subnet" + log_warning "Could not detect LME network. Using default podman subnet" >&2 echo "10.88.0.0/16" return 0 } # Function to detect network interfaces detect_interfaces() { - log_info "Detecting network interfaces..." + log_info "Detecting network interfaces..." >&2 # Get primary network interface (usually eth0, but could be others) local primary_iface @@ -100,10 +94,10 @@ backup_nftables() { log_info "Backing up existing nftables configuration..." local backup_file="/etc/nftables/lme_backup_$(date +%Y%m%d_%H%M%S).nft" - $SUDO mkdir -p /etc/nftables + sudo mkdir -p /etc/nftables - if $SUDO nft list ruleset > /dev/null 2>&1; then - $SUDO nft list ruleset > "$backup_file" || { + if sudo nft list ruleset > /dev/null 2>&1; then + sudo nft list ruleset > "$backup_file" || { log_warning "Could not create backup, continuing anyway..." } log_success "Current ruleset backed up to $backup_file" @@ -121,7 +115,7 @@ create_lme_nftables() { # Create the nftables configuration file local config_file="/etc/nftables/lme.nft" - $SUDO mkdir -p /etc/nftables + sudo mkdir -p /etc/nftables cat > /tmp/lme_nftables.conf << EOF #!/usr/sbin/nft -f @@ -228,8 +222,8 @@ table ip lme_nat { EOF # Move the configuration file to its final location - $SUDO mv /tmp/lme_nftables.conf "$config_file" - $SUDO chmod 640 "$config_file" + sudo mv /tmp/lme_nftables.conf "$config_file" + sudo chmod 640 "$config_file" log_success "nftables configuration created at $config_file" } @@ -241,30 +235,30 @@ apply_nftables() { log_info "Applying nftables configuration..." # Test the configuration first - if ! $SUDO nft -c -f "$config_file"; then + if ! sudo nft -c -f "$config_file"; then log_error "nftables configuration has syntax errors" return 1 fi # Apply the configuration - $SUDO nft -f "$config_file" + sudo nft -f "$config_file" log_success "nftables configuration applied" # Add to main nftables config to persist across reboots local main_config="/etc/nftables.conf" if [[ -f "$main_config" ]]; then if ! grep -q "include.*lme.nft" "$main_config"; then - echo 'include "/etc/nftables/lme.nft"' | $SUDO tee -a "$main_config" > /dev/null + echo 'include "/etc/nftables/lme.nft"' | sudo tee -a "$main_config" > /dev/null log_success "LME configuration added to main nftables config" fi else # Create main config if it doesn't exist - echo 'include "/etc/nftables/lme.nft"' | $SUDO tee "$main_config" > /dev/null + echo 'include "/etc/nftables/lme.nft"' | sudo tee "$main_config" > /dev/null log_success "Created main nftables config with LME rules" fi # Ensure nftables service will start on boot - $SUDO systemctl enable nftables + sudo systemctl enable nftables } # Function to verify configuration @@ -276,15 +270,15 @@ verify_configuration() { echo echo "Filter table (lme_filter):" - $SUDO nft list table inet lme_filter 2>/dev/null || log_warning "lme_filter table not found" + sudo nft list table inet lme_filter 2>/dev/null || log_warning "lme_filter table not found" echo echo "NAT table (lme_nat):" - $SUDO nft list table ip lme_nat 2>/dev/null || log_warning "lme_nat table not found" + sudo nft list table ip lme_nat 2>/dev/null || log_warning "lme_nat table not found" echo echo "=== All Active Rules ===" - $SUDO nft list ruleset | grep -A 10 -B 2 "lme" || log_info "No LME-specific rules found in output" + sudo nft list ruleset | grep -A 10 -B 2 "lme" || log_info "No LME-specific rules found in output" echo } @@ -327,8 +321,8 @@ disable_firewalld() { read -p "Do you want to stop and disable firewalld? (recommended for nftables) (y/N): " -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]]; then - $SUDO systemctl stop firewalld - $SUDO systemctl disable firewalld + sudo systemctl stop firewalld + sudo systemctl disable firewalld log_success "firewalld stopped and disabled" else log_warning "firewalld is still running - this may conflict with nftables rules" From 2b3179498c15d04446d7a2c361e98e7660a3039d Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Mon, 15 Sep 2025 05:49:44 -0400 Subject: [PATCH 43/51] Lme certs required iptables when running in a docker container --- .gitignore | 1 + docker/22.04/Dockerfile | 2 +- docker/24.04/Dockerfile | 2 +- docker/d12.10/Dockerfile | 2 +- docker/rhel9/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 64007bb9..031461ea 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ testing/upgrade_testing/ .cache/ .venv/ *.env +cloud.md # SELinux build artifacts ansible/roles/base/files/selinux/*.mod diff --git a/docker/22.04/Dockerfile b/docker/22.04/Dockerfile index b140e917..ff18bb71 100644 --- a/docker/22.04/Dockerfile +++ b/docker/22.04/Dockerfile @@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ LC_ALL=en_US.UTF-8 RUN apt-get update && apt-get install -y --no-install-recommends \ - locales ca-certificates sudo openssh-client \ + locales ca-certificates sudo openssh-client iptables \ && locale-gen en_US.UTF-8 \ && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ && groupadd -g $GROUP_ID lme-user \ diff --git a/docker/24.04/Dockerfile b/docker/24.04/Dockerfile index 745c30ab..40f5f37e 100644 --- a/docker/24.04/Dockerfile +++ b/docker/24.04/Dockerfile @@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ LC_ALL=en_US.UTF-8 RUN apt-get update && apt-get install -y --no-install-recommends \ - locales ca-certificates sudo openssh-client \ + locales ca-certificates sudo openssh-client iptables \ && locale-gen en_US.UTF-8 \ && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ diff --git a/docker/d12.10/Dockerfile b/docker/d12.10/Dockerfile index 4d450f2e..cd8bdd75 100644 --- a/docker/d12.10/Dockerfile +++ b/docker/d12.10/Dockerfile @@ -7,7 +7,7 @@ ARG GROUP_ID=1002 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ - locales ca-certificates sudo openssh-client \ + locales ca-certificates sudo openssh-client iptables \ && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ && locale-gen en_US.UTF-8 \ && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ diff --git a/docker/rhel9/Dockerfile b/docker/rhel9/Dockerfile index 21058c8e..c15ee604 100644 --- a/docker/rhel9/Dockerfile +++ b/docker/rhel9/Dockerfile @@ -9,7 +9,7 @@ ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 RUN dnf update -y && dnf install -y \ - glibc-langpack-en sudo openssh-clients git vim \ + glibc-langpack-en sudo openssh-clients git vim iptables-nft \ && dnf clean all \ && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ && while getent passwd $USER_ID > /dev/null 2>&1; do USER_ID=$((USER_ID + 1)); done \ From 8f04f0a44d34ed917e4ec2afe01c5de980462398 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Tue, 16 Sep 2025 12:23:06 +0000 Subject: [PATCH 44/51] Adds connectivity tests --- .github/workflows/linux_only.yml | 2 +- .github/workflows/linux_only_redhat.yml | 2 +- .../tests/api_tests/connectivity/__init__.py | 1 + .../tests/api_tests/connectivity/conftest.py | 44 ++++++++++++++ .../connectivity/test_inter_service.py | 38 ++++++++++++ .../connectivity/test_network_isolation.py | 27 +++++++++ .../connectivity/test_port_connectivity.py | 48 +++++++++++++++ .../connectivity/test_service_dependencies.py | 60 +++++++++++++++++++ .../connectivity/test_ssl_connectivity.py | 59 ++++++++++++++++++ testing/tests/requirements.txt | 1 + 10 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 testing/tests/api_tests/connectivity/__init__.py create mode 100644 testing/tests/api_tests/connectivity/conftest.py create mode 100644 testing/tests/api_tests/connectivity/test_inter_service.py create mode 100644 testing/tests/api_tests/connectivity/test_network_isolation.py create mode 100644 testing/tests/api_tests/connectivity/test_port_connectivity.py create mode 100644 testing/tests/api_tests/connectivity/test_service_dependencies.py create mode 100644 testing/tests/api_tests/connectivity/test_ssl_connectivity.py diff --git a/.github/workflows/linux_only.yml b/.github/workflows/linux_only.yml index caaf0417..3ac45b38 100644 --- a/.github/workflows/linux_only.yml +++ b/.github/workflows/linux_only.yml @@ -152,7 +152,7 @@ jobs: echo elastic=\"$ES_PASSWORD\" >> .env && \ cat .env && \ source venv/bin/activate && \ - pytest -v api_tests/linux_only/ selenium_tests/linux_only/' + pytest -v api_tests/linux_only/ selenium_tests/linux_only/ api_tests/connectivity/' " - name: Cleanup Azure resources diff --git a/.github/workflows/linux_only_redhat.yml b/.github/workflows/linux_only_redhat.yml index ec5bd0ff..f33e641a 100644 --- a/.github/workflows/linux_only_redhat.yml +++ b/.github/workflows/linux_only_redhat.yml @@ -153,7 +153,7 @@ jobs: echo elastic=\"$ES_PASSWORD\" >> .env && \ cat .env && \ source venv/bin/activate && \ - pytest -v api_tests/linux_only/ selenium_tests/linux_only/' + pytest -v api_tests/linux_only/ selenium_tests/linux_only/ api_tests/connectivity/' " - name: Cleanup Azure resources diff --git a/testing/tests/api_tests/connectivity/__init__.py b/testing/tests/api_tests/connectivity/__init__.py new file mode 100644 index 00000000..2dadca2b --- /dev/null +++ b/testing/tests/api_tests/connectivity/__init__.py @@ -0,0 +1 @@ +# Connectivity tests for LME services diff --git a/testing/tests/api_tests/connectivity/conftest.py b/testing/tests/api_tests/connectivity/conftest.py new file mode 100644 index 00000000..9dd3fc47 --- /dev/null +++ b/testing/tests/api_tests/connectivity/conftest.py @@ -0,0 +1,44 @@ +# conftest.py for connectivity tests + +import os +import warnings +import pytest +import urllib3 + +# Disable SSL warnings +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +@pytest.fixture(autouse=True) +def suppress_insecure_request_warning(): + warnings.simplefilter("ignore", urllib3.exceptions.InsecureRequestWarning) + +@pytest.fixture +def es_host(): + return os.getenv("ES_HOST", os.getenv("ELASTIC_HOST", "localhost")) + +@pytest.fixture +def es_port(): + return os.getenv("ES_PORT", os.getenv("ELASTIC_PORT", "9200")) + +@pytest.fixture +def username(): + return os.getenv("ES_USERNAME", os.getenv("ELASTIC_USERNAME", "elastic")) + +@pytest.fixture +def password(): + return os.getenv( + "elastic", + os.getenv("ES_PASSWORD", os.getenv("ELASTIC_PASSWORD", "password1")), + ) + +@pytest.fixture +def all_service_ports(): + """Port mapping for all LME services""" + return { + 'elasticsearch': 9200, + 'kibana': [5601, 443], + 'fleet_server': 8220, + 'wazuh_tcp': [1514, 1515, 55000], + 'wazuh_udp': [514] + } + diff --git a/testing/tests/api_tests/connectivity/test_inter_service.py b/testing/tests/api_tests/connectivity/test_inter_service.py new file mode 100644 index 00000000..4b706b7d --- /dev/null +++ b/testing/tests/api_tests/connectivity/test_inter_service.py @@ -0,0 +1,38 @@ +import requests +import pytest +import urllib3 + +# Disable SSL warnings +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +class TestInterServiceCommunication: + """Test communication between LME services""" + + def test_kibana_to_elasticsearch(self, es_host, es_port, username, password): + """Test Kibana can communicate with Elasticsearch""" + # This tests the internal communication path + url = f"https://{es_host}:{es_port}/_cluster/health" + response = requests.get(url, auth=(username, password), verify=False, timeout=10) + assert response.status_code == 200 + health = response.json() + assert health["status"] in ["green", "yellow"], "Elasticsearch cluster unhealthy" + + def test_fleet_server_api_connectivity(self, es_host): + """Test Fleet Server API is responding""" + url = f"https://{es_host}:8220/api/status" + try: + response = requests.get(url, verify=False, timeout=10) + # Fleet server may require authentication, but should respond + assert response.status_code in [200, 401, 403], "Fleet Server not responding" + except requests.exceptions.ConnectionError: + pytest.fail("Fleet Server is not accessible on port 8220") + + def test_wazuh_api_connectivity(self, es_host): + """Test Wazuh Manager API is responding""" + url = f"https://{es_host}:55000" + try: + response = requests.get(url, verify=False, timeout=10) + # Wazuh API should return 401 when not authenticated + assert response.status_code == 401, "Wazuh API not responding correctly" + except requests.exceptions.ConnectionError: + pytest.fail("Wazuh Manager API is not accessible on port 55000") diff --git a/testing/tests/api_tests/connectivity/test_network_isolation.py b/testing/tests/api_tests/connectivity/test_network_isolation.py new file mode 100644 index 00000000..04e2f1e2 --- /dev/null +++ b/testing/tests/api_tests/connectivity/test_network_isolation.py @@ -0,0 +1,27 @@ +import socket +import pytest +from contextlib import closing + +class TestNetworkIsolation: + + def test_expected_ports_accessible(self, es_host): + """Test that only expected LME ports are accessible""" + expected_ports = [443, 5601, 8220, 9200, 1514, 1515, 55000] + + for port in expected_ports: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(10) + result = sock.connect_ex((es_host, port)) + assert result == 0, f"Expected LME port {port} is not accessible" + + def test_udp_port_514_accessible(self, es_host): + """Test that UDP port 514 (syslog) is accessible for Wazuh""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock: + sock.settimeout(5) + try: + # Send a test syslog message + test_message = b"<14>Jan 1 00:00:00 test-host test: connectivity test" + sock.sendto(test_message, (es_host, 514)) + # UDP is connectionless, so we just verify we can send + except Exception as e: + pytest.fail(f"UDP port 514 (syslog) test failed: {e}") diff --git a/testing/tests/api_tests/connectivity/test_port_connectivity.py b/testing/tests/api_tests/connectivity/test_port_connectivity.py new file mode 100644 index 00000000..277a5d50 --- /dev/null +++ b/testing/tests/api_tests/connectivity/test_port_connectivity.py @@ -0,0 +1,48 @@ +import socket +import ssl +import pytest +import requests +from contextlib import closing + +class TestPortConnectivity: + """Test basic port accessibility for all LME services""" + + def test_elasticsearch_port_open(self, es_host, es_port): + """Test Elasticsearch port 9200 is accessible""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(10) + result = sock.connect_ex((es_host, int(es_port))) + assert result == 0, f"Port {es_port} on {es_host} is not accessible" + + def test_kibana_port_open(self, es_host): + """Test Kibana ports 5601 and 443 are accessible""" + for port in [5601, 443]: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(10) + result = sock.connect_ex((es_host, port)) + assert result == 0, f"Port {port} on {es_host} is not accessible" + + def test_fleet_server_port_open(self, es_host): + """Test Fleet Server port 8220 is accessible""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(10) + result = sock.connect_ex((es_host, 8220)) + assert result == 0, f"Fleet Server port 8220 on {es_host} is not accessible" + + def test_wazuh_ports_open(self, es_host): + """Test Wazuh Manager ports are accessible""" + tcp_ports = [1514, 1515, 55000] + for port in tcp_ports: + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.settimeout(10) + result = sock.connect_ex((es_host, port)) + assert result == 0, f"Wazuh port {port} on {es_host} is not accessible" + + # Test UDP port 514 + with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock: + sock.settimeout(5) + try: + sock.sendto(b"test", (es_host, 514)) + # UDP doesn't guarantee delivery, just test socket creation + except Exception as e: + pytest.fail(f"UDP port 514 test failed: {e}") diff --git a/testing/tests/api_tests/connectivity/test_service_dependencies.py b/testing/tests/api_tests/connectivity/test_service_dependencies.py new file mode 100644 index 00000000..db49fae7 --- /dev/null +++ b/testing/tests/api_tests/connectivity/test_service_dependencies.py @@ -0,0 +1,60 @@ +import requests +import pytest +import urllib3 + +# Disable SSL warnings +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +class TestServiceDependencies: + """Test proper service startup order and dependencies""" + + def test_elasticsearch_available_first(self, es_host, es_port, username, password): + """Elasticsearch should be available before other services depend on it""" + url = f"https://{es_host}:{es_port}/_cluster/health" + response = requests.get(url, auth=(username, password), verify=False, timeout=30) + assert response.status_code == 200, "Elasticsearch not ready for dependent services" + + # Ensure cluster is at least yellow (functional) + health = response.json() + assert health["status"] in ["green", "yellow"], f"Cluster status is {health['status']}, should be green or yellow" + + def test_kibana_connects_to_elasticsearch(self, es_host, username, password): + """Test that Kibana successfully connects to Elasticsearch""" + kibana_url = f"https://{es_host}:5601/api/status" + try: + response = requests.get(kibana_url, auth=(username, password), verify=False, timeout=10) + assert response.status_code in [200, 401], "Kibana not properly connected" + except requests.exceptions.ConnectionError: + pytest.fail("Kibana connectivity test failed") + + def test_fleet_server_depends_on_elasticsearch(self, es_host, es_port, username, password): + """Test that Fleet Server can reach Elasticsearch""" + # First verify Elasticsearch is up + es_url = f"https://{es_host}:{es_port}/_cluster/health" + es_response = requests.get(es_url, auth=(username, password), verify=False, timeout=10) + assert es_response.status_code == 200, "Elasticsearch must be up for Fleet Server" + + # Then check Fleet Server is responding + fleet_url = f"https://{es_host}:8220/api/status" + try: + fleet_response = requests.get(fleet_url, verify=False, timeout=10) + # Fleet server should respond, even if auth is required + assert fleet_response.status_code in [200, 401, 403], "Fleet Server should be responding" + except requests.exceptions.ConnectionError: + pytest.fail("Fleet Server not accessible after Elasticsearch is up") + + def test_wazuh_depends_on_elasticsearch(self, es_host, es_port, username, password): + """Test that Wazuh Manager can reach Elasticsearch for log shipping""" + # First verify Elasticsearch is up + es_url = f"https://{es_host}:{es_port}/_cluster/health" + es_response = requests.get(es_url, auth=(username, password), verify=False, timeout=10) + assert es_response.status_code == 200, "Elasticsearch must be up for Wazuh" + + # Check if Wazuh API is responding + wazuh_url = f"https://{es_host}:55000" + try: + wazuh_response = requests.get(wazuh_url, verify=False, timeout=10) + # Wazuh API should return 401 for unauthenticated requests + assert wazuh_response.status_code == 401, "Wazuh API should be responding with 401" + except requests.exceptions.ConnectionError: + pytest.fail("Wazuh Manager API not accessible") diff --git a/testing/tests/api_tests/connectivity/test_ssl_connectivity.py b/testing/tests/api_tests/connectivity/test_ssl_connectivity.py new file mode 100644 index 00000000..11c46ea5 --- /dev/null +++ b/testing/tests/api_tests/connectivity/test_ssl_connectivity.py @@ -0,0 +1,59 @@ +import socket +import ssl +import pytest + +class TestSSLConnectivity: + """Test SSL certificate validation and connectivity""" + + def test_elasticsearch_ssl_connectivity(self, es_host, es_port): + """Test Elasticsearch SSL certificate chain""" + try: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE # For self-signed certs + + with socket.create_connection((es_host, int(es_port)), timeout=10) as sock: + with context.wrap_socket(sock, server_hostname=es_host) as ssock: + assert ssock.version() is not None, "SSL handshake failed" + except ssl.SSLError as e: + pytest.fail(f"SSL connection to Elasticsearch failed: {e}") + + def test_kibana_ssl_connectivity(self, es_host): + """Test Kibana SSL connectivity on both ports""" + for port in [5601, 443]: + try: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with socket.create_connection((es_host, port), timeout=10) as sock: + with context.wrap_socket(sock, server_hostname=es_host) as ssock: + assert ssock.version() is not None, f"SSL handshake failed on port {port}" + except ssl.SSLError as e: + pytest.fail(f"SSL connection to Kibana port {port} failed: {e}") + + def test_fleet_server_ssl_connectivity(self, es_host): + """Test Fleet Server SSL connectivity""" + try: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with socket.create_connection((es_host, 8220), timeout=10) as sock: + with context.wrap_socket(sock, server_hostname=es_host) as ssock: + assert ssock.version() is not None, "Fleet Server SSL handshake failed" + except ssl.SSLError as e: + pytest.fail(f"SSL connection to Fleet Server failed: {e}") + + def test_wazuh_ssl_connectivity(self, es_host): + """Test Wazuh Manager SSL connectivity""" + try: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with socket.create_connection((es_host, 55000), timeout=10) as sock: + with context.wrap_socket(sock, server_hostname=es_host) as ssock: + assert ssock.version() is not None, "Wazuh Manager SSL handshake failed" + except ssl.SSLError as e: + pytest.fail(f"SSL connection to Wazuh Manager failed: {e}") diff --git a/testing/tests/requirements.txt b/testing/tests/requirements.txt index 59af84e1..596c2789 100644 --- a/testing/tests/requirements.txt +++ b/testing/tests/requirements.txt @@ -19,3 +19,4 @@ urllib3>=2.1.0 selenium webdriver-manager pytest-html>=4.1.1 +paramiko>=2.7.2 From 13716d710218f479a419795433cfc06f13acbbc4 Mon Sep 17 00:00:00 2001 From: cbaxley Date: Wed, 17 Sep 2025 11:44:44 +0000 Subject: [PATCH 45/51] Cleans up some stuff in the RHEL9 Dockerfile --- docker/rhel9/Dockerfile | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/docker/rhel9/Dockerfile b/docker/rhel9/Dockerfile index c15ee604..a0f515ec 100644 --- a/docker/rhel9/Dockerfile +++ b/docker/rhel9/Dockerfile @@ -8,13 +8,16 @@ ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 -RUN dnf update -y && dnf install -y \ +# Skip systemd updates to avoid conflicts, install base packages +RUN dnf install -y \ glibc-langpack-en sudo openssh-clients git vim iptables-nft \ && dnf clean all \ - && while getent group $GROUP_ID > /dev/null 2>&1; do GROUP_ID=$((GROUP_ID + 1)); done \ - && while getent passwd $USER_ID > /dev/null 2>&1; do USER_ID=$((USER_ID + 1)); done \ - && groupadd -g $GROUP_ID lme-user \ - && useradd -m -u $USER_ID -g lme-user lme-user \ + && GROUP_ID_VAR=$GROUP_ID \ + && USER_ID_VAR=$USER_ID \ + && while getent group $GROUP_ID_VAR > /dev/null 2>&1; do GROUP_ID_VAR=$((GROUP_ID_VAR + 1)); done \ + && while getent passwd $USER_ID_VAR > /dev/null 2>&1; do USER_ID_VAR=$((USER_ID_VAR + 1)); done \ + && groupadd -g $GROUP_ID_VAR lme-user \ + && useradd -m -u $USER_ID_VAR -g lme-user lme-user \ && usermod -aG wheel lme-user \ && echo "lme-user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ && echo "Defaults:lme-user !requiretty" >> /etc/sudoers \ @@ -32,14 +35,6 @@ WORKDIR $BASE_DIR # Lme stage with full dependencies FROM base AS lme -RUN dnf install -y \ - systemd systemd-sysv \ - && dnf clean all - -# Install EPEL repository -# RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ -# && /usr/bin/crb enable \ -# && dnf clean all RUN cd /lib/systemd/system/sysinit.target.wants/ && \ ls | grep -v systemd-tmpfiles-setup | xargs rm -f $1 && \ From 5b57837ac14fa6849c20025edbc88866bf3b2b99 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 17 Sep 2025 12:40:29 +0000 Subject: [PATCH 46/51] Cleans up the firewall scripts to make them more robust --- scripts/configure_lme_nftables.sh | 176 ++++++++++++++++++++++------ scripts/configure_rhel_firewall.sh | 182 ++++++++++++++++++++++------- 2 files changed, 277 insertions(+), 81 deletions(-) diff --git a/scripts/configure_lme_nftables.sh b/scripts/configure_lme_nftables.sh index 9e039505..be08a981 100755 --- a/scripts/configure_lme_nftables.sh +++ b/scripts/configure_lme_nftables.sh @@ -2,6 +2,8 @@ # LME nftables Configuration Script # Direct nftables configuration equivalent to the firewalld LME setup # Compatible with systems that prefer nftables over firewalld +# +# REQUIRES ROOT ACCESS - Run as: sudo ./configure_lme_nftables.sh set -euo pipefail @@ -18,6 +20,44 @@ log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +# Check if running as root - required for nftables configuration +if [[ $EUID -ne 0 ]]; then + echo -e "${RED}[ERROR]${NC} This script must be run as root" + echo "Please run: sudo $0" + exit 1 +fi + +# Function to check dependencies +check_dependencies() { + log_info "Checking dependencies..." + + # Check if podman is installed + if ! command -v podman &> /dev/null; then + log_error "podman is not installed or not in PATH" + log_error "Please install podman before running this script" + exit 1 + fi + + # Check if jq is installed (needed for network inspection) + if ! command -v jq &> /dev/null; then + log_warning "jq is not installed. Installing jq for network detection..." + # Support multiple package managers + if command -v dnf &> /dev/null; then + dnf install -y jq + elif command -v yum &> /dev/null; then + yum install -y jq + elif command -v zypper &> /dev/null; then + zypper install -y jq + elif command -v apt &> /dev/null; then + apt update && apt install -y jq + else + log_error "Could not install jq automatically. Please install jq manually." + exit 1 + fi + fi + + log_success "All dependencies are available" +} # Function to check if nftables is available check_nftables() { @@ -26,10 +66,15 @@ check_nftables() { # Check if nft command exists if ! command -v nft &> /dev/null; then log_error "nftables is not installed. Installing..." + # Support multiple package managers if command -v dnf &> /dev/null; then - sudo dnf install -y nftables + dnf install -y nftables + elif command -v yum &> /dev/null; then + yum install -y nftables + elif command -v zypper &> /dev/null; then + zypper install -y nftables elif command -v apt &> /dev/null; then - sudo apt update && sudo apt install -y nftables + apt update && apt install -y nftables else log_error "Could not determine package manager to install nftables" exit 1 @@ -37,9 +82,9 @@ check_nftables() { fi # Ensure nftables service is enabled - if ! sudo systemctl is-enabled --quiet nftables; then + if ! systemctl is-enabled --quiet nftables; then log_info "Enabling nftables service..." - sudo systemctl enable nftables + systemctl enable nftables fi log_success "nftables is available" @@ -57,7 +102,7 @@ detect_lme_network() { # Try to get LME network information local lme_subnet - if lme_subnet=$(sudo -i podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then + if lme_subnet=$(podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then if [[ -n "$lme_subnet" && "$lme_subnet" != "null" ]]; then echo "$lme_subnet" return 0 @@ -73,18 +118,70 @@ detect_lme_network() { detect_interfaces() { log_info "Detecting network interfaces..." >&2 - # Get primary network interface (usually eth0, but could be others) - local primary_iface - primary_iface=$(ip route | grep default | head -n1 | awk '{print $5}') + # Get primary network interface with fallback options + local primary_iface="" + + # Try multiple methods to detect primary interface + if primary_iface=$(ip route | grep default | head -n1 | awk '{print $5}' 2>/dev/null); then + if [[ -n "$primary_iface" ]]; then + log_info "Detected primary interface: $primary_iface" >&2 + fi + fi + + # Fallback: look for common interface names if route detection failed + if [[ -z "$primary_iface" ]]; then + for iface in eth0 ens3 ens4 ens5 enp0s3 enp0s8 ens33 ens192; do + if ip link show "$iface" &> /dev/null; then + primary_iface="$iface" + log_warning "Using fallback primary interface: $primary_iface" >&2 + break + fi + done + fi - # Get podman interface + # Final fallback + if [[ -z "$primary_iface" ]]; then + primary_iface="eth0" + log_warning "Could not detect primary interface, using default: $primary_iface" >&2 + fi + + # Comprehensive podman interface detection local podman_iface="" - for iface in podman0 podman1 cni-podman0 cni-podman1; do - if ip link show "$iface" &> /dev/null; then - podman_iface="$iface" - break + + # Try to get LME network interface first (most reliable) + local lme_interface + if lme_interface=$(podman network inspect lme 2>/dev/null | jq -r '.[].network_interface' 2>/dev/null); then + if [[ -n "$lme_interface" && "$lme_interface" != "null" ]]; then + podman_iface="$lme_interface" + log_info "Found LME network interface: $podman_iface" >&2 + fi + fi + + # If no LME interface, look for any podman interfaces + if [[ -z "$podman_iface" ]]; then + # Look for interfaces with podman-related names + local detected_podman + detected_podman=$(ip link show | grep -E '^[0-9]+:' | awk -F': ' '{print $2}' | grep -E '^(podman|cni-podman|veth.*podman|br-.*)' | head -n1) + + if [[ -n "$detected_podman" ]]; then + # Remove any trailing @ and interface suffix + podman_iface=$(echo "$detected_podman" | cut -d'@' -f1) + log_info "Found podman interface: $podman_iface" >&2 + else + # Check common interface names as fallback + for iface in podman0 podman1 podman2 cni-podman0 cni-podman1; do + if ip link show "$iface" &> /dev/null; then + podman_iface="$iface" + log_info "Found podman interface (fallback): $podman_iface" >&2 + break + fi + done fi - done + fi + + if [[ -z "$podman_iface" ]]; then + log_warning "No podman interface detected - container networking may need manual setup" >&2 + fi echo "$primary_iface $podman_iface" } @@ -94,10 +191,10 @@ backup_nftables() { log_info "Backing up existing nftables configuration..." local backup_file="/etc/nftables/lme_backup_$(date +%Y%m%d_%H%M%S).nft" - sudo mkdir -p /etc/nftables + mkdir -p /etc/nftables - if sudo nft list ruleset > /dev/null 2>&1; then - sudo nft list ruleset > "$backup_file" || { + if nft list ruleset > /dev/null 2>&1; then + nft list ruleset > "$backup_file" || { log_warning "Could not create backup, continuing anyway..." } log_success "Current ruleset backed up to $backup_file" @@ -115,7 +212,7 @@ create_lme_nftables() { # Create the nftables configuration file local config_file="/etc/nftables/lme.nft" - sudo mkdir -p /etc/nftables + mkdir -p /etc/nftables cat > /tmp/lme_nftables.conf << EOF #!/usr/sbin/nft -f @@ -222,8 +319,8 @@ table ip lme_nat { EOF # Move the configuration file to its final location - sudo mv /tmp/lme_nftables.conf "$config_file" - sudo chmod 640 "$config_file" + mv /tmp/lme_nftables.conf "$config_file" + chmod 640 "$config_file" log_success "nftables configuration created at $config_file" } @@ -235,30 +332,30 @@ apply_nftables() { log_info "Applying nftables configuration..." # Test the configuration first - if ! sudo nft -c -f "$config_file"; then + if ! nft -c -f "$config_file"; then log_error "nftables configuration has syntax errors" return 1 fi # Apply the configuration - sudo nft -f "$config_file" + nft -f "$config_file" log_success "nftables configuration applied" # Add to main nftables config to persist across reboots local main_config="/etc/nftables.conf" if [[ -f "$main_config" ]]; then if ! grep -q "include.*lme.nft" "$main_config"; then - echo 'include "/etc/nftables/lme.nft"' | sudo tee -a "$main_config" > /dev/null + echo 'include "/etc/nftables/lme.nft"' | tee -a "$main_config" > /dev/null log_success "LME configuration added to main nftables config" fi else # Create main config if it doesn't exist - echo 'include "/etc/nftables/lme.nft"' | sudo tee "$main_config" > /dev/null + echo 'include "/etc/nftables/lme.nft"' | tee "$main_config" > /dev/null log_success "Created main nftables config with LME rules" fi # Ensure nftables service will start on boot - sudo systemctl enable nftables + systemctl enable nftables } # Function to verify configuration @@ -270,15 +367,15 @@ verify_configuration() { echo echo "Filter table (lme_filter):" - sudo nft list table inet lme_filter 2>/dev/null || log_warning "lme_filter table not found" + nft list table inet lme_filter 2>/dev/null || log_warning "lme_filter table not found" echo echo "NAT table (lme_nat):" - sudo nft list table ip lme_nat 2>/dev/null || log_warning "lme_nat table not found" + nft list table ip lme_nat 2>/dev/null || log_warning "lme_nat table not found" echo echo "=== All Active Rules ===" - sudo nft list ruleset | grep -A 10 -B 2 "lme" || log_info "No LME-specific rules found in output" + nft list ruleset | grep -A 10 -B 2 "lme" || log_info "No LME-specific rules found in output" echo } @@ -288,29 +385,29 @@ provide_troubleshooting() { echo echo "If you experience connectivity issues:" echo "1. Check if LME containers are running:" - echo " sudo -i podman ps" + echo " podman ps" echo echo "2. Test container-to-container communication:" - echo " sudo -i podman exec lme-kibana curl -s http://lme-elasticsearch:9200/_cluster/health" + echo " podman exec lme-kibana curl -s http://lme-elasticsearch:9200/_cluster/health" echo echo "3. Test external access (replace with your server IP):" echo " curl -v http://YOUR_SERVER_IP:5601" echo echo "4. Check nftables rules:" - echo " sudo nft list ruleset" + echo " nft list ruleset" echo echo "5. Monitor dropped packets:" - echo " sudo journalctl -f | grep LME_DROPPED" + echo " journalctl -f | grep LME_DROPPED" echo echo "6. Temporarily flush rules for testing:" - echo " sudo nft delete table inet lme_filter" - echo " sudo nft delete table ip lme_nat" + echo " nft delete table inet lme_filter" + echo " nft delete table ip lme_nat" echo " # Test your connections" - echo " sudo nft -f /etc/nftables/lme.nft" + echo " nft -f /etc/nftables/lme.nft" echo echo "7. Restore from backup if needed:" echo " ls -la /etc/nftables/lme_backup_*" - echo " sudo nft -f /etc/nftables/lme_backup_YYYYMMDD_HHMMSS.nft" + echo " nft -f /etc/nftables/lme_backup_YYYYMMDD_HHMMSS.nft" echo } @@ -321,8 +418,8 @@ disable_firewalld() { read -p "Do you want to stop and disable firewalld? (recommended for nftables) (y/N): " -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]]; then - sudo systemctl stop firewalld - sudo systemctl disable firewalld + systemctl stop firewalld + systemctl disable firewalld log_success "firewalld stopped and disabled" else log_warning "firewalld is still running - this may conflict with nftables rules" @@ -340,6 +437,9 @@ main() { # Check if firewalld conflicts disable_firewalld + # Check dependencies first + check_dependencies + # Check prerequisites check_nftables diff --git a/scripts/configure_rhel_firewall.sh b/scripts/configure_rhel_firewall.sh index 42d65947..fa80a2c3 100755 --- a/scripts/configure_rhel_firewall.sh +++ b/scripts/configure_rhel_firewall.sh @@ -2,6 +2,8 @@ # LME Red Hat Firewall Configuration Script # This script automatically detects and configures firewalld rules for LME # Compatible with Red Hat Enterprise Linux, CentOS, and Fedora systems +# +# REQUIRES ROOT ACCESS - Run as: sudo ./configure_rhel_firewall.sh set -euo pipefail @@ -18,11 +20,11 @@ log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } -# Check if running as root -if [[ $EUID -eq 0 ]]; then - SUDO="" -else - SUDO="sudo" +# Check if running as root - required for firewall configuration +if [[ $EUID -ne 0 ]]; then + echo -e "${RED}[ERROR]${NC} This script must be run as root" + echo "Please run: sudo $0" + exit 1 fi # Function to check if firewalld is installed and running @@ -32,17 +34,29 @@ check_firewalld() { # Check if firewalld is installed if ! command -v firewall-cmd &> /dev/null; then log_error "firewalld is not installed. Installing..." - $SUDO dnf install -y firewalld + # Support multiple package managers + if command -v dnf &> /dev/null; then + dnf install -y firewalld + elif command -v yum &> /dev/null; then + yum install -y firewalld + elif command -v zypper &> /dev/null; then + zypper install -y firewalld + elif command -v apt &> /dev/null; then + apt update && apt install -y firewalld + else + log_error "Could not install firewalld automatically. Please install firewalld manually." + exit 1 + fi fi # Check if firewalld service is running - if ! $SUDO systemctl is-active --quiet firewalld; then + if ! systemctl is-active --quiet firewalld; then log_warning "firewalld is not running. Starting..." - $SUDO systemctl enable --now firewalld + systemctl enable --now firewalld fi # Verify firewalld is responding - if ! $SUDO firewall-cmd --state &> /dev/null; then + if ! firewall-cmd --state &> /dev/null; then log_error "firewalld is not responding properly" exit 1 fi @@ -50,19 +64,45 @@ check_firewalld() { log_success "firewalld is installed and running" } -# Function to detect LME container network -detect_lme_network() { - log_info "Detecting LME container network..." +# Function to check dependencies +check_dependencies() { + log_info "Checking dependencies..." # Check if podman is installed if ! command -v podman &> /dev/null; then log_error "podman is not installed or not in PATH" - return 1 + log_error "Please install podman before running this script" + exit 1 + fi + + # Check if jq is installed (needed for network inspection) + if ! command -v jq &> /dev/null; then + log_warning "jq is not installed. Installing jq for network detection..." + # Support multiple package managers + if command -v dnf &> /dev/null; then + dnf install -y jq + elif command -v yum &> /dev/null; then + yum install -y jq + elif command -v zypper &> /dev/null; then + zypper install -y jq + elif command -v apt &> /dev/null; then + apt update && apt install -y jq + else + log_error "Could not install jq automatically. Please install jq manually." + exit 1 + fi fi + log_success "All dependencies are available" +} + +# Function to detect LME container network +detect_lme_network() { + log_info "Detecting LME container network..." + # Try to get LME network information local lme_subnet - if lme_subnet=$($SUDO -i podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then + if lme_subnet=$(podman network inspect lme 2>/dev/null | jq -r '.[].subnets[].subnet' 2>/dev/null); then if [[ -n "$lme_subnet" && "$lme_subnet" != "null" ]]; then echo "$lme_subnet" return 0 @@ -83,17 +123,51 @@ detect_podman_interfaces() { # Try to get LME network interface (most important) local lme_interface - if lme_interface=$($SUDO -i podman network inspect lme 2>/dev/null | jq -r '.[].network_interface' 2>/dev/null); then + if lme_interface=$(podman network inspect lme 2>/dev/null | jq -r '.[].network_interface' 2>/dev/null); then if [[ -n "$lme_interface" && "$lme_interface" != "null" ]]; then interfaces+=("$lme_interface") log_info "Found LME network interface: $lme_interface" fi fi - # Look for common podman interfaces - for iface in podman0 podman1 cni-podman0 cni-podman1; do + # Comprehensive interface detection - look for all podman-related interfaces + local detected_interfaces + detected_interfaces=$(ip link show | grep -E '^[0-9]+:' | awk -F': ' '{print $2}' | grep -E '^(podman|cni-podman|veth.*podman|br-.*)') + + while IFS= read -r iface; do + if [[ -n "$iface" ]]; then + # Remove any trailing @ and interface suffix (e.g., podman1@if2 -> podman1) + iface=$(echo "$iface" | cut -d'@' -f1) + + # Check if this interface is already in our list + local already_added=false + for existing in "${interfaces[@]}"; do + if [[ "$existing" == "$iface" ]]; then + already_added=true + break + fi + done + + if [[ "$already_added" == false ]]; then + interfaces+=("$iface") + log_info "Found podman interface: $iface" + fi + fi + done <<< "$detected_interfaces" + + # Also check for common interface names as fallback + for iface in podman0 podman1 podman2 podman3 cni-podman0 cni-podman1; do if ip link show "$iface" &> /dev/null; then - if [[ ! " ${interfaces[@]} " =~ " ${iface} " ]]; then + # Check if this interface is already in our list + local already_added=false + for existing in "${interfaces[@]}"; do + if [[ "$existing" == "$iface" ]]; then + already_added=true + break + fi + done + + if [[ "$already_added" == false ]]; then interfaces+=("$iface") log_info "Found additional podman interface: $iface" fi @@ -102,6 +176,8 @@ detect_podman_interfaces() { if [[ ${#interfaces[@]} -eq 0 ]]; then log_warning "No podman interfaces detected. Container networking may not be configured." + log_warning "Firewall rules will still be applied for external access" + # Return empty string to prevent array issues echo "" else printf '%s\n' "${interfaces[@]}" @@ -120,7 +196,7 @@ configure_firewall() { log_info "Adding LME ports to public zone..." for port in "${lme_ports[@]}"; do - if $SUDO firewall-cmd --permanent --zone=public --add-port="${port}/tcp" &> /dev/null; then + if firewall-cmd --permanent --zone=public --add-port="${port}/tcp" &> /dev/null; then log_success "Added port ${port}/tcp to public zone" else log_warning "Port ${port}/tcp may already be configured" @@ -131,7 +207,7 @@ configure_firewall() { read -p "Do you want to enable Wazuh API port 55000? (y/N): " -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]]; then - if $SUDO firewall-cmd --permanent --zone=public --add-port=55000/tcp &> /dev/null; then + if firewall-cmd --permanent --zone=public --add-port=55000/tcp &> /dev/null; then log_success "Added Wazuh API port 55000/tcp to public zone" else log_warning "Port 55000/tcp may already be configured" @@ -143,26 +219,30 @@ configure_firewall() { # Add container subnet to trusted zone if [[ -n "$lme_subnet" ]]; then - if $SUDO firewall-cmd --permanent --zone=trusted --add-source="$lme_subnet" &> /dev/null; then + if firewall-cmd --permanent --zone=trusted --add-source="$lme_subnet" &> /dev/null; then log_success "Added container subnet $lme_subnet to trusted zone" else log_warning "Container subnet $lme_subnet may already be configured" fi fi - # Add podman interfaces to trusted zone - for iface in "${interfaces[@]}"; do - if [[ -n "$iface" ]]; then - if $SUDO firewall-cmd --permanent --zone=trusted --add-interface="$iface" &> /dev/null; then - log_success "Added interface $iface to trusted zone" - else - log_warning "Interface $iface may already be configured" + # Add podman interfaces to trusted zone (if any were detected) + if [[ ${#interfaces[@]} -gt 0 && -n "${interfaces[0]}" ]]; then + for iface in "${interfaces[@]}"; do + if [[ -n "$iface" ]]; then + if firewall-cmd --permanent --zone=trusted --add-interface="$iface" &> /dev/null; then + log_success "Added interface $iface to trusted zone" + else + log_warning "Interface $iface may already be configured" + fi fi - fi - done + done + else + log_warning "No podman interfaces to configure - container networking may need manual setup" + fi # Enable masquerading for container traffic - if $SUDO firewall-cmd --permanent --add-masquerade &> /dev/null; then + if firewall-cmd --permanent --add-masquerade &> /dev/null; then log_success "Enabled masquerading for container traffic" else log_warning "Masquerading may already be enabled" @@ -170,7 +250,7 @@ configure_firewall() { # Reload firewall to apply changes log_info "Reloading firewall configuration..." - $SUDO firewall-cmd --reload + firewall-cmd --reload log_success "Firewall configuration reloaded" } @@ -183,15 +263,15 @@ verify_configuration() { echo echo "Public Zone (External Access):" - $SUDO firewall-cmd --zone=public --list-ports + firewall-cmd --zone=public --list-ports echo echo "Trusted Zone (Container Networks):" - $SUDO firewall-cmd --zone=trusted --list-all + firewall-cmd --zone=trusted --list-all echo echo "=== Active Zones ===" - $SUDO firewall-cmd --get-active-zones + firewall-cmd --get-active-zones echo } @@ -201,21 +281,21 @@ provide_troubleshooting() { echo echo "If you experience connectivity issues:" echo "1. Check if LME containers are running:" - echo " sudo -i podman ps" + echo " podman ps" echo echo "2. Test container-to-container communication:" - echo " sudo -i podman exec lme-kibana curl -s http://lme-elasticsearch:9200/_cluster/health" + echo " podman exec lme-kibana curl -s http://lme-elasticsearch:9200/_cluster/health" echo echo "3. Test external access (replace with your server IP):" echo " curl -v http://YOUR_SERVER_IP:5601" echo echo "4. Check firewall logs for blocked connections:" - echo " sudo journalctl -u firewalld | tail -20" + echo " journalctl -u firewalld | tail -20" echo echo "5. Temporarily disable firewall for testing:" - echo " sudo systemctl stop firewalld" + echo " systemctl stop firewalld" echo " # Test your connections" - echo " sudo systemctl start firewalld" + echo " systemctl start firewalld" echo } @@ -229,9 +309,21 @@ main() { # Check prerequisites check_firewalld + # Check dependencies first + check_dependencies + # Detect network configuration lme_subnet=$(detect_lme_network) - readarray -t interfaces <<< "$(detect_podman_interfaces)" + + # Safely handle interface detection + local interface_output + interface_output=$(detect_podman_interfaces) + + # Create interfaces array, handling empty output + local interfaces=() + if [[ -n "$interface_output" ]]; then + readarray -t interfaces <<< "$interface_output" + fi # Display detected configuration echo @@ -252,8 +344,12 @@ main() { exit 0 fi - # Configure firewall - configure_firewall "$lme_subnet" "${interfaces[@]}" + # Configure firewall - handle empty interface array safely + if [[ ${#interfaces[@]} -gt 0 && -n "${interfaces[0]}" ]]; then + configure_firewall "$lme_subnet" "${interfaces[@]}" + else + configure_firewall "$lme_subnet" + fi # Verify configuration verify_configuration From 155cf0ed68acccdbff93dad662cde3d5ce29084d Mon Sep 17 00:00:00 2001 From: cbaxley Date: Wed, 24 Sep 2025 06:53:14 -0400 Subject: [PATCH 47/51] Updates gitignore to ignore all output logs and password files --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 031461ea..470e77b7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,10 +20,10 @@ lme.conf /testing/tests/.env **/.pytest_cache/ **/__pycache__/ -/testing/*.password.txt +/testing/**/*.password.txt /testing/configure/azure_scripts/config.ps1 /testing/configure.zip -/testing/*.output.log +/testing/**/*.output.log /testing/tests/report.html testing/tests/assets/style.css .history/ From 71f46dd0ffbe15777c1223c0b45e1c91bd027e31 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Wed, 24 Sep 2025 15:36:49 +0000 Subject: [PATCH 48/51] Broaden nftables configuration --- scripts/configure_lme_nftables.sh | 42 +++++++++++++++++++------------ 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/scripts/configure_lme_nftables.sh b/scripts/configure_lme_nftables.sh index be08a981..bc5d80fa 100755 --- a/scripts/configure_lme_nftables.sh +++ b/scripts/configure_lme_nftables.sh @@ -204,9 +204,8 @@ backup_nftables() { # Function to create LME nftables configuration create_lme_nftables() { local container_subnet="$1" - local primary_iface="$2" - local podman_iface="$3" - local enable_wazuh_api="$4" + local podman_iface="$2" + local enable_wazuh_api="$3" log_info "Creating LME nftables configuration..." @@ -240,16 +239,19 @@ table inet lme_filter { # Allow SSH (modify port if needed) tcp dport 22 accept - # LME service ports on primary interface - iifname "$primary_iface" tcp dport { 1514, 1515, 8220, 9200, 5601, 443 } accept + # LME service ports - accessible from all interfaces + tcp dport { 1514, 1515, 8220, 9200, 5601, 443 } accept + + # Wazuh syslog UDP port + udp dport 514 accept EOF # Add Wazuh API port if requested if [[ "$enable_wazuh_api" == "yes" ]]; then cat >> /tmp/lme_nftables.conf << EOF - # Wazuh API port - iifname "$primary_iface" tcp dport 55000 accept + # Wazuh API port - accessible from all interfaces + tcp dport 55000 accept EOF fi @@ -309,18 +311,21 @@ table ip lme_nat { chain postrouting { type nat hook postrouting priority srcnat; policy accept; - # Masquerade traffic from container network going out primary interface - ip saddr $container_subnet oifname "$primary_iface" masquerade - - # General masquerading for container traffic (except loopback) - oifname != "lo" masquerade + # Masquerade traffic from container network + ip saddr $container_subnet masquerade } } EOF # Move the configuration file to its final location mv /tmp/lme_nftables.conf "$config_file" - chmod 640 "$config_file" + chmod 644 "$config_file" + + # Fix SELinux context if SELinux is enabled + if command -v restorecon &> /dev/null; then + restorecon "$config_file" 2>/dev/null || true + log_info "SELinux context restored for $config_file" + fi log_success "nftables configuration created at $config_file" } @@ -342,7 +347,13 @@ apply_nftables() { log_success "nftables configuration applied" # Add to main nftables config to persist across reboots + # Handle different distributions (RHEL uses /etc/sysconfig/nftables.conf) local main_config="/etc/nftables.conf" + if [[ -f "/etc/sysconfig/nftables.conf" ]]; then + main_config="/etc/sysconfig/nftables.conf" + log_info "Detected RHEL/CentOS system, using $main_config" + fi + if [[ -f "$main_config" ]]; then if ! grep -q "include.*lme.nft" "$main_config"; then echo 'include "/etc/nftables/lme.nft"' | tee -a "$main_config" > /dev/null @@ -445,13 +456,12 @@ main() { # Detect network configuration container_subnet=$(detect_lme_network) - read -r primary_iface podman_iface <<< "$(detect_interfaces)" + podman_iface=$(detect_interfaces | awk '{print $2}') # Display detected configuration echo log_info "Detected Configuration:" echo " Container Subnet: $container_subnet" - echo " Primary Interface: $primary_iface" echo " Podman Interface: ${podman_iface:-"None detected"}" echo @@ -476,7 +486,7 @@ main() { backup_nftables # Create and apply nftables configuration - create_lme_nftables "$container_subnet" "$primary_iface" "$podman_iface" "$enable_wazuh_api" + create_lme_nftables "$container_subnet" "$podman_iface" "$enable_wazuh_api" apply_nftables # Verify configuration From ec01a81c44f6ab4879b111aa6d4e85d0ed1ccc87 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Thu, 25 Sep 2025 08:34:16 -0400 Subject: [PATCH 49/51] Adds the recommendation to restart the system after applying the firewall configuration --- scripts/configure_lme_nftables.sh | 16 ++++++++++++++++ scripts/configure_rhel_firewall.sh | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/scripts/configure_lme_nftables.sh b/scripts/configure_lme_nftables.sh index bc5d80fa..53ff6177 100755 --- a/scripts/configure_lme_nftables.sh +++ b/scripts/configure_lme_nftables.sh @@ -497,6 +497,22 @@ main() { log_success "LME nftables configuration completed!" log_info "Configuration will persist across reboots via /etc/nftables.conf" + + # Recommend restart for complete activation + echo + log_warning "⚠️ IMPORTANT: System restart recommended for complete nftables activation" + echo "After applying nftables configuration changes, it is highly recommended to reboot" + echo "the machine to ensure all networking and container rules take effect properly." + echo + echo "This is especially important for:" + echo "- Container networking changes - Ensures podman interfaces and bridge networks restart correctly" + echo "- nftables rule persistence - Confirms all rules are properly loaded from configuration files" + echo "- Network interface binding - Ensures proper interface-to-rule assignments" + echo "- Service startup order - Guarantees nftables, networking, and containers start in the correct sequence" + echo + echo "To restart the system:" + echo " sudo reboot" + echo } # Run main function if script is executed directly diff --git a/scripts/configure_rhel_firewall.sh b/scripts/configure_rhel_firewall.sh index fa80a2c3..261fcaa1 100755 --- a/scripts/configure_rhel_firewall.sh +++ b/scripts/configure_rhel_firewall.sh @@ -358,6 +358,22 @@ main() { provide_troubleshooting log_success "LME firewall configuration completed!" + + # Recommend restart for complete activation + echo + log_warning "⚠️ IMPORTANT: System restart recommended for complete firewall activation" + echo "After applying firewall configuration changes, it is highly recommended to reboot" + echo "the machine to ensure all networking and container rules take effect properly." + echo + echo "This is especially important for:" + echo "- Container networking changes - Ensures podman interfaces restart correctly" + echo "- Firewall rule persistence - Confirms all permanent rules are properly loaded" + echo "- Network interface binding - Ensures proper interface-to-zone assignments" + echo "- Service startup order - Guarantees firewall, networking, and containers start correctly" + echo + echo "To restart the system:" + echo " sudo reboot" + echo } # Run main function if script is executed directly From 13bfdecfefc3b7544793692ddef4e9db58b2e082 Mon Sep 17 00:00:00 2001 From: cbaxley Date: Fri, 26 Sep 2025 05:02:45 -0400 Subject: [PATCH 50/51] Remove libsemanage-devel from SELinux policy tools --- ansible/roles/base/tasks/selinux_setup.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/roles/base/tasks/selinux_setup.yml b/ansible/roles/base/tasks/selinux_setup.yml index 6ccd9a8e..0278e286 100644 --- a/ansible/roles/base/tasks/selinux_setup.yml +++ b/ansible/roles/base/tasks/selinux_setup.yml @@ -43,7 +43,6 @@ - policycoreutils - policycoreutils-python-utils - checkpolicy - - libsemanage-devel - selinux-policy - selinux-policy-targeted - libselinux-utils From 1c0680d37b79937ce3268a84a1c9ef2ed05ebc25 Mon Sep 17 00:00:00 2001 From: Clint Baxley Date: Mon, 29 Sep 2025 20:52:58 +0000 Subject: [PATCH 51/51] Allow user to choose ansible installation method on redhat based distros --- install.sh | 61 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/install.sh b/install.sh index a5ec62d4..d71f2b42 100755 --- a/install.sh +++ b/install.sh @@ -191,24 +191,22 @@ install_ansible() { sudo dnf install -y ansible ;; centos|rhel|rocky|almalinux) - # Install EPEL repository first - echo "Installing EPEL repository..." - if ! sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm -E %rhel).noarch.rpm; then - echo -e "${RED}✗ Failed to install EPEL repository${NC}" - exit 1 - fi - echo -e "${GREEN}✓ EPEL repository installed${NC}" - - # For RHEL, you might also need to enable CodeReady Builder repository - if [[ "$DISTRO" == "rhel" ]]; then - sudo subscription-manager repos --enable codeready-builder-for-rhel-$(rpm -E %rhel)-$(arch)-rpms 2>/dev/null || true + # Ask user which installation method to use + use_pip=false + if [ "$NON_INTERACTIVE" != "true" ]; then + echo -e "${YELLOW}Choose Ansible installation method:${NC}" + echo " 1. Fedora EPEL repository (default)" + echo " 2. Python pip" + read -p "Select option (1 or 2): " install_choice + + if [[ "$install_choice" == "2" ]]; then + use_pip=true + fi fi - - # Now try to install ansible via dnf - if sudo dnf install -y ansible; then - echo -e "${GREEN}✓ Ansible installed via dnf${NC}" - else - echo -e "${YELLOW}⚠ dnf installation failed, trying pip installation...${NC}" + + if [ "$use_pip" = true ]; then + # Install via pip + echo -e "${YELLOW}Installing Ansible via pip...${NC}" sudo dnf install -y python3-pip sudo pip3 install ansible # Create symlink to make pip-installed ansible available in PATH @@ -217,6 +215,35 @@ install_ansible() { sudo ln -sf /usr/local/bin/ansible-vault /usr/bin/ansible-vault echo -e "${GREEN}✓ Created symlink for ansible in /usr/bin${NC}" fi + echo -e "${GREEN}✓ Ansible installed via pip${NC}" + else + # Install EPEL repository first + echo "Installing EPEL repository..." + if ! sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm -E %rhel).noarch.rpm; then + echo -e "${RED}✗ Failed to install EPEL repository${NC}" + exit 1 + fi + echo -e "${GREEN}✓ EPEL repository installed${NC}" + + # For RHEL, you might also need to enable CodeReady Builder repository + if [[ "$DISTRO" == "rhel" ]]; then + sudo subscription-manager repos --enable codeready-builder-for-rhel-$(rpm -E %rhel)-$(arch)-rpms 2>/dev/null || true + fi + + # Now try to install ansible via dnf + if sudo dnf install -y ansible; then + echo -e "${GREEN}✓ Ansible installed via dnf${NC}" + else + echo -e "${YELLOW}⚠ dnf installation failed, trying pip installation...${NC}" + sudo dnf install -y python3-pip + sudo pip3 install ansible + # Create symlink to make pip-installed ansible available in PATH + if [ -f /usr/local/bin/ansible ] && [ ! -f /usr/bin/ansible ]; then + sudo ln -sf /usr/local/bin/ansible /usr/bin/ansible + sudo ln -sf /usr/local/bin/ansible-vault /usr/bin/ansible-vault + echo -e "${GREEN}✓ Created symlink for ansible in /usr/bin${NC}" + fi + fi fi ;; arch|manjaro)