diff --git a/images/rhel/assets/post-gen/cleanup-logs.sh b/images/rhel/assets/post-gen/cleanup-logs.sh new file mode 100755 index 0000000..335f848 --- /dev/null +++ b/images/rhel/assets/post-gen/cleanup-logs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# journalctl +if command -v journalctl; then + journalctl --rotate + journalctl --vacuum-time=1s +fi + +# delete all .gz and rotated file +find /var/log -type f -regex ".*\.gz$" -delete +find /var/log -type f -regex ".*\.[0-9]$" -delete + +# wipe log files +find /var/log/ -type f -exec cp /dev/null {} \; \ No newline at end of file diff --git a/images/rhel/assets/post-gen/environment-variables.sh b/images/rhel/assets/post-gen/environment-variables.sh new file mode 100755 index 0000000..975c8a4 --- /dev/null +++ b/images/rhel/assets/post-gen/environment-variables.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Replace $HOME with the default user's home directory for environmental variables related to the default user home directory + +homeDir=$(cut -d: -f6 /etc/passwd | tail -1) +sed -i "s|\$HOME|$homeDir|g" /etc/environment \ No newline at end of file diff --git a/images/rhel/assets/post-gen/systemd-linger.sh b/images/rhel/assets/post-gen/systemd-linger.sh new file mode 100755 index 0000000..79003d9 --- /dev/null +++ b/images/rhel/assets/post-gen/systemd-linger.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Enable user session on boot, not on login +UserId=$(cut -d: -f3 /etc/passwd | tail -1) +loginctl enable-linger "$UserId" diff --git a/images/rhel/scripts/build/cleanup.sh b/images/rhel/scripts/build/cleanup.sh new file mode 100755 index 0000000..1860fe9 --- /dev/null +++ b/images/rhel/scripts/build/cleanup.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e +################################################################################ +## File: cleanup.sh +## Desc: Perform cleanup for RHEL +################################################################################ + +# before cleanup +before=$(df / -Pm | awk 'NR==2{print $4}') + +# Clear the local repository of retrieved package files +yum clean all +rm -rf /var/cache/yum/* +rm -rf /tmp/* +rm -rf /root/.cache + +# Rotate and vacuum journal logs if `journalctl` is available +if command -v journalctl; then + journalctl --rotate + journalctl --vacuum-time=1s +fi + +# Delete all .gz and rotated files +find /var/log -type f -regex ".*\.gz$" -delete +find /var/log -type f -regex ".*\.[0-9]$" -delete + +# Wipe log files +find /var/log/ -type f -exec cp /dev/null {} \; + +# Remove mock binaries for yum/dnf +prefix=/usr/local/bin +for tool in yum dnf; do + rm -f $prefix/$tool +done + +# after cleanup +after=$(df / -Pm | awk 'NR==2{print $4}') + +# Display size +echo "Before: $before MB" +echo "After : $after MB" +# shellcheck disable=SC2004 +echo "Delta : $(($after - $before)) MB" diff --git a/images/rhel/scripts/build/configure-dnf.sh b/images/rhel/scripts/build/configure-dnf.sh new file mode 100755 index 0000000..db67aa5 --- /dev/null +++ b/images/rhel/scripts/build/configure-dnf.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e +################################################################################ +## File: configure-dnf.sh +## Desc: Configure dnf/yum, install jq package, and improve package management behavior. +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh +# Enable retries for DNF (maximum retries set to 10) +# shellcheck disable=SC2129 +echo "retries=10" >> /etc/dnf/dnf.conf + +# Automatically assume 'yes' for prompts in DNF +echo "assumeyes=True" >> /etc/dnf/dnf.conf + +# Configure DNF to always consider phased updates +echo "phased_updates=1" >> /etc/dnf/dnf.conf + +# Fix potential bad proxy or HTTP headers settings +cat <> /etc/dnf/dnf.conf +http_caching=none +EOF + +# Remove unattended-upgrade equivalents if present (e.g., dnf-automatic) +dnf remove -y dnf-automatic + +# Display DNF repository configurations +echo 'DNF/YUM repositories:' +dnf repolist + +# Update repositories and install jq +install_dnfpkgs jq + +# Optional: Configure parallel downloads to speed up package installation +echo "max_parallel_downloads=10" >> /etc/dnf/dnf.conf diff --git a/images/rhel/scripts/build/configure-dnfpkg.sh b/images/rhel/scripts/build/configure-dnfpkg.sh new file mode 100755 index 0000000..6aff56d --- /dev/null +++ b/images/rhel/scripts/build/configure-dnfpkg.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e +################################################################################ +## File: configure-dnfpkg.sh +## Desc: Configure dnf and package management settings +################################################################################ + +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/etc-environment.sh + +# Configure dnf to automatically answer 'yes' for package installation +# This replaces the non-interactive mode typically set in DEBIAN_FRONTEND +# shellcheck disable=SC2129 +echo "assumeyes=True" >> /etc/dnf/dnf.conf + +# Prevent dnf from prompting for confirmation on replacing configuration files +# Equivalent to dpkg's --force-confdef --force-confold +echo "override_install_langs=en_US.UTF-8" >> /etc/dnf/dnf.conf + +# Hide information about packages that are no longer required +# dnf has an autoremove feature, but it can be configured to prevent auto removal prompts +echo "clean_requirements_on_remove=True" >> /etc/dnf/dnf.conf + +# Configure dnf to automatically clean up unused packages and dependencies +echo "autoclean_metadata=True" >> /etc/dnf/dnf.conf diff --git a/images/rhel/scripts/build/configure-environment.sh b/images/rhel/scripts/build/configure-environment.sh new file mode 100755 index 0000000..bde2d5a --- /dev/null +++ b/images/rhel/scripts/build/configure-environment.sh @@ -0,0 +1,59 @@ +#!/bin/bash -e +################################################################################ +## File: configure-environment.sh +## Desc: Configure system and environment +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/os.sh +source "$HELPER_SCRIPTS"/etc-environment.sh + +# Set ImageVersion and ImageOS env variables +set_etc_environment_variable "ImageVersion" "${IMAGE_VERSION}" +set_etc_environment_variable "ImageOS" "${IMAGE_OS}" + +# Set the ACCEPT_EULA variable to Y value to confirm your acceptance of the End-User Licensing Agreement +set_etc_environment_variable "ACCEPT_EULA" "Y" + +# This directory is supposed to be created in $HOME and owned by user(https://github.com/actions/runner-images/issues/491) +mkdir -p /etc/skel/.config/configstore +# shellcheck disable=SC2016 +set_etc_environment_variable "XDG_CONFIG_HOME" '$HOME/.config' + +# Change waagent entries to use /mnt for swap file +# sed -i 's/ResourceDisk.Format=n/ResourceDisk.Format=y/g' /etc/waagent.conf +# sed -i 's/ResourceDisk.EnableSwap=n/ResourceDisk.EnableSwap=y/g' /etc/waagent.conf +# sed -i 's/ResourceDisk.SwapSizeMB=0/ResourceDisk.SwapSizeMB=4096/g' /etc/waagent.conf + +# Add localhost alias to ::1 IPv6 +sed -i 's/::1 ip6-localhost ip6-loopback/::1 localhost ip6-localhost ip6-loopback/g' /etc/hosts + +# Prepare directory and env variable for toolcache +AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache +mkdir -p $AGENT_TOOLSDIRECTORY && echo "Directory created." || echo "Directory already exists." +set_etc_environment_variable "AGENT_TOOLSDIRECTORY" "${AGENT_TOOLSDIRECTORY}" +set_etc_environment_variable "RUNNER_TOOL_CACHE" "${AGENT_TOOLSDIRECTORY}" +chmod -R 777 $AGENT_TOOLSDIRECTORY + +# https://github.com/orgs/community/discussions/47563 +echo 'net.ipv6.conf.all.disable_ipv6=1' | tee -a /etc/sysctl.conf +echo 'net.ipv6.conf.default.disable_ipv6=1' | tee -a /etc/sysctl.conf +echo 'net.ipv6.conf.lo.disable_ipv6=1' | tee -a /etc/sysctl.conf + +# https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html +# https://www.suse.com/support/kb/doc/?id=000016692 +echo 'vm.max_map_count=262144' | tee -a /etc/sysctl.conf + +# https://kind.sigs.k8s.io/docs/user/known-issues/#pod-errors-due-to-too-many-open-files +echo 'fs.inotify.max_user_watches=655360' | tee -a /etc/sysctl.conf +echo 'fs.inotify.max_user_instances=1280' | tee -a /etc/sysctl.conf + +# https://github.com/actions/runner-images/issues/9491 +echo 'vm.mmap_rnd_bits=28' | tee -a /etc/sysctl.conf + +# https://github.com/actions/runner-images/pull/7860 +netfilter_rule='/etc/udev/rules.d/50-netfilter.rules' +rules_directory="$(dirname "${netfilter_rule}")" +mkdir -p "$rules_directory" +touch $netfilter_rule +echo 'ACTION=="add", SUBSYSTEM=="module", KERNEL=="nf_conntrack", RUN+="/usr/sbin/sysctl net.netfilter.nf_conntrack_tcp_be_liberal=1"' | tee -a $netfilter_rule diff --git a/images/rhel/scripts/build/configure-image-data.sh b/images/rhel/scripts/build/configure-image-data.sh new file mode 100755 index 0000000..6ac1032 --- /dev/null +++ b/images/rhel/scripts/build/configure-image-data.sh @@ -0,0 +1,60 @@ +#!/bin/bash -e +################################################################################ +## File: configure-image-data.sh +## Desc: Create a file with image data and documentation links +################################################################################ +# shellcheck disable=SC2153 +imagedata_file="$IMAGEDATA_FILE" +image_version="$IMAGE_VERSION" +image_version_major=${image_version/.*/} # Extract the major version +image_version_minor=$(echo "$image_version" | cut -d "." -f 2) # Extract the minor version + +# Determine OS name and version for CentOS +# shellcheck disable=SC2002 +os_name=$(cat /etc/redhat-release | sed "s/ /\\\n/g") # Get OS name +# shellcheck disable=SC1083 +os_version=$(rpm -E %{rhel}) # Get CentOS version +image_label="centos-${os_version}" # Set image label + +REPO_OWNER="IBM" +REPO_NAME="action-runner-image-pz" +BRANCH="main" + +api_release_response=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases/latest") +git_tag=$(echo "$api_release_response" | jq -r .tag_name) + +build_sha=$(curl -s ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits/${BRANCH}" | jq -r .sha) + +github_url="https://github.com/${REPO_OWNER}/${REPO_NAME}/blob/${BRANCH}/images" +software_url="${github_url}/centos/toolsets/toolset-${image_version_major}${image_version_minor}.json" + +if [ "$git_tag" != "null" ] && [ -n "$git_tag" ]; then + echo "Release found: ${git_tag}" + tag_slug=${git_tag//\//%2F} # URL encode slashes + releaseUrl="https://github.com/${REPO_OWNER}/${REPO_NAME}/releases/tag/${tag_slug}" +else + echo "Warning: No release found. Falling back to commit SHA: ${build_sha}" + releaseUrl="https://github.com/${REPO_OWNER}/${REPO_NAME}/tree/${build_sha}" +fi + +runner_image_version="$(date +%Y%m%d)" +image_build_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +image_builder_id=$(cat /etc/machine-id 2>/dev/null || hostname -s 2>/dev/null) + +# Create the image data JSON file +cat < "$imagedata_file" +[ + { + "group": "Runner Image Provisioner", + "detail": "Commit: ${build_sha}\nBuild Date: ${image_build_date}\nBuilder ID: ${image_builder_id}" + }, + { + "group": "Operating System", + "detail": "${os_name}" + }, + { + "group": "Runner Image", + "detail": "Image: ${image_label}\nVersion: ${runner_image_version}\nIncluded Software: ${software_url}\nImage Release: ${releaseUrl}" + } +] +EOF diff --git a/images/rhel/scripts/build/configure-limits.sh b/images/rhel/scripts/build/configure-limits.sh new file mode 100755 index 0000000..d7ca9c4 --- /dev/null +++ b/images/rhel/scripts/build/configure-limits.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +################################################################################ +## File: configure-limits.sh +## Desc: Configure limits +################################################################################ +echo 'session required pam_limits.so' >> /etc/pam.d/system-auth +echo 'session required pam_limits.so' >> /etc/pam.d/password-auth +echo 'DefaultLimitNOFILE=65536' >> /etc/systemd/system.conf +echo 'DefaultLimitSTACK=16M:infinity' >> /etc/systemd/system.conf + +# Raise Number of File Descriptors +# shellcheck disable=SC2129 +echo '* soft nofile 65536' >> /etc/security/limits.conf +echo '* hard nofile 65536' >> /etc/security/limits.conf + +# Double stack size from default 8192KB +echo '* soft stack 16384' >> /etc/security/limits.conf +echo '* hard stack 16384' >> /etc/security/limits.conf diff --git a/images/rhel/scripts/build/configure-runner.sh b/images/rhel/scripts/build/configure-runner.sh new file mode 100755 index 0000000..c1e9ba7 --- /dev/null +++ b/images/rhel/scripts/build/configure-runner.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -euo pipefail + +header() { + TS=$(date +"%Y-%m-%dT%H:%M:%S%:z") + echo "${TS} +--------------------------------------------+" + echo "${TS} | $*" + echo "${TS} +--------------------------------------------+" + echo +} + +msg() { + # shellcheck disable=SC2046 + echo $(date +"%Y-%m-%dT%H:%M:%S%:z") "$*" +} + +check_idempotency() { + header "Checking Idempotency" + + msg "Fetching latest upstream version from ${RUNNERREPO}..." + UPSTREAM_TAG=$(git ls-remote --tags --refs --sort='v:refname' "${RUNNERREPO}" | tail -n1 | awk -F/ '{print $NF}') + + UPSTREAM_VER="${UPSTREAM_TAG#v}" + msg "Latest Upstream Version: ${UPSTREAM_VER}" + + CURRENT_VER="none" + if [ -f "/opt/runner-cache/bin/Runner.Listener" ]; then + CURRENT_VER=$(/opt/runner-cache/bin/Runner.Listener --version 2>/dev/null || echo "error") + fi + + msg "Current Installed Version: ${CURRENT_VER}" + + if [ "${UPSTREAM_VER}" == "${CURRENT_VER}" ]; then + header "Versions match (${UPSTREAM_VER}). Skipping build." + exit 0 + else + msg "Versions do not match or runner not installed. Proceeding with build..." + fi +} + +patch_runner() { + header "Cloning repo and Patching runner" + cd /tmp + git clone --tags -q "${RUNNERREPO}" + cd runner + # shellcheck disable=SC2046 + git checkout $(git tag --sort=-v:refname | grep '^v[0-9]' | head -n1) + git apply --whitespace=nowarn "${IMAGE_FOLDER}"/runner-sdk-8.patch + sed -i'' -e '/version/s/8......"$/8.0.100"/' src/global.json +} + +build_runner() { + export DOTNET_NUGET_SIGNATURE_VERIFICATION=false + header "Building runner binary" + cd src + + msg "Running dev layout" + ./dev.sh layout Release + + msg "Creating package" + ./dev.sh package Release + + msg "Running tests" + ./dev.sh test +} + +install_runner() { + header "Installing runner" + mkdir -p /opt/runner-cache + tar -xf /tmp/runner/_package/*.tar.gz -C /opt/runner-cache +} + +pre_cleanup() { + rm -rf /tmp/runner /opt/runner-cache +} + +post_cleanup() { + rm -rf "${IMAGE_FOLDER}"/runner-sdk-8.patch \ + /tmp/preseed-yaml /home/ubuntu/.nuget \ + /home/runner/.local/share +} + +run() { + check_idempotency + pre_cleanup + patch_runner + build_runner + install_runner + post_cleanup +} + +RUNNERREPO="https://github.com/actions/runner" + +# Parse arguments +while getopts "a:" opt; do + case ${opt} in + a) RUNNERREPO=${OPTARG} ;; + *) exit 1 ;; + esac +done +shift $(( OPTIND - 1 )) + +run \ No newline at end of file diff --git a/images/rhel/scripts/build/configure-snap.sh b/images/rhel/scripts/build/configure-snap.sh new file mode 100755 index 0000000..048688d --- /dev/null +++ b/images/rhel/scripts/build/configure-snap.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e +################################################################################ +## File: configure-snap.sh +## Desc: Configure snap +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/etc-environment.sh + +# Update /etc/environment to include /snap/bin in PATH +# because /etc/profile.d is ignored by `--norc` shell launch option +if [[ ":$PATH:" == *"/snap/bin"* ]]; then + echo "/snap/bin is already in the PATH" +else + echo "/snap/bin is not in the PATH. Adding it now..." + export PATH=/snap/bin:$PATH + echo "export PATH=/snap/bin:$PATH" >> ~/.bashrc # Persist for future sessions + echo "/snap/bin has been added to the PATH" +fi +# Put snapd auto refresh on hold +# as it may generate too much traffic on Canonical's snap server +# when they are rolling a new major update out. +# Hold is calculated as today's date + 60 days + +# snapd is started automatically, but during image generation +# a unix socket may die, restart snapd.service (and therefore snapd.socket) +# to make sure the socket is alive. + +systemctl restart snapd.socket +systemctl restart snapd +snap set system refresh.hold="$(date --date='today+60 days' +%Y-%m-%dT%H:%M:%S%:z)" diff --git a/images/rhel/scripts/build/configure-system.sh b/images/rhel/scripts/build/configure-system.sh new file mode 100755 index 0000000..dc9b769 --- /dev/null +++ b/images/rhel/scripts/build/configure-system.sh @@ -0,0 +1,48 @@ +#!/bin/bash -e +################################################################################ +## File: configure-system.sh +## Desc: Post deployment system configuration actions for CentOS +################################################################################ + +# Source helper scripts +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/etc-environment.sh +source "$HELPER_SCRIPTS"/os.sh + +if [ -d "/opt/post-generation" ]; then + rm -rf "/opt/post-generation" +fi +mv -f "${IMAGE_FOLDER}/post-generation" /opt + +# Adjust permissions +echo "chmod -R 777 /opt" +chmod -R 777 /opt +echo "chmod -R 777 /usr/share" +chmod -R 777 /usr/share + +chmod 755 "$IMAGE_FOLDER" + +# Remove quotes around PATH in /etc/environment and ensure system directories are present +ENVPATH=$(grep 'PATH=' /etc/environment | head -n 1 | sed -z 's/^PATH=*//') +ENVPATH=${ENVPATH#"\""} +ENVPATH=${ENVPATH%"\""} + +# Append standard system directories so that login shells via PAM (e.g. sudo su -c) +# can find basic commands like find, grep, etc. +for dir in /usr/local/sbin /usr/local/bin /usr/sbin /usr/bin /sbin /bin; do + if [[ ":${ENVPATH}:" != *":${dir}:"* ]]; then + ENVPATH="${ENVPATH}:${dir}" + fi +done + +replace_etc_environment_variable "PATH" "${ENVPATH}" +echo "Updated /etc/environment: $(cat /etc/environment)" + +# Clean yarn and npm cache if installed +if command -v yarn > /dev/null; then + yarn cache clean +fi + +if command -v npm > /dev/null; then + npm cache clean --force +fi diff --git a/images/rhel/scripts/build/configure-yum-mock.sh b/images/rhel/scripts/build/configure-yum-mock.sh new file mode 100755 index 0000000..1401011 --- /dev/null +++ b/images/rhel/scripts/build/configure-yum-mock.sh @@ -0,0 +1,54 @@ +#!/bin/bash -e +################################################################################ +## File: configure-yum-mock.sh +## Desc: A temporary workaround to handle transient issues with DNF/YUM. +## Cleaned up during cleanup.sh. +################################################################################ +prefix=/usr/local/bin + +for real_tool in /usr/bin/yum /usr/bin/dnf; do + tool=$(basename $real_tool) + cat >$prefix/"$tool" <\$err + rc=\$? + + if [ \$rc -eq 0 ]; then + rm -f \$err + exit 0 + fi + + cat \$err >&2 + + retry=false + + if grep -q 'Could not get lock' \$err;then + retry=true + elif grep -q 'Temporary failure in name resolution' \$err;then + retry=true + elif grep -q 'Package is being held by another process' \$err;then + retry=true + elif grep -q 'Failed to download metadata' \$err && ! grep -q 'Status code: 404' \$err;then + retry=true + fi + + rm -f \$err + if [ \$retry = false ]; then + exit \$rc + fi + + sleep 5 + echo "...retry \$i/\$max_retries" + i=\$((i + 1)) +done + +echo "ERROR: Exhausted \$max_retries retries, giving up." +exit \$rc +EOT + chmod +x $prefix/"$tool" +done diff --git a/images/rhel/scripts/build/install-actions-cache.sh b/images/rhel/scripts/build/install-actions-cache.sh new file mode 100755 index 0000000..2cde76a --- /dev/null +++ b/images/rhel/scripts/build/install-actions-cache.sh @@ -0,0 +1,22 @@ +#!/bin/bash -e +################################################################################ +## File: install-actions-cache.sh +## Desc: Download latest release from https://github.com/actions/action-versions +## Maintainer: #actions-runtime and @TingluoHuang +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh +source "$HELPER_SCRIPTS"/etc-environment.sh + +# Prepare directory and env variable for ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE +ACTION_ARCHIVE_CACHE_DIR=/opt/actionarchivecache +mkdir -p $ACTION_ARCHIVE_CACHE_DIR +chmod -R 777 $ACTION_ARCHIVE_CACHE_DIR +echo "Setting up ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE variable to ${ACTION_ARCHIVE_CACHE_DIR}" +set_etc_environment_variable "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE" "${ACTION_ARCHIVE_CACHE_DIR}" + +# Download latest release from github.com/actions/action-versions and untar to /opt/actionarchivecache +download_url=$(resolve_github_release_asset_url "actions/action-versions" "endswith(\"action-versions.tar.gz\")" "latest") +archive_path=$(download_with_retry "$download_url") +tar -xzf "$archive_path" -C $ACTION_ARCHIVE_CACHE_DIR diff --git a/images/rhel/scripts/build/install-dnf-common.sh b/images/rhel/scripts/build/install-dnf-common.sh new file mode 100755 index 0000000..8c62326 --- /dev/null +++ b/images/rhel/scripts/build/install-dnf-common.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e +################################################################################ +## File: install-dnf-common.sh +## Desc: Install basic command-line utilities and development packages +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +common_packages=$(get_toolset_value .dnf.common_packages[]) +cmd_packages=$(get_toolset_value .dnf.cmd_packages[]) + +for package in $common_packages $cmd_packages; do + echo "Install $package" + install_dnfpkgs --setopt=install_weak_deps=False "$package" +done \ No newline at end of file diff --git a/images/rhel/scripts/build/install-dnf-vital.sh b/images/rhel/scripts/build/install-dnf-vital.sh new file mode 100755 index 0000000..c2692c2 --- /dev/null +++ b/images/rhel/scripts/build/install-dnf-vital.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e +################################################################################ +## File: install-dnf-vital.sh +## Desc: Install vital command-line utilities +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +vital_packages=$(get_toolset_value .dnf.vital_packages[]) + +# Install vital packages using dnf +install_dnfpkgs --setopt=install_weak_deps=False "$vital_packages" diff --git a/images/rhel/scripts/build/install-docker.sh b/images/rhel/scripts/build/install-docker.sh new file mode 100755 index 0000000..4092097 --- /dev/null +++ b/images/rhel/scripts/build/install-docker.sh @@ -0,0 +1,151 @@ +#!/bin/bash -e +################################################################################ +## File: install-docker.sh +## Desc: Install docker onto the image +## Supply chain security: amazon-ecr-credential-helper - dynamic checksum validation +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh +# shellcheck disable=SC2086 +source $HELPER_SCRIPTS/os.sh + +# Set architecture-specific variables using a case statement for clarity +case "$ARCH" in + "x86_64") + package_arch="amd64" + ;; + *) + package_arch="$ARCH" + ;; +esac + +os_codename=$(. /etc/os-release && echo "$VERSION_ID") +os_major="${os_codename%%.*}" +dnf -y install dnf-plugins-core + +# Remove any stale docker-ce.repo from a previous build run +rm -f /etc/yum.repos.d/docker-ce.repo + +# Docker does not publish RHEL packages for ppc64le; use the CentOS repo instead +# (CentOS 9 and RHEL 9 are binary-compatible) +if [[ "$ARCH" == "ppc64le" ]]; then + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo +else + dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo +fi + +# Install docker components via dnf +# Using toolsets keep installation order to install dependencies before the package in order to control versions + +components=$(get_toolset_value '.docker.components[] .package') +for package in $components; do + version=$(get_toolset_value ".docker.components[] | select(.package == \"$package\") | .version") + if [[ $version == "latest" ]]; then + install_dnfpkgs --setopt=install_weak_deps=False "$package" + else + version_string=$(dnf --showduplicates list "$package" | awk '{ print $2 }' | grep "$version" | grep "el${os_major}" | head -1) + if [[ -z "$version_string" ]]; then + echo "WARNING: version $version not found for $package, installing latest available" + install_dnfpkgs --setopt=install_weak_deps=False "$package" + else + install_dnfpkgs --setopt=install_weak_deps=False "${package}-${version_string}" + fi + fi +done + +# Install plugins that are best installed from the GitHub repository +# Be aware that `url` built from github repo name and plugin name because of current repo naming for those plugins + +plugins=$(get_toolset_value '.docker.plugins[] .plugin') +for plugin in $plugins; do + version=$(get_toolset_value ".docker.plugins[] | select(.plugin == \"$plugin\") | .version") + filter=$(get_toolset_value ".docker.plugins[] | select(.plugin == \"$plugin\") | .asset_map[\"$ARCH\"]") + url=$(resolve_github_release_asset_url "docker/$plugin" "endswith(\"$filter\")" "$version") + binary_path=$(download_with_retry "$url" "/tmp/docker-$plugin") + mkdir -pv "/usr/libexec/docker/cli-plugins" + install "$binary_path" "/usr/libexec/docker/cli-plugins/docker-$plugin" +done + +# docker from official repo introduced different GID generation: https://github.com/actions/runner-images/issues/8157 +gid=$(cut -d ":" -f 3 /etc/group | grep "^1..$" | sort -n | tail -n 1 | awk '{ print $1+1 }') +groupmod -g "$gid" docker + +[ -f /usr/share/bash-completion/completions/docker ] && \ + cp -f /usr/share/bash-completion/completions/docker /etc/bash_completion.d/ + +[ ! -d /etc/docker ] && mkdir /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "data-root": "/var/lib/docker", + "log-driver": "local", + "log-opts": { + "max-size": "100m", + "max-file": "6", + "compress": "true", + "mode": "non-blocking", + "max-buffer-size": "4m" + }, + "default-ulimits": { + "nofile": { + "Name": "nofile", + "Hard": 65536, + "Soft": 65536 + }, + "nproc": { + "Name": "nproc", + "Hard": 65536, + "Soft": 65536 + } + }, + "live-restore": true, + "max-concurrent-downloads": 20, + "max-concurrent-uploads": 10, + "storage-driver": "overlay2", + "exec-opts": [ + "native.cgroupdriver=systemd" + ] +} +EOF + +# Create systemd-tmpfiles configuration for Docker +cat </dev/null 2>&1; then + if [[ "$(systemd-detect-virt)" == "lxc" ]]; then + ENV_TYPE="container" + fi +# Fallback check: Look at process 1 environment for container flag +elif grep -qa "container=lxc" /proc/1/environ; then + ENV_TYPE="container" +fi + +CONFIG_FILENAME="lxd_init_${ENV_TYPE}_${ARCH}.yml" +CONFIG_PATH="$INSTALLER_SCRIPT_FOLDER/$CONFIG_FILENAME" + +echo "----------------------------------------" +echo "LXD Initialization Setup" +echo "Detected Architecture : $ARCH" +echo "Detected Environment : $ENV_TYPE" +echo "Target Config File : $CONFIG_FILENAME" +echo "----------------------------------------" + +echo "Initializing LXD with preseed configuration..." + +if [[ -f "$CONFIG_PATH" ]]; then + # shellcheck disable=SC2002 + cat "$CONFIG_PATH" | /snap/bin/lxd init --preseed + + # Check if the command succeeded + # shellcheck disable=SC2181 + if [[ $? -eq 0 ]]; then + echo "Success: LXD initialized using $CONFIG_FILENAME" + else + echo "Error: LXD initialization failed." + exit 1 + fi +else + echo "Warning: $CONFIG_FILENAME not found at $INSTALLER_SCRIPT_FOLDER." + echo "Falling back to default auto initialization..." + /snap/bin/lxd init --auto +fi + +echo "LXD installation and initialization are complete!" \ No newline at end of file diff --git a/images/rhel/scripts/build/install-pipx-packages.sh b/images/rhel/scripts/build/install-pipx-packages.sh new file mode 100755 index 0000000..7c5674b --- /dev/null +++ b/images/rhel/scripts/build/install-pipx-packages.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e +################################################################################ +## File: install-pipx-packages.sh +## Desc: Install tools via pipx +################################################################################ + +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +export PATH="$PATH:/opt/pipx_bin" + +pipx_packages=$(get_toolset_value ".pipx[] .package") + +for package in $pipx_packages; do + echo "Install $package into default python" + pipx install "$package" + + # https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html + # Install ansible into an existing ansible-core Virtual Environment + if [[ $package == "ansible-core" ]]; then + pipx inject "$package" ansible + fi +done diff --git a/images/rhel/scripts/build/install-podman.sh b/images/rhel/scripts/build/install-podman.sh new file mode 100755 index 0000000..83a94ea --- /dev/null +++ b/images/rhel/scripts/build/install-podman.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e +################################################################################ +## File: install-snap.sh +## Desc: Install snapd +################################################################################ +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +dnf -y install podman \ No newline at end of file diff --git a/images/rhel/scripts/build/install-python.sh b/images/rhel/scripts/build/install-python.sh new file mode 100644 index 0000000..e2600af --- /dev/null +++ b/images/rhel/scripts/build/install-python.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e +################################################################################ +## File: install-python.sh +## Desc: Install Python 3 +################################################################################ + +set -e +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/etc-environment.sh +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/os.sh +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +# Install Python, Python 3, pip +install_dnfpkgs --setopt=install_weak_deps=False python3 python3-devel python3-pip + +# Install pipx +# Set pipx custom directory +export PIPX_BIN_DIR=/opt/pipx_bin +export PIPX_HOME=/opt/pipx + +python3 -m pip install pipx +python3 -m pipx ensurepath + +# Update /etc/environment +set_etc_environment_variable "PIPX_BIN_DIR" $PIPX_BIN_DIR +set_etc_environment_variable "PIPX_HOME" $PIPX_HOME +prepend_etc_environment_path $PIPX_BIN_DIR + +# Test pipx +if ! command -v pipx; then + echo "pipx was not installed or not found on PATH" + exit 1 +fi + +# Adding this dir to PATH will make installed pip commands are immediately available. +# shellcheck disable=SC2016 +prepend_etc_environment_path '$HOME/.local/bin' diff --git a/images/rhel/scripts/build/install-runner-package.sh b/images/rhel/scripts/build/install-runner-package.sh new file mode 100755 index 0000000..99eeedf --- /dev/null +++ b/images/rhel/scripts/build/install-runner-package.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e +################################################################################ +## File: install-runner-package.sh +## Desc: Download and Install runner package +################################################################################ + +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Set architecture-specific variables using a case statement for clarity +case "$ARCH" in + "ppc64le" | "s390x") + source "${DIR}/configure-runner.sh" + exit 0 + ;; + "x86_64") + package_arch="x64" + ;; + *) + package_arch="$ARCH" + ;; +esac + +download_url=$(resolve_github_release_asset_url "actions/runner" "test(\"actions-runner-linux-${package_arch}-[0-9]+\\\\.[0-9]{3}\\\\.[0-9]+\\\\.tar\\\\.gz$\")" "latest") +archive_name="${download_url##*/}" +archive_path=$(download_with_retry "$download_url") + +rm -rf /opt/runner-cache +mkdir -p /opt/runner-cache +mv "$archive_path" "/opt/runner-cache/$archive_name" +tar -xf /opt/runner-cache/*.tar.gz -C /opt/runner-cache diff --git a/images/rhel/scripts/build/install-snap.sh b/images/rhel/scripts/build/install-snap.sh new file mode 100755 index 0000000..8684414 --- /dev/null +++ b/images/rhel/scripts/build/install-snap.sh @@ -0,0 +1,47 @@ +#!/bin/bash -e +################################################################################ +## File: install-snap.sh +## Desc: Install snapd +################################################################################ +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +# Install snapd if not already installed +echo "Installing snapd..." +if ! rpm -q snapd &>/dev/null; then + update_dnfpkgs + install_dnfpkgs snapd +else + echo "snapd is already installed." +fi + +# Enable and start snapd.socket +echo "Enabling and starting snapd.socket..." +systemctl enable --now snapd.socket + +# Create symbolic link for snap directory if not already exists +if [ -L /snap ]; then + echo "Symbolic link for /snap already exists." +elif [ -e /snap ]; then + echo "/snap exists but is a directory/file, not a symbolic link." +else + echo "Creating symbolic link for /snap..." + ln -sf /var/lib/snapd/snap /snap +fi + +# Ensure /snap/bin is in the PATH +echo "Checking if /snap/bin is in the PATH..." +if [[ "$PATH" != *"/snap/bin"* ]]; then + echo "/snap/bin is not in the PATH. Adding it now..." + export PATH=/snap/bin:$PATH + echo "export PATH=/snap/bin:$PATH" >> ~/.bashrc # Persist for future sessions + echo "/snap/bin has been added to the PATH." +else + echo "/snap/bin is already in the PATH." +fi + +echo "Checking the status of snapd.seeded.service..." +ensure_service_is_active snapd.seeded.service +ensure_service_is_active snapd.service + +echo "Snapd setup and initialization completed successfully." \ No newline at end of file diff --git a/images/rhel/scripts/build/install-zstd.sh b/images/rhel/scripts/build/install-zstd.sh new file mode 100755 index 0000000..58764b8 --- /dev/null +++ b/images/rhel/scripts/build/install-zstd.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e +################################################################################ +## File: install-zstd.sh +## Desc: Install zstd +## Supply chain security: zstd - checksum validation +################################################################################ +# Source the helpers for use with the script +# shellcheck disable=SC1091 +source "$HELPER_SCRIPTS"/install.sh + +# Download zstd +release_tag=$(curl -fsSL ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} https://api.github.com/repos/facebook/zstd/releases/latest | jq -r '.tag_name') +release_name="zstd-${release_tag//v}" +download_url="https://github.com/facebook/zstd/releases/download/${release_tag}/${release_name}.tar.gz" +archive_path=$(download_with_retry "${download_url}") + +# Supply chain security - zstd +external_hash=$(get_checksum_from_url "${download_url}.sha256" "${release_name}.tar.gz" "SHA256") +use_checksum_comparison "$archive_path" "$external_hash" + +# Install dependencies +install_dnfpkgs lz4-devel gcc make + +# Extract and build zstd +tar xzf "$archive_path" -C /tmp + +make -C "/tmp/${release_name}/contrib/pzstd" all +make -C "/tmp/${release_name}" zstd-release + +# Copy binaries +for copyprocess in zstd zstdless zstdgrep; do + cp "/tmp/${release_name}/programs/${copyprocess}" /usr/local/bin/ +done + +cp "/tmp/${release_name}/contrib/pzstd/pzstd" /usr/local/bin/ + +# Create symlinks +for symlink in zstdcat zstdmt unzstd; do + ln -sf /usr/local/bin/zstd /usr/local/bin/${symlink} +done diff --git a/images/rhel/scripts/helpers/etc-environment.sh b/images/rhel/scripts/helpers/etc-environment.sh new file mode 100755 index 0000000..a6946bb --- /dev/null +++ b/images/rhel/scripts/helpers/etc-environment.sh @@ -0,0 +1,102 @@ +#!/bin/bash -e +################################################################################ +## File: etc-environment.sh +## Desc: Helper functions for source and modify /etc/environment +################################################################################ + +# Ensure standard directories are in PATH (sudo bash -c strips inherited PATH) +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}" + +# Ensure /etc/environment exists (RHEL doesn't always create it by default) +touch /etc/environment + +# NB: sed expression use '%' as a delimiter in order to simplify handling +# values containing slashes (i.e. directory path) +# The values containing '%' will break the functions + +get_etc_environment_variable() { + local variable_name=$1 + + # remove `variable_name=` and possible quotes from the line + grep "^${variable_name}=" /etc/environment | sed -E "s%^${variable_name}=\"?([^\"]+)\"?.*$%\1%" +} + +add_etc_environment_variable() { + local variable_name=$1 + local variable_value=$2 + + echo "${variable_name}=${variable_value}" | tee -a /etc/environment +} + +replace_etc_environment_variable() { + local variable_name=$1 + local variable_value=$2 + + # modify /etc/environemnt in place by replacing a string that begins with variable_name + sed -i -e "s%^${variable_name}=.*$%${variable_name}=${variable_value}%" /etc/environment +} + +set_etc_environment_variable() { + local variable_name=$1 + local variable_value=$2 + + if grep "^${variable_name}=" /etc/environment > /dev/null; then + replace_etc_environment_variable "$variable_name" "$variable_value" + else + add_etc_environment_variable "$variable_name" "$variable_value" + fi +} + +prepend_etc_environment_variable() { + local variable_name=$1 + local element=$2 + + if grep "^${variable_name}=" /etc/environment > /dev/null 2>&1; then + existing_value=$(get_etc_environment_variable "${variable_name}") + set_etc_environment_variable "${variable_name}" "${element}:${existing_value}" + else + add_etc_environment_variable "${variable_name}" "${element}" + fi +} + +append_etc_environment_variable() { + local variable_name=$1 + local element=$2 + + if grep "^${variable_name}=" /etc/environment > /dev/null 2>&1; then + existing_value=$(get_etc_environment_variable "${variable_name}") + set_etc_environment_variable "${variable_name}" "${existing_value}:${element}" + else + add_etc_environment_variable "${variable_name}" "${element}" + fi +} + +prepend_etc_environment_path() { + local element=$1 + + prepend_etc_environment_variable PATH "${element}" +} + +append_etc_environment_path() { + local element=$1 + + append_etc_environment_variable PATH "${element}" +} + +# Process /etc/environment as if it were shell script with `export VAR=...` expressions +# The PATH variable is handled specially in order to do not override the existing PATH +# variable. The value of PATH variable read from /etc/environment is added to the end +# of value of the exiting PATH variable exactly as it would happen with real PAM app read +# /etc/environment +# +# TODO: there might be the others variables to be processed in the same way as "PATH" variable +# ie MANPATH, INFOPATH, LD_*, etc. In the current implementation the values from /etc/evironments +# replace the values of the current environment +reload_etc_environment() { + # add `export ` to every variable of /etc/environemnt except PATH and eval the result shell script + # shellcheck disable=SC2046 + eval $(grep -v '^PATH=' /etc/environment | sed -e 's%^%export %') + # handle PATH specially + etc_path=$(get_etc_environment_variable PATH) + export PATH="$PATH:$etc_path" +} diff --git a/images/rhel/scripts/helpers/install.sh b/images/rhel/scripts/helpers/install.sh new file mode 100755 index 0000000..9078800 --- /dev/null +++ b/images/rhel/scripts/helpers/install.sh @@ -0,0 +1,287 @@ +#!/bin/bash -e +################################################################################ +## File: install.sh +## Desc: Helper functions for installing tools +################################################################################ + +download_with_retry() { + local url=$1 + local download_path=$2 + + if [ -z "$download_path" ]; then + download_path="/tmp/$(basename "$url")" + fi + + echo "Downloading package from $url to $download_path..." >&2 + + interval=30 + download_start_time=$(date +%s) + + for ((retries=20; retries>0; retries--)); do + attempt_start_time=$(date +%s) + if http_code=$(curl -4sSLo "$download_path" "$url" -w '%{http_code}'); then + attempt_seconds=$(($(date +%s) - attempt_start_time)) + if [ "$http_code" -eq 200 ]; then + echo "Package downloaded in $attempt_seconds seconds" >&2 + break + else + echo "Received HTTP status code $http_code after $attempt_seconds seconds" >&2 + fi + else + attempt_seconds=$(($(date +%s) - attempt_start_time)) + echo "Package download failed in $attempt_seconds seconds" >&2 + fi + + if [ "$retries" -le 1 ]; then + total_seconds=$(($(date +%s) - download_start_time)) + echo "Package download failed after $total_seconds seconds" >&2 + exit 1 + fi + + echo "Waiting $interval seconds before retrying (retries left: $retries)..." >&2 + sleep $interval + done + + echo "$download_path" +} + +get_toolset_value() { + local toolset_path="${INSTALLER_SCRIPT_FOLDER}/toolset.json" + local query=$1 + + # shellcheck disable=SC2005 + echo "$(jq -r "$query" "$toolset_path")" +} + +get_github_releases_by_version() { + local repo=$1 + local version=${2:-".+"} + local allow_pre_release=${3:-false} + local with_assets_only=${4:-false} + + page_size="100" + + json=$(curl -fsSL ${GITHUB_TOKEN:+-H "Authorization: Bearer ${GITHUB_TOKEN}"} "https://api.github.com/repos/${repo}/releases?per_page=${page_size}") + + if [[ -z "$json" ]]; then + echo "Failed to get releases" >&2 + exit 1 + fi + + if [[ $with_assets_only == "true" ]]; then + json=$(echo "$json" | jq -r '.[] | select(.assets | length > 0)') + else + json=$(echo "$json" | jq -r '.[]') + fi + + if [[ $allow_pre_release == "true" ]]; then + json=$(echo "$json" | jq -r '.') + else + json=$(echo "$json" | jq -r '. | select(.prerelease==false)') + fi + + # Filter out rc/beta/etc releases, convert to numeric version and sort + json=$(echo "$json" | jq '. | select(.tag_name | test(".*-[a-z]|beta") | not)' | jq '.tag_name |= gsub("[^\\d.]"; "")' | jq -s 'sort_by(.tag_name | split(".") | map(tonumber))') + + # Select releases matching version + if [[ $version == "latest" ]]; then + json_filtered=$(echo "$json" | jq .[-1]) + elif [[ $version == *"+"* ]] || [[ $version == *"*"* ]]; then + json_filtered=$(echo "$json" | jq --arg version "$version" '.[] | select(.tag_name | test($version))') + else + json_filtered=$(echo "$json" | jq --arg version "$version" '.[] | select(.tag_name | contains($version))') + fi + + if [[ -z "$json_filtered" ]]; then + echo "Failed to get releases from ${repo} matching version ${version}" >&2 + echo "Available versions: $(echo "$json" | jq -r '.tag_name')" >&2 + exit 1 + fi + + echo "$json_filtered" +} + +resolve_github_release_asset_url() { + local repo=$1 + local url_filter=$2 + local version=${3:-".+"} + local allow_pre_release=${4:-false} + local allow_multiple_matches=${5:-false} + + matching_releases=$(get_github_releases_by_version "${repo}" "${version}" "${allow_pre_release}" "true") + matched_url=$(echo "$matching_releases" | jq -r ".assets[].browser_download_url | select(${url_filter})") + + if [[ -z "$matched_url" ]]; then + echo "Found no download urls matching pattern: ${url_filter}" >&2 + echo "Available download urls: $(echo "$matching_releases" | jq -r '.assets[].browser_download_url')" >&2 + exit 1 + fi + + if [[ "$(echo "$matched_url" | wc -l)" -gt 1 ]]; then + if [[ $allow_multiple_matches == "true" ]]; then + matched_url=$(echo "$matched_url" | tail -n 1) + else + echo "Multiple matches found for ${version} version and ${url_filter} URL filter. Please make filters more specific" >&2 + exit 1 + fi + fi + + echo "$matched_url" +} + +get_checksum_from_github_release() { + local repo=$1 + local file_name=$2 + local version=${3:-".+"} + local hash_type=$4 + local allow_pre_release=${5:-false} + + if [[ -z "$file_name" ]]; then + echo "File name is not specified." >&2 + exit 1 + fi + + if [[ "$hash_type" == "SHA256" ]]; then + hash_pattern="[A-Fa-f0-9]{64}" + elif [[ "$hash_type" == "SHA512" ]]; then + hash_pattern="[A-Fa-f0-9]{128}" + else + echo "Unknown hash type: ${hash_type}" >&2 + exit 1 + fi + + matching_releases=$(get_github_releases_by_version "${repo}" "${version}" "${allow_pre_release}" "true") + # shellcheck disable=SC2059 + matched_line=$(printf "$(echo "$matching_releases" | jq '.body')\n" | grep "$file_name") + + if [[ -z "$matched_line" ]]; then + echo "File name ${file_name} not found in release body" >&2 + exit 1 + fi + + if [[ "$(echo "$matched_line" | wc -l)" -gt 1 ]]; then + echo "Multiple matches found for ${file_name} in release body: ${matched_line}" >&2 + exit 1 + fi + + hash=$(echo "$matched_line" | grep -oE "$hash_pattern") + + if [[ -z "$hash" ]]; then + echo "Found ${file_name} in body of release, but failed to get hash from it: ${matched_line}" >&2 + exit 1 + fi + + echo "$hash" +} + +get_checksum_from_url() { + local url=$1 + local file_name=$2 + local hash_type=$3 + local use_custom_search_pattern=${4:-false} + local delimiter=${5:-' '} + local word_number=${6:-1} + + if [[ "$hash_type" == "SHA256" ]]; then + hash_pattern="[A-Fa-f0-9]{64}" + elif [[ "$hash_type" == "SHA512" ]]; then + hash_pattern="[A-Fa-f0-9]{128}" + else + echo "Unknown hash type: ${hash_type}" >&2 + exit 1 + fi + + checksums_file_path=$(download_with_retry "$url") + checksums=$(cat "$checksums_file_path") + rm "$checksums_file_path" + + # shellcheck disable=SC2059 + matched_line=$(printf "$checksums\n" | grep "$file_name") + + if [[ "$(echo "$matched_line" | wc -l)" -gt 1 ]]; then + echo "Found multiple lines matching file name ${file_name} in checksum file." >&2 + exit 1 + fi + + if [[ -z "$matched_line" ]]; then + echo "File name ${file_name} not found in checksum file." >&2 + exit 1 + fi + + if [[ $use_custom_search_pattern == "true" ]]; then + hash=$(echo "$matched_line" | sed 's/ */ /g' | cut -d "$delimiter" -f "$word_number" | tr -d -c '[:alnum:]') + else + hash=$(echo "$matched_line" | grep -oE "$hash_pattern") + fi + + if [[ -z "$hash" ]]; then + echo "Found ${file_name} in checksum file, but failed to get hash from it: ${matched_line}" >&2 + exit 1 + fi + + echo "$hash" +} + +use_checksum_comparison() { + local file_path=$1 + local checksum=$2 + local sha_type=${3:-"256"} + + echo "Performing checksum verification" + + if [[ ! -f "$file_path" ]]; then + echo "File not found: $file_path" + exit 1 + fi + + if command -v shasum &>/dev/null; then + local_file_hash=$(shasum --algorithm "$sha_type" "$file_path" | awk '{print $1}') + else + local_file_hash=$(sha256sum "$file_path" | awk '{print $1}') + fi + + if [[ "$local_file_hash" != "$checksum" ]]; then + echo "Checksum verification failed. Expected hash: $checksum; Actual hash: $local_file_hash." + exit 1 + else + echo "Checksum verification passed" + fi +} + +# Ensures a systemd service is active, starting it if necessary. +ensure_service_is_active() { + local service="$1" + # 'systemctl is-active' is quiet and returns 0 if active. + if ! systemctl is-active --quiet "$service"; then + echo "Service '$service' is not running. Attempting to start it..." + systemctl restart "$service" + fi +} + +install_dnfpkgs() +{ + FLAGS="" + PKGS="" + # shellcheck disable=SC2048 + for pkg in $* + do + case ${pkg} in + -*) + FLAGS="${FLAGS} ${pkg}" + ;; + *) + PKGS="${PKGS} ${pkg}" + ;; + esac + done + for pkg in ${PKGS} + do + # shellcheck disable=SC2086 + dnf install -y -qq ${FLAGS} ${pkg} 2>&1 >/dev/null | tee -a /tmp/install.errors + done +} + +update_dnfpkgs() +{ + dnf -qq update -y 2>&1 >/dev/null | tee -a /tmp/install.errors +} diff --git a/images/rhel/scripts/helpers/os.sh b/images/rhel/scripts/helpers/os.sh new file mode 100755 index 0000000..97fa95e --- /dev/null +++ b/images/rhel/scripts/helpers/os.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e +################################################################################ +## File: os.sh +## Desc: Helper functions for OS releases +################################################################################ +is_rhel9() { + source /etc/os-release && [[ "$VERSION_ID" == 9* ]] +} + +is_rhel10() { + source /etc/os-release && [[ "$VERSION_ID" == 10* ]] +} diff --git a/images/rhel/toolsets/toolset-10.json b/images/rhel/toolsets/toolset-10.json new file mode 100644 index 0000000..5b4688a --- /dev/null +++ b/images/rhel/toolsets/toolset-10.json @@ -0,0 +1,304 @@ +{ + "toolcache": [ + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json", + "platform" : "linux", + "platform_version": "10", + "arch": "x64", + "versions": [ + "3.10.*", + "3.11.*", + "3.12.*", + "3.13.*", + "3.14.*" + ] + }, + { + "name": "PyPy", + "arch": "x64", + "platform" : "linux", + "versions": [ + "3.9", + "3.10", + "3.11" + ] + }, + { + "name": "node", + "url" : "https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json", + "platform" : "linux", + "arch": "x64", + "versions": [ + "20.*", + "22.*", + "24.*" + ] + }, + { + "name": "go", + "url" : "https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json", + "arch": "x64", + "platform" : "linux", + "versions": [ + "1.22.*", + "1.23.*", + "1.24.*", + "1.25.*" + ], + "default": "1.24.*" + }, + { + "name": "Ruby", + "platform_version": "10", + "arch": "x64", + "versions": [ + "3.2.*", + "3.3.*", + "3.4.*", + "4.0.*" + ] + }, + { + "name": "CodeQL", + "platform" : "linux", + "arch": "x64", + "versions": [ + "*" + ] + } + ], + "java": { + "default": "17", + "versions": [ "11", "17", "21", "25"], + "maven": "3.9.15" + }, + "android": { + "cmdline-tools": "commandlinetools-linux-11076708_latest.zip", + "platform_min_version": "34", + "build_tools_min_version": "34.0.0", + "extra_list": [ + "android;m2repository", + "google;m2repository", + "google;google_play_services" + ], + "addon_list": [ + ], + "additional_tools": [ + "cmake;3.31.5", + "cmake;4.1.2" + ], + "ndk": { + "default": "27", + "versions": [ + "27", "28", "29" + ] + } + }, + "powershellModules": [ + {"name": "Microsoft.Graph"}, + {"name": "Pester"}, + {"name": "PSScriptAnalyzer"} + ], + "azureModules": [ + { + "name": "az", + "versions": [ + "14.6.0" + ] + } + ], + "dnf": { + "vital_packages": [ + "bzip2", + "curl", + "gcc-c++", + "gcc", + "gzip", + "make", + "jq", + "tar", + "unzip", + "wget" + ], + "common_packages": [ + "autoconf", + "automake", + "dbus", + "bind-utils", + "gnupg2", + "libtool", + "pkg-config", + "python-unversioned-command", + "rpm", + "tk", + "tree", + "tzdata" + ], + "cmd_packages": [ + "acl", + "binutils", + "bison", + "brotli", + "coreutils", + "file", + "findutils", + "flex", + "ftp", + "lz4", + "m4", + "nmap-ncat", + "net-tools", + "pigz", + "rsync", + "sqlite", + "sshpass", + "sudo", + "telnet", + "time", + "zip" + ] + }, + "brew": [ + ], + "docker": { + "components": [ + { + "package": "containerd.io", + "version": "1.7.27-1" + }, + { + "package": "docker-ce-cli", + "version": "28.0.4" + }, + { + "package": "docker-ce", + "version": "28.0.4" + } + ], + "plugins": [ + { + "plugin": "buildx", + "version": "latest", + "asset_map": { + "x86_64": "linux-amd64", + "ppc64le": "linux-ppc64le", + "s390x": "linux-s390x" + } + }, + { + "plugin": "compose", + "version": "2.38.2", + "asset_map": { + "x86_64": "linux-x86_64", + "ppc64le": "linux-ppc64le", + "s390x": "linux-s390x" + } + } + ] + }, + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + }, + { + "package": "ansible-core", + "cmd": "ansible" + } + ], + "dotnet": { + "versions": [ + "8.0", + "9.0", + "10.0" + ], + "tools": [ + { "name": "nbgv", "test": "nbgv --version", "getversion" : "nbgv --version" } + ] + }, + "clang": { + "versions": [ + "16", + "17", + "18" + ], + "default_version": "18" + }, + "gcc": { + "versions": [ + "g++-12", + "g++-13", + "g++-14" + ] + }, + "gfortran": { + "versions": [ + "gfortran-12", + "gfortran-13", + "gfortran-14" + ] + }, + "php": { + "versions": [ + "8.3" + ] + }, + "rubygems": [ + {"name": "fastlane"} + ], + "selenium": { + "version": "4" + }, + "node": { + "default": "20" + }, + "node_modules": [ + { + "name": "grunt", + "command": "grunt" + }, + { + "name": "gulp", + "command": "gulp" + }, + { + "name": "n", + "command": "n" + }, + { + "name": "parcel", + "command": "parcel" + }, + { + "name": "typescript", + "command": "tsc" + }, + { + "name": "newman", + "command": "newman" + }, + { + "name": "webpack", + "command": "webpack" + }, + { + "name": "webpack-cli", + "command": "webpack-cli" + }, + { + "name": "lerna", + "command": "lerna" + }, + { + "name": "yarn", + "command": "yarn" + } + ], + "postgresql": { + "version": "16" + }, + "pwsh": { + "version": "7.4" + } +} \ No newline at end of file diff --git a/images/rhel/toolsets/toolset-9.json b/images/rhel/toolsets/toolset-9.json new file mode 100755 index 0000000..ddf7d14 --- /dev/null +++ b/images/rhel/toolsets/toolset-9.json @@ -0,0 +1,304 @@ +{ + "toolcache": [ + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json", + "platform" : "linux", + "platform_version": "9", + "arch": "x64", + "versions": [ + "3.10.*", + "3.11.*", + "3.12.*", + "3.13.*", + "3.14.*" + ] + }, + { + "name": "PyPy", + "arch": "x64", + "platform" : "linux", + "versions": [ + "3.9", + "3.10", + "3.11" + ] + }, + { + "name": "node", + "url" : "https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json", + "platform" : "linux", + "arch": "x64", + "versions": [ + "20.*", + "22.*", + "24.*" + ] + }, + { + "name": "go", + "url" : "https://raw.githubusercontent.com/actions/go-versions/main/versions-manifest.json", + "arch": "x64", + "platform" : "linux", + "versions": [ + "1.22.*", + "1.23.*", + "1.24.*", + "1.25.*" + ], + "default": "1.24.*" + }, + { + "name": "Ruby", + "platform_version": "9", + "arch": "x64", + "versions": [ + "3.2.*", + "3.3.*", + "3.4.*", + "4.0.*" + ] + }, + { + "name": "CodeQL", + "platform" : "linux", + "arch": "x64", + "versions": [ + "*" + ] + } + ], + "java": { + "default": "17", + "versions": [ "11", "17", "21", "25"], + "maven": "3.9.15" + }, + "android": { + "cmdline-tools": "commandlinetools-linux-11076708_latest.zip", + "platform_min_version": "34", + "build_tools_min_version": "34.0.0", + "extra_list": [ + "android;m2repository", + "google;m2repository", + "google;google_play_services" + ], + "addon_list": [ + ], + "additional_tools": [ + "cmake;3.31.5", + "cmake;4.1.2" + ], + "ndk": { + "default": "27", + "versions": [ + "27", "28", "29" + ] + } + }, + "powershellModules": [ + {"name": "Microsoft.Graph"}, + {"name": "Pester"}, + {"name": "PSScriptAnalyzer"} + ], + "azureModules": [ + { + "name": "az", + "versions": [ + "14.6.0" + ] + } + ], + "dnf": { + "vital_packages": [ + "bzip2", + "curl", + "gcc-c++", + "gcc", + "gzip", + "make", + "jq", + "tar", + "unzip", + "wget" + ], + "common_packages": [ + "autoconf", + "automake", + "dbus", + "bind-utils", + "gnupg2", + "libtool", + "pkg-config", + "python-unversioned-command", + "rpm", + "tk", + "tree", + "tzdata" + ], + "cmd_packages": [ + "acl", + "binutils", + "bison", + "brotli", + "coreutils", + "file", + "findutils", + "flex", + "ftp", + "lz4", + "m4", + "nmap-ncat", + "net-tools", + "pigz", + "rsync", + "sqlite", + "sshpass", + "sudo", + "telnet", + "time", + "zip" + ] + }, + "brew": [ + ], + "docker": { + "components": [ + { + "package": "containerd.io", + "version": "1.7.27-1" + }, + { + "package": "docker-ce-cli", + "version": "28.0.4" + }, + { + "package": "docker-ce", + "version": "28.0.4" + } + ], + "plugins": [ + { + "plugin": "buildx", + "version": "latest", + "asset_map": { + "x86_64": "linux-amd64", + "ppc64le": "linux-ppc64le", + "s390x": "linux-s390x" + } + }, + { + "plugin": "compose", + "version": "2.38.2", + "asset_map": { + "x86_64": "linux-x86_64", + "ppc64le": "linux-ppc64le", + "s390x": "linux-s390x" + } + } + ] + }, + "pipx": [ + { + "package": "yamllint", + "cmd": "yamllint" + }, + { + "package": "ansible-core", + "cmd": "ansible" + } + ], + "dotnet": { + "versions": [ + "8.0", + "9.0", + "10.0" + ], + "tools": [ + { "name": "nbgv", "test": "nbgv --version", "getversion" : "nbgv --version" } + ] + }, + "clang": { + "versions": [ + "16", + "17", + "18" + ], + "default_version": "18" + }, + "gcc": { + "versions": [ + "g++-12", + "g++-13", + "g++-14" + ] + }, + "gfortran": { + "versions": [ + "gfortran-12", + "gfortran-13", + "gfortran-14" + ] + }, + "php": { + "versions": [ + "8.3" + ] + }, + "rubygems": [ + {"name": "fastlane"} + ], + "selenium": { + "version": "4" + }, + "node": { + "default": "20" + }, + "node_modules": [ + { + "name": "grunt", + "command": "grunt" + }, + { + "name": "gulp", + "command": "gulp" + }, + { + "name": "n", + "command": "n" + }, + { + "name": "parcel", + "command": "parcel" + }, + { + "name": "typescript", + "command": "tsc" + }, + { + "name": "newman", + "command": "newman" + }, + { + "name": "webpack", + "command": "webpack" + }, + { + "name": "webpack-cli", + "command": "webpack-cli" + }, + { + "name": "lerna", + "command": "lerna" + }, + { + "name": "yarn", + "command": "yarn" + } + ], + "postgresql": { + "version": "16" + }, + "pwsh": { + "version": "7.4" + } +} \ No newline at end of file diff --git a/run.sh b/run.sh index 4e7f0e5..ebb9397 100755 --- a/run.sh +++ b/run.sh @@ -95,7 +95,7 @@ get_os_details() { local os_options=() case "$env" in lxd_container|lxd_vm) os_options=("Ubuntu" "Back");; - *) os_options=("Ubuntu" "CentOS/AlmaLinux" "Back");; + *) os_options=("Ubuntu" "CentOS/AlmaLinux" "RHEL" "Back");; esac local os_choice @@ -114,6 +114,12 @@ get_os_details() { [[ "$version" == "Back" ]] && return 1 echo "centos $version" ;; + "RHEL") + local version + version=$(select_menu "Select RHEL version: " "9" "10" "Back") + [[ "$version" == "Back" ]] && return 1 + echo "rhel $version" + ;; "Back") return 1 ;; @@ -124,7 +130,7 @@ get_os_details() { get_setup_type() { local env="$1" os="$2" - if [[ "$os" == "centos" || "$env" == "docker" || "$env" == "podman" ]]; then + if [[ "$os" == "centos" || "$os" == "rhel" || "$env" == "docker" || "$env" == "podman" ]]; then echo "minimal" # These combinations only support minimal setup return fi diff --git a/scripts/helpers/run_script.sh b/scripts/helpers/run_script.sh index c6505c4..55c6f6b 100755 --- a/scripts/helpers/run_script.sh +++ b/scripts/helpers/run_script.sh @@ -10,6 +10,10 @@ run_script() { local env_vars=() local env_vars_display=() + # Always pass through PATH so child scripts can find basic commands + env_vars+=("PATH=${PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}") + env_vars_display+=("PATH=...") + # Always include GITHUB_TOKEN if it's set in the environment if [[ -n "${GITHUB_TOKEN}" ]]; then env_vars+=("GITHUB_TOKEN=${GITHUB_TOKEN}") diff --git a/scripts/helpers/setup_install.sh b/scripts/helpers/setup_install.sh index 1a1f217..8d17cea 100755 --- a/scripts/helpers/setup_install.sh +++ b/scripts/helpers/setup_install.sh @@ -2,6 +2,8 @@ set -e # Exit on any error set -o pipefail # Fail if any command in a pipeline fails +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}" + CURRENT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" # shellcheck disable=SC1091 @@ -27,19 +29,19 @@ if [[ "$IMAGE_OS" == *"ubuntu"* ]]; then run_script "${INSTALLER_SCRIPT_FOLDER}/install-apt-common.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" "INSTALLER_SCRIPT_FOLDER" "ARCH" run_script "${INSTALLER_SCRIPT_FOLDER}/configure-dpkg.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" "INSTALLER_SCRIPT_FOLDER" "ARCH" -elif [[ "$IMAGE_OS" == *"centos"* ]]; then - # Add apt wrapper to implement retries +elif [[ "$IMAGE_OS" == *"centos"* || "$IMAGE_OS" == *"rhel"* ]]; then + # Add yum/dnf wrapper to implement retries run_script "${INSTALLER_SCRIPT_FOLDER}/configure-yum-mock.sh" - echo "Setting user ubuntu with sudo privileges" + echo "Setting user with sudo privileges" - # Install Configure apt + # Configure dnf run_script "${INSTALLER_SCRIPT_FOLDER}/configure-dnf.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" run_script "${INSTALLER_SCRIPT_FOLDER}/install-dnf-vital.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" "INSTALLER_SCRIPT_FOLDER" "ARCH" run_script "${INSTALLER_SCRIPT_FOLDER}/install-dnf-common.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" "INSTALLER_SCRIPT_FOLDER" "ARCH" - run_script "${INSTALLER_SCRIPT_FOLDER}/configure-dnfpkg.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" "INSTALLER_SCRIPT_FOLDER" "ARCH" + run_script "${INSTALLER_SCRIPT_FOLDER}/configure-dnfpkg.sh" "DEBIAN_FRONTEND" "HELPER_SCRIPTS" "INSTALLER_SCRIPT_FOLDER" "ARCH" fi diff --git a/scripts/helpers/setup_vars.sh b/scripts/helpers/setup_vars.sh index 576d4b2..5a4498c 100755 --- a/scripts/helpers/setup_vars.sh +++ b/scripts/helpers/setup_vars.sh @@ -124,7 +124,11 @@ fi # Define Dependent Variables --- # shellcheck disable=SC2001 # shellcheck disable=SC2034 -toolset_file_name="toolset-$(echo "$IMAGE_VERSION" | sed 's/\.//g').json" +if [[ "$IMAGE_OS" == "rhel" || "$IMAGE_OS" == "centos" ]]; then + toolset_file_name="toolset-${IMAGE_VERSION%%.*}.json" +else + toolset_file_name="toolset-$(echo "$IMAGE_VERSION" | sed 's/\.//g').json" +fi image_folder="/var/tmp/imagegeneration-${IMAGE_OS}-${IMAGE_VERSION}" helper_script_folder="${image_folder}/helpers" installer_script_folder="${image_folder}/installers" diff --git a/scripts/vm.sh b/scripts/vm.sh index 09838d2..5a84bce 100755 --- a/scripts/vm.sh +++ b/scripts/vm.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e # Exit on any error +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}" + HELPERS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/helpers" # shellcheck disable=SC1091 @@ -45,8 +47,23 @@ chmod -R 0755 /etc/systemd/system/gha-runner.service # shellcheck disable=SC2154 sudo bash -c 'exec "$@"' _ "${HELPER_SCRIPTS}/setup_install.sh" "${clean_args[@]}" "${forward_args[@]}" -sudo bash -c 'id -u runner >/dev/null 2>&1 || (useradd -c "Action Runner" -m -s /bin/bash runner && usermod -L runner && echo "runner ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/runner && chmod 440 /etc/sudoers.d/runner)' +if ! id -u runner >/dev/null 2>&1; then + sudo useradd -c "Action Runner" -m -s /bin/bash runner + sudo usermod -L runner + echo 'runner ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/runner > /dev/null + sudo chmod 440 /etc/sudoers.d/runner +fi -sudo bash -c "usermod -aG adm,users,systemd-journal,docker,lxd runner" +groups_to_add="" +for g in adm users systemd-journal docker lxd; do + getent group "$g" >/dev/null 2>&1 && groups_to_add="${groups_to_add:+$groups_to_add,}$g" +done +if [ -n "$groups_to_add" ]; then + sudo usermod -aG "$groups_to_add" runner +fi + +if [[ "$IMAGE_OS" == "rhel" ]]; then + sudo sed -i "s/^User=ubuntu/User=runner/" /etc/systemd/system/gha-runner.service +fi sudo su -c "find /opt/post-generation -mindepth 1 -maxdepth 1 -type f -name '*.sh' -exec bash {} \;" \ No newline at end of file