Revert^2 "Add Werk #19573: FIX Fix system executables failing due to Checkmk's OpenSSL being loaded via LD_LIBRARY_PATH"

nigelgbanks · nigelgbanks · commit 51613204e1ee · 2026-04-22T19:54:32.000+02:00
This reverts commit c08a432. Reason for revert: Testing now the cache issues should be resolved. CMK-33201 Change-Id: I569362bac88fab98dc03b2497186c2a7ccd8c3fa
diff --git a/.werks/19573.md b/.werks/19573.md
@@ -0,0 +1,36 @@
+[//]: # (werk v3)
+# Fix system executables failing due to Checkmk's OpenSSL being loaded via LD_LIBRARY_PATH
+
+key        | value
+---------- | ---
+date       | 2026-04-10T17:16:28.506025+00:00
+version    | 2.6.0b1
+class      | fix
+edition    | community
+component  | omd
+level      | 2
+compatible | yes
+
+Previously, Checkmk's site environment included `$OMD_ROOT/lib` in `LD_LIBRARY_PATH`
+so that Checkmk's own processes could find the bundled shared libraries. The unintended
+side effect: system executables invoked from within the site environment (e.g. `ssh`,
+`curl`, `openssl`, `zypper`, `systemctl`, `php`) would also pick up Checkmk's bundled
+OpenSSL instead of the system's version. On distributions where the two versions were
+incompatible, those commands would fail with errors such as:
+
+```
+/usr/bin/systemctl: /omd/sites/mysite/lib/libcrypto.so.3: version OPENSSL_3.4.0 not found
+```
+
+As a workaround, Checkmk shipped wrapper scripts that unset `LD_LIBRARY_PATH` before
+invoking the real system executable for a list of commonly affected commands.
+
+We have now fixed the root cause: all Checkmk-shipped ELF binaries have their RUNPATH
+correctly set so they locate their bundled libraries directly, without relying on
+`LD_LIBRARY_PATH`. As a result, `$OMD_ROOT/lib` is no longer added to `LD_LIBRARY_PATH`,
+and the workaround wrapper scripts have been removed.
+
+`LD_LIBRARY_PATH` still includes `$OMD_ROOT/local/lib` to support native libraries
+installed by site users (e.g. via MKPs). If you have placed a native shared library in
+`local/lib` that is incompatible with a system executable, that executable may still fail
+— but this is a rare edge case and can be resolved by removing the conflicting library.
diff --git a/agents/plugins/mk_site_object_counts b/agents/plugins/mk_site_object_counts
@@ -25,6 +25,10 @@ _query_site() {
     site_lib() {
         printf "/omd/versions/%s/lib" "$(realpath "/omd/sites/${1}/version" | sed 's|.*/||')"
     }
+    # LD_LIBRARY_PATH is set inline (not exported) as a per-subprocess override pointing to the
+    # queried site's own lib directory. This is required for backward compatibility with older
+    # installed site versions whose unixcat binary may lack RPATH and cannot locate its shared
+    # libraries otherwise. Do not remove until all supported older versions have RPATH set.
     echo -e "${2}" | LD_LIBRARY_PATH="$(site_lib "${1}"):${LD_LIBRARY_PATH}" waitmax 3 "$(site_cmd "${1}" unixcat)" "/omd/sites/${1}/tmp/run/live"
 }
 
diff --git a/bazel/cmk/distro/BUILD.bazel b/bazel/cmk/distro/BUILD.bazel
@@ -110,18 +110,3 @@ selects.config_setting_group(
         ":ubuntu-26.04",
     ],
 )
-
-# These distros either ship too old or too new of a OpenSSL than us.
-#
-# See omd/packages/omd/BUILD
-# See MODULE.bazel for the current version.
-selects.config_setting_group(
-    name = "incompatible-openssl",
-    match_any = [
-        ":almalinux-8",
-        ":almalinux-9",
-        ":sles-15sp6",
-        ":sles-15sp7",
-        ":ubuntu-22.04",
-    ],
-)
diff --git a/bin/cmk-monitor-core b/bin/cmk-monitor-core
@@ -5,8 +5,6 @@
 
 OMD_SITE=$(whoami)
 SITE_VERSION="$(realpath ~/version | sed 's|.*/||')"
-LD_LIBRARY_PATH="/omd/versions/${SITE_VERSION}/lib:${LD_LIBRARY_PATH}"
-export LD_LIBRARY_PATH
 
 which timeout >/dev/null 2>&1 && waitmax() { timeout "$@"; }
 
diff --git a/omd/packages/omd/BUILD b/omd/packages/omd/BUILD
@@ -7,7 +7,6 @@ load("//bazel/rules:file_from_flag.bzl", "file_from_flag")
 load("//bazel/rules:replace_string_in_file.bzl", "replace_string_in_file")
 
 exports_files([
-    "use_system_openssl",
     "skel.permissions",
     "skel/.profile",
     "skel/.bashrc",
@@ -41,38 +40,6 @@ pkg_files(
     prefix = "bin",
 )
 
-# SUP-10161: our openssl is incompatible with some system executables on various sles15sp*
-# SUP-26565: Getting an exception error when baking agent almalinux-9+
-# SUP-26879: Wrap other tools commonly used by users that require linking to the system openssl
-# etc.
-WRAPPED_EXECUTABLES = [
-    "curl",
-    "dig",
-    "host",
-    "nc",
-    "nmap",
-    "nslookup",
-    "pdftoppm",
-    "php",
-    "php-cgi",
-    "php5",
-    "php5-cgi",
-    "resolvectl",
-    "rpmbuild",
-    "scp",
-    "ssh",
-    "systemctl",
-    "zypper",
-]
-
-[pkg_files(
-    name = "wrapper_" + executable,
-    srcs = ["use_system_openssl"],
-    attributes = pkg_attributes(mode = "0755"),
-    prefix = "bin",
-    renames = {"use_system_openssl": executable},
-) for executable in WRAPPED_EXECUTABLES]
-
 genrule(
     name = "omd_man_gz",
     srcs = ["omd.8"],
@@ -370,10 +337,7 @@ pkg_tar(
         ":omdlib_pkg",
         ":skel",
         ":xinetd_link",
-    ] + select({
-        "@cmk//distro:incompatible-openssl": ["wrapper_" + executable for executable in WRAPPED_EXECUTABLES],
-        "//conditions:default": [],
-    }),
+    ],
     stamp = 1,
     visibility = ["//omd:__pkg__"],
 )
diff --git a/omd/packages/omd/omdlib/site_user.py b/omd/packages/omd/omdlib/site_user.py
@@ -29,7 +29,7 @@ def set_environment(site_name: str, config: Config) -> None:
     os.environ["USER"] = site_name
     os.environ["HOME"] = site_home
 
-    os.environ["LD_LIBRARY_PATH"] = f"{site_home}/local/lib:{site_home}/lib"
+    os.environ["LD_LIBRARY_PATH"] = f"{site_home}/local/lib"
 
     # Special agents / active checks environment
     os.environ["PASSWORD_STORE_SECRET_FILE"] = f"{site_home}/etc/password_store.secret"
diff --git a/omd/packages/omd/skel/.profile b/omd/packages/omd/skel/.profile
@@ -4,7 +4,7 @@ export OMD_SITE=###SITE###
 export OMD_ROOT=###ROOT###
 
 PATH=$OMD_ROOT/local/bin:$OMD_ROOT/bin:$OMD_ROOT/local/lib/perl5/bin:$PATH
-export LD_LIBRARY_PATH=$OMD_ROOT/local/lib:$OMD_ROOT/lib
+export LD_LIBRARY_PATH=$OMD_ROOT/local/lib
 
 # export special agent / active check environment
 export PASSWORD_STORE_SECRET_FILE="${OMD_ROOT}/etc/password_store.secret"
diff --git a/omd/packages/omd/use_system_openssl b/omd/packages/omd/use_system_openssl
diff --git a/omd/packages/python3-modules/BUILD b/omd/packages/python3-modules/BUILD
@@ -21,8 +21,8 @@ config_setting(
     name = module,
     srcs = [
         "%s_requirements.txt" % module,
+        ":git_wrapper",
         "@freetds",
-        "@omd_packages//omd/packages/omd:use_system_openssl",
         "@openssl",
         "@python",
     ],
@@ -41,8 +41,8 @@ config_setting(
 build_python_module(
     name = "netapp-ontap",
     srcs = [
+        ":git_wrapper",
         "@freetds",
-        "@omd_packages//omd/packages/omd:use_system_openssl",
         "@openssl",
         "@python",
     ],
diff --git a/omd/packages/python3-modules/build-python3-modules.bzl b/omd/packages/python3-modules/build-python3-modules.bzl
@@ -79,9 +79,10 @@ build_cmd = """
     # Python binary supplied by bazel build process
     export PYTHON_EXECUTABLE=$$PWD/$$EXT_DEPS_PATH/{python_dir}/python/bin/python3
 
-    # Workaround for git execution issue
+    # Workaround for git execution issue: pip may call git for VCS deps, but LD_LIBRARY_PATH
+    # is set to our OpenSSL which conflicts with system git. The wrapper unsets it.
     mkdir -p $$TMPDIR/workdir/$$MODULE_NAME
-    install -m 755 "$(execpath @omd_packages//omd/packages/omd:use_system_openssl)" "$$TMPDIR/workdir/$$MODULE_NAME/git"
+    install -m 755 "$(execpath :git_wrapper)" "$$TMPDIR/workdir/$$MODULE_NAME/git"
     export PATH="$$TMPDIR/workdir/$$MODULE_NAME:$$PATH"
 
     # Build directory
diff --git a/omd/packages/python3-modules/git_wrapper b/omd/packages/python3-modules/git_wrapper
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Wrapper for git that unsets LD_LIBRARY_PATH before execution.
+# During Python module builds, LD_LIBRARY_PATH is set to point at our bundled
+# OpenSSL and Python libraries. This conflicts with system git, which is
+# dynamically linked against the system OpenSSL. This wrapper unsets
+# LD_LIBRARY_PATH so that git uses the system libraries as intended.
+
+_self=$(readlink -f "$0")
+_name=$(basename "${_self}")
+for _exec in $(type -ap "${_name}" 2>/dev/null); do
+    if [ "${_exec}" != "${_self}" ]; then
+        unset LD_LIBRARY_PATH
+        exec "${_exec}" "$@"
+    fi
+done
+
+echo "No ${_name} executable found other than wrapper" >&2
+exit 1
diff --git a/tests/integration/omd/test_openssl_using_cmds.py b/tests/integration/omd/test_openssl_using_cmds.py
@@ -30,12 +30,12 @@ def test_command(site: Site, cmd: str) -> None:
     """
     Ensures that commands using OpenSSL, such as ssh and curl, are working.
 
-    When executing ssh as a site user, usually, the system ssh is used. This binary is usually
-    dynamically linked against OpenSSL. We ship OpenSSL with OMD and set LD_LIBRARY_PATH to within
-    the site environment. Hence, when calling ssh as a site user, the OMD OpenSSL is used. There is
-    no garantuee that this version is compatible with the ssh compilation available on the system
-    (which is intended to compatible with the system OpenSSL). See also SUP-10161 and
-    omd/packages/omd/ssh_system_openssl.
+    In the past we set LD_LIBRARY_PATH to use the installed OMD site's library folder.
+    This was to force our binaries/shared libraries to use our OpenSSL.
+    We no longer do this, since our dependencies have the RPATH set correctly.
+
+    This test is to ensure that we do not regress to the previous behaviour.
+    See also SUP-10161.
 
     We execute this test in the CI containers of all supported distros to ensure that commands using
     OpenSSL which are available there can be executed.
@@ -47,11 +47,13 @@ def test_scp(site: Site) -> None:
     """
     Ensures that scp is working.
 
-    When executing scp as a site user, usually, the system scp is used. This binary is usually
-    dynamically linked against OpenSSL. We ship OpenSSL with OMD and set LD_LIBRARY_PATH to within
-    the site environment. Hence, when calling scp as a site user, the OMD OpenSSL is used. There is
-    no garantuee that this version is compatible with the scp compilation available on the system
-    (which is intended to compatible with the system OpenSSL). See also SUP-17682.
+    In the past we set LD_LIBRARY_PATH to use the installed OMD site's library folder.
+    This was to force our binaries/shared libraries to use our OpenSSL.
+    We no longer do this, since our dependencies have the RPATH set correctly.
+
+    This test is to ensure that we do not regress to the previous behaviour.
+
+    See also SUP-17682.
 
     We execute this test in the CI containers of all supported distros to ensure that commands using
     OpenSSL which are available there can be executed.

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,10 @@ _query_site() {`
`25`	`25`	`site_lib() {`
`26`	`26`	`printf "/omd/versions/%s/lib" "$(realpath "/omd/sites/${1}/version" \| sed 's\|.*/\|\|')"`
`27`	`27`	`}`
	`28`	`+ # LD_LIBRARY_PATH is set inline (not exported) as a per-subprocess override pointing to the`
	`29`	`+ # queried site's own lib directory. This is required for backward compatibility with older`
	`30`	`+ # installed site versions whose unixcat binary may lack RPATH and cannot locate its shared`
	`31`	`+ # libraries otherwise. Do not remove until all supported older versions have RPATH set.`
`28`	`32`	`echo -e "${2}" \| LD_LIBRARY_PATH="$(site_lib "${1}"):${LD_LIBRARY_PATH}" waitmax 3 "$(site_cmd "${1}" unixcat)" "/omd/sites/${1}/tmp/run/live"`
`29`	`33`	`}`
`30`	`34`