Implement DOCA kernel driver installation into the netboot image

dhilst · dhilst · commit a3e7612ab5b7 · 2025-03-19T16:56:55.000-03:00
diff --git a/include/cloysterhpc/functions.h b/include/cloysterhpc/functions.h
@@ -228,6 +228,7 @@ void copyFile(std::filesystem::path source, std::filesystem::path destination);
  * @param data The contents of the file to install
  */
 void installFile(const std::filesystem::path& path, std::istream& data);
+void installFile(const std::filesystem::path& path, std::string&& data);
 
 } // namespace cloyster
 
@@ -289,7 +290,7 @@ std::string enumToString(T enumValue)
  */
 template <typename T>
 requires std::is_enum_v<T>
-std::optional<T> enumOfStringOpt(const std::string& str)
+std::optional<T> enumOfStringOpt(std::string_view str)
 {
     return magic_enum::enum_cast<T>(str, magic_enum::case_insensitive);
 }
diff --git a/include/cloysterhpc/models/answerfile.h b/include/cloysterhpc/models/answerfile.h
@@ -159,7 +159,7 @@ class AnswerFile {
     };
 
     struct AFOFED {
-        std::string kind = static_cast<std::string>(magic_enum::enum_name(OFED::Kind::Inbox));
+        std::string kind = cloyster::utils::enumToString(OFED::Kind::Inbox);
         std::string version = "latest";
         bool enabled = false;
     };
diff --git a/include/cloysterhpc/ofed.h b/include/cloysterhpc/ofed.h
@@ -6,11 +6,11 @@
 #ifndef CLOYSTERHPC_OFED_H_
 #define CLOYSTERHPC_OFED_H_
 
-#include "services/repos.h"
 #include <string>
-
 #include <utility>
 
+#include <cloysterhpc/services/repos.h>
+
 
 /**
  * @class OFED
@@ -51,7 +51,7 @@ class OFED {
      * This method installs the appropriate OFED components based on the
      * specified kind.
      */
-    void install(cloyster::services::repos::RepoManager& repoManager) const;
+    void install() const;
 };
 
 #endif // CLOYSTERHPC_OFED_H_
diff --git a/include/cloysterhpc/services/shell.h b/include/cloysterhpc/services/shell.h
@@ -145,8 +145,8 @@ class Shell final : public Execution {
      *
      * This function sets up the InfiniBand interconnect settings.
      */
-    void configureInfiniband();
-    void configureMailSystem();
+    static void configureInfiniband();
+    static void configureMailSystem();
 
     /**
      * @brief Removes memory lock limits.
diff --git a/src/functions.cpp b/src/functions.cpp
@@ -374,4 +374,10 @@ void installFile(const std::filesystem::path& path, std::istream& data)
     fil << data.rdbuf();
 }
 
+void installFile(const std::filesystem::path& path, std::string&& data)
+{
+    std::istringstream stringData(std::move(data));
+    installFile(path, stringData);
+}
+
 }; // namespace cloyster
diff --git a/src/main.cpp b/src/main.cpp
@@ -29,32 +29,33 @@
 namespace {
 void initializeSingletons(auto&& cluster)
 {
-        using cloyster::models::Cluster;
-        cloyster::Singleton<Cluster>::init(std::forward<decltype(cluster)>(cluster));
+    using cloyster::models::Cluster;
+    cloyster::Singleton<Cluster>::init(
+        std::forward<decltype(cluster)>(cluster));
 
-        cloyster::Singleton<cloyster::services::BaseRunner>::init([](){
-            using cloyster::services::BaseRunner;
-            using cloyster::services::DryRunner;
-            using cloyster::services::Runner;
+    cloyster::Singleton<cloyster::services::BaseRunner>::init([](){
+        using cloyster::services::BaseRunner;
+        using cloyster::services::DryRunner;
+        using cloyster::services::Runner;
 
-            if (cloyster::dryRun) {
-                return cloyster::makeUniqueDerived<BaseRunner, DryRunner>();
-            }
+        if (cloyster::dryRun) {
+            return cloyster::makeUniqueDerived<BaseRunner, DryRunner>();
+        }
 
-            return cloyster::makeUniqueDerived<BaseRunner, Runner>();
-        });
+        return cloyster::makeUniqueDerived<BaseRunner, Runner>();
+    });
 
-        using cloyster::services::repos::RepoManager;
-        cloyster::Singleton<RepoManager>::init([]() {
-            auto clusterPtr = cloyster::Singleton<Cluster>::get();
-            const auto& osinfo = clusterPtr->getHeadnode().getOS();
-            return std::make_unique<RepoManager>(osinfo);
-        });
+    using cloyster::services::repos::RepoManager;
+    cloyster::Singleton<RepoManager>::init([]() {
+        auto clusterPtr = cloyster::Singleton<Cluster>::get();
+        const auto& osinfo = clusterPtr->getHeadnode().getOS();
+        return std::make_unique<RepoManager>(osinfo);
+    });
 
-        cloyster::Singleton<MessageBus>::init([]() {
-            return cloyster::makeUniqueDerived<MessageBus, DBusClient>(
-                "org.freedesktop.systemd1", "/org/freedesktop/systemd1");
-        });
+    cloyster::Singleton<MessageBus>::init([]() {
+        return cloyster::makeUniqueDerived<MessageBus, DBusClient>(
+            "org.freedesktop.systemd1", "/org/freedesktop/systemd1");
+    });
 }
 }; // anonymous namespace 
 
@@ -73,9 +74,6 @@ int main(int argc, const char** argv)
 
     app.add_flag("-t, --tui", cloyster::enableTUI, "Enable TUI");
 
-#if 0    
-    app.add_flag("-c, --cli", cloyster::enableCLI, "Enable CLI");
-#endif
 
     cloyster::logLevelInput
         = fmt::format("{}", magic_enum::enum_name(Log::Level::Info));
@@ -147,13 +145,15 @@ int main(int argc, const char** argv)
         }
     }());
 
+#ifndef NDEBUG
     if (!loadConfFile.empty()) {
         LOG_INFO("Loading file {}", loadConfFile);
         auto file = cloyster::services::files::KeyFile(loadConfFile);
         LOG_INFO("Groups: {}", fmt::join(file.getGroups(), ","));
         LOG_INFO("Contents: {}", file.toData());
         return EXIT_SUCCESS;
     }
+#endif
 
     if (cloyster::showVersion) {
         fmt::print("{}: Version {}\n", productName, productVersion);
diff --git a/src/models/cluster.cpp b/src/models/cluster.cpp
@@ -322,8 +322,7 @@ void Cluster::printData()
     LOG_DEBUG("DomainName: {}", getDomainName());
     LOG_DEBUG("FQDN: {}", this->m_headnode.getFQDN());
     if (m_ofed) {
-         auto ofed = m_ofed.value();
-         LOG_DEBUG("OFED: {} {}", utils::enumToString(ofed.getKind()), ofed.getVersion());
+         LOG_DEBUG("OFED: {} {}", utils::enumToString(m_ofed->getKind()), m_ofed->getVersion());
      }
         
     printNetworks(m_network);
@@ -593,19 +592,20 @@ void Cluster::fillData(const std::filesystem::path& answerfilePath)
         "{0}.{1}", this->m_headnode.getHostname(), getDomainName()));
 
     if (answerfil.ofed.enabled) {
+         // Install the cofigured OFED variant
          LOG_DEBUG("Loading OFED {}", answerfil.ofed.kind);
          auto kind = utils::enumOfStringOpt<OFED::Kind>(answerfil.ofed.kind);
          if (!kind) {
             throw std::runtime_error(
-                fmt::format("Invalid OFED kind, expected one of {}, found {}",
+                fmt::format("Invalid OFED kind, expected one of {}, found {}. Edit the anwerfile {} [ofed] sectino and try again.",
+                            cloyster::answerfile,
                             fmt::join(magic_enum::enum_names<OFED::Kind>(), ", "),
                             answerfil.ofed.kind
                             ));
          }
-         auto version = answerfil.ofed.version;
-         setOFED(kind.value(), version);
+         setOFED(kind.value(), answerfil.ofed.version);
     } else {
-         // @FIXME: Is this correct? It installs the Inbox infiniband stack by default
+         // Install Inbox OFED by default
          setOFED(OFED::Kind::Inbox);
     }
 
diff --git a/src/models/os.cpp b/src/models/os.cpp
@@ -4,6 +4,7 @@
  */
 
 #include <cloysterhpc/cloyster.h>
+#include <cloysterhpc/functions.h>
 #include <cloysterhpc/models/os.h>
 #include <cloysterhpc/services/dnf.h>
 #include <cloysterhpc/services/package_manager.h>
@@ -161,20 +162,9 @@ void OS::setDistro(OS::Distro distro) { m_distro = distro; }
 
 void OS::setDistro(std::string_view distro)
 {
-    // This code block is left for future reference, if an insensitive
-    // comparison in magic_enum would be implemented it may easily replace the
-    // lambda block. Reference: https://github.com/Neargye/magic_enum/pull/139
-
-#if 1
-    if (const auto& rv
-        = magic_enum::enum_cast<Distro>(distro, magic_enum::case_insensitive)) {
-#else
-    if (const auto &rv
-        = magic_enum::enum_cast<Distro>(distro, [](char lhs, char rhs) {
-              return std::tolower(lhs) == std::tolower(rhs);
-          }))
-#endif
-        setDistro(rv.value());
+    if (const auto& rval
+        = cloyster::utils::enumOfStringOpt<OS::Distro>(std::string(distro))) {
+        setDistro(rval.value());
     } else {
         throw std::runtime_error(
             fmt::format("Unsupported Distribution: {}", distro));
diff --git a/src/ofed.cpp b/src/ofed.cpp
@@ -4,6 +4,7 @@
  */
 
 #include <cloysterhpc/functions.h>
+#include <cloysterhpc/services/repos.h>
 #include <cloysterhpc/ofed.h>
 #include <utility>
 
@@ -48,47 +49,16 @@ std::string headnodeDistroName()
     std::unreachable();
 }
 
-void installMellanoxDoca(cloyster::services::repos::RepoManager& repoManager, const OFED& ofed)
+void installMellanoxDoca(const OFED& ofed)
 {
-    auto runner = cloyster::Singleton<cloyster::services::BaseRunner>::get();
-
-    if (runner->executeCommand("modprobe mlx5_core") == 0) {
-        LOG_WARN("mlx5_core module loaded, skiping DOCA setup");
-        return;
-    }
-
-    // @FIMXE deduce "rockylinux9.2"
-    auto repoData = docaRepoTemplate(ofed.getVersion(), headnodeDistroName());
-    std::filesystem::path path = "/etc/yum.repos.d/mlx-doca.repo";
-
-    // Install the repository and enable it
-    cloyster::installFile(path, repoData);
-    repoManager.install(path);
-    repoManager.enable("doca");
-
-    // Install the required packages
-    runner->executeCommand("dnf makecache");
-    runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
-
-    // Run the Mellanox script, this generates an RPM at tmp
-    assert(runner->executeCommand("/opt/mellanox/doca/tools/doca-kernel-support -k $(rpm -q --qf \"%{VERSION}-%{RELEASE}.%{ARCH}\n\" kernel-devel") == 0);
-
-    // Install the generated rpm
-    runner->executeCommand("rpm -ivh $(find /tmp/DOCA.*/ -name '*.rpm' -printf \"%T@ %p\n\" | sort -nrk1 | tail -1 | awk '{print $2}')");
-
-    runner->executeCommand("dnf makecache");
-    runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
-    if (runner->executeCommand("lsmod | grep mlx5_core") != 0) {
-        runner->executeCommand("modprobe mlx_core");
-    }
 }
 };
 
 void OFED::setKind(Kind kind) { m_kind = kind; }
 
 OFED::Kind OFED::getKind() const { return m_kind; }
 
-void OFED::install(cloyster::services::repos::RepoManager& repoManager) const
+void OFED::install() const
 {
     switch (m_kind) {
         case OFED::Kind::Inbox:
@@ -97,8 +67,40 @@ void OFED::install(cloyster::services::repos::RepoManager& repoManager) const
             break;
 
         case OFED::Kind::Mellanox:
-            installMellanoxDoca(repoManager, *this);
+            {
+                auto runner = cloyster::Singleton<cloyster::services::BaseRunner>::get();
+                auto repoManager = cloyster::Singleton<cloyster::services::repos::RepoManager>::get();
+
+                if (runner->executeCommand("modprobe mlx5_core") == 0) {
+                    LOG_WARN("mlx5_core module loaded, skiping DOCA setup");
+                    return;
+                }
+
+                auto repoData = docaRepoTemplate(getVersion(), headnodeDistroName());
+                std::filesystem::path path = "/etc/yum.repos.d/mlx-doca.repo";
+
+                // Install the repository and enable it
+                cloyster::installFile(path, repoData);
+                repoManager->install(path);
+                repoManager->enable("doca");
+
+                // Install the required packages
+                runner->executeCommand("dnf makecache");
+                runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
+
+                // Run the Mellanox script, this generates an RPM at tmp
+                assert(runner->executeCommand("/opt/mellanox/doca/tools/doca-kernel-support -k $(rpm -q --qf \"%{VERSION}-%{RELEASE}.%{ARCH}\n\" kernel-devel") == 0);
+
+                // Install the (last) generated rpm
+                runner->executeCommand("rpm -ivh $(find /tmp/DOCA.*/ -name '*.rpm' -printf \"%T@ %p\n\" | sort -nrk1 | tail -1 | awk '{print $2}')");
+
+                runner->executeCommand("dnf makecache");
+                runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
+                if (runner->executeCommand("lsmod | grep mlx5_core") != 0) {
+                    runner->executeCommand("modprobe mlx_core");
+                }
 
+            }
             break;
 
         case OFED::Kind::Oracle:
diff --git a/src/services/repos.cpp b/src/services/repos.cpp
@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <filesystem>
 #include <functional>
+#include <glibmm/keyfile.h>
 #include <memory>
 #include <ranges>
 #include <sstream>
@@ -68,8 +69,8 @@ class RPMRepository final : public IRepository {
     std::string m_name;
     std::optional<std::string> m_baseurl;
     std::optional<std::string> m_metalink;
+    std::optional<std::string> m_gpgkey;
     bool m_gpgcheck = true;
-    std::string m_gpgkey;
     std::filesystem::path m_source;
     std::string m_group;
 
@@ -102,7 +103,7 @@ class RPMRepository final : public IRepository {
         return m_metalink;
     };
     [[nodiscard]] bool gpgcheck() const { return m_gpgcheck; };
-    [[nodiscard]] std::string gpgkey() const { return m_gpgkey; };
+    [[nodiscard]] std::optional<std::string> gpgkey() const { return m_gpgkey; };
 
     void id(std::string value) override { m_id = value; };
     void enabled(bool enabled) override { m_enabled = enabled; };
@@ -119,7 +120,7 @@ class RPMRepository final : public IRepository {
         m_metalink = std::move(metalink);
     };
     void gpgcheck(bool gpgcheck) { m_gpgcheck = gpgcheck; };
-    void gpgkey(std::string gpgkey) { m_gpgkey = std::move(gpgkey); };
+    void gpgkey(std::optional<std::string> gpgkey) { m_gpgkey = std::move(gpgkey); };
 
     void valid() const;
 
@@ -171,7 +172,7 @@ class RPMRepositoryParser final {
             auto baseurl = file.getStringOpt(repogroup, "baseurl");
             auto enabled = file.getBoolean(repogroup, "enabled");
             auto gpgcheck = file.getBoolean(repogroup, "gpgcheck");
-            auto gpgkey = file.getString(repogroup, "gpgkey");
+            auto gpgkey = file.getStringOpt(repogroup, "gpgkey");
 
             RPMRepository repo;
             repo.group(repogroup);
@@ -282,19 +283,8 @@ class RPMRepoManager final {
     void install(std::filesystem::directory_iterator&& dirIter)
     {
         for (const auto& fil : std::move(dirIter)) {
-            auto fname = fil.path().filename().string();
-            // Return true if the repository should not be loaded
-            constexpr auto blacklisted = [](const std::string& repo) {
-                if (repo.starts_with("doca-kernel-")) {
-                    // @FIXME: This is the repositories created by the doca scripts
-                    //   Skipping them for now because they break the glib parser
-                    LOG_DEBUG("Skipping DOCA local repositories {}", repo);
-                    return true; // doca repositories break glib parser
-                }
-
-                return false;
-            };
-            if (fname.ends_with(".repo") && !blacklisted(fname)) {
+            std::string fname = fil.path().filename().string();
+            if (fname.ends_with(".repo")) {
                 install(fil);
             }
         }
diff --git a/src/services/shell.cpp b/src/services/shell.cpp
@@ -380,10 +380,9 @@ void Shell::configureMailSystem()
 
 void Shell::configureInfiniband()
 {
-    auto repos = cloyster::Singleton<repos::RepoManager>::get();
     if (const auto& ofed = cluster()->getOFED()) {
         LOG_INFO("Setting up Infiniband support")
-        ofed->install(*repos); // shared pointer
+        ofed->install(); // shared pointer
     }
 }
 
diff --git a/src/services/xcat.cpp b/src/services/xcat.cpp

Original file line number	Diff line number	Diff line change
`@@ -228,6 +228,7 @@ void copyFile(std::filesystem::path source, std::filesystem::path destination);`
`228`	`228`	`* @param data The contents of the file to install`
`229`	`229`	`*/`
`230`	`230`	`void installFile(const std::filesystem::path& path, std::istream& data);`
	`231`	`+void installFile(const std::filesystem::path& path, std::string&& data);`
`231`	`232`
`232`	`233`	`} // namespace cloyster`
`233`	`234`
`@@ -289,7 +290,7 @@ std::string enumToString(T enumValue)`
`289`	`290`	`*/`
`290`	`291`	`template <typename T>`
`291`	`292`	`requires std::is_enum_v<T>`
`292`		`-std::optional<T> enumOfStringOpt(const std::string& str)`
	`293`	`+std::optional<T> enumOfStringOpt(std::string_view str)`
`293`	`294`	`{`
`294`	`295`	`return magic_enum::enum_cast<T>(str, magic_enum::case_insensitive);`
`295`	`296`	`}`