Implement DOCA kernel driver installation into the netboot image

Author: dhilst

include/cloysterhpc/functions.h

@@ -228,6 +228,7 @@ void copyFile(std::filesystem::path source, std::filesystem::path destination);
  * @param data The contents of the file to install
  */
 void installFile(const std::filesystem::path& path, std::istream& data);
+void installFile(const std::filesystem::path& path, std::string&& data);
 
 } // namespace cloyster
 

include/cloysterhpc/ofed.h

@@ -6,11 +6,11 @@
 #ifndef CLOYSTERHPC_OFED_H_
 #define CLOYSTERHPC_OFED_H_
 
-#include "services/repos.h"
 #include <string>
-
 #include <utility>
 
+#include <cloysterhpc/services/repos.h>
+
 
 /**
  * @class OFED
@@ -51,7 +51,7 @@ class OFED {
      * This method installs the appropriate OFED components based on the
      * specified kind.
      */
-    void install(cloyster::services::repos::RepoManager& repoManager) const;
+    void install() const;
 };
 
 #endif // CLOYSTERHPC_OFED_H_

include/cloysterhpc/services/shell.h

@@ -145,8 +145,8 @@ class Shell final : public Execution {
      *
      * This function sets up the InfiniBand interconnect settings.
      */
-    void configureInfiniband();
-    void configureMailSystem();
+    static void configureInfiniband();
+    static void configureMailSystem();
 
     /**
      * @brief Removes memory lock limits.

src/functions.cpp

@@ -374,4 +374,10 @@ void installFile(const std::filesystem::path& path, std::istream& data)
     fil << data.rdbuf();
 }
 
+void installFile(const std::filesystem::path& path, std::string&& data)
+{
+    std::istringstream stringData(std::move(data));
+    installFile(path, stringData);
+}
+
 }; // namespace cloyster

src/main.cpp

@@ -147,13 +147,15 @@ int main(int argc, const char** argv)
         }
     }());
 
+#ifndef NDEBUG
     if (!loadConfFile.empty()) {
         LOG_INFO("Loading file {}", loadConfFile);
         auto file = cloyster::services::files::KeyFile(loadConfFile);
         LOG_INFO("Groups: {}", fmt::join(file.getGroups(), ","));
         LOG_INFO("Contents: {}", file.toData());
         return EXIT_SUCCESS;
     }
+#endif
 
     if (cloyster::showVersion) {
         fmt::print("{}: Version {}\n", productName, productVersion);

src/ofed.cpp

@@ -4,6 +4,7 @@
  */
 
 #include <cloysterhpc/functions.h>
+#include <cloysterhpc/services/repos.h>
 #include <cloysterhpc/ofed.h>
 #include <utility>
 
@@ -48,47 +49,16 @@ std::string headnodeDistroName()
     std::unreachable();
 }
 
-void installMellanoxDoca(cloyster::services::repos::RepoManager& repoManager, const OFED& ofed)
+void installMellanoxDoca(const OFED& ofed)
 {
-    auto runner = cloyster::Singleton<cloyster::services::BaseRunner>::get();
-
-    if (runner->executeCommand("modprobe mlx5_core") == 0) {
-        LOG_WARN("mlx5_core module loaded, skiping DOCA setup");
-        return;
-    }
-
-    // @FIMXE deduce "rockylinux9.2"
-    auto repoData = docaRepoTemplate(ofed.getVersion(), headnodeDistroName());
-    std::filesystem::path path = "/etc/yum.repos.d/mlx-doca.repo";
-
-    // Install the repository and enable it
-    cloyster::installFile(path, repoData);
-    repoManager.install(path);
-    repoManager.enable("doca");
-
-    // Install the required packages
-    runner->executeCommand("dnf makecache");
-    runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
-
-    // Run the Mellanox script, this generates an RPM at tmp
-    assert(runner->executeCommand("/opt/mellanox/doca/tools/doca-kernel-support -k $(rpm -q --qf \"%{VERSION}-%{RELEASE}.%{ARCH}\n\" kernel-devel") == 0);
-
-    // Install the generated rpm
-    runner->executeCommand("rpm -ivh $(find /tmp/DOCA.*/ -name '*.rpm' -printf \"%T@ %p\n\" | sort -nrk1 | tail -1 | awk '{print $2}')");
-
-    runner->executeCommand("dnf makecache");
-    runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
-    if (runner->executeCommand("lsmod | grep mlx5_core") != 0) {
-        runner->executeCommand("modprobe mlx_core");
-    }
 }
 };
 
 void OFED::setKind(Kind kind) { m_kind = kind; }
 
 OFED::Kind OFED::getKind() const { return m_kind; }
 
-void OFED::install(cloyster::services::repos::RepoManager& repoManager) const
+void OFED::install() const
 {
     switch (m_kind) {
         case OFED::Kind::Inbox:
@@ -97,8 +67,40 @@ void OFED::install(cloyster::services::repos::RepoManager& repoManager) const
             break;
 
         case OFED::Kind::Mellanox:
-            installMellanoxDoca(repoManager, *this);
+            {
+                auto runner = cloyster::Singleton<cloyster::services::BaseRunner>::get();
+                auto repoManager = cloyster::Singleton<cloyster::services::repos::RepoManager>::get();
+
+                if (runner->executeCommand("modprobe mlx5_core") == 0) {
+                    LOG_WARN("mlx5_core module loaded, skiping DOCA setup");
+                    return;
+                }
+
+                auto repoData = docaRepoTemplate(getVersion(), headnodeDistroName());
+                std::filesystem::path path = "/etc/yum.repos.d/mlx-doca.repo";
+
+                // Install the repository and enable it
+                cloyster::installFile(path, repoData);
+                repoManager->install(path);
+                repoManager->enable("doca");
+
+                // Install the required packages
+                runner->executeCommand("dnf makecache");
+                runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
+
+                // Run the Mellanox script, this generates an RPM at tmp
+                assert(runner->executeCommand("/opt/mellanox/doca/tools/doca-kernel-support -k $(rpm -q --qf \"%{VERSION}-%{RELEASE}.%{ARCH}\n\" kernel-devel") == 0);
+
+                // Install the (last) generated rpm
+                runner->executeCommand("rpm -ivh $(find /tmp/DOCA.*/ -name '*.rpm' -printf \"%T@ %p\n\" | sort -nrk1 | tail -1 | awk '{print $2}')");
+
+                runner->executeCommand("dnf makecache");
+                runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
+                if (runner->executeCommand("lsmod | grep mlx5_core") != 0) {
+                    runner->executeCommand("modprobe mlx_core");
+                }
 
+            }
             break;
 
         case OFED::Kind::Oracle:

src/services/repos.cpp

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <filesystem>
 #include <functional>
+#include <glibmm/keyfile.h>
 #include <memory>
 #include <ranges>
 #include <sstream>
@@ -68,8 +69,8 @@ class RPMRepository final : public IRepository {
     std::string m_name;
     std::optional<std::string> m_baseurl;
     std::optional<std::string> m_metalink;
+    std::optional<std::string> m_gpgkey;
     bool m_gpgcheck = true;
-    std::string m_gpgkey;
     std::filesystem::path m_source;
     std::string m_group;
 
@@ -102,7 +103,7 @@ class RPMRepository final : public IRepository {
         return m_metalink;
     };
     [[nodiscard]] bool gpgcheck() const { return m_gpgcheck; };
-    [[nodiscard]] std::string gpgkey() const { return m_gpgkey; };
+    [[nodiscard]] std::optional<std::string> gpgkey() const { return m_gpgkey; };
 
     void id(std::string value) override { m_id = value; };
     void enabled(bool enabled) override { m_enabled = enabled; };
@@ -119,7 +120,7 @@ class RPMRepository final : public IRepository {
         m_metalink = std::move(metalink);
     };
     void gpgcheck(bool gpgcheck) { m_gpgcheck = gpgcheck; };
-    void gpgkey(std::string gpgkey) { m_gpgkey = std::move(gpgkey); };
+    void gpgkey(std::optional<std::string> gpgkey) { m_gpgkey = std::move(gpgkey); };
 
     void valid() const;
 
@@ -171,7 +172,7 @@ class RPMRepositoryParser final {
             auto baseurl = file.getStringOpt(repogroup, "baseurl");
             auto enabled = file.getBoolean(repogroup, "enabled");
             auto gpgcheck = file.getBoolean(repogroup, "gpgcheck");
-            auto gpgkey = file.getString(repogroup, "gpgkey");
+            auto gpgkey = file.getStringOpt(repogroup, "gpgkey");
 
             RPMRepository repo;
             repo.group(repogroup);
@@ -282,19 +283,8 @@ class RPMRepoManager final {
     void install(std::filesystem::directory_iterator&& dirIter)
     {
         for (const auto& fil : std::move(dirIter)) {
-            auto fname = fil.path().filename().string();
-            // Return true if the repository should not be loaded
-            constexpr auto blacklisted = [](const std::string& repo) {
-                if (repo.starts_with("doca-kernel-")) {
-                    // @FIXME: This is the repositories created by the doca scripts
-                    //   Skipping them for now because they break the glib parser
-                    LOG_DEBUG("Skipping DOCA local repositories {}", repo);
-                    return true; // doca repositories break glib parser
-                }
-
-                return false;
-            };
-            if (fname.ends_with(".repo") && !blacklisted(fname)) {
+            std::string fname = fil.path().filename().string();
+            if (fname.ends_with(".repo")) {
                 install(fil);
             }
         }

src/services/shell.cpp

@@ -380,10 +380,9 @@ void Shell::configureMailSystem()
 
 void Shell::configureInfiniband()
 {
-    auto repos = cloyster::Singleton<repos::RepoManager>::get();
     if (const auto& ofed = cluster()->getOFED()) {
         LOG_INFO("Setting up Infiniband support")
-        ofed->install(*repos); // shared pointer
+        ofed->install(); // shared pointer
     }
 }