Skip to content
This repository was archived by the owner on Apr 16, 2026. It is now read-only.

Commit 2cb9272

Browse files
dhilstdhilst
committed
Fix OFED installation in headnode
1 parent 63b127b commit 2cb9272

11 files changed

Lines changed: 171 additions & 49 deletions

File tree

‎include/cloysterhpc/functions.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,10 @@ void copyFile(std::filesystem::path source, std::filesystem::path destination);
230230
void installFile(const std::filesystem::path& path, std::istream& data);
231231
void installFile(const std::filesystem::path& path, std::string&& data);
232232

233+
bool exists(const std::filesystem::path& path);
234+
235+
void createHTTPRepo(const std::string_view repoName);
236+
233237
} // namespace cloyster
234238

235239
/**

‎include/cloysterhpc/ofed.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ class OFED {
5252
* specified kind.
5353
*/
5454
void install() const;
55+
56+
/**
57+
* @brief Returns true if OFED is already installed
58+
*/
59+
[[nodiscard]] bool installed() const;
5560
};
5661

5762
#endif // CLOYSTERHPC_OFED_H_

‎include/cloysterhpc/services/files.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010

1111
namespace cloyster::services::files {
1212

13+
constexpr std::size_t CHUNK_SIZE = 16384;
14+
1315
template <typename File>
1416
concept IsKeyFileReadable = requires(
1517
const File& file, const std::string& group, const std::string& key) {
@@ -91,9 +93,8 @@ static_assert(cloyster::concepts::IsSaveable<KeyFile>);
9193
static_assert(concepts::IsMoveable<KeyFile>);
9294
static_assert(!concepts::IsCopyable<KeyFile>);
9395

94-
9596
std::string checksum(const std::string& data);
96-
std::string checksum(const std::filesystem::path& path, const std::size_t chunkSize = 16384);
97+
std::string checksum(const std::filesystem::path& path, const std::size_t chunkSize = CHUNK_SIZE);
9798

9899
};
99100

‎include/cloysterhpc/services/runner.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ namespace cloyster::services {
1717
class BaseRunner {
1818
public:
1919
virtual int executeCommand(const std::string&) = 0;
20+
virtual void checkCommand(const std::string&) = 0;
21+
virtual std::vector<std::string> checkOutput(const std::string&) = 0;
2022

2123
virtual int downloadFile(const std::string& url, const std::string& file);
2224

@@ -26,20 +28,26 @@ class BaseRunner {
2628
class Runner : public BaseRunner {
2729
public:
2830
int executeCommand(const std::string&) override;
31+
void checkCommand(const std::string&) override;
32+
std::vector<std::string> checkOutput(const std::string&) override;
2933

3034
virtual ~Runner() = default;
3135
};
3236

3337
class DryRunner : public BaseRunner {
3438
public:
3539
int executeCommand(const std::string&) override;
40+
void checkCommand(const std::string&) override;
41+
std::vector<std::string> checkOutput(const std::string&) override;
3642

3743
virtual ~DryRunner() = default;
3844
};
3945

4046
class MockRunner : public BaseRunner {
4147
public:
4248
int executeCommand(const std::string&) override;
49+
void checkCommand(const std::string&) override;
50+
std::vector<std::string> checkOutput(const std::string&) override;
4351

4452
virtual ~MockRunner() = default;
4553

‎rpmspecs/opencattus.spec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,5 @@ install -m 755 build/src/cloysterhpc %{buildroot}/usr/bin/cloysterhpc
3636
/usr/bin/cloysterhpc
3737

3838
%changelog
39-
* Tue Feb 25 2025 Daniel Hilst <danielhilst@versatushpc.com> - 1.0-1
39+
* Tue Feb 25 2025 Daniel Hilst <daniel@versatushpc.com.br> - 1.0-1
4040
- Initial release

‎src/functions.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,4 +380,29 @@ void installFile(const std::filesystem::path& path, std::string&& data)
380380
installFile(path, stringData);
381381
}
382382

383+
bool exists(const std::filesystem::path& path)
384+
{
385+
return std::filesystem::exists(path);
386+
}
387+
388+
void createHTTPRepo(const std::string_view repoName)
389+
{
390+
auto runner = cloyster::Singleton<BaseRunner>::get();
391+
auto repoFolder = fmt::format("/var/www/html/{}", repoName);
392+
cloyster::createDirectory(repoFolder);
393+
cloyster::installFile(
394+
fmt::format("/etc/httpd.d/conf.d/{}.conf", repoName),
395+
fmt::format(
396+
R"(Alias "/rpmrepo" "{0}"
397+
<Directory "{0}">
398+
Options +Indexes +FollowSymLinks
399+
AllowOverride None
400+
Require all granted
401+
IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8
402+
</Directory>
403+
)", repoFolder));
404+
runner->checkCommand("apachectl configtest");
405+
runner->checkCommand("systemctl restart httpd");
406+
}
407+
383408
}; // namespace cloyster

‎src/main.cpp‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,10 +125,13 @@ int main(int argc, const char** argv)
125125
"-u, --unattended", unattended, "Perform an unattended installation");
126126

127127
#ifndef NDEBUG
128-
std::string loadConfFile {};
128+
std::string loadConfFile{};
129129
app.add_option("--test-conf-file", loadConfFile,
130130
"Hook for testing configuration file loading");
131131

132+
std::string testCommand{};
133+
app.add_option("--test-command", testCommand,
134+
"Run a command for testing purposes");
132135
#endif
133136

134137
CLI11_PARSE(app, argc, argv)
@@ -229,6 +232,16 @@ int main(int argc, const char** argv)
229232

230233
initializeSingletons(std::move(model));
231234

235+
#ifndef NDEBUG
236+
if (!testCommand.empty()) {
237+
LOG_INFO("Running test command {}", testCommand);
238+
auto runner = cloyster::Singleton<cloyster::BaseRunner>::get();
239+
runner->checkCommand(testCommand);
240+
return EXIT_SUCCESS;
241+
}
242+
#endif
243+
244+
232245
std::unique_ptr<Execution> executionEngine
233246
= std::make_unique<cloyster::services::Shell>();
234247

‎src/ofed.cpp‎

Lines changed: 61 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,22 @@
88
#include <cloysterhpc/ofed.h>
99
#include <utility>
1010

11+
using cloyster::BaseRunner;
1112
using cloyster::runCommand;
1213

1314
namespace {
1415

15-
auto docaRepoTemplate(std::string version, std::string distro)
16+
auto docaRepoTemplate(std::string version, std::string distro, std::string arch)
1617
{
1718
static constexpr std::string_view templ = R"(
1819
[doca]
1920
name=NVIDIA DOCA Repository - RHEL {1}
20-
baseurl=https://linux.mellanox.com/public/repo/doca/{0}/{1}/x86_64/
21+
baseurl=https://linux.mellanox.com/public/repo/doca/{0}/{1}/{2}/
2122
enabled=1
2223
gpgcheck=1
23-
gpgkey=https://linux.mellanox.com/public/repo/doca/{0}/GPG-KEY-Mellanox
24+
gpgkey=https://linux.mellanox.com/public/repo/doca/{0}/{1}/{2}/GPG-KEY-Mellanox.pub
2425
)";
25-
std::istringstream data(fmt::format(templ, version, distro));
26+
std::istringstream data(fmt::format(templ, version, distro, arch));
2627
return data;
2728
}
2829

@@ -58,25 +59,51 @@ void OFED::setKind(Kind kind) { m_kind = kind; }
5859

5960
OFED::Kind OFED::getKind() const { return m_kind; }
6061

62+
bool OFED::installed() const
63+
{
64+
if (cloyster::getEnvironmentVariable("CATTUS_FORCE_INFINIBAND_INSTALL") == "1") {
65+
return false;
66+
}
67+
68+
// Return false so the installation runs on dry run
69+
if (cloyster::dryRun) {
70+
return false;
71+
}
72+
73+
auto runner = cloyster::Singleton<BaseRunner>::get();
74+
switch (m_kind) {
75+
case OFED::Kind::Mellanox:
76+
return cloyster::exists("/opt/mellanox/doca/tools/doca-kernel-support");
77+
case OFED::Kind::Inbox:
78+
return runner->executeCommand("dnf group info \"Infiniband Support\"") == 0;
79+
case OFED::Kind::Oracle:
80+
throw std::logic_error("Not implemented");
81+
}
82+
83+
std::unreachable();
84+
}
85+
6186
void OFED::install() const
6287
{
88+
// Idempotency check
89+
if (installed()) {
90+
return;
91+
}
92+
6393
switch (m_kind) {
6494
case OFED::Kind::Inbox:
6595
runCommand("dnf -y groupinstall \"Infiniband Support\"");
66-
6796
break;
6897

6998
case OFED::Kind::Mellanox:
7099
{
100+
auto cluster = cloyster::Singleton<cloyster::models::Cluster>::get();
71101
auto runner = cloyster::Singleton<cloyster::services::BaseRunner>::get();
72102
auto repoManager = cloyster::Singleton<cloyster::services::repos::RepoManager>::get();
73103

74-
if (runner->executeCommand("modprobe mlx5_core") == 0) {
75-
LOG_WARN("mlx5_core module loaded, skiping DOCA setup");
76-
return;
77-
}
78-
79-
auto repoData = docaRepoTemplate(getVersion(), headnodeDistroName());
104+
auto repoData = docaRepoTemplate(
105+
getVersion(), headnodeDistroName(),
106+
cloyster::utils::enumToString(cluster->getHeadnode().getOS().getArch()));
80107
std::filesystem::path path = "/etc/yum.repos.d/mlx-doca.repo";
81108

82109
// Install the repository and enable it
@@ -85,21 +112,33 @@ void OFED::install() const
85112
repoManager->enable("doca");
86113

87114
// Install the required packages
88-
runner->executeCommand("dnf makecache");
89-
runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
115+
runner->checkCommand("dnf makecache");
116+
runner->checkCommand("dnf -y install kernel kernel-devel doca-extra");
117+
118+
LOG_INFO("Compiling OFED DOCA drivers, this may take a while");
119+
// Run the Mellanox script, this generates an RPM at tmp.
120+
//
121+
// Use the kernel-devel version instead of the booted kernel
122+
// version, this is to handle the case where a new kernel is
123+
// installed but no reboot was done yet. After compiling the
124+
// drivers the headnode should be rebooted to reload the new kernel.
125+
// The driver may support weak updates modules and load without
126+
// need for reboot.
127+
if (cloyster::getEnvironmentVariable("CATTUS_SKIP_INFINIBAND_COMPILE_DOCA_DRIVER") != "1") {
128+
runner->checkCommand("bash -c \"/opt/mellanox/doca/tools/doca-kernel-support -k $(rpm -q --qf \"%{VERSION}-%{RELEASE}.%{ARCH}\n\" kernel-devel)\"");
129+
}
90130

91-
// Run the Mellanox script, this generates an RPM at tmp
92-
assert(runner->executeCommand("/opt/mellanox/doca/tools/doca-kernel-support -k $(rpm -q --qf \"%{VERSION}-%{RELEASE}.%{ARCH}\n\" kernel-devel") == 0);
131+
// Get the last rpm in /tmp/DOCA*/ folder
132+
auto rpm = runner->checkOutput("bash -c \"find /tmp/DOCA*/ -name '*.rpm' -printf '%T@ %p\n' | sort -nk1 | tail -1 | awk '{print $2}'\"");
133+
assert(rpm.size() > 0); // at last one line
93134

94135
// Install the (last) generated rpm
95-
runner->executeCommand("rpm -ivh $(find /tmp/DOCA.*/ -name '*.rpm' -printf \"%T@ %p\n\" | sort -nrk1 | tail -1 | awk '{print $2}')");
96-
97-
runner->executeCommand("dnf makecache");
98-
runner->executeCommand("dnf install –y kernel kernel-devel doca-extra");
99-
if (runner->executeCommand("lsmod | grep mlx5_core") != 0) {
100-
runner->executeCommand("modprobe mlx_core");
101-
}
136+
runner->executeCommand(fmt::format("rpm -vih {}", rpm[0]));
102137

138+
runner->checkCommand("dnf makecache");
139+
// @NOTE: Are these packages correct/good default?
140+
runner->checkCommand("dnf install -y doca-ofed mlnx-fw-updater");
141+
runner->checkCommand("modprobe mlx5_core");
103142
}
104143
break;
105144

‎src/services/runner.cpp‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <cloysterhpc/services/runner.h>
55

66
#include <fmt/format.h>
7+
#include <ranges>
78

89
namespace cloyster::services {
910

@@ -18,18 +19,55 @@ int Runner::executeCommand(const std::string& cmd)
1819
return cloyster::runCommand(cmd);
1920
}
2021

22+
void Runner::checkCommand(const std::string& cmd)
23+
{
24+
if (cloyster::runCommand(cmd) != 0) {
25+
throw std::runtime_error(fmt::format("ERROR: Command failed '{}'", cmd));
26+
}
27+
}
28+
29+
std::vector<std::string> Runner::checkOutput(const std::string& cmd)
30+
{
31+
std::list<std::string> output;
32+
if (cloyster::runCommand(cmd, output) != 0) {
33+
throw std::runtime_error(fmt::format("ERROR: Command failed '{}'", cmd));
34+
}
35+
return output | std::ranges::to<std::vector>();
36+
}
37+
38+
2139
int DryRunner::executeCommand(const std::string& cmd)
2240
{
2341
LOG_WARN("Dry Run: Would execute command: {}", cmd);
2442
return OK;
2543
}
2644

45+
void DryRunner::checkCommand(const std::string& cmd)
46+
{
47+
LOG_WARN("Dry Run: Would execute command: {}", cmd);
48+
}
49+
50+
std::vector<std::string> DryRunner::checkOutput(const std::string& cmd)
51+
{
52+
LOG_WARN("Dry Run: Would check output of command: {}", cmd);
53+
return {};
54+
}
55+
2756
int MockRunner::executeCommand(const std::string& cmd)
2857
{
2958
m_cmds.push_back(cmd);
3059
return OK;
3160
}
3261

62+
void MockRunner::checkCommand(const std::string& cmd)
63+
{
64+
}
65+
66+
std::vector<std::string> MockRunner::checkOutput(const std::string& cmd)
67+
{
68+
return {};
69+
}
70+
3371
const std::vector<std::string>& MockRunner::listCommands() const
3472
{
3573
return m_cmds;

‎src/services/shell.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/*
22
* Copyright 2021 Vinícius Ferrão <vinicius@ferrao.net.br>
3+
*
34
* SPDX-License-Identifier: Apache-2.0
45
*/
56

@@ -281,7 +282,7 @@ void Shell::installRequiredPackages()
281282
{
282283
LOG_INFO("Installing required system packages")
283284

284-
runCommand("dnf -y install wget dnf-plugins-core");
285+
runCommand("dnf -y install wget dnf-plugins-core chkconfig");
285286
}
286287

287288
void Shell::disallowSSHRootPasswordLogin()

0 commit comments

Comments
 (0)