Skip to content
This repository was archived by the owner on Jul 25, 2022. It is now read-only.

Commit 3e84581

Browse files
johnsonwjohnsonw
committed
EX-1476 Create step to create and populate ldev config
Create the step that will populate the ldev config. Signed-off-by: johnsonw <[email protected]>
1 parent 724e50e commit 3e84581

File tree

21 files changed

+536
-61
lines changed

21 files changed

+536
-61
lines changed

‎chroma_core/migrations/0028_configureldevjob.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.11.23 on 2020-10-01 17:00
3+
from __future__ import unicode_literals
4+
5+
import django.contrib.postgres.fields.jsonb
6+
from django.db import migrations, models
7+
import django.db.models.deletion
8+
9+
10+
class Migration(migrations.Migration):
11+
12+
dependencies = [
13+
("chroma_core", "0027_add_scan_mdt_jobs"),
14+
]
15+
16+
operations = [
17+
migrations.CreateModel(
18+
name="ConfigureLDevJob",
19+
fields=[
20+
(
21+
"job_ptr",
22+
models.OneToOneField(
23+
auto_created=True,
24+
on_delete=django.db.models.deletion.CASCADE,
25+
parent_link=True,
26+
primary_key=True,
27+
serialize=False,
28+
to="chroma_core.Job",
29+
),
30+
),
31+
("ldev_entries", django.contrib.postgres.fields.jsonb.JSONField(default={})),
32+
],
33+
options={
34+
"ordering": ["id"],
35+
},
36+
bases=("chroma_core.job",),
37+
),
38+
]

‎chroma_core/models/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from repo import *
3232
from copytool import *
3333
from client_mount import *
34+
from ldev_configuration import *
3435
from lnet_configuration import *
3536
from sparse_model import *
3637
from stratagem import *

‎chroma_core/models/ldev_configuration.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# -*- coding: utf-8 -*-
2+
# Copyright (c) 2020 DDN. All rights reserved.
3+
# Use of this source code is governed by a MIT-style
4+
# license that can be found in the LICENSE file.
5+
6+
from chroma_core.lib.job import Step
7+
from chroma_core.models import Job
8+
from chroma_help.help import help_text
9+
from django.contrib.postgres import fields
10+
11+
import json
12+
13+
14+
class ConfigureLDevStep(Step):
15+
def run(self, kwargs):
16+
ldev_entries = kwargs["ldev_entries"]
17+
ldev_entries = json.loads(ldev_entries)
18+
19+
for (fqdn, entries) in ldev_entries.items():
20+
self.invoke_rust_agent_expect_result(fqdn, "create_ldev_conf", entries)
21+
22+
23+
class ConfigureLDevJob(Job):
24+
verb = "Configure LDev"
25+
ldev_entries = fields.JSONField(default={})
26+
27+
class Meta:
28+
app_label = "chroma_core"
29+
ordering = ["id"]
30+
31+
@classmethod
32+
def long_description(cls):
33+
return help_text["configure_ldev"]
34+
35+
def description(self):
36+
return self.long_description()
37+
38+
def get_steps(self):
39+
return [(ConfigureLDevStep, {"ldev_entries": self.ldev_entries})]

‎chroma_help/help.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@
118118
"local_diff": "This row has changed locally. Click to reset value to %(initial)s",
119119
"remote_diff": "This row has changed remotely. Click to set value to %(remote)s.",
120120
"configure_lnet": "Configure LNet for %s",
121+
"configure_ldev": "Configure LDev",
121122
"change_host_profile": "Changing host %s to profile %s",
122123
"change_host_state": "Changing host %s to state %s",
123124
"configure_lnet_not_allowed": "LNet can only be configured in managed mode.",

‎iml-agent/src/action_plugins/ldev.rs

Lines changed: 118 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ async fn read_ldev_config() -> Result<String, ImlAgentError> {
2828
}
2929

3030
fn parse_entries(ldev_config: String) -> BTreeSet<LdevEntry> {
31-
ldev_config.lines().map(LdevEntry::from).collect()
31+
ldev_config
32+
.lines()
33+
.filter(|x| !x.trim_start().starts_with("#"))
34+
.map(LdevEntry::from)
35+
.collect()
3236
}
3337

3438
fn convert(entries: &[LdevEntry]) -> String {
@@ -44,6 +48,7 @@ pub async fn create(entries: Vec<LdevEntry>) -> Result<(), ImlAgentError> {
4448
let ldev_config = read_ldev_config().await?;
4549
let existing_entries = parse_entries(ldev_config);
4650
let entries_set = entries.iter().cloned().collect::<BTreeSet<LdevEntry>>();
51+
4752
if existing_entries != entries_set {
4853
let data = convert(&entries);
4954
write_to_file(data).await?;
@@ -63,7 +68,69 @@ mod tests {
6368
use iml_wire_types::FsType;
6469

6570
#[test]
66-
fn test_create() -> Result<(), ImlAgentError> {
71+
fn test_ldiskfs_create() -> Result<(), ImlAgentError> {
72+
let entries = vec![
73+
LdevEntry {
74+
primary: "mds1".into(),
75+
failover: Some("mds2".into()),
76+
label: "MGS".into(),
77+
device: "/mnt/mgt".into(),
78+
fs_type: Some(FsType::Ldiskfs),
79+
},
80+
LdevEntry {
81+
primary: "mds1".into(),
82+
failover: Some("mds2".into()),
83+
label: "fs-MDT0000".into(),
84+
device: "/mnt/mdt0".into(),
85+
fs_type: Some(FsType::Ldiskfs),
86+
},
87+
LdevEntry {
88+
primary: "mds2".into(),
89+
failover: Some("mds1".into()),
90+
label: "fs-MDT0001".into(),
91+
device: "/mnt/mdt1".into(),
92+
fs_type: Some(FsType::Ldiskfs),
93+
},
94+
LdevEntry {
95+
primary: "oss1".into(),
96+
failover: Some("oss2".into()),
97+
label: "fs-OST0000".into(),
98+
device: "/mnt/ost0".into(),
99+
fs_type: Some(FsType::Ldiskfs),
100+
},
101+
LdevEntry {
102+
primary: "oss2".into(),
103+
failover: Some("oss1".into()),
104+
label: "fs-OST0001".into(),
105+
device: "/mnt/ost1".into(),
106+
fs_type: Some(FsType::Ldiskfs),
107+
},
108+
LdevEntry {
109+
primary: "oss1".into(),
110+
failover: Some("oss2".into()),
111+
label: "fs-OST0002".into(),
112+
device: "/mnt/ost2".into(),
113+
fs_type: Some(FsType::Ldiskfs),
114+
},
115+
LdevEntry {
116+
primary: "oss2".into(),
117+
failover: Some("oss1".into()),
118+
label: "fs-OST0003".into(),
119+
device: "/mnt/ost3".into(),
120+
fs_type: Some(FsType::Ldiskfs),
121+
},
122+
]
123+
.into_iter()
124+
.collect::<Vec<LdevEntry>>();
125+
126+
let data = convert(&entries);
127+
insta::assert_snapshot!(data);
128+
129+
Ok(())
130+
}
131+
132+
#[test]
133+
fn test_zfs_create() -> Result<(), ImlAgentError> {
67134
let entries = vec![
68135
LdevEntry {
69136
primary: "mds1".into(),
@@ -208,21 +275,21 @@ mod tests {
208275
LdevEntry {
209276
primary: "oss2".into(),
210277
failover: Some("oss1".into()),
211-
label: "zfsmo-OST00011".into(),
278+
label: "zfsmo-OST0011".into(),
212279
device: "zfs:ost17/ost17".into(),
213280
fs_type: Some(FsType::Zfs),
214281
},
215282
LdevEntry {
216283
primary: "oss2".into(),
217284
failover: Some("oss1".into()),
218-
label: "zfsmo-OST00012".into(),
285+
label: "zfsmo-OST0012".into(),
219286
device: "zfs:ost18/ost18".into(),
220287
fs_type: Some(FsType::Zfs),
221288
},
222289
LdevEntry {
223290
primary: "oss2".into(),
224291
failover: Some("oss1".into()),
225-
label: "zfsmo-OST00013".into(),
292+
label: "zfsmo-OST0013".into(),
226293
device: "zfs:ost19/ost19".into(),
227294
fs_type: Some(FsType::Zfs),
228295
},
@@ -262,4 +329,50 @@ mod tests {
262329

263330
Ok(())
264331
}
332+
333+
#[test]
334+
fn test_parsing_commented_data() -> Result<(), ImlAgentError> {
335+
let content: String = r#"# example /etc/ldev.conf
336+
#
337+
#local foreign/- label [md|zfs:]device-path [journal-path]/- [raidtab]
338+
#
339+
#zeno-mds1 - zeno-MDT0000 zfs:lustre-zeno-mds1/mdt1
340+
#
341+
#zeno1 zeno5 zeno-OST0000 zfs:lustre-zeno1/ost1
342+
#zeno2 zeno6 zeno-OST0001 zfs:lustre-zeno2/ost1
343+
#zeno3 zeno7 zeno-OST0002 zfs:lustre-zeno3/ost1
344+
#zeno4 zeno8 zeno-OST0003 zfs:lustre-zeno4/ost1
345+
#zeno5 zeno1 zeno-OST0004 zfs:lustre-zeno5/ost1
346+
#zeno6 zeno2 zeno-OST0005 zfs:lustre-zeno6/ost1
347+
#zeno7 zeno3 zeno-OST0006 zfs:lus tre-zeno7/ost1
348+
#zeno8 zeno4 zeno-OST0007 zfs:lustre-zeno8/ost1"#
349+
.into();
350+
351+
let data: BTreeSet<LdevEntry> = parse_entries(content);
352+
353+
assert!(data.is_empty());
354+
355+
Ok(())
356+
}
357+
358+
#[test]
359+
fn test_parsing_data() -> Result<(), ImlAgentError> {
360+
let content: String = r#"#zeno-mds1 - zeno-MDT0000 zfs:lustre-zeno-mds1/mdt1
361+
# Random comment
362+
zeno1 zeno5 zeno-OST0000 zfs:lustre-zeno1/ost1
363+
zeno2 - zeno-OST0001 zfs:lustre-zeno2/ost1
364+
zeno3 zeno7 zeno-OST0002 zfs:lustre-zeno3/ost1
365+
zeno4 zeno8 zeno-OST0003 zfs:lustre-zeno4/ost1
366+
zeno5 zeno1 zeno-OST0004 zfs:lustre-zeno5/ost1
367+
zeno6 - zeno-OST0005 zfs:lustre-zeno6/ost1
368+
zeno7 zeno3 zeno-OST0006 zfs:lustre-zeno7/ost1
369+
zeno8 zeno4 zeno-OST0007 zfs:lustre-zeno8/ost1"#
370+
.into();
371+
372+
let data: BTreeSet<LdevEntry> = parse_entries(content);
373+
374+
insta::assert_debug_snapshot!(data);
375+
376+
Ok(())
377+
}
265378
}

‎iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__ldiskfs_create.snap

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
source: iml-agent/src/action_plugins/ldev.rs
3+
expression: data
4+
---
5+
mds1 mds2 MGS ldiskfs:/mnt/mgt
6+
mds1 mds2 fs-MDT0000 ldiskfs:/mnt/mdt0
7+
mds2 mds1 fs-MDT0001 ldiskfs:/mnt/mdt1
8+
oss1 oss2 fs-OST0000 ldiskfs:/mnt/ost0
9+
oss2 oss1 fs-OST0001 ldiskfs:/mnt/ost1
10+
oss1 oss2 fs-OST0002 ldiskfs:/mnt/ost2
11+
oss2 oss1 fs-OST0003 ldiskfs:/mnt/ost3

‎iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__parsing_data.snap

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
---
2+
source: iml-agent/src/action_plugins/ldev.rs
3+
expression: data
4+
---
5+
{
6+
LdevEntry {
7+
primary: "zeno1",
8+
failover: Some(
9+
"zeno5",
10+
),
11+
label: "zeno-OST0000",
12+
device: "zfs:lustre-zeno1/ost1",
13+
fs_type: Some(
14+
Zfs,
15+
),
16+
},
17+
LdevEntry {
18+
primary: "zeno2",
19+
failover: None,
20+
label: "zeno-OST0001",
21+
device: "zfs:lustre-zeno2/ost1",
22+
fs_type: Some(
23+
Zfs,
24+
),
25+
},
26+
LdevEntry {
27+
primary: "zeno3",
28+
failover: Some(
29+
"zeno7",
30+
),
31+
label: "zeno-OST0002",
32+
device: "zfs:lustre-zeno3/ost1",
33+
fs_type: Some(
34+
Zfs,
35+
),
36+
},
37+
LdevEntry {
38+
primary: "zeno4",
39+
failover: Some(
40+
"zeno8",
41+
),
42+
label: "zeno-OST0003",
43+
device: "zfs:lustre-zeno4/ost1",
44+
fs_type: Some(
45+
Zfs,
46+
),
47+
},
48+
LdevEntry {
49+
primary: "zeno5",
50+
failover: Some(
51+
"zeno1",
52+
),
53+
label: "zeno-OST0004",
54+
device: "zfs:lustre-zeno5/ost1",
55+
fs_type: Some(
56+
Zfs,
57+
),
58+
},
59+
LdevEntry {
60+
primary: "zeno6",
61+
failover: None,
62+
label: "zeno-OST0005",
63+
device: "zfs:lustre-zeno6/ost1",
64+
fs_type: Some(
65+
Zfs,
66+
),
67+
},
68+
LdevEntry {
69+
primary: "zeno7",
70+
failover: Some(
71+
"zeno3",
72+
),
73+
label: "zeno-OST0006",
74+
device: "zfs:lustre-zeno7/ost1",
75+
fs_type: Some(
76+
Zfs,
77+
),
78+
},
79+
LdevEntry {
80+
primary: "zeno8",
81+
failover: Some(
82+
"zeno4",
83+
),
84+
label: "zeno-OST0007",
85+
device: "zfs:lustre-zeno8/ost1",
86+
fs_type: Some(
87+
Zfs,
88+
),
89+
},
90+
}

‎iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__create.snap renamed to ‎iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__zfs_create.snap

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,6 @@ oss2 oss1 zfsmo-OST000d zfs:ost13/ost13
2222
oss2 oss1 zfsmo-OST000e zfs:ost14/ost14
2323
oss2 oss1 zfsmo-OST000f zfs:ost15/ost15
2424
oss2 oss1 zfsmo-OST0010 zfs:ost16/ost16
25-
oss2 oss1 zfsmo-OST00011 zfs:ost17/ost17
26-
oss2 oss1 zfsmo-OST00012 zfs:ost18/ost18
27-
oss2 oss1 zfsmo-OST00013 zfs:ost19/ost19
25+
oss2 oss1 zfsmo-OST0011 zfs:ost17/ost17
26+
oss2 oss1 zfsmo-OST0012 zfs:ost18/ost18
27+
oss2 oss1 zfsmo-OST0013 zfs:ost19/ost19

‎iml-api/src/error.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ pub enum ImlApiError {
3030
FilesystemNotFound,
3131
#[error("Filesystem Not Found")]
3232
MgsNotFound,
33+
#[error("All targets must be mounted when configuring ldev conf.")]
34+
TargetsNotMounted,
3335
}
3436

3537
impl reject::Reject for ImlApiError {}

0 commit comments

Comments
 (0)