Merge pull request #36 from RSE-Cambridge/mgt

JohnGarbutt · web-flow · commit fa884b310d52 · 2018-12-18T16:55:15.000Z
Use nvme0 by offloading lustre mgt to sdb
diff --git a/dac-ansible/roles/data-acc/tasks/systemd.yml b/dac-ansible/roles/data-acc/tasks/systemd.yml
@@ -92,6 +92,7 @@
     - "{{data_acc_data_dir}}/fake_nvme2"
     - "{{data_acc_data_dir}}/fake_nvme3"
     - "{{data_acc_data_dir}}/fake_nvme4"
+    - "{{data_acc_data_dir}}/fake_sdb"
 
 - name: create loopback
   become: yes
@@ -110,6 +111,8 @@
       file: "{{data_acc_data_dir}}/fake_nvme3"
     - loop: 4
       file: "{{data_acc_data_dir}}/fake_nvme4"
+    - loop: 5
+      file: "{{data_acc_data_dir}}/fake_sdb"
 
 - name: create nvme symlinks
   become: yes
@@ -130,6 +133,14 @@
     - loop: 4
       file: "{{data_acc_data_dir}}/fake_nvme4"
 
+- name: create nvme symlinks
+  become: yes
+  become_user: root
+  file:
+    src: "/dev/loop5"
+    dest: "/dev/sdb"
+    state: link
+
 - name: launching data_acc...
   when: data_acc_launch
   become: yes
diff --git a/dac-ansible/roles/data-acc/templates/dacd.conf.j2 b/dac-ansible/roles/data-acc/templates/dacd.conf.j2
@@ -5,4 +5,10 @@ ETCDCTL_CERT_FILE=/etc/data-acc/pki/{{ inventory_hostname }}.pem
 ETCDCTL_KEY_FILE=/etc/data-acc/pki/{{ inventory_hostname }}-key.pem
 ETCDCTL_CA_FILE=/etc/data-acc/pki/ca.pem
 DAC_ANSIBLE_DIR={{data_acc_install_dir}}/{{data_acc_name}}/fs-ansible/
+#DAC_SKIP_ANSIBLE=True
+DAC_HOST_GROUP=dac-fake
 DEVICE_COUNT=5
+DAC_POOL_NAME=default
+DAC_DEVICE_CAPACITY_GB=1600
+DACCTL_LOG=/var/log/dacctl.log
+#DAC_LNET_SUFFIX="-opa@o2ib1"
diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml
@@ -23,13 +23,13 @@
   tags: [ 'never', 'reformat_mgs']
 
 - name: Reformat MDT
-  command: /usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index=0 --mgsnode={{ mgsnode }} /dev/{{ mdt }}
+  command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index=0 --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ mdt }}"
   when:
     - mdt is defined
   tags: [ 'never', 'reformat_mdts', 'format']
 
 - name: Reformat OSTs
-  command: /usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }} /dev/{{ item.key }}
+  command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}"
   loop: "{{ osts|dict2items }}"
   when:
     - osts is defined
diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml
@@ -77,7 +77,7 @@
         state: directory
         recurse: yes
     - name: mount lustre fs
-      command: "mount -t lustre {{ mgsnode }}:/{{ fs_name }} /mnt/lustre/{{ fs_name }}"
+      command: "mount -t lustre {{ mgsnode }}{{ lnet_suffix }}:/{{ fs_name }} /mnt/lustre/{{ fs_name }}"
       register: command_result
       failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)"
       changed_when: "command_result.rc == 0"
diff --git a/fs-ansible/test-inventory-lustre b/fs-ansible/test-inventory-lustre
@@ -11,6 +11,7 @@ dac-fake:
       vars:
         fs1_mgsnode: dac1
         fs1_client_port: 1001
+        lnet_suffix: ""
     fs2:
       hosts:
         dac1:
@@ -22,3 +23,4 @@ dac-fake:
       vars:
         fs2_mgsnode: dac1
         fs2_client_port: 1002
+        lnet_suffix: ""
diff --git a/internal/pkg/dacctl/actions/actions_test.go b/internal/pkg/dacctl/actions/actions_test.go
@@ -167,9 +167,9 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) {
 		Owner:     1001,
 		CreatedAt: uint(time.Now().Unix()),
 		Paths: map[string]string{
-			"DW_PERSISTENT_STRIPED_mybuffer": "/dac/token/persistent/mybuffer",
-			"DW_JOB_PRIVATE":                 "/dac/token/job_private",
-			"DW_JOB_STRIPED":                 "/dac/token/job/global",
+			"DW_PERSISTENT_STRIPED_mybuffer": "/dac/token_persistent_mybuffer",
+			"DW_JOB_PRIVATE":                 "/dac/token_job_private",
+			"DW_JOB_STRIPED":                 "/dac/token_job/global",
 		},
 		JobVolume:       registry.VolumeName("token"),
 		MultiJobVolumes: []registry.VolumeName{"mybuffer"},
diff --git a/internal/pkg/dacctl/buffer.go b/internal/pkg/dacctl/buffer.go
@@ -108,15 +108,15 @@ func setPaths(perJobVolume *registry.Volume, job registry.Job) map[string]string
 	paths := make(map[string]string)
 	if perJobVolume != nil {
 		if perJobVolume.AttachPrivateNamespace {
-			paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s/job_private", job.Name)
+			paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s_job_private", job.Name)
 		}
 		if perJobVolume.AttachGlobalNamespace {
-			paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s/job/global", job.Name)
+			paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s_job/global", job.Name)
 		}
 	}
 	for _, multiJobVolume := range job.MultiJobVolumes {
 		paths[fmt.Sprintf("DW_PERSISTENT_STRIPED_%s", multiJobVolume)] = fmt.Sprintf(
-			"/dac/%s/persistent/%s", job.Name, multiJobVolume)
+			"/dac/%s_persistent_%s", job.Name, multiJobVolume)
 	}
 	return paths
 }
diff --git a/internal/pkg/dacctl/buffer_test.go b/internal/pkg/dacctl/buffer_test.go
@@ -52,15 +52,15 @@ func TestSetPaths(t *testing.T) {
 
 	assert.Equal(t, 4, len(paths))
 	assert.Equal(t,
-		"/dac/job1/job_private",
+		"/dac/job1_job_private",
 		paths["DW_JOB_PRIVATE"])
 	assert.Equal(t,
-		"/dac/job1/job/global",
+		"/dac/job1_job/global",
 		paths["DW_JOB_STRIPED"])
 	assert.Equal(t,
-		"/dac/job1/persistent/multi1",
+		"/dac/job1_persistent_multi1",
 		paths["DW_PERSISTENT_STRIPED_multi1"])
 	assert.Equal(t,
-		"/dac/job1/persistent/multi2",
+		"/dac/job1_persistent_multi2",
 		paths["DW_PERSISTENT_STRIPED_multi2"])
 }
diff --git a/internal/pkg/dacctl/persistent.go b/internal/pkg/dacctl/persistent.go
@@ -73,8 +73,8 @@ func getPoolAndBrickCount(poolRegistry registry.PoolRegistry, capacity string) (
 	}
 
 	bricksRequired = uint(math.Ceil(float64(capacityGB) / float64(pool.GranularityGB)))
-	// Add one more for the metadata... TODO: lustre specific?
-	if bricksRequired != 0 {
+	// Add one more for the metadata, but only if request only one brick... TODO: lustre specific?
+	if bricksRequired == 1 {
 		bricksRequired += 1
 	}
 	return
diff --git a/internal/pkg/lifecycle/brickmanager/manager.go b/internal/pkg/lifecycle/brickmanager/manager.go
@@ -2,7 +2,6 @@ package brickmanager
 
 import (
 	"fmt"
-	"github.com/RSE-Cambridge/data-acc/internal/pkg/pfsprovider/ansible"
 	"github.com/RSE-Cambridge/data-acc/internal/pkg/registry"
 	"log"
 	"os"
@@ -65,10 +64,6 @@ func getDevices(devicesStr string) []string {
 
 	var bricks []string
 	for i := 0; i < count; i++ {
-		if i == 0 && FSType == ansible.Lustre {
-			// TODO: we should use another disk for MGS
-			continue
-		}
 		device := fmt.Sprintf(DefaultDeviceAddress, i)
 		bricks = append(bricks, device)
 	}
@@ -100,7 +95,7 @@ func updateBricks(poolRegistry registry.PoolRegistry, hostname string) {
 	devicesStr := os.Getenv("DEVICE_COUNT")
 	devices := getDevices(devicesStr)
 
-	capacityStr := os.Getenv("DAC_DEVICE_COUNT")
+	capacityStr := os.Getenv("DAC_DEVICE_CAPACITY_GB")
 	poolName := os.Getenv("DAC_POOL_NAME")
 	bricks := getBricks(devices, hostname, capacityStr, poolName)
 
diff --git a/internal/pkg/lifecycle/brickmanager/manager_test.go b/internal/pkg/lifecycle/brickmanager/manager_test.go
@@ -8,14 +8,14 @@ import (
 
 func TestGetDevices(t *testing.T) {
 	devices := getDevices("5")
-	assert.Equal(t, 4, len(devices))
-	assert.Equal(t, "nvme1n1", devices[0])
-	assert.Equal(t, "nvme4n1", devices[3])
+	assert.Equal(t, 5, len(devices))
+	assert.Equal(t, "nvme0n1", devices[0])
+	assert.Equal(t, "nvme4n1", devices[4])
 
 	devices = getDevices("asdf")
-	assert.Equal(t, 11, len(devices))
-	assert.Equal(t, "nvme1n1", devices[0])
-	assert.Equal(t, "nvme11n1", devices[10])
+	assert.Equal(t, 12, len(devices))
+	assert.Equal(t, "nvme0n1", devices[0])
+	assert.Equal(t, "nvme11n1", devices[11])
 }
 
 func TestGetBricks(t *testing.T) {
diff --git a/internal/pkg/pfsprovider/ansible/ansible.go b/internal/pkg/pfsprovider/ansible/ansible.go
@@ -34,9 +34,15 @@ type Wrapper struct {
 	All FileSystems
 }
 
-var DefaultHostGroup = "dac-fake"
+var DefaultHostGroup = "dac-prod"
 
 func getInventory(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) string {
+	// NOTE: only used by lustre
+	mgsDevice := os.Getenv("DAC_MGS_DEV")
+	if mgsDevice == "" {
+		mgsDevice = "sdb"
+	}
+
 	var mdt registry.BrickAllocation
 	osts := make(map[string][]registry.BrickAllocation)
 	for _, allocation := range brickAllocations {
@@ -62,7 +68,7 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi
 		if mdt.Hostname == host {
 			hostInfo.MDTS = mdt.Device
 			if fsType == Lustre {
-				hostInfo.MGS = "nvme0n1" // TODO: horrible hack!!
+				hostInfo.MGS = mgsDevice // TODO: horrible hack!!
 			} else {
 				hostInfo.MGS = mdt.Device
 			}
@@ -78,7 +84,9 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi
 	fsinfo := FSInfo{
 		Vars: map[string]string{
 			"mgsnode":     mdt.Hostname,
-			"client_port": fmt.Sprintf("%d", volume.ClientPort)},
+			"client_port": fmt.Sprintf("%d", volume.ClientPort),
+			"lnet_suffix": getLnetSuffix(),
+		},
 		Hosts: hosts,
 	}
 	fsname := fmt.Sprintf("%s", volume.UUID)
diff --git a/internal/pkg/pfsprovider/ansible/ansible_test.go b/internal/pkg/pfsprovider/ansible/ansible_test.go
@@ -22,7 +22,7 @@ func TestPlugin_GetInventory(t *testing.T) {
 		{Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 4},
 	}
 	result := getInventory(BeegFS, volume, brickAllocations)
-	expected := `dac-fake:
+	expected := `dac-prod:
   children:
     abcdefgh:
       hosts:
@@ -36,6 +36,7 @@ func TestPlugin_GetInventory(t *testing.T) {
           abcdefgh_osts: {nvme2n1: 3, nvme3n1: 4}
       vars:
         abcdefgh_client_port: "10002"
+        lnet_suffix: ""
         abcdefgh_mgsnode: dac1
 `
 	assert.Equal(t, expected, result)
@@ -48,17 +49,18 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) {
 		{Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 1},
 	}
 	result := getInventory(Lustre, volume, brickAllocations)
-	expected := `dac-fake:
+	expected := `dac-prod:
   children:
     abcdefgh:
       hosts:
         dac1:
-          abcdefgh_mgs: nvme0n1
+          abcdefgh_mgs: sdb
           abcdefgh_mdt: nvme1n1
         dac2:
           abcdefgh_osts: {nvme2n1: 1}
       vars:
         abcdefgh_client_port: "10002"
+        lnet_suffix: ""
         abcdefgh_mgsnode: dac1
 `
 	assert.Equal(t, expected, result)
diff --git a/internal/pkg/pfsprovider/ansible/mount.go b/internal/pkg/pfsprovider/ansible/mount.go
diff --git a/internal/pkg/pfsprovider/ansible/mount_test.go b/internal/pkg/pfsprovider/ansible/mount_test.go
diff --git a/tools/dac-reset.sh b/tools/dac-reset.sh
diff --git a/tools/slurm-test.sh b/tools/slurm-test.sh

Original file line number	Diff line number	Diff line change
`@@ -108,15 +108,15 @@ func setPaths(perJobVolume *registry.Volume, job registry.Job) map[string]string`
`108`	`108`	`paths := make(map[string]string)`
`109`	`109`	`if perJobVolume != nil {`
`110`	`110`	`if perJobVolume.AttachPrivateNamespace {`
`111`		`- paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s/job_private", job.Name)`
	`111`	`+ paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s_job_private", job.Name)`
`112`	`112`	`}`
`113`	`113`	`if perJobVolume.AttachGlobalNamespace {`
`114`		`- paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s/job/global", job.Name)`
	`114`	`+ paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s_job/global", job.Name)`
`115`	`115`	`}`
`116`	`116`	`}`
`117`	`117`	`for _, multiJobVolume := range job.MultiJobVolumes {`
`118`	`118`	`paths[fmt.Sprintf("DW_PERSISTENT_STRIPED_%s", multiJobVolume)] = fmt.Sprintf(`
`119`		`- "/dac/%s/persistent/%s", job.Name, multiJobVolume)`
	`119`	`+ "/dac/%s_persistent_%s", job.Name, multiJobVolume)`
`120`	`120`	`}`
`121`	`121`	`return paths`
`122`	`122`	`}`
Original file line number	Diff line number	Diff line change
`@@ -73,8 +73,8 @@ func getPoolAndBrickCount(poolRegistry registry.PoolRegistry, capacity string) (`
`73`	`73`	`}`
`74`	`74`
`75`	`75`	`bricksRequired = uint(math.Ceil(float64(capacityGB) / float64(pool.GranularityGB)))`
`76`		`- // Add one more for the metadata... TODO: lustre specific?`
`77`		`- if bricksRequired != 0 {`
	`76`	`+ // Add one more for the metadata, but only if request only one brick... TODO: lustre specific?`
	`77`	`+ if bricksRequired == 1 {`
`78`	`78`	`bricksRequired += 1`
`79`	`79`	`}`
`80`	`80`	`return`