Slurm nodes installation under Ubuntu16.04

README.md

@@ -3,9 +3,9 @@
 
 Salt formula provisioning a Slurm cluster
 
-Availables states:
-  * Munge
-  * Screen
-  * Slurm
-  * Slurm Database
-  * SSH
+To install Slurm nodes, you need to copy (on Slurm mater node)
+
+- munge.key from /etc/munge/munge.key to /srv/salt/munge.key
+- slurm.cert from /etc/slurm-llnl/slurm.cert to /srv/salt/slurm.cert
+- slurm.conf from files/etc/slurm-llnl/slurm.conf to /srv/salt/slurm.conf
+- create empty cgroup.conf and gres.conf in /srv/salt/

files/etc/apt/sources.list.d/xenial.list

@@ -0,0 +1 @@
+deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse

files/etc/security/access.conf

@@ -9,4 +9,4 @@
 # who wants to be able to SSH in as root via public-key on Biomedia servers. 
 # disable SSH for anybody but root
 +:root:ALL
--:ALL EXCEPT (csg) dr jpassera bglocker:ALL
+-:ALL EXCEPT (csg) (biomedia) dr jpassera bglocker jgao:ALL

files/etc/slurm-llnl/cgroup.conf

@@ -16,7 +16,8 @@ CgroupReleaseAgentDir=/var/spool/slurm-llnl/cgroup
 ConstrainCores=yes
 TaskAffinity=yes
 #ConstrainRAMSpace=no
-### not used yet
-#ConstrainDevices=no
-#AllowedDevicesFile=/etc/slurm-llnl/cgroup_allowed_devices_file.conf
-
+ConstrainSwapSpace=yes
+AllowedSwapSpace=10.0
+# Not well supported until Slurm v14.11.4 https://groups.google.com/d/msg/slurm-devel/oKAUed7AETs/Eb6thh9Lc0YJ
+#ConstrainDevices=yes
+#AllowedDevicesFile=/etc/slurm-llnl/cgroup_allowed_devices_file.conf

files/etc/slurm-llnl/bardolph/gres.conf → files/etc/slurm-llnl/gres.conf_gpu_l

@@ -1,2 +1,4 @@
 Name=gpu File=/dev/nvidia0
 Name=gpu File=/dev/nvidia1
+Name=gpu File=/dev/nvidia2
+Name=gpu File=/dev/nvidia3

files/etc/slurm-llnl/slurm.conf

@@ -31,6 +31,7 @@ JobCredentialPublicCertificate=/etc/slurm-llnl/slurm.cert
 #Licenses=foo*4,bar
 MailProg=/usr/bin/mail
 MaxJobCount=25000
+MaxArraySize=32000
 #MaxStepCount=40000
 #MaxTasksPerNode=128
 MpiDefault=none
@@ -119,9 +120,9 @@ PreemptMode=OFF
 #
 # LOGGING AND ACCOUNTING
 DefaultStorageType=slurmdbd
-DefaultStorageUser={{ pillar['slurm']['db']['user'] }}
+DefaultStorageUser=slurm
 #DefaultStorageLoc=/var/log/slurm-llnl/job_completions.log
-DefaultStorageHost={{ pillar['slurm']['controller'] }}
+DefaultStorageHost=biomedia03
 DefaultStoragePort=6819
 AccountingStorageEnforce=associations,limits
 #AccountingStorageHost=
@@ -132,11 +133,11 @@ AccountingStorageEnforce=associations,limits
 #AccountingStorageUser=
 AccountingStoreJobComment=YES
 ClusterName=biomediacluster
-#DebugFlags=
-#JobCompHost={{ pillar['slurm']['controller'] }}
+DebugFlags=Gres
+#JobCompHost=biomedia03
 #JobCompLoc=
-#JobCompUser={{ pillar['slurm']['db']['user'] }}
-#JobCompPass={{ pillar['slurm']['db']['password'] }}
+#JobCompUser=slurm
+#JobCompPass=1BUy4eVv7X
 #JobCompPort=
 JobCompType=jobcomp/none
 JobAcctGatherFrequency=30
@@ -164,6 +165,7 @@ SlurmSchedLogFile=/var/log/slurm-llnl/sched.log
 #
 #
 # GRes configuration
+# GresTypes=gpu
 GresTypes={{ ','.join(pillar['slurm']['gres']) }}
 # COMPUTE NODES
 {% for node, values in pillar['slurm']['nodes']['batch']['cpus'].items() %}
@@ -177,13 +179,12 @@ NodeName={{ node }} RealMemory={{ values.mem }} CPUs={{ values.cores }} Gres={{
 {% endfor %}
 # Partitions
 PartitionName=long Nodes={{ ','.join(pillar['slurm']['nodes']['batch']['cpus']) }} Default=YES MaxTime=43200
-PartitionName=short Nodes={{ ','.join(pillar['slurm']['nodes']['batch']['cpus']) }} Default=NO MaxTime=60 Priority=5000
-PartitionName=gpus  Nodes={{ ','.join(pillar['slurm']['nodes']['batch']['gpus']) }} Default=NO MaxTime=10080
-PartitionName=interactive Nodes={{ ','.join(pillar['slurm']['nodes']['interactive']['cpus']) }} Default=NO MaxTime=4320 Priority=7000 PreemptMode=OFF
+#PartitionName=short Nodes={{ ','.join(pillar['slurm']['nodes']['batch']['cpus']) }} Default=NO MaxTime=60 Priority=5000
+PartitionName=gpus  Nodes={{ ','.join(pillar['slurm']['nodes']['batch']['gpus']) }} Default=NO MaxTime=10080 MaxCPUsPerNode=4 MaxMemPerNode=30720
+#PartitionName=interactive Nodes={{ ','.join(pillar['slurm']['nodes']['interactive']['cpus']) }} Default=NO MaxTime=4320 Priority=7000 PreemptMode=OFF
 
-{% set rocsList = [] %}
-{% for node, values in pillar['slurm']['nodes']['batch']['cpus'].items() %} {% if node.startswith('roc') %} {% set rocsListTrash = rocsList.append(node) %} {% endif %} {% endfor %}
-
-PartitionName=rocsLong  Nodes={{ ','.join(rocsList) }} Default=NO MaxTime=43200
-PartitionName=rocsShort  Nodes={{ ','.join(rocsList) }} Default=NO MaxTime=60 Priority=5000
+#{% set rocsList = [] %}
+#{% for node, values in pillar['slurm']['nodes']['batch']['cpus'].items() %} {% if node.startswith('roc') %} {% set rocsListTrash = rocsList.append(node) %} {% endif %} {% endfor %}
 
+#PartitionName=rocsLong  Nodes={{ ','.join(rocsList) }} Default=NO MaxTime=43200
+#PartitionName=rocsShort  Nodes={{ ','.join(rocsList) }} Default=NO MaxTime=60 Priority=5000

files/etc/slurm-llnl/slurmdbd.conf

@@ -6,7 +6,7 @@ ArchiveSuspend=no
 #ArchiveScript=/usr/sbin/slurm.dbd.archive
 #AuthInfo=/var/run/munge/munge.socket.2
 AuthType=auth/munge
-DbdHost={{ pillar['slurm']['controller'] }}
+DbdHost=biomedia03
 DbdPort=6819
 DebugLevel=info
 PurgeEventAfter=1month
@@ -16,10 +16,10 @@ PurgeSuspendAfter=1month
 LogFile=/var/log/slurm-llnl/slurmdbd.log
 PidFile=/var/run/slurm-llnl/slurmdbd.pid
 SlurmUser=slurm
-#StorageHost={{ pillar['slurm']['controller'] }}
+#StorageHost=biomedia03
 StorageHost=localhost
 StorageType=accounting_storage/mysql
 StoragePort=3306
-StorageLoc={{ pillar['slurm']['db']['name'] }}
-StorageUser={{ pillar['slurm']['db']['user'] }}
-StoragePass={{ pillar['slurm']['db']['password'] }}
+StorageLoc=slurmdb
+StorageUser=slurm
+StoragePass=1BUy4eVv7X

get_slurm_ver.sls

@@ -0,0 +1,3 @@
+get slurmd ver:
+  cmd.run:
+    - name: dpkg -s slurmd |grep "^Version:" > /tmp/local_slurm_ver.txt

files/etc/slurm-llnl/monal02/gres.conf → gres.conf_gpu

@@ -1,6 +1,8 @@
 Name=gpu File=/dev/nvidia0
 Name=gpu File=/dev/nvidia1
+Name=gpu File=/dev/nvidia2
 Name=gpu File=/dev/nvidia3
 Name=gpu File=/dev/nvidia4
 Name=gpu File=/dev/nvidia5
+Name=gpu File=/dev/nvidia6
 Name=gpu File=/dev/nvidia7