Skip to content
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion lib/ood_core/job/adapters/slurm.rb
Original file line number Diff line number Diff line change
Expand Up @@ -871,8 +871,18 @@ def get_state(st)
STATE_MAP.fetch(st, :undetermined)
end

# Helper to parse the memory string returned by Slurm
def parse_memory(mem_str)
# Convert MB to bytes
mem_str.to_i * 1024 * 1024
end

# Parse hash describing Slurm job status
def parse_job_info(v)
min_memory = nil
# per cpu or per node
memory_per = nil

allocated_nodes = parse_nodes(v[:node_list])
if allocated_nodes.empty?
if v[:scheduled_nodes] && v[:scheduled_nodes] != "(null)"
Expand All @@ -882,6 +892,20 @@ def parse_job_info(v)
end
end

if v[:min_memory] && !v[:min_memory].empty?
# Slurm uses per CPU memory if --mem-per-cpu
# or uses per node if --mem
if v[:min_memory].end_with?('c')
min_memory = parse_memory(v[:min_memory].chomp('c'))
# memory per CPU
memory_per = :cpu
else
min_memory = parse_memory(v[:min_memory])
# memory per node
memory_per = :node
end
end

Info.new(
id: v[:job_id],
status: get_state(v[:state_compact]),
Expand All @@ -897,7 +921,7 @@ def parse_job_info(v)
cpu_time: nil,
submission_time: v[:submit_time] ? Time.parse(v[:submit_time]) : nil,
dispatch_time: (v[:start_time].nil? || v[:start_time] == "N/A") ? nil : Time.parse(v[:start_time]),
native: v,
native: v.merge(min_memory: min_memory, memory_per: memory_per),
gpus: self.class.gpus_from_gres(v[:gres])
)
end
Expand Down
17 changes: 17 additions & 0 deletions lib/ood_core/job/info.rb
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,23 @@ def gpu?
gpus.positive?
end

# Compute the total memory being used by a job
# @return [Integer] total memory in use
def total_memory
# Ensure the scheduler has needed fields for computation
return nil unless native[:min_memory] && allocated_nodes&.any?

# Using the adapter, key off whether the job is CPU or node calculated
case native[:memory_per]
when :cpu
native[:min_memory] * procs
when :node
native[:min_memory] * allocated_nodes.count
else
nil
end
end

# The comparison operator
# @param other [#to_h] object to compare against
# @return [Boolean] whether objects are equivalent
Expand Down
50 changes: 50 additions & 0 deletions spec/job/info_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -215,4 +215,54 @@ def build_info(opts = {})
it { is_expected.to eq(OodCore::Job::Status.new(state: :running)) }
end
end

describe "#total_memory" do
context "when memory is per node" do
subject do
build_info(
native: {
min_memory: 1 * 1024 * 1024 * 1024, # 1 GiB in bytes
memory_per: :node
},
allocated_nodes: [
double(to_h: {name: "node1", procs: 2}),
double(to_h: {name: "node2", procs: 2})
]
)
end

it "calculates total memory based on node count" do
expect(subject.total_memory).to eq(2 * 1024 * 1024 * 1024) # 2 nodes * 1 GiB
end
end

context "when memory is per cpu" do
subject do
build_info(
native: {
min_memory: 2 * 1024 * 1024 * 1024, # 2 GiB in bytes
memory_per: :cpu
},
procs: 4
)
end

it "calculates total memory based on procs" do
expect(subject.total_memory).to eq(8 * 1024 * 1024 * 1024) # 4 procs * 2 GiB
end
end

context "when memory info is missing" do
subject do
build_info(
native: {},
allocated_nodes: [double(to_h: {name: "node1"})]
)
end

it "returns nil" do
expect(subject.total_memory).to be_nil
end
end
end
end
Loading