Skip to content

Commit 2991b03

Browse files
abcdvvvvabcdvvvv
committed
modified: .github/workflows/Benchmarks.yml
modified: benchmark/runbench.jl
1 parent c6d56fc commit 2991b03

2 files changed

Lines changed: 199 additions & 35 deletions

File tree

‎.github/workflows/Benchmarks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Benchmarks
22

33
env:
4-
BENCHLEDGER_VERSION: "0.6.0"
4+
BENCHLEDGER_VERSION: "0.6.1"
55
TARGET_REPOSITORY: ""
66
TARGET_REF: ""
77
BENCH_DB_BRANCH: "gh-pages"

‎benchmark/runbench.jl‎

Lines changed: 198 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,120 @@ function detect_kernel_version()
127127
return ""
128128
end
129129

130+
function loaded_module_by_name(name::Symbol)
131+
try
132+
for _module in values(Base.loaded_modules)
133+
nameof(_module) == name && return _module
134+
end
135+
catch
136+
end
137+
nothing
138+
end
139+
140+
function module_version_string(_module::Module)
141+
try
142+
version = Base.pkgversion(_module)
143+
version === nothing ? "" : string(version)
144+
catch
145+
""
146+
end
147+
end
148+
149+
function module_call_version(_module::Module, name::Symbol)
150+
isdefined(_module, name) || return ""
151+
try
152+
string(getfield(_module, name)())
153+
catch
154+
""
155+
end
156+
end
157+
158+
function detect_nvidia_gpu()
159+
output = try
160+
readchomp(`nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader,nounits`)
161+
catch
162+
return (gpus=Dict{String,Any}[], driver_version="")
163+
end
164+
isempty(strip(output)) && return (gpus=Dict{String,Any}[], driver_version="")
165+
166+
counts = Dict{Tuple{String,Int},Int}()
167+
driver_versions = Set{String}()
168+
for line in split(output, '\n'; keepempty=false)
169+
fields = strip.(split(line, ','; limit=3))
170+
length(fields) >= 2 || continue
171+
model = fields[1]
172+
memory_mib = tryparse(Int, fields[2])
173+
memory_mib === nothing && continue
174+
key = (model, memory_mib * 1024^2)
175+
counts[key] = get(counts, key, 0) + 1
176+
length(fields) == 3 && !isempty(fields[3]) && push!(driver_versions, fields[3])
177+
end
178+
179+
gpus = Dict{String,Any}[]
180+
for (model, memory_bytes) in sort!(collect(keys(counts)); by=x -> (x[1], x[2]))
181+
push!(gpus, Dict{String,Any}(
182+
"vendor" => "NVIDIA",
183+
"model" => model,
184+
"memory_bytes" => memory_bytes,
185+
"count" => counts[(model, memory_bytes)],
186+
))
187+
end
188+
driver_version = isempty(driver_versions) ? "" : first(sort!(collect(driver_versions)))
189+
(; gpus, driver_version)
190+
end
191+
192+
function visible_gpu_count(detected_count::Int)
193+
for name in ("CUDA_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES")
194+
haskey(ENV, name) || continue
195+
raw = strip(ENV[name])
196+
isempty(raw) && return 0
197+
lowercase(raw) in ("-1", "none", "nodevfiles") && return 0
198+
devices = filter(!isempty, strip.(split(raw, ','; keepempty=false)))
199+
return length(devices)
200+
end
201+
detected_count > 0 ? detected_count : nothing
202+
end
203+
204+
function detect_gpu_interface()
205+
for (module_name, display_name) in ((:CUDA, "CUDA.jl"), (:AMDGPU, "AMDGPU.jl"), (:Metal, "Metal.jl"), (:oneAPI, "oneAPI.jl"))
206+
_module = loaded_module_by_name(module_name)
207+
_module === nothing && continue
208+
interface = Dict{String,Any}("name" => display_name)
209+
version = module_version_string(_module)
210+
!isempty(version) && (interface["version"] = version)
211+
return interface
212+
end
213+
Dict{String,Any}()
214+
end
215+
216+
function detect_gpu_runtime(nvidia_driver_version::AbstractString)
217+
runtime = Dict{String,Any}()
218+
cuda = loaded_module_by_name(:CUDA)
219+
amdgpu = loaded_module_by_name(:AMDGPU)
220+
221+
if cuda !== nothing
222+
runtime["backend"] = "CUDA"
223+
driver_version = module_call_version(cuda, :driver_version)
224+
isempty(driver_version) && (driver_version = String(nvidia_driver_version))
225+
!isempty(driver_version) && (runtime["driver"] = Dict{String,Any}("version" => driver_version))
226+
227+
runtime_version = module_call_version(cuda, :runtime_version)
228+
runtime_info = Dict{String,Any}("name" => "CUDA")
229+
!isempty(runtime_version) && (runtime_info["version"] = runtime_version)
230+
runtime["runtime"] = runtime_info
231+
elseif amdgpu !== nothing
232+
runtime["backend"] = "ROCm"
233+
runtime_info = Dict{String,Any}("name" => "ROCm")
234+
runtime_version = module_call_version(amdgpu, :runtime_version)
235+
!isempty(runtime_version) && (runtime_info["version"] = runtime_version)
236+
runtime["runtime"] = runtime_info
237+
elseif !isempty(nvidia_driver_version)
238+
runtime["driver"] = Dict{String,Any}("version" => String(nvidia_driver_version))
239+
end
240+
241+
runtime
242+
end
243+
130244
function normalize_code_state_id(value::AbstractString)
131245
startswith(value, "code-") && return String(value)
132246
startswith(value, "local+") && return string("code-local-", replace(String(value), "local+" => ""; count=1))
@@ -147,13 +261,17 @@ function detect_code_state_label(commit::AbstractString)
147261
isempty(commit) ? "local" : first(commit, min(7, ncodeunits(commit)))
148262
end
149263

150-
function merge_metadata!(metadata::AbstractDict, override::AbstractDict)
264+
function merge_metadata!(metadata::AbstractDict, override::AbstractDict; path::AbstractString="")
151265
for (key, value) in pairs(override)
152266
key_string = String(key)
153-
if haskey(metadata, key_string) && metadata[key_string] isa AbstractDict && value isa AbstractDict
154-
merge_metadata!(metadata[key_string], value)
155-
else
267+
key_path = isempty(path) ? key_string : string(path, ".", key_string)
268+
269+
if !haskey(metadata, key_string)
156270
metadata[key_string] = value
271+
elseif metadata[key_string] isa AbstractDict && value isa AbstractDict
272+
merge_metadata!(metadata[key_string], value; path=key_path)
273+
elseif metadata[key_string] != value
274+
error("Conflicting metadata value at $(key_path): existing=$(repr(metadata[key_string])), new=$(repr(value)).")
157275
end
158276
end
159277
metadata
@@ -237,45 +355,52 @@ function make_environment()
237355
cpu_model = detect_cpu_model()
238356
os_release = detect_os_release()
239357
kernel_version = detect_kernel_version()
358+
nvidia_gpu = detect_nvidia_gpu()
240359

241360
os_identity = Dict{String,Any}("name" => os_release.name)
242361
!isempty(os_release.version) && (os_identity["version"] = os_release.version)
362+
kernel_identity = Dict{String,Any}("name" => lowercase(string(Sys.KERNEL)))
363+
!isempty(kernel_version) && (kernel_identity["version"] = kernel_version)
243364
runtime_identity = Dict{String,Any}("name" => "Julia", "version" => string(VERSION))
365+
366+
hardware_identity = Dict{String,Any}(
367+
"cpu" => Dict{String,Any}(
368+
"model" => cpu_model,
369+
"logical_threads" => Sys.CPU_THREADS,
370+
),
371+
)
372+
!isempty(nvidia_gpu.gpus) && (hardware_identity["gpu"] = nvidia_gpu.gpus)
373+
374+
execution_identity = Dict{String,Any}(
375+
"processes" => 1,
376+
"threads" => Threads.nthreads(),
377+
)
378+
detected_gpu_count = sum(Int(gpu["count"]) for gpu in nvidia_gpu.gpus; init=0)
379+
visible_gpus = visible_gpu_count(detected_gpu_count)
380+
visible_gpus === nothing || (execution_identity["gpu_devices"] = Dict{String,Any}("visible" => visible_gpus))
381+
244382
identity = Dict{String,Any}(
245383
"runtime" => runtime_identity,
246384
"platform" => Dict{String,Any}(
247385
"os" => os_identity,
386+
"kernel" => kernel_identity,
248387
"architecture" => string(Sys.ARCH),
249388
),
250-
"hardware" => Dict{String,Any}(
251-
"cpu" => Dict{String,Any}(
252-
"model" => cpu_model,
253-
"logical_threads" => Sys.CPU_THREADS,
254-
),
255-
),
256-
"execution" => Dict{String,Any}(
257-
"processes" => 1,
258-
"threads" => Threads.nthreads(),
259-
),
389+
"hardware" => hardware_identity,
390+
"execution" => execution_identity,
260391
)
392+
gpu_runtime = detect_gpu_runtime(nvidia_gpu.driver_version)
393+
!isempty(gpu_runtime) && (identity["gpu_runtime"] = gpu_runtime)
394+
395+
# hardware.gpu, hardware.tpu, and hardware.npu are optional identity fields.
396+
# TPU, NPU, and non-NVIDIA GPU details can be supplied through BENCH_ENVIRONMENT.identity.
261397
merge_metadata!(identity, object_field(override, "identity", "BENCH_ENVIRONMENT"))
262398

263-
kernel_metadata = Dict{String,Any}("name" => lowercase(string(Sys.KERNEL)))
264-
!isempty(kernel_version) && (kernel_metadata["version"] = kernel_version)
265-
metadata = Dict{String,Any}(
266-
"benchmark" => Dict{String,Any}(
267-
"framework" => Dict{String,Any}(
268-
"name" => "BenchmarkTools.jl",
269-
"version" => string(Base.pkgversion(BenchmarkTools)),
270-
),
271-
),
272-
"platform" => Dict{String,Any}("kernel" => kernel_metadata),
273-
)
274-
merge_metadata!(metadata, object_field(override, "metadata", "BENCH_ENVIRONMENT"))
399+
metadata = object_field(override, "metadata", "BENCH_ENVIRONMENT")
275400

276401
identity_json = canonical_json(identity)
277402
id = string("env-", bytes2hex(sha256(codeunits(identity_json))))
278-
label = haskey(override, "label") ? String(override["label"]) : gethostname()
403+
label = haskey(override, "label") ? String(override["label"]) : string(cpu_model, " / Julia ", VERSION, " / ", Threads.nthreads(), " threads")
279404
(; id, label, identity=identity_json, metadata=canonical_json(metadata))
280405
end
281406

@@ -285,11 +410,22 @@ function make_run_context(source, code_state, environment, measured_at::Abstract
285410
!isempty(source.branch) && (source_metadata["branch"] = source.branch)
286411
!isempty(source.tags) && (source_metadata["tags"] = source.tags)
287412
metadata = Dict{String,Any}(
413+
"benchmark" => Dict{String,Any}(
414+
"framework" => Dict{String,Any}(
415+
"name" => "BenchmarkTools.jl",
416+
"version" => string(Base.pkgversion(BenchmarkTools)),
417+
),
418+
),
419+
"host" => Dict{String,Any}(
420+
"hostname" => gethostname(),
421+
),
288422
"writer" => Dict{String,Any}(
289423
"name" => "BenchLedger Julia template",
290424
"version" => Benchledger_Schema_Version,
291425
),
292426
)
427+
gpu_interface = detect_gpu_interface()
428+
!isempty(gpu_interface) && (metadata["gpu"] = Dict{String,Any}("interface" => gpu_interface))
293429
!isempty(source_metadata) && (metadata["source"] = source_metadata)
294430
merge_metadata!(metadata, object_field(override, "metadata", "BENCH_RUN"))
295431
return (
@@ -542,7 +678,9 @@ function split_v4_environment_metadata(metadata::AbstractDict)
542678
platform_identity = Dict{String,Any}()
543679
if platform isa AbstractDict
544680
os_identity = take_child_fields!(platform, "os", ("name", "version"))
681+
kernel_identity = take_child_fields!(platform, "kernel", ("name", "version"))
545682
!isempty(os_identity) && (platform_identity["os"] = os_identity)
683+
!isempty(kernel_identity) && (platform_identity["kernel"] = kernel_identity)
546684
haskey(platform, "architecture") && (platform_identity["architecture"] = pop!(platform, "architecture"))
547685
isempty(platform) && delete!(remaining, "platform")
548686
end
@@ -552,17 +690,34 @@ function split_v4_environment_metadata(metadata::AbstractDict)
552690
if hardware isa AbstractDict
553691
cpu_identity = take_child_fields!(hardware, "cpu", ("model", "logical_threads"))
554692
!isempty(cpu_identity) && (hardware_identity["cpu"] = cpu_identity)
693+
haskey(hardware, "gpu") && (hardware_identity["gpu"] = pop!(hardware, "gpu"))
694+
haskey(hardware, "tpu") && (hardware_identity["tpu"] = pop!(hardware, "tpu"))
695+
haskey(hardware, "npu") && (hardware_identity["npu"] = pop!(hardware, "npu"))
555696
isempty(hardware) && delete!(remaining, "hardware")
556697
end
557698

558-
execution_identity = take_child_fields!(remaining, "execution", ("processes", "threads"))
699+
execution_identity = take_child_fields!(remaining, "execution", ("processes", "threads", "gpu_devices", "tpu_devices", "npu_devices"))
700+
gpu_runtime_identity = haskey(remaining, "gpu_runtime") ? pop!(remaining, "gpu_runtime") : nothing
701+
702+
run_metadata = Dict{String,Any}()
703+
benchmark = get(remaining, "benchmark", nothing)
704+
if benchmark isa AbstractDict && haskey(benchmark, "framework")
705+
run_metadata["benchmark"] = Dict{String,Any}("framework" => pop!(benchmark, "framework"))
706+
isempty(benchmark) && delete!(remaining, "benchmark")
707+
end
708+
gpu = get(remaining, "gpu", nothing)
709+
if gpu isa AbstractDict && haskey(gpu, "interface")
710+
run_metadata["gpu"] = Dict{String,Any}("interface" => pop!(gpu, "interface"))
711+
isempty(gpu) && delete!(remaining, "gpu")
712+
end
559713

560714
identity = Dict{String,Any}()
561715
!isempty(runtime_identity) && (identity["runtime"] = runtime_identity)
562716
!isempty(platform_identity) && (identity["platform"] = platform_identity)
563717
!isempty(hardware_identity) && (identity["hardware"] = hardware_identity)
564718
!isempty(execution_identity) && (identity["execution"] = execution_identity)
565-
identity, remaining
719+
gpu_runtime_identity !== nothing && (identity["gpu_runtime"] = gpu_runtime_identity)
720+
identity, remaining, run_metadata
566721
end
567722

568723
function migrate_v4_code_state_id(old_id::AbstractString)
@@ -588,6 +743,11 @@ function migrate_v4_to_v5!(db::SQLite.DB, path::AbstractString)
588743
validate_v4_layout!(db, path)
589744
println("Migrating BenchLedger database from schema v4 to v5: $(path)")
590745

746+
backup_path = string(path, "_v4")
747+
SQLite.execute(db, "PRAGMA wal_checkpoint(FULL)")
748+
cp(path, backup_path; force=true)
749+
println("Backed up schema v4 database to: $(backup_path)")
750+
591751
SQLite.execute(db, "PRAGMA foreign_keys=OFF")
592752
SQLite.execute(db, "BEGIN IMMEDIATE TRANSACTION")
593753
try
@@ -611,24 +771,29 @@ function migrate_v4_to_v5!(db::SQLite.DB, path::AbstractString)
611771
end
612772

613773
environment_id_map = Dict{String,String}()
774+
environment_run_metadata = Dict{String,Dict{String,Any}}()
614775
for row in DBInterface.execute(db, "SELECT environment_id, label, metadata FROM benchmark_environments_v4")
615776
old_id = String(row.environment_id)
616777
parsed_metadata = JSON.parse(String(row.metadata); dicttype=Dict{String,Any})
617-
identity, metadata = split_v4_environment_metadata(parsed_metadata)
778+
identity, metadata, run_metadata = split_v4_environment_metadata(parsed_metadata)
618779
identity_json = canonical_json(identity)
619780
new_id = string("env-", bytes2hex(sha256(codeunits(identity_json))))
620781
persist_labeled_entity!(db, "benchmark_environments", new_id, identity_json, canonical_json(metadata), String(row.label))
621782
environment_id_map[old_id] = new_id
783+
run_metadata["host"] = Dict{String,Any}("hostname" => String(row.label))
784+
environment_run_metadata[old_id] = run_metadata
622785
end
623786

624787
for row in DBInterface.execute(db, "SELECT run_id, code_state_id, environment_id, measured_at, notes, metadata FROM benchmark_runs_v4")
625788
old_code_state_id = String(row.code_state_id)
626789
old_environment_id = String(row.environment_id)
627790
haskey(code_state_id_map, old_code_state_id) || error("Missing migrated code state for $(old_code_state_id).")
628791
haskey(environment_id_map, old_environment_id) || error("Missing migrated environment for $(old_environment_id).")
792+
run_metadata = deepcopy(environment_run_metadata[old_environment_id])
793+
merge_metadata!(run_metadata, JSON.parse(String(row.metadata); dicttype=Dict{String,Any}))
629794
DBInterface.execute(db,
630795
"INSERT INTO benchmark_runs (id, code_state_id, environment_id, measured_at, notes, metadata) VALUES (?, ?, ?, ?, ?, ?)",
631-
(String(row.run_id), code_state_id_map[old_code_state_id], environment_id_map[old_environment_id], String(row.measured_at), String(row.notes), String(row.metadata)))
796+
(String(row.run_id), code_state_id_map[old_code_state_id], environment_id_map[old_environment_id], String(row.measured_at), String(row.notes), canonical_json(run_metadata)))
632797
end
633798

634799
SQLite.execute(db,
@@ -799,9 +964,8 @@ function persist_labeled_entity!(db::SQLite.DB, table::AbstractString, id_value:
799964
row_iter = iterate(result)
800965
row_iter === nothing && return nothing
801966
value = row_iter[1]
802-
code_date === nothing ?
803-
(label=String(value.label), identity=String(value.identity), metadata=String(value.metadata)) :
804-
(label=String(value.label), code_date=String(value.code_date), identity=String(value.identity), metadata=String(value.metadata))
967+
code_date === nothing ? (label=String(value.label), identity=String(value.identity), metadata=String(value.metadata)) :
968+
(label=String(value.label), code_date=String(value.code_date), identity=String(value.identity), metadata=String(value.metadata))
805969
end
806970
row === nothing && error("Failed to persist $(table) $(id_value).")
807971
code_date === nothing || row.code_date == code_date || error("Conflicting code_date for id=$(id_value) in $(table).")

0 commit comments

Comments
 (0)