Skip to content

Commit 059b925

Browse files
fpjentzschfpjentzsch
authored
Merge pull request #100 from eki-project/ci/driver_rework
2 parents c4e3f53 + 4846472 commit 059b925

31 files changed

+2521
-2205
lines changed

‎ci/.gitlab-bench.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ Measurement:
5959
script:
6060
# Build power measurement driver (TODO: support other boards)
6161
- make -C ci/power_measurement/boards/rfsoc2x2 shared
62+
- make -C ci/power_measurement/boards/rfsoc4x2 shared
63+
# Install dependencies for FINN driver
64+
- sudo bash -c "source /etc/profile.d/pynq_venv.sh && chmod +x scripts/driver_install_dependencies.sh && scripts/driver_install_dependencies.sh"
6265
# Run as root and activate the PYNQ venv manually to use PYNQ outside of the typical Jupyter environment
6366
- sudo bash -c "source /etc/profile.d/pynq_venv.sh && export XILINX_XRT=/usr && python ci/measure.py"
6467
artifacts:

‎ci/cfg/regression_basic.yml‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,32 @@
22
# VGG-10 (1D CNN) (RadioML)
33
{
44
"dut": ["vgg10"],
5-
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]]
5+
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]],
6+
"enable_instrumentation": [False],
7+
"board": ["RFSoC2x2"],
8+
"experiments_config": ["../experiments/driver_rework_ex_config_dma.json"],
69
},
7-
8-
# MobileNetV1 (ImageNet)
910
{
10-
"dut": ["mobilenetv1"],
11-
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]]
11+
"dut": ["vgg10"],
12+
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]],
13+
"enable_instrumentation": [True],
14+
"instrumentation_no_dma": [True],
15+
"board": ["RFSoC2x2"],
16+
"experiments_config": ["../experiments/driver_rework_ex_config_instr.json"],
1217
},
13-
14-
# Cybersecurity (MLP)
1518
{
16-
"dut": ["cybsec"],
17-
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]]
19+
"dut": ["vgg10"],
20+
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]],
21+
"enable_instrumentation": [True],
22+
"live_fifo_sizing": [True],
23+
"board": ["RFSoC2x2"],
24+
"experiments_config": ["../experiments/driver_rework_ex_config_fifo.json"],
1825
},
26+
{
27+
"dut": ["vgg10"],
28+
"generate_outputs": [["estimate_reports", "bitfile", "pynq_driver", "deployment_package"]],
29+
"enable_instrumentation": [True],
30+
"board": ["RFSoC2x2"],
31+
"experiments_config": ["../experiments/driver_rework_ex_config_instr_dma.json"]
32+
}
1933
]

‎ci/collect.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
1+
"""Collect and log benchmark results to DVC."""
2+
13
import json
24
import os
35
import shutil
46
import sys
57
from datetime import date
6-
from dvclive.live import Live
8+
from dvclive import Live
79

810

911
def delete_dir_contents(dir):
12+
"""Delete all contents of a directory."""
1013
for filename in os.listdir(dir):
1114
file_path = os.path.join(dir, filename)
1215
try:
@@ -19,6 +22,7 @@ def delete_dir_contents(dir):
1922

2023

2124
def open_json_report(id, report_name):
25+
"""Open JSON report from build or measurement artifacts."""
2226
# look in both, build & measurement, artifacts
2327
path1 = os.path.join("build_artifacts", "runs_output", "run_%d" % (id), "reports", report_name)
2428
path2 = os.path.join(
@@ -38,7 +42,10 @@ def open_json_report(id, report_name):
3842

3943
# Wrapper around DVC Live object
4044
class DVCLoggerHelper:
45+
"""Wrapper around DVC Live for logging experiments."""
46+
4147
def __init__(self, experiment_name, experiment_msg, id, params):
48+
"""Initialize DVC logger with experiment details."""
4249
self.id = id
4350

4451
# extract logging settings from params
@@ -62,20 +69,24 @@ def __init__(self, experiment_name, experiment_msg, id, params):
6269
self.log_params(params)
6370

6471
def __enter__(self):
72+
"""Enter context manager."""
6573
return self
6674

6775
def __exit__(self, exc_type, exc_value, traceback):
76+
"""Exit context manager and end DVC session."""
6877
if self.store_as_experiment:
6978
# End DVC Live experiment session
7079
self.live.end()
7180

7281
def log_params(self, params):
82+
"""Log parameters to DVC."""
7383
if self.store_as_experiment:
7484
self.live.log_params(params)
7585
if self.store_as_data:
7686
self.data_dict.update(params)
7787

7888
def log_metric(self, prefix, name, value):
89+
"""Log a single metric to DVC."""
7990
# sanitize '/' in name because DVC uses it to nest metrics (which we do via prefix)
8091
name = name.replace("/", "-")
8192

@@ -95,27 +106,32 @@ def log_metric(self, prefix, name, value):
95106
_dict[name] = value
96107

97108
def log_image(self, image_name, image_path):
109+
"""Log an image artifact to DVC."""
98110
if self.store_as_experiment:
99111
self.live.log_image(image_name, image_path)
100112

101113
def log_artifact(self, artifact_path):
114+
"""Log an artifact to DVC."""
102115
if self.store_as_experiment:
103116
self.live.log_artifact(artifact_path)
104117

105118
def log_all_metrics_from_report(self, report_name, prefix=""):
119+
"""Log all metrics from a JSON report."""
106120
report = open_json_report(self.id, report_name)
107121
if report:
108122
for key in report:
109123
self.log_metric(prefix, key, report[key])
110124

111125
def log_metrics_from_report(self, report_name, keys, prefix=""):
126+
"""Log specific metrics from a JSON report."""
112127
report = open_json_report(self.id, report_name)
113128
if report:
114129
for key in keys:
115130
if key in report:
116131
self.log_metric(prefix, key, report[key])
117132

118133
def log_nested_metrics_from_report(self, report_name, key_top, keys, prefix=""):
134+
"""Log nested metrics from a JSON report."""
119135
report = open_json_report(self.id, report_name)
120136
if report:
121137
if key_top in report:
@@ -125,7 +141,10 @@ def log_nested_metrics_from_report(self, report_name, key_top, keys, prefix=""):
125141

126142

127143
if __name__ == "__main__":
128-
# Go through all runs found in the artifacts and log their results to DVC
144+
"""Go through all runs found in the artifacts and log their results to DVC."""
145+
print("FIXME: collect.py")
146+
sys.exit(0)
147+
129148
run_dir_list = os.listdir(os.path.join("build_artifacts", "runs_output"))
130149
print("Looking for runs in build artifacts")
131150
run_ids = []

‎ci/experiments/driver_rework_ex_config_dma.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"global": {
3+
"driver": "driver/driver.py",
4+
"bitfile_name": "bitfile/finn-accel.bit",
5+
"report_path": "report",
6+
7+
"PAF": {
8+
"rails": [
9+
"0V85",
10+
"3V3"
11+
],
12+
"sensors": [],
13+
"board": "rfsoc2x2"
14+
}
15+
},
16+
"experiments": [
17+
{
18+
"name": "idle_experiment",
19+
"warmup": 1,
20+
"num_runs": 1,
21+
"functions": [
22+
{
23+
"function_name": "idle",
24+
"kwargs" : {"time" : 10}
25+
}
26+
]
27+
},
28+
{
29+
"name": "throughput_experiment",
30+
"warmup": 1,
31+
"num_runs": 1,
32+
"functions": [
33+
{
34+
"function_name": "run_throughput_test",
35+
"kwargs" : {}
36+
}
37+
]
38+
},
39+
{
40+
"name": "validate_experiment",
41+
"warmup": 1,
42+
"num_runs": 1,
43+
"functions": [
44+
{
45+
"function_name": "validate",
46+
"kwargs" : {
47+
"dataset_root" : "/home/xilinx/datasets",
48+
"dataset" : "mnist"
49+
}
50+
}
51+
]
52+
}
53+
]
54+
}

‎ci/experiments/driver_rework_ex_config_fifo.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"global": {
3+
"driver": "driver/driver.py",
4+
"bitfile_name": "bitfile/finn-accel.bit",
5+
"report_path": "report",
6+
7+
"PAF": {
8+
"rails": [
9+
"0V85",
10+
"3V3"
11+
],
12+
"sensors": [],
13+
"board": "rfsoc2x2"
14+
}
15+
},
16+
"experiments": [
17+
{
18+
"name": "experiment_fifosizing",
19+
"warmup": 1,
20+
"num_runs": 1,
21+
"functions": [
22+
{
23+
"function_name": "experiment_fifosizing",
24+
"kwargs" : {}
25+
}
26+
]
27+
}
28+
]
29+
}

‎ci/experiments/driver_rework_ex_config_instr.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"global": {
3+
"driver": "driver/driver.py",
4+
"bitfile_name": "bitfile/finn-accel.bit",
5+
"report_path": "report",
6+
7+
"PAF": {
8+
"rails": [
9+
"0V85",
10+
"3V3"
11+
],
12+
"sensors": [],
13+
"board": "rfsoc2x2"
14+
}
15+
},
16+
"experiments": [
17+
{
18+
"name": "idle_experiment",
19+
"warmup": 1,
20+
"num_runs": 1,
21+
"functions": [
22+
{
23+
"function_name": "idle",
24+
"kwargs" : {"time" : 10}
25+
}
26+
]
27+
},
28+
{
29+
"name": "instrumentation_experiment",
30+
"warmup": 1,
31+
"num_runs": 1,
32+
"functions": [
33+
{
34+
"function_name": "experiment_instrumentation",
35+
"kwargs" : {"runtime": 20}
36+
}
37+
]
38+
}
39+
]
40+
}

‎ci/experiments/driver_rework_ex_config_instr_dma.json‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
{
2+
"global": {
3+
"driver": "driver/driver.py",
4+
"bitfile_name": "bitfile/finn-accel.bit",
5+
"report_path": "report",
6+
7+
"PAF": {
8+
"rails": [
9+
"0V85",
10+
"3V3"
11+
],
12+
"sensors": [],
13+
"board": "rfsoc2x2"
14+
}
15+
},
16+
"experiments": [
17+
{
18+
"name": "throughput_experiment",
19+
"warmup": 1,
20+
"num_runs": 1,
21+
"functions": [
22+
{
23+
"function_name": "throughput_test",
24+
"kwargs" : {}
25+
}
26+
]
27+
},
28+
{
29+
"name": "instrumentation_experiment",
30+
"warmup": 1,
31+
"num_runs": 1,
32+
"functions": [
33+
{
34+
"function_name": "experiment_instrumentation",
35+
"kwargs" : {"runtime": 20}
36+
}
37+
]
38+
},
39+
{
40+
"name": "throughput_experiment_2",
41+
"warmup": 1,
42+
"num_runs": 1,
43+
"functions": [
44+
{
45+
"function_name": "throughput_test",
46+
"kwargs" : {}
47+
}
48+
]
49+
}
50+
]
51+
}

0 commit comments

Comments
 (0)