Skip to content

Commit 5e32e02

Browse files
csziklaicsziklai
authored
Queues Compiler (#2457)
This PR adds a compiler that takes in a JSON file generated from the [Packet Scheduling repo](https://github.com/cucapra/packet-scheduling), which represent a Rio program. - The file `queues_compile` contains the logic for translating the JSON file into a Calyx program. - Six example tests have been included in `frontend/queues/tests/compiler` and a corresponding stanza has been added to runt. - The commands to generate the data and expect files can be found at the end of `gen_test_data.sh`.
1 parent fabf065 commit 5e32e02

26 files changed

+407
-12
lines changed

‎.gitignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ results.xml
5353

5454
!cider/dap/calyxDebug/package.json
5555
!cider/dap/calyxDebug/tsconfig.json
56+
!frontends/queues/tests/compiler/jsons/complex_tree.json
57+
!frontends/queues/tests/compiler/jsons/fifo.json
58+
!frontends/queues/tests/compiler/jsons/pifo_tree.json
59+
!frontends/queues/tests/compiler/jsons/rr_2flows.json
60+
!frontends/queues/tests/compiler/jsons/rr_union.json
61+
!frontends/queues/tests/compiler/jsons/strict_3flows.json
5662

5763
# btor2i ignore
5864
tools/btor2/btor2i/build/

‎frontends/queues/queues/compile_queue.py

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# pylint: disable=import-error
2+
import sys
3+
import calyx.builder as cb
4+
import queues.queue_call as qc
5+
import queues.strict_or_rr as strict_or_rr
6+
import queues.fifo as fifo
7+
import queues.flow_inference as fi
8+
import json
9+
import os
10+
11+
rr_id = 0
12+
strict_id = 0
13+
14+
15+
def create(data, lower, upper, prog, fifo_queue):
16+
"""
17+
Recursively creates the PIFO by crawling the json file `data`, which represents
18+
a Rio program. `lower' and `upper` are the bounds for which a flow is restricted
19+
to.
20+
"""
21+
global rr_id
22+
global strict_id
23+
assert isinstance(data, dict)
24+
for key, val in data.items():
25+
if key == "FIFO":
26+
return fifo_queue
27+
elif key == "RR" or key == "Strict":
28+
num_children = len(val)
29+
if num_children == 1:
30+
return create(val[0], lower, upper, prog, fifo_queue)
31+
else:
32+
interval = (upper - lower) // num_children
33+
boundaries = []
34+
for i in range(1, num_children):
35+
boundaries.append(lower + (interval * i))
36+
boundaries.append(upper)
37+
38+
children = []
39+
lo = lower
40+
u = upper
41+
for i in range(num_children):
42+
u = lo + interval
43+
if i == num_children - 1:
44+
u = upper
45+
children.append(create(val[i], lo, u, prog, fifo_queue))
46+
lo = u
47+
48+
if key == "RR":
49+
rr_id += 1
50+
return strict_or_rr.insert_queue(
51+
prog,
52+
f"pifo_rr{rr_id}",
53+
True,
54+
children,
55+
fi.insert_boundary_flow_inference(
56+
prog, f"fi_rr{rr_id}", boundaries
57+
),
58+
)
59+
elif key == "Strict":
60+
strict_id += 1
61+
return strict_or_rr.insert_queue(
62+
prog,
63+
f"pifo_strict{strict_id}",
64+
False,
65+
children,
66+
fi.insert_boundary_flow_inference(
67+
prog, f"fi_strict{strict_id}", boundaries
68+
),
69+
order=[n for n in range(num_children)],
70+
)
71+
72+
73+
def build(json_file):
74+
"""
75+
Top-level function to build the program. Requires `json_file` to be in the same
76+
directory as this program.
77+
"""
78+
num_cmds = int(sys.argv[1])
79+
keepgoing = "--keepgoing" in sys.argv
80+
prog = cb.Builder()
81+
82+
base_dir = os.path.dirname(__file__)
83+
json_subdir = "../tests/compiler/jsons"
84+
file_path = os.path.join(base_dir, json_subdir, json_file)
85+
with open(file_path) as f:
86+
data = json.load(f)
87+
88+
fifo_queue = fifo.insert_fifo(prog, "fifo")
89+
root = create(data, 0, 400, prog, fifo_queue)
90+
91+
qc.insert_main(prog, root, num_cmds, keepgoing=keepgoing)
92+
return prog.program
93+
94+
95+
if __name__ == "__main__":
96+
build().emit()

‎frontends/queues/test_data_gen/gen_test_data.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,16 @@ for n in {2..7}; do
112112
[[ $? -eq 0 ]] && echo "Generated strict/strict_${n}flow_test.expect"
113113
done
114114

115+
# Tests for a specific ordering different from the default
116+
python3 ${data_gen_dir}/gen_oracle_data.py $num_cmds \
117+
> ${tests_dir}/strict/strict_order_test.data
118+
[[ $? -eq 0 ]] && echo "Generated strict/strict_order_test.data"
119+
120+
cat ${tests_dir}/strict/strict_order_test.data \
121+
| python3 ${data_gen_dir}/strict_oracle.py $num_cmds $queue_size 3 --keepgoing --order 2,0,1 \
122+
> ${tests_dir}/strict/strict_order_test.expect
123+
[[ $? -eq 0 ]] && echo "Generated strict/strict_order_test.expect"
124+
115125

116126
# Copying into binheap/ for heap-based implementations of previous queues: namely,
117127
# - FIFO
@@ -144,3 +154,55 @@ for sched_algo in round_robin strict; do
144154
[[ $? -eq 0 ]] && echo "Generated binheap/${sched_algo}/${name}.expect"
145155
done
146156
done
157+
158+
# For the sample compiled programs (provided the corresponding JSON file exists)
159+
# - FIFO
160+
# - PIFO tree
161+
# - Round robin with 2 flows
162+
# - Strict with 3 flows
163+
# - Round robin union, which essentially behaves like a FIFO
164+
165+
python3 ${data_gen_dir}/gen_oracle_data.py $num_cmds \
166+
> ${tests_dir}/compiler/fifo_compile.data
167+
[[ $? -eq 0 ]] && echo "Generated compiler/fifo_compile.data"
168+
169+
cat ${tests_dir}/compiler/fifo_compile.data \
170+
| python3 ${data_gen_dir}/fifo_oracle.py $num_cmds $queue_size --keepgoing \
171+
> ${tests_dir}/compiler/fifo_compile.expect
172+
[[ $? -eq 0 ]] && echo "Generated compiler/fifo_compile.expect"
173+
174+
python3 ${data_gen_dir}/gen_oracle_data.py $num_cmds \
175+
> ${tests_dir}/compiler/pifo_tree_compile.data
176+
[[ $? -eq 0 ]] && echo "Generated compiler/pifo_tree_compile.data"
177+
178+
cat ${tests_dir}/compiler/pifo_tree_compile.data \
179+
| python3 ${data_gen_dir}/pifo_tree_oracle.py $num_cmds $queue_size --keepgoing \
180+
> ${tests_dir}/compiler/pifo_tree_compile.expect
181+
[[ $? -eq 0 ]] && echo "Generated compiler/pifo_tree_compile.expect"
182+
183+
python3 ${data_gen_dir}/gen_oracle_data.py $num_cmds \
184+
> ${tests_dir}/compiler/rr_compile.data
185+
[[ $? -eq 0 ]] && echo "Generated compiler/rr_compile.data"
186+
187+
cat ${tests_dir}/compiler/rr_compile.data \
188+
| python3 ${data_gen_dir}/rr_oracle.py $num_cmds $queue_size 2 --keepgoing \
189+
> ${tests_dir}/compiler/rr_compile.expect
190+
[[ $? -eq 0 ]] && echo "Generated compiler/rr_compile.expect"
191+
192+
python3 ${data_gen_dir}/gen_oracle_data.py $num_cmds \
193+
> ${tests_dir}/compiler/strict_compile.data
194+
[[ $? -eq 0 ]] && echo "Generated compiler/strict_compile.data"
195+
196+
cat ${tests_dir}/compiler/strict_compile.data \
197+
| python3 ${data_gen_dir}/strict_oracle.py $num_cmds $queue_size 3 --keepgoing \
198+
> ${tests_dir}/compiler/strict_compile.expect
199+
[[ $? -eq 0 ]] && echo "Generated compiler/strict_compile.expect"
200+
201+
python3 ${data_gen_dir}/gen_oracle_data.py $num_cmds \
202+
> ${tests_dir}/compiler/fifo_union_compile.data
203+
[[ $? -eq 0 ]] && echo "Generated compiler/fifo_union_compile.data"
204+
205+
cat ${tests_dir}/compiler/fifo_union_compile.data \
206+
| python3 ${data_gen_dir}/fifo_oracle.py $num_cmds $queue_size --keepgoing \
207+
> ${tests_dir}/compiler/fifo_union_compile.expect
208+
[[ $? -eq 0 ]] && echo "Generated compiler/fifo_union_compile.expect"

‎frontends/queues/test_data_gen/strict_oracle.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,29 +8,37 @@
88
if __name__ == "__main__":
99
num_cmds, len, numflows = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3])
1010
keepgoing = "--keepgoing" in sys.argv
11+
order_override = "--order" in sys.argv
1112
commands, values, _, _ = util.parse_json()
1213

1314
if numflows == 2:
1415
boundaries = [200, 400]
15-
order = [1, 0]
16+
order = [0, 1]
1617
elif numflows == 3:
1718
boundaries = [133, 266, 400]
18-
order = [1, 2, 0]
19+
order = [0, 1, 2]
1920
elif numflows == 4:
2021
boundaries = [100, 200, 300, 400]
21-
order = [3, 0, 2, 1]
22+
order = [0, 1, 2, 3]
2223
elif numflows == 5:
2324
boundaries = [80, 160, 240, 320, 400]
2425
order = [0, 1, 2, 3, 4]
2526
elif numflows == 6:
2627
boundaries = [66, 100, 200, 220, 300, 400]
27-
order = [3, 1, 5, 2, 4, 0]
28+
order = [0, 1, 2, 3, 4, 5]
2829
elif numflows == 7:
2930
boundaries = [50, 100, 150, 200, 250, 300, 400]
3031
order = [0, 1, 2, 3, 4, 5, 6]
3132
else:
3233
raise ValueError("Unsupported number of flows")
3334

35+
if order_override:
36+
# --order expects an argument where the ints are comma-separated,
37+
# for example "2,3,1,0" for [2, 3, 1, 0]
38+
order_idx = sys.argv.index("--order") + 1
39+
lst = sys.argv[order_idx]
40+
order = list(map(int, lst.split(",")))
41+
3442
subqueues = [queues.Fifo(len) for _ in range(numflows)]
3543

3644
# Our Strict queue orchestrates n subqueues. It takes in a list of

‎frontends/queues/tests/binheap/strict/strict_2flow_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
prog = cb.Builder()
1818

19-
order = [1, 0]
19+
order = [0, 1]
2020
if sim_pcap:
2121
flow_infer = fi.insert_tuple_flow_inference(prog, "flow_inference", NUMFLOWS)
2222
pifo = st.insert_binheap_strict(prog, "pifo", NUMFLOWS, order, flow_infer)

‎frontends/queues/tests/binheap/strict/strict_3flow_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
prog = cb.Builder()
1818

19-
order = [1, 2, 0]
19+
order = [0, 1, 2]
2020
if sim_pcap:
2121
flow_infer = fi.insert_tuple_flow_inference(prog, "flow_inference", NUMFLOWS)
2222
pifo = st.insert_binheap_strict(prog, "pifo", NUMFLOWS, order, flow_infer)

‎frontends/queues/tests/binheap/strict/strict_4flow_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
prog = cb.Builder()
1818

19-
order = [3, 0, 2, 1]
19+
order = [0, 1, 2, 3]
2020
if sim_pcap:
2121
flow_infer = fi.insert_tuple_flow_inference(prog, "flow_inference", NUMFLOWS)
2222
pifo = st.insert_binheap_strict(prog, "pifo", NUMFLOWS, order, flow_infer)

‎frontends/queues/tests/binheap/strict/strict_6flow_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
prog = cb.Builder()
1818

19-
order = [3, 1, 5, 2, 4, 0]
19+
order = [0, 1, 2, 3, 4, 5]
2020
if sim_pcap:
2121
flow_infer = fi.insert_tuple_flow_inference(prog, "flow_inference", NUMFLOWS)
2222
pifo = st.insert_binheap_strict(prog, "pifo", NUMFLOWS, order, flow_infer)

‎frontends/queues/tests/compiler/complex_tree_compile.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from queues.compile_queue import build
2+
3+
"""A test for the compiler-generated version of the same complex tree as complex_tree_test.py"""
4+
5+
if __name__ == "__main__":
6+
build("complex_tree.json").emit()

‎frontends/queues/tests/compiler/fifo_compile.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from queues.compile_queue import build
2+
3+
"""
4+
A test for the compiler-generated version of the same fifo as in fifo_test.py
5+
(and the rio program fifo_1_class_sugar.sched in the packet scheduling repo)
6+
"""
7+
8+
if __name__ == "__main__":
9+
build("fifo.json").emit()

‎frontends/queues/tests/compiler/fifo_union_compile.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from queues.compile_queue import build
2+
3+
"""
4+
A test for the compiler-generated version of the round robin with only one child
5+
that is just a fifo of the union of 2 classes. This is the Rio program rr_2_classes.sched
6+
in the packet scheduling repo, which is rr[fifo[union[A, B]]]).
7+
"""
8+
9+
if __name__ == "__main__":
10+
build("rr_union.json").emit()

‎frontends/queues/tests/compiler/jsons/complex_tree.json

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{
2+
"RR": [
3+
{
4+
"Strict": [
5+
{
6+
"FIFO": [
7+
"A"
8+
]
9+
},
10+
{
11+
"FIFO": [
12+
"B"
13+
]
14+
},
15+
{
16+
"FIFO": [
17+
"C"
18+
]
19+
}
20+
]
21+
},
22+
{
23+
"RR": [
24+
{
25+
"FIFO": [
26+
"D"
27+
]
28+
},
29+
{
30+
"FIFO": [
31+
"E"
32+
]
33+
},
34+
{
35+
"FIFO": [
36+
"F"
37+
]
38+
}
39+
]
40+
},
41+
{
42+
"Strict": [
43+
{
44+
"FIFO": [
45+
"G"
46+
]
47+
},
48+
{
49+
"FIFO": [
50+
"H"
51+
]
52+
}
53+
]
54+
}
55+
]
56+
}

‎frontends/queues/tests/compiler/jsons/fifo.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"FIFO": [
3+
"A"
4+
]
5+
}

‎frontends/queues/tests/compiler/jsons/pifo_tree.json

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"RR": [
3+
{
4+
"RR": [
5+
{
6+
"FIFO": [
7+
"RP"
8+
]
9+
},
10+
{
11+
"FIFO": [
12+
"RT"
13+
]
14+
}
15+
]
16+
},
17+
{
18+
"FIFO": [
19+
"B"
20+
]
21+
}
22+
]
23+
}

‎frontends/queues/tests/compiler/jsons/rr_2flows.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"RR": [
3+
{
4+
"FIFO": [
5+
"A"
6+
]
7+
},
8+
{
9+
"FIFO": [
10+
"B"
11+
]
12+
}
13+
]
14+
}

0 commit comments

Comments
 (0)