Skip to content

Commit db56bb8

Browse files
JCoymJCoym
committed
Add Python interface
1 parent 9f2e9c5 commit db56bb8

28 files changed

+1965
-1
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,6 @@ waf-extras/__pycache__/
2424
.lock-waf*
2525
.waf-*/
2626
.waf3-*/
27+
28+
# Python
29+
python/**/__pycache__/

‎Doxyfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ WARN_LOGFILE =
112112
#---------------------------------------------------------------------------
113113
# Configuration options related to the input files
114114
#---------------------------------------------------------------------------
115-
INPUT = include lib
115+
INPUT = include lib python
116116
INPUT_ENCODING = UTF-8
117117
FILE_PATTERNS = *.c \
118118
*.cc \

‎python/benchmark.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# JULEA - Flexible storage framework
2+
# Copyright (C) 2019 Johannes Coym
3+
#
4+
# This program is free software: you can redistribute it and/or modify
5+
# it under the terms of the GNU Lesser General Public License as published by
6+
# the Free Software Foundation, either version 3 of the License, or
7+
# (at your option) any later version.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU Lesser General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU Lesser General Public License
15+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
from julea.benchmark import (
18+
distributed_object_bench, item_bench, kv_bench, object_bench)
19+
from julea import juleadir
20+
import subprocess
21+
22+
setupfile = juleadir.joinpath('scripts', 'setup.sh')
23+
24+
subprocess.run([setupfile, "start"])
25+
26+
kv_bench()
27+
distributed_object_bench()
28+
object_bench()
29+
item_bench()
30+
31+
subprocess.run([setupfile, "stop"])

‎python/hello_world.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# JULEA - Flexible storage framework
2+
# Copyright (C) 2019 Johannes Coym
3+
#
4+
# This program is free software: you can redistribute it and/or modify
5+
# it under the terms of the GNU Lesser General Public License as published by
6+
# the Free Software Foundation, either version 3 of the License, or
7+
# (at your option) any later version.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU Lesser General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU Lesser General Public License
15+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
from julea.object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
18+
19+
20+
def main():
21+
object = JObject("hello", "world")
22+
hello_world = "Hello World!"
23+
24+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
25+
object.create(batch)
26+
bw = object.write(hello_world.encode('utf-8'), 0, batch)
27+
28+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
29+
out = object.read(bw.value, 0, batch)
30+
31+
print("Object contains: {buffer} ({length} bytes)".format(
32+
buffer=out[0].raw.decode('utf-8'), length=out[1].value))
33+
34+
35+
if __name__ == "__main__":
36+
main()

‎python/julea/__init__.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# JULEA - Flexible storage framework
2+
# Copyright (C) 2019 Johannes Coym
3+
#
4+
# This program is free software: you can redistribute it and/or modify
5+
# it under the terms of the GNU Lesser General Public License as published by
6+
# the Free Software Foundation, either version 3 of the License, or
7+
# (at your option) any later version.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU Lesser General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU Lesser General Public License
15+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
import ctypes
18+
import pathlib
19+
20+
juleadir = pathlib.Path(__file__).parent.parent.parent
21+
libdir = juleadir.joinpath('build', 'lib')
22+
23+
JULEA = ctypes.CDLL(libdir.joinpath('libjulea.so'))
24+
JULEA_ITEM = ctypes.CDLL(libdir.joinpath('libjulea-item.so'))
25+
JULEA_KV = ctypes.CDLL(libdir.joinpath('libjulea-kv.so'))
26+
JULEA_OBJECT = ctypes.CDLL(libdir.joinpath('libjulea-object.so'))

‎python/julea/benchmark/__init__.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# JULEA - Flexible storage framework
2+
# Copyright (C) 2019 Johannes Coym
3+
#
4+
# This program is free software: you can redistribute it and/or modify
5+
# it under the terms of the GNU Lesser General Public License as published by
6+
# the Free Software Foundation, either version 3 of the License, or
7+
# (at your option) any later version.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU Lesser General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU Lesser General Public License
15+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
from .bench_distributed_object import distributed_object_bench
18+
from .bench_item import item_bench
19+
from .bench_kv import kv_bench
20+
from .bench_object import object_bench

‎python/julea/benchmark/bench_distributed_object.py

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
# JULEA - Flexible storage framework
2+
# Copyright (C) 2019 Johannes Coym
3+
# Copyright (C) 2017-2019 Michael Kuhn
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU Lesser General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU Lesser General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU Lesser General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
18+
from ..object import (JDistributedObject, JBatch, JDistribution,
19+
J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN)
20+
from ..lib.common import format_benchmark
21+
import time
22+
23+
24+
def bench_create(use_batch):
25+
if use_batch:
26+
n = 100000
27+
else:
28+
n = 1000
29+
30+
distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
31+
32+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
33+
start = time.time()
34+
35+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
36+
for i in range(0, n):
37+
name = "benchmark-" + str(i)
38+
o = JDistributedObject("python", name, distribution)
39+
o.create(batch)
40+
o.delete(delete_batch)
41+
if not use_batch:
42+
batch.execute()
43+
44+
end = time.time()
45+
46+
if use_batch:
47+
return "/object/distributed-object/create-batch", end - start, n
48+
else:
49+
return "/object/distributed-object/create", end - start, n
50+
51+
52+
def bench_delete(use_batch):
53+
n = 10000
54+
distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
55+
56+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
57+
for i in range(0, n):
58+
name = "benchmark-" + str(i)
59+
o = JDistributedObject("python", name, distribution)
60+
o.create(batch)
61+
62+
start = time.time()
63+
64+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
65+
for i in range(0, n):
66+
name = "benchmark-" + str(i)
67+
o = JDistributedObject("python", name, distribution)
68+
o.delete(batch)
69+
if not use_batch:
70+
batch.execute()
71+
72+
end = time.time()
73+
74+
if use_batch:
75+
return "/object/distributed-object/delete-batch", end - start, n
76+
else:
77+
return "/object/distributed-object/delete", end - start, n
78+
79+
80+
def bench_status(use_batch):
81+
n = 1000
82+
distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
83+
dummy = bytearray(1)
84+
85+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
86+
o = JDistributedObject("python", "benchmark", distribution)
87+
o.delete(batch)
88+
o.create(batch)
89+
o.write(dummy, 0, batch)
90+
91+
start = time.time()
92+
93+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
94+
for _ in range(0, n):
95+
o.status(batch)
96+
if not use_batch:
97+
batch.execute()
98+
99+
end = time.time()
100+
101+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
102+
o.delete(batch)
103+
104+
if use_batch:
105+
return "/object/distributed-object/status-batch", end - start, n
106+
else:
107+
return "/object/distributed-object/status", end - start, n
108+
109+
110+
def bench_read(use_batch, block_size):
111+
n = 25000
112+
distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
113+
dummy = bytearray(block_size)
114+
115+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
116+
o = JDistributedObject("python", "benchmark", distribution)
117+
o.delete(batch)
118+
o.create(batch)
119+
120+
for i in range(0, n):
121+
o.write(dummy, i * block_size, batch)
122+
123+
start = time.time()
124+
125+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
126+
for i in range(0, n):
127+
o.read(block_size, i * block_size, batch)
128+
if not use_batch:
129+
batch.execute()
130+
131+
end = time.time()
132+
133+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
134+
o.delete(batch)
135+
136+
if use_batch:
137+
name = "/object/distributed-object/read-batch"
138+
return name, end - start, n, n * block_size
139+
else:
140+
name = "/object/distributed-object/read"
141+
return name, end - start, n, n * block_size
142+
143+
144+
def bench_write(use_batch, block_size):
145+
n = 25000
146+
distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
147+
dummy = bytearray(block_size)
148+
149+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
150+
o = JDistributedObject("python", "benchmark", distribution)
151+
o.delete(batch)
152+
o.create(batch)
153+
154+
start = time.time()
155+
156+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
157+
for i in range(0, n):
158+
o.write(dummy, i * block_size, batch)
159+
if not use_batch:
160+
batch.execute()
161+
162+
end = time.time()
163+
164+
with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
165+
o.delete(batch)
166+
167+
if use_batch:
168+
name = "/object/distributed-object/write-batch"
169+
return name, end - start, n, n * block_size
170+
else:
171+
name = "/object/distributed-object/write"
172+
return name, end - start, n, n * block_size
173+
174+
175+
def distributed_object_bench():
176+
create = bench_create(False)
177+
print(format_benchmark(create))
178+
create_batch = bench_create(True)
179+
print(format_benchmark(create_batch))
180+
181+
delete = bench_delete(False)
182+
print(format_benchmark(delete))
183+
delete_batch = bench_delete(True)
184+
print(format_benchmark(delete_batch))
185+
186+
status = bench_status(False)
187+
print(format_benchmark(status))
188+
status_batch = bench_status(True)
189+
print(format_benchmark(status_batch))
190+
191+
read = bench_read(False, 4 * 1024)
192+
print(format_benchmark(read))
193+
read_batch = bench_read(True, 4 * 1024)
194+
print(format_benchmark(read_batch))
195+
196+
write = bench_write(False, 4 * 1024)
197+
print(format_benchmark(write))
198+
write_batch = bench_write(True, 4 * 1024)
199+
print(format_benchmark(write_batch))

0 commit comments

Comments
 (0)