Skip to content

Commit da217bb

Browse files
authored
Merge pull request #18 from NLeSC/develop
Develop
2 parents d10e826 + c38d27d commit da217bb

File tree

16 files changed

+1082
-2118
lines changed

16 files changed

+1082
-2118
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
*.pyc
2-
__pycache__/
2+
**/__pycache__/
3+
**/.ipynb_checkpoints/
34
/build
45
/.coverage
56
/coverage.xml
@@ -14,3 +15,4 @@ __pycache__/
1415
*.swp
1516
*~
1617

18+
xenon-grpc-shadow*

bin/xenon-grpc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ case "`uname`" in
6464
;;
6565
esac
6666

67-
CLASSPATH=$APP_HOME/lib/xenon-grpc-1.0.0-all.jar
67+
CLASSPATH=$APP_HOME/lib/xenon-grpc-2.0.0-all.jar
6868

6969
# Determine the Java command to use to start the JVM.
7070
if [ -n "$JAVA_HOME" ] ; then

bin/xenon-grpc.bat

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ set CMD_LINE_ARGS=%*
6363
:execute
6464
@rem Setup the command line
6565

66-
set CLASSPATH=%APP_HOME%\lib\xenon-grpc-1.0.0-all.jar
66+
set CLASSPATH=%APP_HOME%\lib\xenon-grpc-2.0.0-all.jar
6767

6868
@rem Execute xenon-grpc
6969
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %XENON_GRPC_OPTS% -jar "%CLASSPATH%" %CMD_LINE_ARGS%

docs/adaptors.rst

Lines changed: 497 additions & 0 deletions
Large diffs are not rendered by default.

docs/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ Welcome to PyXenon's documentation!
1212
self
1313
quick-start
1414
streaming
15+
adaptors
1516
api
1617

1718
The PyXenon module interfaces with Xenon-GRPC to get an interface to the Xenon

examples/tutorial.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
xenon.init(log_level='INFO')
88

99
tmpdir = Path('.')
10-
location = '10.0.2.2:10022'
10+
location = 'localhost:10022'
1111

1212
#
1313
# step 0: write our script

scripts/print_adaptor_docs.py

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import xenon
2+
import textwrap
3+
4+
xenon.init()
5+
6+
7+
class Table:
8+
def __init__(self, headings, data):
9+
self.headings = headings
10+
self.data = data
11+
self.n_cols = len(self.headings)
12+
self.n_rows = len(self.data)
13+
14+
@staticmethod
15+
def from_sequence(seq, **getters):
16+
headings = list(getters.keys())
17+
data = [
18+
tuple(getters[k](x) for k in headings)
19+
for x in seq
20+
]
21+
return Table(headings, data)
22+
23+
def print_rst(self, fmt_width=70):
24+
column_max = [
25+
max(max(len(row[c]) for row in self.data), len(self.headings[c]))
26+
for c in range(self.n_cols)
27+
]
28+
29+
# take all columns that are less than 10 characters max, and see how much space we have left
30+
space_left = fmt_width - sum(n for n in column_max if n <= 25)
31+
space_needed = sum(n > 25 for n in column_max) * 25
32+
33+
if space_needed > space_left:
34+
raise ValueError("Table is too wide.")
35+
36+
large_column_width = space_left // max(1, sum(n > 25 for n in column_max))
37+
column_width = [
38+
min(large_column_width, n) for n in column_max
39+
]
40+
41+
def hline(c):
42+
return '+' + c + (c + '+' + c).join(c*w for w in column_width) + c + '+'
43+
44+
def row_height(row):
45+
return max(
46+
len(textwrap.wrap(x, width=w))
47+
for x, w in zip(row, column_width))
48+
49+
def row_strings(row):
50+
texts = [
51+
textwrap.wrap(x, width=w)
52+
for x, w in zip(row, column_width)
53+
]
54+
row_height = max(len(t) for t in texts)
55+
56+
def extend_text(text, width):
57+
n = len(text)
58+
r = [t + ' ' * (width - len(t)) for t in text] + \
59+
[' ' * width] * (row_height - n)
60+
return r
61+
62+
extended_texts = list(map(extend_text, texts, column_width))
63+
64+
return [
65+
'| ' + ' | '.join(t[l] for t in extended_texts) + ' |'
66+
for l in range(row_height)
67+
]
68+
69+
print(hline('-'))
70+
for l in row_strings(self.headings):
71+
print(l)
72+
print(hline('='))
73+
for row in self.data:
74+
for l in row_strings(row):
75+
print(l)
76+
print(hline('-'))
77+
78+
79+
def property_table(d):
80+
def prop_typename(p):
81+
typemap = { v: k for k, v in p.Type.items() }
82+
return typemap[p.type].lower()
83+
84+
return Table.from_sequence(
85+
d.supported_properties,
86+
name=lambda p: p.name.split('.')[-1],
87+
description=lambda p: p.description,
88+
data_type=prop_typename,
89+
default=lambda p: '`{}`'.format(p.default_value) if p.default_value else '(empty)')
90+
91+
92+
def print_adaptor_descriptions(desc, extra=[]):
93+
for d in desc:
94+
print(d.name.title())
95+
print('~' * len(d.name))
96+
print(textwrap.fill(d.description))
97+
print()
98+
if extra:
99+
Table.from_sequence(
100+
extra,
101+
field=lambda x: x,
102+
value=lambda x: str(getattr(d, x))).print_rst()
103+
print()
104+
print('location string:')
105+
print('\n'.join(' * `{}`'.format(l) for l in d.supported_locations))
106+
print()
107+
print('supported properties:')
108+
print()
109+
property_table(d).print_rst()
110+
print()
111+
112+
113+
scheduler_adaptor_props = [
114+
'is_embedded',
115+
'supports_interactive',
116+
'supports_batch',
117+
'uses_file_system'
118+
]
119+
120+
file_system_adaptor_props = [
121+
'supports_third_party_copy',
122+
'can_create_symboliclinks',
123+
'can_read_symboliclinks',
124+
'is_connectionless'
125+
]
126+
127+
print("""Adaptors
128+
========
129+
This section contains the adaptor documentation which is generated from the
130+
information provided by the adaptors themselves.
131+
""")
132+
133+
print("File System")
134+
print("-----------")
135+
print()
136+
print_adaptor_descriptions(
137+
xenon.FileSystem.get_adaptor_descriptions(),
138+
extra=file_system_adaptor_props)
139+
print()
140+
141+
print("Scheduler")
142+
print("---------")
143+
print()
144+
print_adaptor_descriptions(
145+
xenon.Scheduler.get_adaptor_descriptions(),
146+
extra=scheduler_adaptor_props)

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
'Topic :: System :: Distributed Computing',
4848
],
4949
data_files=[
50-
('lib', ['lib/xenon-grpc-1.0.0-all.jar']),
50+
('lib', ['lib/xenon-grpc-2.0.0-all.jar']),
5151
('bin', [{'posix': 'bin/xenon-grpc',
5252
'nt': 'bin/xenon-grpc.bat'}[os.name]])],
5353
install_requires=['grpcio', 'grpcio-tools', 'pyxdg', 'pyopenssl'],

tests/conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@
55
@pytest.fixture(scope="session")
66
def xenon_server(request):
77
print("============== Starting Xenon-GRPC server ================")
8-
m = xenon.init(do_not_exit=True, disable_tls=False)
8+
m = xenon.init(do_not_exit=True, disable_tls=False, log_level='INFO')
99
request.addfinalizer(lambda: m.__exit__(None, None, None))

0 commit comments

Comments
 (0)