Skip to content

Commit 727a4f6

Browse files
authored
Merge branch 'master' into filter
2 parents 17ffca3 + 7d29791 commit 727a4f6

File tree

7 files changed

+242
-20
lines changed

7 files changed

+242
-20
lines changed

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ HOST_UID=1000 # Unix UID associated with non-root login
3131
PY_VER=3.8 # Python version: 3.6|3.7|3.8
3232
IMAGE=djtest # Image type: djbase|djtest|djlab|djlabhub
3333
DISTRO=alpine # Distribution: alpine|debian
34-
AS_SCRIPT=
34+
AS_SCRIPT= # If 'TRUE', will not keep container alive but run tests and exit
3535
```
3636
- Navigate to `LNX-docker-compose.yaml` and check first comment which will provide best instruction on how to start the service. Yes, the command is a bit long...
3737

dj_gui_api_server/DJConnector.py

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,18 @@
22
import datajoint as dj
33
import datetime
44
import numpy as np
5+
56
from .dj_connector_exceptions import InvalidDeleteRequest, InvalidRestriction, \
67
UnsupportedTableType
78
from functools import reduce
9+
from datajoint.errors import AccessError
10+
import re
11+
from .errors import InvalidDeleteRequest, InvalidRestriction, UnsupportedTableType
812

913
DAY = 24 * 60 * 60
14+
DEFAULT_FETCH_LIMIT = 1000 # Stop gap measure to deal with super large tables
15+
TABLE_INFO_REGEX = re.compile(
16+
r'.*?FROM\s+`(?P<schema>\w+)`.*?name\s*?=\s*?"(?P<table>.*?)".*?')
1017

1118

1219
class DJConnector():
@@ -281,6 +288,43 @@ def insert_tuple(jwt_payload: dict, schema_name: str, table_name: str,
281288
schema_virtual_module = dj.create_virtual_module(schema_name, schema_name)
282289
getattr(schema_virtual_module, table_name).insert1(tuple_to_insert)
283290

291+
@staticmethod
292+
def record_dependency(jwt_payload: dict, schema_name: str, table_name: str,
293+
primary_restriction: dict) -> list:
294+
"""
295+
Return summary of dependencies associated with a restricted table
296+
:param jwt_payload: Dictionary containing databaseAddress, username and password
297+
strings
298+
:type jwt_payload: dict
299+
:param schema_name: Name of schema
300+
:type schema_name: str
301+
:param table_name: Table name under the given schema; must be in camel case
302+
:type table_name: str
303+
:param primary_restriction: Restriction to be applied to table
304+
:type primary_restriction: dict
305+
:return: Tables that are dependant on specific records. Includes accessibility and,
306+
if accessible, how many rows are affected.
307+
:rtype: list
308+
"""
309+
DJConnector.set_datajoint_config(jwt_payload)
310+
virtual_module = dj.VirtualModule(schema_name, schema_name)
311+
table = getattr(virtual_module, table_name)
312+
# Retrieve dependencies of related to retricted
313+
dependencies = [dict(schema=descendant.database, table=descendant.table_name,
314+
accessible=True, count=len(descendant & primary_restriction))
315+
for descendant in table().descendants(as_objects=True)]
316+
# Determine first issue regarding access
317+
# Start transaction, try to delete, catch first occurrence, rollback
318+
virtual_module.schema.connection.start_transaction()
319+
try:
320+
(table & primary_restriction).delete(safemode=False, transaction=False)
321+
except AccessError as errors:
322+
dependencies = dependencies + [dict(TABLE_INFO_REGEX.match(
323+
errors.args[2]).groupdict(), accessible=False)]
324+
finally:
325+
virtual_module.schema.connection.cancel_transaction()
326+
return dependencies
327+
284328
@staticmethod
285329
def update_tuple(jwt_payload: dict, schema_name: str, table_name: str,
286330
tuple_to_update: dict):
@@ -371,7 +415,6 @@ def set_datajoint_config(jwt_payload: dict):
371415
dj.config['database.host'] = jwt_payload['databaseAddress']
372416
dj.config['database.user'] = jwt_payload['username']
373417
dj.config['database.password'] = jwt_payload['password']
374-
375418
dj.conn(reset=True)
376419

377420
@staticmethod

dj_gui_api_server/DJGUIAPIServer.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,31 @@ def insert_tuple(jwt_payload: dict):
229229
return str(e), 500
230230

231231

232+
@app.route('/api/record/dependency', methods=['GET'])
233+
@protected_route
234+
def record_dependency(jwt_payload: dict) -> dict:
235+
"""
236+
Route to insert record. Expects:
237+
(html:GET:Authorization): Must include in format of: bearer <JWT-Token>
238+
(html:query_params): {"schemaName": <schema_name>, "tableName": <table_name>,
239+
"restriction": <b64 JSON restriction>}
240+
NOTE: Table name must be in CamalCase
241+
:param jwt_payload: Dictionary containing databaseAddress, username and password
242+
strings.
243+
:type jwt_payload: dict
244+
:return: If sucessfuly sends back a list of dependencies otherwise returns error
245+
:rtype: dict
246+
"""
247+
# Get dependencies
248+
try:
249+
dependencies = DJConnector.record_dependency(
250+
jwt_payload, request.args.get('schemaName'), request.args.get('tableName'),
251+
loads(b64decode(request.args.get('restriction').encode('utf-8')).decode('utf-8')))
252+
return dict(dependencies=dependencies)
253+
except Exception as e:
254+
return str(e), 500
255+
256+
232257
@app.route('/api/update_tuple', methods=['POST'])
233258
@protected_route
234259
def update_tuple(jwt_payload: dict):
File renamed without changes.

requirements.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
flask
2-
datajoint==0.13.dev2
32
pyjwt[crypto]
3+
datajoint==0.13.dev4
4+
datajoint_connection_hub

tests/test_attributes.py

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ class Decimal(dj.Manual):
7676
yield Decimal
7777
Decimal.drop()
7878

79+
7980
@pytest.fixture
8081
def String(schema):
8182
@schema
@@ -88,6 +89,7 @@ class String(dj.Manual):
8889
yield String
8990
String.drop()
9091

92+
9193
@pytest.fixture
9294
def Bool(schema):
9395
@schema
@@ -100,6 +102,7 @@ class Bool(dj.Manual):
100102
yield Bool
101103
Bool.drop()
102104

105+
103106
@pytest.fixture
104107
def Date(schema):
105108
@schema
@@ -190,6 +193,7 @@ class Uuid(dj.Manual):
190193
yield Uuid
191194
Uuid.drop()
192195

196+
193197
@pytest.fixture
194198
def ParentPart(schema):
195199
@schema
@@ -199,30 +203,29 @@ class ScanData(dj.Manual):
199203
---
200204
data: int unsigned
201205
"""
202-
206+
203207
@schema
204208
class ProcessScanData(dj.Computed):
205209
definition = """
206210
-> ScanData # Forigen Key Reference
207211
---
208212
processed_scan_data : int unsigned
209213
"""
210-
214+
211215
class ProcessScanDataPart(dj.Part):
212216
definition = """
213217
-> ProcessScanData
214218
---
215219
processed_scan_data_part : int unsigned
216220
"""
217-
218-
221+
219222
def make(self, key):
220223
scan_data_dict = (ScanData & key).fetch1()
221224
self.insert1(dict(key, processed_scan_data=scan_data_dict['data']))
222225
self.ProcessScanDataPart.insert1(
223226
dict(key, processed_scan_data_part=scan_data_dict['data'] * 2))
224227

225-
yield dict(ScanData=ScanData, ProcessScanData=ProcessScanData)
228+
yield ScanData, ProcessScanData
226229
ScanData.drop()
227230

228231

@@ -267,6 +270,7 @@ def test_decimal(token, client, Decimal):
267270
token=token,
268271
)
269272

273+
270274
def test_string(token, client, String):
271275
validate(
272276
table=String,
@@ -277,6 +281,7 @@ def test_string(token, client, String):
277281
token=token,
278282
)
279283

284+
280285
def test_bool(token, client, Bool):
281286
validate(
282287
table=Bool,
@@ -287,6 +292,7 @@ def test_bool(token, client, Bool):
287292
token=token,
288293
)
289294

295+
290296
def test_date(token, client, Date):
291297
validate(
292298
table=Date,
@@ -363,23 +369,26 @@ def test_uuid(token, client, Uuid):
363369
token=token,
364370
)
365371

372+
366373
def test_part_table(token, client, ParentPart):
367-
ParentPart['ScanData'].insert1(dict(scan_id=0, data=5))
368-
ParentPart['ProcessScanData'].populate()
369-
374+
ScanData, ProcessScanData = ParentPart
375+
ScanData.insert1(dict(scan_id=0, data=5))
376+
ProcessScanData.populate()
377+
370378
# Test Parent
371379
REST_value = client.post('/api/fetch_tuples',
372-
headers=dict(Authorization=f'Bearer {token}'),
373-
json=dict(schemaName='add_types',
374-
tableName=ParentPart['ProcessScanData'].__name__)).json['tuples'][0]
375-
380+
headers=dict(Authorization=f'Bearer {token}'),
381+
json=dict(schemaName='add_types',
382+
tableName=ProcessScanData.__name__)).json['tuples'][0]
383+
376384
assert REST_value == [0, 5]
377385

378386
# Test Child
379-
REST_value = client.post('/api/fetch_tuples',
380-
headers=dict(Authorization=f'Bearer {token}'),
381-
json=dict(schemaName='add_types',
382-
tableName=ParentPart['ProcessScanData'].__name__ + '.' +
383-
ParentPart['ProcessScanData'].ProcessScanDataPart.__name__)).json['tuples'][0]
387+
REST_value = client.post(
388+
'/api/fetch_tuples',
389+
headers=dict(Authorization=f'Bearer {token}'),
390+
json=dict(schemaName='add_types',
391+
tableName=(ProcessScanData.__name__ + '.' +
392+
ProcessScanData.ProcessScanDataPart.__name__))).json['tuples'][0]
384393

385394
assert REST_value == [0, 10]

tests/test_dependencies.py

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
from os import getenv
2+
import pytest
3+
from dj_gui_api_server.DJGUIAPIServer import app
4+
import datajoint as dj
5+
from base64 import b64encode
6+
from json import dumps
7+
8+
9+
@pytest.fixture
10+
def client():
11+
with app.test_client() as client:
12+
yield client
13+
14+
15+
@pytest.fixture
16+
def token(client):
17+
yield client.post('/api/login', json=dict(databaseAddress=getenv('TEST_DB_SERVER'),
18+
username=getenv('TEST_DB_USER'),
19+
password=getenv('TEST_DB_PASS'))).json['jwt']
20+
21+
22+
@pytest.fixture
23+
def connection():
24+
dj.config['safemode'] = False
25+
connection = dj.conn(host=getenv('TEST_DB_SERVER'),
26+
user=getenv('TEST_DB_USER'),
27+
password=getenv('TEST_DB_PASS'), reset=True)
28+
connection.query("""
29+
CREATE USER IF NOT EXISTS 'underprivileged'@'%%'
30+
IDENTIFIED BY 'datajoint';
31+
""")
32+
connection.query("GRANT ALL PRIVILEGES ON `deps`.* TO 'underprivileged'@'%%';")
33+
deps_secret = dj.VirtualModule('deps_secret', 'deps_secret', create_tables=True)
34+
deps = dj.VirtualModule('deps', 'deps', create_tables=True)
35+
@deps.schema
36+
class TableA(dj.Lookup):
37+
definition = """
38+
a_id: int
39+
---
40+
a_name: varchar(30)
41+
"""
42+
contents = [(0, 'Raphael',), (1, 'Bernie',)]
43+
44+
@deps.schema
45+
class TableB(dj.Lookup):
46+
definition = """
47+
-> TableA
48+
b_id: int
49+
---
50+
b_number: float
51+
"""
52+
contents = [(0, 10, 22.12), (0, 11, -1.21,), (1, 21, 7.77,)]
53+
deps = dj.VirtualModule('deps', 'deps', create_tables=True)
54+
55+
@deps_secret.schema
56+
class DiffTableB(dj.Lookup):
57+
definition = """
58+
-> deps.TableA
59+
bs_id: int
60+
---
61+
bs_number: float
62+
"""
63+
contents = [(0, -10, -99.99), (0, -11, 287.11,)]
64+
65+
@deps.schema
66+
class TableC(dj.Lookup):
67+
definition = """
68+
-> TableB
69+
c_id: int
70+
---
71+
c_int: int
72+
"""
73+
contents = [(0, 10, 100, -8), (0, 11, 200, -9,), (0, 11, 300, -7,)]
74+
75+
yield connection
76+
77+
deps_secret.schema.drop()
78+
deps.schema.drop()
79+
connection.query("DROP USER 'underprivileged'@'%%';")
80+
connection.close()
81+
dj.config['safemode'] = True
82+
83+
84+
@pytest.fixture
85+
def underprivileged_token(client, connection):
86+
yield client.post('/api/login', json=dict(databaseAddress=getenv('TEST_DB_SERVER'),
87+
username='underprivileged',
88+
password='datajoint')).json['jwt']
89+
90+
91+
def test_dependencies_underprivileged(underprivileged_token, client):
92+
schema_name = 'deps'
93+
table_name = 'TableA'
94+
restriction = b64encode(dumps(dict(a_id=0)).encode('utf-8')).decode('utf-8')
95+
REST_dependencies = client.get(
96+
f"""/api/record/dependency?schemaName={
97+
schema_name}&tableName={table_name}&restriction={restriction}""",
98+
headers=dict(Authorization=f'Bearer {underprivileged_token}')).json['dependencies']
99+
REST_records = client.post('/api/fetch_tuples',
100+
headers=dict(Authorization=f'Bearer {underprivileged_token}'),
101+
json=dict(schemaName=schema_name,
102+
tableName=table_name)).json['tuples']
103+
assert len(REST_records) == 2
104+
assert len(REST_dependencies) == 4
105+
table_a = [el for el in REST_dependencies
106+
if el['schema'] == 'deps' and 'table_a' in el['table']][0]
107+
assert table_a['accessible'] and table_a['count'] == 1
108+
table_b = [el for el in REST_dependencies
109+
if el['schema'] == 'deps' and 'table_b' in el['table']][0]
110+
assert table_b['accessible'] and table_b['count'] == 2
111+
table_c = [el for el in REST_dependencies
112+
if el['schema'] == 'deps' and 'table_c' in el['table']][0]
113+
assert table_c['accessible'] and table_c['count'] == 3
114+
diff_table_b = [el for el in REST_dependencies
115+
if el['schema'] == 'deps_secret' and 'diff_table_b' in el['table']][0]
116+
assert not diff_table_b['accessible']
117+
118+
119+
def test_dependencies_admin(token, client, connection):
120+
schema_name = 'deps'
121+
table_name = 'TableA'
122+
restriction = b64encode(dumps(dict(a_id=0)).encode('utf-8')).decode('utf-8')
123+
REST_dependencies = client.get(
124+
f"""/api/record/dependency?schemaName={
125+
schema_name}&tableName={table_name}&restriction={restriction}""",
126+
headers=dict(Authorization=f'Bearer {token}')).json['dependencies']
127+
REST_records = client.post('/api/fetch_tuples',
128+
headers=dict(Authorization=f'Bearer {token}'),
129+
json=dict(schemaName=schema_name,
130+
tableName=table_name)).json['tuples']
131+
assert len(REST_records) == 2
132+
assert len(REST_dependencies) == 4
133+
table_a = [el for el in REST_dependencies
134+
if el['schema'] == 'deps' and 'table_a' in el['table']][0]
135+
assert table_a['accessible'] and table_a['count'] == 1
136+
table_b = [el for el in REST_dependencies
137+
if el['schema'] == 'deps' and 'table_b' in el['table']][0]
138+
assert table_b['accessible'] and table_b['count'] == 2
139+
table_c = [el for el in REST_dependencies
140+
if el['schema'] == 'deps' and 'table_c' in el['table']][0]
141+
assert table_c['accessible'] and table_c['count'] == 3
142+
diff_table_b = [el for el in REST_dependencies
143+
if el['schema'] == 'deps_secret' and 'diff_table_b' in el['table']][0]
144+
assert diff_table_b['accessible'] and diff_table_b['count'] == 2

0 commit comments

Comments
 (0)