Skip to content

Commit eded881

Browse files
author
Stephen Hoover
authored
[PARO-720] ENH Model sharing helpers (#315)
A CivisML model has File and Project run outputs, each of which need to be shared to effectively share the "model". This helper will transparently handle sharing the run outputs. Include JSONValue in case a future version of CivisML uses that. The functions are patterned after the autogenerated API sharing endpoints.
1 parent a5b5821 commit eded881

File tree

4 files changed

+431
-2
lines changed

4 files changed

+431
-2
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
77
- Add helper function (run_template) to run a template given its id and return
88
either the JSON output or the associated file ids. (#318)
99
- Add helper function to list CivisML models. (#314)
10+
- Added helper functions to share CivisML models with users or groups,
11+
patterned after the existing API sharing endpoints. (#315)
1012
- Allow the base URL of the CLI to be configured through the
1113
`CIVIS_API_ENDPOINT` environment variable, like the civis Python module. (#312)
1214
- Allow the CLI log level to be configured with the `CIVIS_LOG_LEVEL`

‎civis/ml/_helper.py

Lines changed: 263 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
from collections import namedtuple
2+
import logging
23

34
from civis import APIClient
45
from civis.ml._model import _PRED_TEMPLATES
56

6-
__all__ = ['list_models']
7+
__all__ = ['list_models', 'put_models_shares_groups',
8+
'put_models_shares_users', 'delete_models_shares_groups',
9+
'delete_models_shares_users']
10+
log = logging.getLogger(__name__)
711

812
# sentinel value for default author value
913
SENTINEL = namedtuple('Sentinel', [])()
@@ -58,3 +62,261 @@ def list_models(job_type="train", author=SENTINEL, client=None, **kwargs):
5862
author=author,
5963
**kwargs)
6064
return models
65+
66+
67+
def put_models_shares_users(id, user_ids, permission_level,
68+
client=None,
69+
share_email_body='DEFAULT',
70+
send_shared_email='DEFAULT'):
71+
"""Set the permissions users have on this object
72+
73+
Use this on both training and scoring jobs.
74+
If used on a training job, note that "read" permission is
75+
sufficient to score the model.
76+
77+
Parameters
78+
----------
79+
id : integer
80+
The ID of the resource that is shared.
81+
user_ids : list
82+
An array of one or more user IDs.
83+
permission_level : string
84+
Options are: "read", "write", or "manage".
85+
client : :class:`civis.APIClient`, optional
86+
If not provided, an :class:`civis.APIClient` object will be
87+
created from the :envvar:`CIVIS_API_KEY`.
88+
share_email_body : string, optional
89+
Custom body text for e-mail sent on a share.
90+
send_shared_email : boolean, optional
91+
Send email to the recipients of a share.
92+
93+
Returns
94+
-------
95+
readers : dict::
96+
- users : list::
97+
- id : integer
98+
- name : string
99+
- groups : list::
100+
- id : integer
101+
- name : string
102+
writers : dict::
103+
- users : list::
104+
- id : integer
105+
- name : string
106+
- groups : list::
107+
- id : integer
108+
- name : string
109+
owners : dict::
110+
- users : list::
111+
- id : integer
112+
- name : string
113+
- groups : list::
114+
- id : integer
115+
- name : string
116+
total_user_shares : integer
117+
For owners, the number of total users shared. For writers and readers,
118+
the number of visible users shared.
119+
total_group_shares : integer
120+
For owners, the number of total groups shared. For writers and readers,
121+
the number of visible groups shared.
122+
"""
123+
kwargs = {}
124+
if send_shared_email != 'DEFAULT':
125+
kwargs['send_shared_email'] = send_shared_email
126+
if share_email_body != 'DEFAULT':
127+
kwargs['share_email_body'] = share_email_body
128+
return _share_model(id, user_ids, permission_level, entity_type='users',
129+
client=client, **kwargs)
130+
131+
132+
def put_models_shares_groups(id, group_ids, permission_level,
133+
client=None,
134+
share_email_body='DEFAULT',
135+
send_shared_email='DEFAULT'):
136+
"""Set the permissions groups have on this model.
137+
138+
Use this on both training and scoring jobs.
139+
If used on a training job, note that "read" permission is
140+
sufficient to score the model.
141+
142+
Parameters
143+
----------
144+
id : integer
145+
The ID of the resource that is shared.
146+
group_ids : list
147+
An array of one or more group IDs.
148+
permission_level : string
149+
Options are: "read", "write", or "manage".
150+
client : :class:`civis.APIClient`, optional
151+
If not provided, an :class:`civis.APIClient` object will be
152+
created from the :envvar:`CIVIS_API_KEY`.
153+
share_email_body : string, optional
154+
Custom body text for e-mail sent on a share.
155+
send_shared_email : boolean, optional
156+
Send email to the recipients of a share.
157+
158+
Returns
159+
-------
160+
readers : dict::
161+
- users : list::
162+
- id : integer
163+
- name : string
164+
- groups : list::
165+
- id : integer
166+
- name : string
167+
writers : dict::
168+
- users : list::
169+
- id : integer
170+
- name : string
171+
- groups : list::
172+
- id : integer
173+
- name : string
174+
owners : dict::
175+
- users : list::
176+
- id : integer
177+
- name : string
178+
- groups : list::
179+
- id : integer
180+
- name : string
181+
total_user_shares : integer
182+
For owners, the number of total users shared. For writers and readers,
183+
the number of visible users shared.
184+
total_group_shares : integer
185+
For owners, the number of total groups shared. For writers and readers,
186+
the number of visible groups shared.
187+
"""
188+
kwargs = {}
189+
if send_shared_email != 'DEFAULT':
190+
kwargs['send_shared_email'] = send_shared_email
191+
if share_email_body != 'DEFAULT':
192+
kwargs['share_email_body'] = share_email_body
193+
return _share_model(id, group_ids, permission_level, entity_type='groups',
194+
client=client, **kwargs)
195+
196+
197+
def _share_model(job_id, entity_ids, permission_level, entity_type,
198+
client=None, **kwargs):
199+
"""Share a container job and all run outputs with requested entities"""
200+
client = client or APIClient()
201+
if entity_type not in ['groups', 'users']:
202+
raise ValueError("'entity_type' must be one of ['groups', 'users']. "
203+
"Got '{0}'.".format(entity_type))
204+
205+
log.debug("Sharing object %d with %s %s at permission level %s.",
206+
job_id, entity_type, entity_ids, permission_level)
207+
_func = getattr(client.scripts, "put_containers_shares_" + entity_type)
208+
result = _func(job_id, entity_ids, permission_level, **kwargs)
209+
210+
# CivisML relies on several run outputs attached to each model run.
211+
# Go through and share all outputs on each run.
212+
runs = client.scripts.list_containers_runs(job_id, iterator=True)
213+
for run in runs:
214+
log.debug("Sharing outputs on %d, run %s.", job_id, run.id)
215+
outputs = client.scripts.list_containers_runs_outputs(job_id, run.id)
216+
for _output in outputs:
217+
if _output['object_type'] == 'File':
218+
_func = getattr(client.files, "put_shares_" + entity_type)
219+
obj_permission = permission_level
220+
elif _output['object_type'] == 'Project':
221+
_func = getattr(client.projects, "put_shares_" + entity_type)
222+
if permission_level == 'read':
223+
# Users must be able to add to projects to use the model
224+
obj_permission = 'write'
225+
else:
226+
obj_permission = permission_level
227+
elif _output['object_type'] == 'JSONValue':
228+
_func = getattr(client.json_values,
229+
"put_shares_" + entity_type)
230+
obj_permission = permission_level
231+
else:
232+
log.debug("Found a run output of type %s, ID %s; not sharing "
233+
"it.", _output['object_type'], _output['object_id'])
234+
continue
235+
_oid = _output['object_id']
236+
# Don't send share emails for any of the run outputs.
237+
_func(_oid, entity_ids, obj_permission, send_shared_email=False)
238+
239+
return result
240+
241+
242+
def delete_models_shares_users(id, user_id, client=None):
243+
"""Revoke the permissions a user has on this object
244+
245+
Use this function on both training and scoring jobs.
246+
247+
Parameters
248+
----------
249+
id : integer
250+
The ID of the resource that is shared.
251+
user_id : integer
252+
The ID of the user.
253+
client : :class:`civis.APIClient`, optional
254+
If not provided, an :class:`civis.APIClient` object will be
255+
created from the :envvar:`CIVIS_API_KEY`.
256+
257+
Returns
258+
-------
259+
None
260+
Response code 204: success
261+
"""
262+
return _unshare_model(id, user_id, entity_type='users', client=client)
263+
264+
265+
def delete_models_shares_groups(id, group_id, client=None):
266+
"""Revoke the permissions a group has on this object
267+
268+
Use this function on both training and scoring jobs.
269+
270+
Parameters
271+
----------
272+
id : integer
273+
The ID of the resource that is shared.
274+
group_id : integer
275+
The ID of the group.
276+
client : :class:`civis.APIClient`, optional
277+
If not provided, an :class:`civis.APIClient` object will be
278+
created from the :envvar:`CIVIS_API_KEY`.
279+
280+
Returns
281+
-------
282+
None
283+
Response code 204: success
284+
"""
285+
return _unshare_model(id, group_id, entity_type='groups', client=client)
286+
287+
288+
def _unshare_model(job_id, entity_id, entity_type, client=None):
289+
"""Revoke permissions on a container job and all run outputs
290+
for the requested entity (singular)
291+
"""
292+
client = client or APIClient()
293+
if entity_type not in ['groups', 'users']:
294+
raise ValueError("'entity_type' must be one of ['groups', 'users']. "
295+
"Got '{0}'.".format(entity_type))
296+
297+
log.debug("Revoking permissions on object %d for %s %s.",
298+
job_id, entity_type, entity_id)
299+
_func = getattr(client.scripts, "delete_containers_shares_" + entity_type)
300+
result = _func(job_id, entity_id)
301+
302+
# CivisML relies on several run outputs attached to each model run.
303+
# Go through and revoke permissions for outputs on each run.
304+
runs = client.scripts.list_containers_runs(job_id, iterator=True)
305+
endpoint_name = "delete_shares_" + entity_type
306+
for run in runs:
307+
log.debug("Unsharing outputs on %d, run %s.", job_id, run.id)
308+
outputs = client.scripts.list_containers_runs_outputs(job_id, run.id)
309+
for _output in outputs:
310+
if _output['object_type'] == 'File':
311+
_func = getattr(client.files, endpoint_name)
312+
elif _output['object_type'] == 'Project':
313+
_func = getattr(client.projects, endpoint_name)
314+
elif _output['object_type'] == 'JSONValue':
315+
_func = getattr(client.json_values, endpoint_name)
316+
else:
317+
log.debug("Found run output of type %s, ID %s; not unsharing "
318+
"it.", _output['object_type'], _output['object_id'])
319+
continue
320+
_func(_output['object_id'], entity_id)
321+
322+
return result

0 commit comments

Comments
 (0)