diff --git a/girderformindlogger/api/v1/applet.py b/girderformindlogger/api/v1/applet.py index fee3e09e4..8424ce471 100644 --- a/girderformindlogger/api/v1/applet.py +++ b/girderformindlogger/api/v1/applet.py @@ -627,6 +627,7 @@ def getProtocolVersions(self, applet, retrieveDate=False): items = list(ItemModel().find({ 'folderId': protocol['meta'].get('contentId', None), + 'version': {'$exists': True}, }, fields=['version', 'created'], sort=[("created", DESCENDING)])) if 'contentId' in protocol['meta'] else [] if retrieveDate: diff --git a/girderformindlogger/external/convert_applets.py b/girderformindlogger/external/convert_applets.py index 61f012ab1..ef59c87bb 100644 --- a/girderformindlogger/external/convert_applets.py +++ b/girderformindlogger/external/convert_applets.py @@ -8,11 +8,12 @@ from girderformindlogger.utility import jsonld_expander from bson.objectid import ObjectId -# '_id': ObjectId('5f0e35523477de8b4a528dd0'), -applets = Applet().find(query={ 'meta.applet': { '$exists': True } }, fields= {"_id": 1}) +# '_id': ObjectId('633fc958b7ee9765ba5447a6') +# github: 'meta.protocol.url': {'$exists': True}, 'meta.applet': {'$exists': True}, 'meta.applet.deleted': {'$exists': False}, 'parentId': ObjectId('5ea689a286d25a5dbb14e82c') +applets = Applet().find(query={'_id': ObjectId('633fc958b7ee9765ba5447a6')}, fields= {"_id": 1}) appletsCount = applets.count() print('total', appletsCount) -skipUntil = None +skipUntil = None # ObjectId('60a398c9acd96cf825f7679d') for index, appletId in enumerate(applets, start=1): if skipUntil == appletId['_id']: skipUntil = None @@ -43,8 +44,18 @@ g = [] activityIRIs = dict.keys(formatted['activities'].copy()) for activityIRI in activityIRIs: + activityLink = formatted['activities'][activityIRI] + if isinstance(activityLink, ObjectId): + activityId = activityLink + elif isinstance(activityLink, str): + activityId = ObjectId(activityLink) + elif '_id' in activityLink: + activityId = ObjectId(activityLink['_id'].split('/').pop()) + else: + continue + activity = Activity().findOne({ - '_id': ObjectId(formatted['activities'][activityIRI]) + '_id': activityId }) if not activity: diff --git a/girderformindlogger/external/fix_activity_flow_order.py b/girderformindlogger/external/fix_activity_flow_order.py new file mode 100644 index 000000000..1e7d8f6a9 --- /dev/null +++ b/girderformindlogger/external/fix_activity_flow_order.py @@ -0,0 +1,68 @@ +# fix broken links in protocol.activityFlowOrder + +from girderformindlogger.models.applet import Applet +from girderformindlogger.models.folder import Folder as FolderModel +from girderformindlogger.utility import jsonld_expander +from bson.objectid import ObjectId + + +def findFlowsStringIds(applet): + flowIds = [] + + protocolId = applet['meta']['protocol'].get('_id').split('/').pop() + activityFlows = FolderModel().find({'meta.protocolId': ObjectId(protocolId), 'meta.activityFlow': {'$exists': True} }, fields={"_id": 1}) + for af in activityFlows: + flowIds.append(str(af['_id'])) + + return flowIds + + +def main(applets): + appletsCount = applets.count() + print('total', appletsCount) + skipUntil = None # ObjectId('60a398c9acd96cf825f7679d') + for index, appletId in enumerate(applets, start=1): + if skipUntil == appletId['_id']: + skipUntil = None + if skipUntil is not None: + continue + + applet = Applet().findOne(appletId) + protocolId = ObjectId(applet['meta']['protocol'].get('_id').split('/').pop()) + protocol = FolderModel().findOne(query={'_id': protocolId}) + + existingFlowsIds = findFlowsStringIds(applet) + + # find and attach orphan flows + for afId in applet['meta']['protocol']['activityFlows']: + if str(afId) in existingFlowsIds: + continue # exclude that already refer to us + if FolderModel().find(query={'meta.protocol.activityFlows': afId, '_id': {'$ne': appletId['_id']}}).count() > 0: + continue # if no other protocols refer to this flow + flow = FolderModel().findOne(query={'_id': afId}) + if flow is None: + continue + print('attaching orphan flow '+str(afId)+' to applet ' + str(applet['_id'])) + flow['meta']['protocolId'] = protocolId + FolderModel().setMetadata(flow, flow['meta']) + existingFlowsIds.append(str(afId)) + + + flowOrder = protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] if 'reprolib:terms/activityFlowOrder' in protocol['meta']['protocol'] else [] + flowOrderIdsMap = [fo['@id'] for fo in flowOrder] + for afId in existingFlowsIds: + if not str(afId) in flowOrderIdsMap: + flowOrder.append({'@id': str(afId)}) + + filteredFlowOrder = [fo for fo in flowOrder if fo['@id'] in existingFlowsIds] + flowOrderChanged = flowOrder != filteredFlowOrder or len(flowOrder) != len(flowOrderIdsMap) + if flowOrderChanged: + print('fixing activityFlowOrder for applet ' + str(applet['_id'])) + protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] = filteredFlowOrder + FolderModel().setMetadata(protocol, protocol['meta']) + jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) + + +if __name__ == '__main__': + applets = Applet().find(query={'_id': ObjectId('633e855131f2c2777e5e1bb6')}, fields={"_id": 1}) + main(applets) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py new file mode 100644 index 000000000..0f23ea581 --- /dev/null +++ b/girderformindlogger/external/fix_flankers.py @@ -0,0 +1,172 @@ +# Update flanker activities from old to new protocol +# Old: https://raw.githubusercontent.com/mtg137/Flanker_applet/master/protocols/flanker/flanker_schema +# New: https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/protocols/flanker/flanker_schema + +from girderformindlogger.models.item import Item +from girderformindlogger.models.activity import Activity +from girderformindlogger.models.folder import Folder +from girderformindlogger.models.user import User +from girderformindlogger.models.applet import Applet +from girderformindlogger.models.account_profile import AccountProfile +from girderformindlogger.models.cache import Cache +from girderformindlogger.utility import jsonld_expander +from bson.objectid import ObjectId +import json + +def prepare_items(activityId): + items = Item().find(query={'meta.activityId': activityId, 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) + itemsCount = items.count() + print('total', itemsCount) + affectedActivityIds = [] + for index, itemId in enumerate(items, start=1): + item = Item().findOne(itemId) + print('processing', item['_id'], index, '/', itemsCount) + affectedActivityIds.append(item['meta']['activityId']) + + item['meta']['screen']['url'] = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/items/{}'.format(item['meta']['screen']['@id']) + item['meta']['identifier'] = '{}/{}'.format(str(item['meta']['activityId']), str(itemId['_id'])) #after import + Item().setMetadata(item, item['meta']) + + affectedActivityIds = list(set(affectedActivityIds)) + + if (len(affectedActivityIds)): + print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) + + return affectedActivityIds + + +def findInput(name, inputs): + return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) + + +def fix_q1_issue_in_json(id, model): + if not 'reprolib:terms/inputs' in model: + return False + inputs = model['reprolib:terms/inputs'] + trials = findInput('trials', inputs) + if trials is None or not 'schema:itemListElement' in trials: + return False + print('processing item:', id, model['@id']) + itemChanged = False + for trial in trials['schema:itemListElement']: + if 'q1' == trial['schema:name'][0]['@value']: + trial['schema:name'][0]['@value'] = trial['schema:image'] + trial['schema:image'] = '' + itemChanged = True + + return itemChanged + + +def fix_q1_issue(activityId): + activityChanged = False + items = Item().find(query={'meta.activityId': activityId, 'meta.screen': {'$exists': True}}) + for item in items: + if fix_q1_issue_in_json(item['_id'], item['meta']['screen']): + Item().setMetadata(item, item['meta'], validate=False) + activityChanged = True + + if 'cached' in item: + cache = Cache().getFromSourceID('item', item['_id']) + if cache is not None and fix_q1_issue_in_json(item['_id'], cache): + Cache().updateCache(item['cached'], 'item', item['_id'], 'screen', cache) + + return activityChanged + + +def fix_q1_issue_in_versions(activityId): + print('fix_q1_issue_in_versions') + hActivities = Folder().find(query={'meta.originalId': activityId, 'meta.activity': {'$exists': True}}) + hCount = hActivities.count() + if hCount == 0: + return + print('Fixing ' + str(hCount) + ' historical version(s) of the activity id=' + str(activityId)) + for hActivity in hActivities: + print('processing activity: ', hActivity['_id']) + fix_q1_issue(hActivity['_id']) + + +def find_applet_by_activity(activity): + protocolId = activity['meta']['protocolId'] + applet = Folder().findOne(query={'meta.applet': {'$exists': True}, 'meta.applet.deleted': {'$exists': False}, 'meta.protocol._id': "protocol/{}".format(str(protocolId))}) + return applet + + +def fix_editing(applet): + if 'editing' in applet['meta']['applet'] and applet['meta']['applet']['editing']: + applet['meta']['applet']['editing'] = False + Folder().setMetadata(folder=applet, metadata=applet['meta']) + return True + return False + + +def fix_flankers(activityId, reImport = True): + activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema' + print('Refreshing affected activity id=' + str(activityId)) + activity = Folder().findOne(query={'_id': activityId}) + activity['meta']['activity']['url'] = activityUrl + activity['meta']['activity']['_id'] = "activity/{}".format(str(activityId)) #after import + if not 'identifier' in activity['meta']: + activity['meta']['identifier'] = str(activityId) + + Folder().setMetadata(folder=activity, metadata=activity['meta']) + + user = User().findOne({'_id': activity['creatorId']}) + searchCriteria = {'identifier': activity['meta']['identifier'], 'protocolId': activity['meta']['protocolId']} + if reImport: + res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta=searchCriteria) # comment out after first run + # refresh cache for the affected activity + jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) + applet = find_applet_by_activity(activity) + print('Refreshing affected applet id:', str(applet['_id'])) + fix_editing(applet) + jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) + + +def main(activityId): + affectedActivityIds = prepare_items(activityId) + + for activityId in affectedActivityIds: + fix_q1_issue_in_versions(activityId) + + for activityId in affectedActivityIds: + fix_flankers(activityId, True) + + # fix some fields after import + affectedActivityIds = prepare_items(ObjectId('6290ed45e50eef5716db579c')) + for activityId in affectedActivityIds: + fix_flankers(activityId, False) + +def get_activities_for_account(email): + print('get_activities_for_account for', email) + user = User().findOne({'email': User().hash(email), 'email_encrypted': True}) + + if user is None: + user = User().findOne({'email': email, 'email_encrypted': {'$ne': True}}) + + if user is None: + raise AccessException('user not found') + + activities = [] + applets = Applet().getAppletsForUser('manager', user, active=True) + for applet in applets: + for activityId in applet['meta']['protocol']['activities']: + activity = Activity().findOne({'_id': activityId}) + if activity is None or not '@id' in activity['meta']['activity']: + continue + if activity['meta']['activity']['@id'] == 'Flanker_360': + print('applet', applet['name'], applet['_id'], 'activity', activity['name'], activity['_id']) + activities.append(activity) + print('activities to process', len(activities)) + return activities + + +if __name__ == '__main__': + activities = get_activities_for_account('jeligi9407@zneep.com') + for activity in activities: + main(activity['_id']) + # activityId = ObjectId('6290ed45e50eef5716db579c') + # main(activityId) + # fix_editing for applet + # applet = Folder().findOne(query={'_id': ObjectId('627d1e2de50eef3d5567ee55')}) + # if fix_editing(applet): + # jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) diff --git a/girderformindlogger/models/__init__.py b/girderformindlogger/models/__init__.py index dfe988c22..87c6ee8d2 100644 --- a/girderformindlogger/models/__init__.py +++ b/girderformindlogger/models/__init__.py @@ -136,6 +136,9 @@ def pluralize(modelType): def cycleModels(IRIset, modelType=None, meta={}): + # import pandas as pd + from pandas.io.json import json_normalize + from girderformindlogger.constants import REPROLIB_TYPES from girderformindlogger.models.folder import Folder as FolderModel from girderformindlogger.models.item import Item as ItemModel @@ -164,8 +167,9 @@ def cycleModels(IRIset, modelType=None, meta={}): } } - for key in meta: - query['meta.{}'.format(key)] = meta[key] + flattenedMeta = json_normalize(meta).to_dict(orient='records')[0] + for key in flattenedMeta: + query['meta.{}'.format(key)] = flattenedMeta[key] cachedDoc = ItemModel().findOne(query) if modelType == 'screen' else FolderModel().findOne(query) @@ -189,10 +193,13 @@ def smartImport(IRI, user=None, refreshCache=False, modelType=None, meta={}): reprolibCanonize MODELS = MODELS() - mt1 = "screen" if modelType in [ - None, - "external JSON-LD document" - ] else modelType + if modelType in [None, "external JSON-LD document"]: + mt1 = "screen" + if '/' in IRI: + meta['screen'] = {'@id': IRI.split('/')[-1]} + else: + mt1 = modelType + model, modelType = MODELS[mt1]().getFromUrl( IRI, user=user, diff --git a/girderformindlogger/models/applet.py b/girderformindlogger/models/applet.py index a95e3f082..26633047f 100644 --- a/girderformindlogger/models/applet.py +++ b/girderformindlogger/models/applet.py @@ -1943,6 +1943,9 @@ def getNextAppletData(self, activities, nextActivity, bufferSize): activity, 'activity' ) + if not formattedActivity: + print('formattedActivity is empty. ActivityId=' + str(activity['_id'])) + continue buffer['activities'][activityIRI] = formattedActivity['activity'] buffer['items'].update(formattedActivity['items']) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index 67dd5fe36..4f7ceb79e 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -572,12 +572,13 @@ def cacheProtocolContent(protocol, document, user, editExisting=False): cacheModel = CacheModel() - if editExisting and 'baseVersion' in document: - latestItem = ItemModel().findOne({ - 'folderId': contentFolder['_id'], - 'version': document['baseVersion'] - }) + latestItem = ItemModel().findOne({ + 'folderId': contentFolder['_id'], + 'version': document['baseVersion'], + 'content': {'$exists': True} + }) if editExisting and 'baseVersion' in document else None + if latestItem is not None: latestDocument = json_util.loads(latestItem['content']) for key in dict.keys(latestDocument['protocol']['activities']): @@ -817,6 +818,7 @@ def loadFromSingleFile(document, user, editExisting=False): def importAndCompareModelType(model, url, user, modelType, meta={}, existing=None): import threading from girderformindlogger.utility import firstLower + from functools import reduce if model is None: return(None, None) @@ -870,15 +872,15 @@ def importAndCompareModelType(model, url, user, modelType, meta={}, existing=Non if modelClass.name=='folder': newModel = modelClass.setMetadata( docFolder, - { - modelType: { + reduce(merge, [ + {modelType: { **model, 'schema:url': url, 'url': url - }, - **meta, - 'schema': APPLET_SCHEMA_VERSION - } + }}, + meta, + {'schema': APPLET_SCHEMA_VERSION} + ]) ) elif modelClass.name=='item': item = None @@ -909,15 +911,15 @@ def importAndCompareModelType(model, url, user, modelType, meta={}, existing=Non newModel = modelClass.setMetadata( item, - { - modelType: { + reduce(merge, [ + {modelType: { **model, 'schema:url': url, 'url': url - }, - **meta, - 'schema': APPLET_SCHEMA_VERSION - } + }}, + meta, + {'schema': APPLET_SCHEMA_VERSION} + ]) ) modelClass.update( @@ -937,6 +939,15 @@ def importAndCompareModelType(model, url, user, modelType, meta={}, existing=Non createCache(newModel, formatted, modelType, user) return(formatted, modelType) +def merge(a, b, path=None): + "merges b into a" + if path is None: path = [] + for key in b: + if key in a and isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + else: + a[key] = b[key] + return a def _createContextForStr(s): sp = s.split('/') @@ -1656,7 +1667,12 @@ def formatLdObject( oc is not None ]): if mesoPrefix == 'item' or mesoPrefix == 'screen' or obj.get('meta', {}).get('schema', '') == APPLET_SCHEMA_VERSION: - return(loadCache(oc)) + cached = loadCache(oc) + if cached is not None: + return cached + else: + refreshCache=True + reimportFromUrl=False else: return {} @@ -1669,6 +1685,7 @@ def formatLdObject( o, mesoPrefix, refreshCache=refreshCache, + reimportFromUrl=reimportFromUrl, user=user ) for o in obj if o is not None ])) @@ -1731,7 +1748,8 @@ def formatLdObject( protocolObj, 'protocol', user, - refreshCache=refreshCache + refreshCache=refreshCache, + reimportFromUrl=reimportFromUrl ) applet = {} @@ -1796,13 +1814,13 @@ def formatLdObject( modelClasses = {} - if obj.get('loadedFromSingleFile', False): + if obj.get('loadedFromSingleFile', False) or not reimportFromUrl: activities = ActivityModel().find({'meta.protocolId': obj['_id'], 'meta.activity': { '$exists': True }}) activityIDMapping = {} for activity in activities: - formatted = formatLdObject(activity, 'activity', user, refreshCache=refreshCache) + formatted = formatLdObject(activity, 'activity', user, refreshCache=refreshCache, reimportFromUrl=False) if refreshCache: createCache(activity, formatted, 'activity', user, modelClasses) @@ -1818,7 +1836,7 @@ def formatLdObject( activityFlowIdMapping = {} for activityFlow in activityFlows: - formatted = formatLdObject(activityFlow, 'activityFlow', user, refreshCache=refreshCache) + formatted = formatLdObject(activityFlow, 'activityFlow', user, refreshCache=refreshCache, reimportFromUrl=False) if refreshCache: createCache(activityFlow, formatted, 'activityFlow', user, modelClasses) @@ -1851,8 +1869,12 @@ def formatLdObject( return protocol elif mesoPrefix=='activity': + if refreshCache and 'activity' in obj['meta'] and not '_id' in obj['meta']['activity']: + obj['meta']['activity']['_id'] = "activity/{}".format(str(obj['_id'])) + ActivityModel().setMetadata(obj, obj['meta']) + itemIDMapping = {} - if obj.get('loadedFromSingleFile', False): + if obj.get('loadedFromSingleFile', False) or not reimportFromUrl: items = ScreenModel().find({'meta.activityId': obj['_id']}) activity = { @@ -1860,15 +1882,22 @@ def formatLdObject( } for item in items: + if not 'identifier' in item['meta']: + item['meta']['identifier'] = '{}/{}'.format(str(obj['_id']), str(item['_id'])) identifier = item['meta']['identifier'] - activity['items'][identifier] = formatLdObject(item, 'screen', user) + itemFormatted = formatLdObject(item, 'screen', user, refreshCache=refreshCache, reimportFromUrl=False) + activity['items'][identifier] = itemFormatted key = '{}/{}'.format(str(item['meta']['activityId']), str(item['_id'])) - itemIDMapping['{}/{}'.format(str(item['meta']['activityId']), activity['items'][identifier]['@id'])] = key + itemId = activity['items'][identifier]['@id'] if '@id' in activity['items'][identifier] else activity['items'][identifier]['_id'] + itemIDMapping['{}/{}'.format(str(item['meta']['activityId']), itemId)] = key if item.get('duplicateOf', None): itemIDMapping['{}/{}'.format(str(item['meta']['activityId']), str(item['duplicateOf']))] = key + if refreshCache: + createCache(item, itemFormatted, 'item', user) + if refreshCache and fixUpList(obj, 'activity', itemIDMapping, 'reprolib:terms/order'): ActivityModel().setMetadata(obj, obj['meta']) @@ -1888,6 +1917,9 @@ def formatLdObject( meta={'protocolId': obj['meta']['protocolId'], 'activityId': obj['_id']} ) + if refreshCache: + createCache(obj, activity, 'activity', user) + return activity else: return (_fixUpFormat(newObj)) @@ -1900,6 +1932,7 @@ def formatLdObject( keepUndefined, dropErrors, refreshCache=True, + reimportFromUrl=reimportFromUrl, responseDates=responseDates ))) import sys, traceback