Fix missing fields for flanker activity 910

girderformindlogger/api/v1/applet.py

@@ -627,6 +627,7 @@ def getProtocolVersions(self, applet, retrieveDate=False):
 
         items = list(ItemModel().find({
             'folderId': protocol['meta'].get('contentId', None),
+            'version': {'$exists': True},
         }, fields=['version', 'created'], sort=[("created", DESCENDING)])) if 'contentId' in protocol['meta'] else []
 
         if retrieveDate:

girderformindlogger/external/convert_applets.py

@@ -8,11 +8,12 @@
 from girderformindlogger.utility import jsonld_expander
 from bson.objectid import ObjectId
 
-# '_id': ObjectId('5f0e35523477de8b4a528dd0'),
-applets = Applet().find(query={ 'meta.applet': { '$exists': True } }, fields= {"_id": 1})
+# '_id': ObjectId('633fc958b7ee9765ba5447a6')
+# github: 'meta.protocol.url': {'$exists': True}, 'meta.applet': {'$exists': True}, 'meta.applet.deleted': {'$exists': False}, 'parentId': ObjectId('5ea689a286d25a5dbb14e82c')
+applets = Applet().find(query={'_id': ObjectId('633fc958b7ee9765ba5447a6')}, fields= {"_id": 1})
 appletsCount = applets.count()
 print('total', appletsCount)
-skipUntil = None
+skipUntil = None # ObjectId('60a398c9acd96cf825f7679d')
 for index, appletId in enumerate(applets, start=1):
     if skipUntil == appletId['_id']:
         skipUntil = None
@@ -43,8 +44,18 @@
     g = []
     activityIRIs = dict.keys(formatted['activities'].copy())
     for activityIRI in activityIRIs:
+        activityLink = formatted['activities'][activityIRI]
+        if isinstance(activityLink, ObjectId):
+            activityId = activityLink
+        elif isinstance(activityLink, str):
+            activityId = ObjectId(activityLink)
+        elif '_id' in activityLink:
+            activityId = ObjectId(activityLink['_id'].split('/').pop())
+        else:
+            continue
+
         activity = Activity().findOne({
-            '_id': ObjectId(formatted['activities'][activityIRI])
+            '_id': activityId
         })
 
         if not activity:

girderformindlogger/external/fix_activity_flow_order.py

@@ -0,0 +1,68 @@
+# fix broken links in protocol.activityFlowOrder
+
+from girderformindlogger.models.applet import Applet
+from girderformindlogger.models.folder import Folder as FolderModel
+from girderformindlogger.utility import jsonld_expander
+from bson.objectid import ObjectId
+
+
+def findFlowsStringIds(applet):
+    flowIds = []
+
+    protocolId = applet['meta']['protocol'].get('_id').split('/').pop()
+    activityFlows = FolderModel().find({'meta.protocolId': ObjectId(protocolId), 'meta.activityFlow': {'$exists': True} }, fields={"_id": 1})
+    for af in activityFlows:
+        flowIds.append(str(af['_id']))
+
+    return flowIds
+
+
+def main(applets):
+    appletsCount = applets.count()
+    print('total', appletsCount)
+    skipUntil = None # ObjectId('60a398c9acd96cf825f7679d')
+    for index, appletId in enumerate(applets, start=1):
+        if skipUntil == appletId['_id']:
+            skipUntil = None
+        if skipUntil is not None:
+            continue
+
+        applet = Applet().findOne(appletId)
+        protocolId = ObjectId(applet['meta']['protocol'].get('_id').split('/').pop())
+        protocol = FolderModel().findOne(query={'_id': protocolId})
+
+        existingFlowsIds = findFlowsStringIds(applet)
+
+        # find and attach orphan flows
+        for afId in applet['meta']['protocol']['activityFlows']:
+            if str(afId) in existingFlowsIds:
+                continue # exclude that already refer to us
+            if FolderModel().find(query={'meta.protocol.activityFlows': afId, '_id': {'$ne': appletId['_id']}}).count() > 0:
+                continue # if no other protocols refer to this flow
+            flow = FolderModel().findOne(query={'_id': afId})
+            if flow is None:
+                continue
+            print('attaching orphan flow '+str(afId)+' to applet ' + str(applet['_id']))
+            flow['meta']['protocolId'] = protocolId
+            FolderModel().setMetadata(flow, flow['meta'])
+            existingFlowsIds.append(str(afId))
+
+
+        flowOrder = protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] if 'reprolib:terms/activityFlowOrder' in protocol['meta']['protocol'] else []
+        flowOrderIdsMap = [fo['@id'] for fo in flowOrder]
+        for afId in existingFlowsIds:
+            if not str(afId) in flowOrderIdsMap:
+                flowOrder.append({'@id': str(afId)})
+
+        filteredFlowOrder = [fo for fo in flowOrder if fo['@id'] in existingFlowsIds]
+        flowOrderChanged = flowOrder != filteredFlowOrder or len(flowOrder) != len(flowOrderIdsMap)
+        if flowOrderChanged:
+            print('fixing activityFlowOrder for applet ' + str(applet['_id']))
+            protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] = filteredFlowOrder
+            FolderModel().setMetadata(protocol, protocol['meta'])
+            jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False)
+
+
+if __name__ == '__main__':
+    applets = Applet().find(query={'_id': ObjectId('633e855131f2c2777e5e1bb6')}, fields={"_id": 1})
+    main(applets)

girderformindlogger/external/fix_flankers.py

@@ -0,0 +1,172 @@
+# Update flanker activities from old to new protocol
+# Old: https://raw.githubusercontent.com/mtg137/Flanker_applet/master/protocols/flanker/flanker_schema
+# New: https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/protocols/flanker/flanker_schema
+
+from girderformindlogger.models.item import Item
+from girderformindlogger.models.activity import Activity
+from girderformindlogger.models.folder import Folder
+from girderformindlogger.models.user import User
+from girderformindlogger.models.applet import Applet
+from girderformindlogger.models.account_profile import AccountProfile
+from girderformindlogger.models.cache import Cache
+from girderformindlogger.utility import jsonld_expander
+from bson.objectid import ObjectId
+import json
+
+def prepare_items(activityId):
+    items = Item().find(query={'meta.activityId': activityId, '[email protected]': 'reprolib:schemas/Field'}, fields= {"_id": 1})
+    itemsCount = items.count()
+    print('total', itemsCount)
+    affectedActivityIds = []
+    for index, itemId in enumerate(items, start=1):
+        item = Item().findOne(itemId)
+        print('processing', item['_id'], index, '/', itemsCount)
+        affectedActivityIds.append(item['meta']['activityId'])
+
+        item['meta']['screen']['url'] = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/items/{}'.format(item['meta']['screen']['@id'])
+        item['meta']['identifier'] = '{}/{}'.format(str(item['meta']['activityId']), str(itemId['_id'])) #after import
+        Item().setMetadata(item, item['meta'])
+
+    affectedActivityIds = list(set(affectedActivityIds))
+
+    if (len(affectedActivityIds)):
+        print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds))
+
+    return affectedActivityIds
+
+
+def findInput(name, inputs):
+    return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None)
+
+
+def fix_q1_issue_in_json(id, model):
+    if not 'reprolib:terms/inputs' in model:
+        return False
+    inputs = model['reprolib:terms/inputs']
+    trials = findInput('trials', inputs)
+    if trials is None or not 'schema:itemListElement' in trials:
+        return False
+    print('processing item:', id, model['@id'])
+    itemChanged = False
+    for trial in trials['schema:itemListElement']:
+        if 'q1' == trial['schema:name'][0]['@value']:
+            trial['schema:name'][0]['@value'] = trial['schema:image']
+            trial['schema:image'] = ''
+            itemChanged = True
+
+    return itemChanged
+
+
+def fix_q1_issue(activityId):
+    activityChanged = False
+    items = Item().find(query={'meta.activityId': activityId, 'meta.screen': {'$exists': True}})
+    for item in items:
+        if fix_q1_issue_in_json(item['_id'], item['meta']['screen']):
+            Item().setMetadata(item, item['meta'], validate=False)
+            activityChanged = True
+
+        if 'cached' in item:
+            cache = Cache().getFromSourceID('item', item['_id'])
+            if cache is not None and fix_q1_issue_in_json(item['_id'], cache):
+                Cache().updateCache(item['cached'], 'item', item['_id'], 'screen', cache)
+
+    return activityChanged
+
+
+def fix_q1_issue_in_versions(activityId):
+    print('fix_q1_issue_in_versions')
+    hActivities = Folder().find(query={'meta.originalId': activityId, 'meta.activity': {'$exists': True}})
+    hCount = hActivities.count()
+    if hCount == 0:
+        return
+    print('Fixing ' + str(hCount) + ' historical version(s) of the activity id=' + str(activityId))
+    for hActivity in hActivities:
+        print('processing activity: ', hActivity['_id'])
+        fix_q1_issue(hActivity['_id'])
+
+
+def find_applet_by_activity(activity):
+    protocolId = activity['meta']['protocolId']
+    applet = Folder().findOne(query={'meta.applet': {'$exists': True}, 'meta.applet.deleted': {'$exists': False}, 'meta.protocol._id': "protocol/{}".format(str(protocolId))})
+    return applet
+
+
+def fix_editing(applet):
+    if 'editing' in applet['meta']['applet'] and applet['meta']['applet']['editing']:
+        applet['meta']['applet']['editing'] = False
+        Folder().setMetadata(folder=applet, metadata=applet['meta'])
+        return True
+    return False
+
+
+def fix_flankers(activityId, reImport = True):
+    activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema'
+    print('Refreshing affected activity id=' + str(activityId))
+    activity = Folder().findOne(query={'_id': activityId})
+    activity['meta']['activity']['url'] = activityUrl
+    activity['meta']['activity']['_id'] = "activity/{}".format(str(activityId)) #after import
+    if not 'identifier' in activity['meta']:
+        activity['meta']['identifier'] = str(activityId)
+
+    Folder().setMetadata(folder=activity, metadata=activity['meta'])
+
+    user = User().findOne({'_id': activity['creatorId']})
+    searchCriteria = {'identifier': activity['meta']['identifier'], 'protocolId': activity['meta']['protocolId']}
+    if reImport:
+        res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta=searchCriteria) # comment out after first run
+    # refresh cache for the affected activity
+    jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False)
+    applet = find_applet_by_activity(activity)
+    print('Refreshing affected applet id:', str(applet['_id']))
+    fix_editing(applet)
+    jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False)
+
+
+def main(activityId):
+    affectedActivityIds = prepare_items(activityId)
+
+    for activityId in affectedActivityIds:
+        fix_q1_issue_in_versions(activityId)
+
+    for activityId in affectedActivityIds:
+        fix_flankers(activityId, True)
+
+    # fix some fields after import
+    affectedActivityIds = prepare_items(ObjectId('6290ed45e50eef5716db579c'))
+    for activityId in affectedActivityIds:
+        fix_flankers(activityId, False)
+
+def get_activities_for_account(email):
+    print('get_activities_for_account for', email)
+    user = User().findOne({'email': User().hash(email), 'email_encrypted': True})
+
+    if user is None:
+        user = User().findOne({'email': email, 'email_encrypted': {'$ne': True}})
+
+    if user is None:
+        raise AccessException('user not found')
+
+    activities = []
+    applets = Applet().getAppletsForUser('manager', user, active=True)
+    for applet in applets:
+        for activityId in applet['meta']['protocol']['activities']:
+            activity = Activity().findOne({'_id': activityId})
+            if activity is None or not '@id' in activity['meta']['activity']:
+                continue
+            if activity['meta']['activity']['@id'] == 'Flanker_360':
+                print('applet', applet['name'], applet['_id'], 'activity', activity['name'], activity['_id'])
+                activities.append(activity)
+    print('activities to process', len(activities))
+    return activities
+
+
+if __name__ == '__main__':
+    activities = get_activities_for_account('[email protected]')
+    for activity in activities:
+        main(activity['_id'])
+    # activityId = ObjectId('6290ed45e50eef5716db579c')
+    # main(activityId)
+    # fix_editing for applet
+    # applet = Folder().findOne(query={'_id': ObjectId('627d1e2de50eef3d5567ee55')})
+    # if fix_editing(applet):
+    #     jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False)

girderformindlogger/models/__init__.py

@@ -136,6 +136,9 @@ def pluralize(modelType):
 
 
 def cycleModels(IRIset, modelType=None, meta={}):
+    # import pandas as pd
+    from pandas.io.json import json_normalize
+
     from girderformindlogger.constants import REPROLIB_TYPES
     from girderformindlogger.models.folder import Folder as FolderModel
     from girderformindlogger.models.item import Item as ItemModel
@@ -164,8 +167,9 @@ def cycleModels(IRIset, modelType=None, meta={}):
         }
     }
 
-    for key in meta:
-        query['meta.{}'.format(key)] = meta[key]
+    flattenedMeta = json_normalize(meta).to_dict(orient='records')[0]
+    for key in flattenedMeta:
+        query['meta.{}'.format(key)] = flattenedMeta[key]
 
     cachedDoc = ItemModel().findOne(query) if modelType == 'screen' else FolderModel().findOne(query)
 
@@ -189,10 +193,13 @@ def smartImport(IRI, user=None, refreshCache=False, modelType=None, meta={}):
         reprolibCanonize
 
     MODELS = MODELS()
-    mt1 = "screen" if modelType in [
-        None,
-        "external JSON-LD document"
-    ] else modelType
+    if modelType in [None, "external JSON-LD document"]:
+        mt1 = "screen"
+        if '/' in IRI:
+            meta['screen'] = {'@id': IRI.split('/')[-1]}
+    else:
+        mt1 = modelType
+
     model, modelType = MODELS[mt1]().getFromUrl(
         IRI,
         user=user,

girderformindlogger/models/applet.py

@@ -1943,6 +1943,9 @@ def getNextAppletData(self, activities, nextActivity, bufferSize):
                 activity,
                 'activity'
             )
+            if not formattedActivity:
+                print('formattedActivity is empty. ActivityId=' + str(activity['_id']))
+                continue
 
             buffer['activities'][activityIRI] = formattedActivity['activity']
             buffer['items'].update(formattedActivity['items'])