From 2f22ebd65dfd896d7523d665ba115f23b10c3950 Mon Sep 17 00:00:00 2001 From: karser Date: Sat, 24 Sep 2022 20:11:38 +0300 Subject: [PATCH 01/16] Fix missing fields for flanker activity ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 71 +++++++++++++++++++ .../utility/jsonld_expander.py | 12 +++- 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 girderformindlogger/external/fix_flankers.py diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py new file mode 100644 index 000000000..9ca7129fa --- /dev/null +++ b/girderformindlogger/external/fix_flankers.py @@ -0,0 +1,71 @@ +# Add missing properties to flanker items: blocks, buttons, fixationDuration, fixationScreen, blockType + +from girderformindlogger.models.item import Item +from girderformindlogger.models.activity import Activity +from girderformindlogger.utility import jsonld_expander +from bson.objectid import ObjectId + + +def inputExists(name, inputs): + return any(i['schema:name'][0]['@value'] == name for i in inputs) + +def findInput(name, inputs): + return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) + +items = Item().find(query={'meta.activityId': ObjectId('628e3d6be50eef3353e63813'), 'meta.screen.reprolib:terms/inputType.0.@value': 'visual-stimulus-response'}, fields= {"_id": 1}) +itemsCount = items.count() +print('total', itemsCount) +skipUntil = None +affectedActivityIds = [] +for index, itemId in enumerate(items, start=1): + if skipUntil == itemId['_id']: + skipUntil = None + if skipUntil is not None: + continue + + item = Item().findOne(itemId) + itemChanged = False + activityId = item['meta']['activityId'] + print('processing', item['_id'], index, '/', itemsCount) + inputs = item['meta']['screen']['reprolib:terms/inputs'] + if (findInput('fixationScreen', inputs) is None): + print('adding fixationScreen') + inputs.append({"@type":["http://schema.org/Text"],"schema:image":"","schema:name":[{"@language":"en","@value":"fixationScreen"}],"schema:value":[{"@language":"en","@value":""}]}) + itemChanged = True + + if (findInput('fixationDuration', inputs) is None): + print('adding fixationDuration') + inputs.append({"@type":["http://schema.org/Number"],"schema:name":[{"@language":"en","@value":"fixationDuration"}],"schema:value":[{"@value":0}]}) + itemChanged = True + + if (findInput('buttons', inputs) is None): + print('adding buttons') + inputs.append({"schema:itemListElement":[{"schema:image":"","schema:name":[{"@language":"en","@value":"left"}],"schema:value":[{"@value":0}]},{"schema:image":"","schema:name":[{"@language":"en","@value":"right"}],"schema:value":[{"@value":0}]}],"schema:name":[{"@language":"en","@value":"buttons"}]}) + itemChanged = True + + if (findInput('blocks', inputs) is None): + print('adding blocks') + inputs.append({"@type":["http://schema.org/ItemList"],"schema:itemListElement":[],"schema:name":[{"@language":"en","@value":"blocks"}]}) + itemChanged = True + + if (findInput('blockType', inputs) is None): + value = 'test' if 'test' in item['name'].casefold() else 'practice' + print('adding blockType', value) + inputs.append({"@type":["http://schema.org/Text"],"schema:name":[{"@language":"en","@value":"blockType"}],"schema:value":[{"@language":"en","@value":value}]}) + itemChanged = True + + if itemChanged: + Item().setMetadata(item, item['meta'], validate=False) + affectedActivityIds.append(activityId) + + +affectedActivityIds = list(set(affectedActivityIds)) + +if (len(affectedActivityIds)): + print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) + +# refresh cache for the affected activities +for activityId in affectedActivityIds: + print('Refreshing affected activity id=' + str(activityId)) + activity = Activity().findOne({'_id': activityId}) + jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index 67dd5fe36..ba8c9eb90 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -1861,14 +1861,19 @@ def formatLdObject( for item in items: identifier = item['meta']['identifier'] - activity['items'][identifier] = formatLdObject(item, 'screen', user) + itemFormatted = formatLdObject(item, 'screen', user, refreshCache=refreshCache) + activity['items'][identifier] = itemFormatted key = '{}/{}'.format(str(item['meta']['activityId']), str(item['_id'])) - itemIDMapping['{}/{}'.format(str(item['meta']['activityId']), activity['items'][identifier]['@id'])] = key + itemId = activity['items'][identifier]['@id'] if '@id' in activity['items'][identifier] else activity['items'][identifier]['_id'] + itemIDMapping['{}/{}'.format(str(item['meta']['activityId']), itemId)] = key if item.get('duplicateOf', None): itemIDMapping['{}/{}'.format(str(item['meta']['activityId']), str(item['duplicateOf']))] = key + if refreshCache: + createCache(item, itemFormatted, 'item', user) + if refreshCache and fixUpList(obj, 'activity', itemIDMapping, 'reprolib:terms/order'): ActivityModel().setMetadata(obj, obj['meta']) @@ -1888,6 +1893,9 @@ def formatLdObject( meta={'protocolId': obj['meta']['protocolId'], 'activityId': obj['_id']} ) + if refreshCache: + createCache(obj, activity, 'activity', user) + return activity else: return (_fixUpFormat(newObj)) From d504e0572a642381feec9720a32d8ff7aa2e2d3c Mon Sep 17 00:00:00 2001 From: karser Date: Thu, 29 Sep 2022 13:01:36 +0300 Subject: [PATCH 02/16] re-import flanker activity ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 149 +++++++++++++----- girderformindlogger/models/__init__.py | 19 ++- .../utility/jsonld_expander.py | 42 +++-- 3 files changed, 149 insertions(+), 61 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index 9ca7129fa..e8cdc79ab 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -2,17 +2,13 @@ from girderformindlogger.models.item import Item from girderformindlogger.models.activity import Activity +from girderformindlogger.models.folder import Folder +from girderformindlogger.models.user import User from girderformindlogger.utility import jsonld_expander from bson.objectid import ObjectId -def inputExists(name, inputs): - return any(i['schema:name'][0]['@value'] == name for i in inputs) - -def findInput(name, inputs): - return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) - -items = Item().find(query={'meta.activityId': ObjectId('628e3d6be50eef3353e63813'), 'meta.screen.reprolib:terms/inputType.0.@value': 'visual-stimulus-response'}, fields= {"_id": 1}) +items = Item().find(query={'meta.activityId': ObjectId('628e3d6be50eef3353e63813'), 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) itemsCount = items.count() print('total', itemsCount) skipUntil = None @@ -24,39 +20,11 @@ def findInput(name, inputs): continue item = Item().findOne(itemId) - itemChanged = False - activityId = item['meta']['activityId'] print('processing', item['_id'], index, '/', itemsCount) - inputs = item['meta']['screen']['reprolib:terms/inputs'] - if (findInput('fixationScreen', inputs) is None): - print('adding fixationScreen') - inputs.append({"@type":["http://schema.org/Text"],"schema:image":"","schema:name":[{"@language":"en","@value":"fixationScreen"}],"schema:value":[{"@language":"en","@value":""}]}) - itemChanged = True - - if (findInput('fixationDuration', inputs) is None): - print('adding fixationDuration') - inputs.append({"@type":["http://schema.org/Number"],"schema:name":[{"@language":"en","@value":"fixationDuration"}],"schema:value":[{"@value":0}]}) - itemChanged = True - - if (findInput('buttons', inputs) is None): - print('adding buttons') - inputs.append({"schema:itemListElement":[{"schema:image":"","schema:name":[{"@language":"en","@value":"left"}],"schema:value":[{"@value":0}]},{"schema:image":"","schema:name":[{"@language":"en","@value":"right"}],"schema:value":[{"@value":0}]}],"schema:name":[{"@language":"en","@value":"buttons"}]}) - itemChanged = True - - if (findInput('blocks', inputs) is None): - print('adding blocks') - inputs.append({"@type":["http://schema.org/ItemList"],"schema:itemListElement":[],"schema:name":[{"@language":"en","@value":"blocks"}]}) - itemChanged = True - - if (findInput('blockType', inputs) is None): - value = 'test' if 'test' in item['name'].casefold() else 'practice' - print('adding blockType', value) - inputs.append({"@type":["http://schema.org/Text"],"schema:name":[{"@language":"en","@value":"blockType"}],"schema:value":[{"@language":"en","@value":value}]}) - itemChanged = True - - if itemChanged: - Item().setMetadata(item, item['meta'], validate=False) - affectedActivityIds.append(activityId) + affectedActivityIds.append(item['meta']['activityId']) + + item['meta']['screen']['url'] = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/items/{}'.format(item['meta']['screen']['@id']) + Item().setMetadata(item, item['meta']) affectedActivityIds = list(set(affectedActivityIds)) @@ -64,8 +32,109 @@ def findInput(name, inputs): if (len(affectedActivityIds)): print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) +activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema' + # refresh cache for the affected activities for activityId in affectedActivityIds: print('Refreshing affected activity id=' + str(activityId)) - activity = Activity().findOne({'_id': activityId}) + activity = Folder().findOne(query={'_id': activityId}) + activity['meta']['activity']['url'] = activityUrl + Folder().setMetadata(folder=activity, metadata=activity['meta']) + protocolId = activity['meta']['protocolId'] + + user = User().findOne({'_id': activity['creatorId']}) + res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta={'identifier': activityId, 'protocolId': protocolId}) # meta.activity._id=activity/628e3d6be50eef3353e63813 + jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) + +exit(0) + + +# def inputExists(name, inputs): +# return any(i['schema:name'][0]['@value'] == name for i in inputs) +# +# def findInput(name, inputs): +# return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) +# +# items = Item().find(query={'meta.activityId': ObjectId('628e3d6be50eef3353e63813'), 'meta.screen.reprolib:terms/inputType.0.@value': 'visual-stimulus-response'}, fields= {"_id": 1}) +# itemsCount = items.count() +# print('total', itemsCount) +# skipUntil = None +# affectedActivityIds = [] +# for index, itemId in enumerate(items, start=1): +# if skipUntil == itemId['_id']: +# skipUntil = None +# if skipUntil is not None: +# continue +# +# item = Item().findOne(itemId) +# itemChanged = False +# activityId = item['meta']['activityId'] +# print('processing', item['_id'], index, '/', itemsCount) +# inputs = item['meta']['screen']['reprolib:terms/inputs'] +# +# +# itemChanged = True +# +# trials = findInput('trials', inputs) +# trialsIdx = inputs.index(trials) +# inputs[trialsIdx] = {"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"@id":"left-con","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"<<<<<"}],"schema:value":[{"@value":0}]},{"@id":"right-inc","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"<<><<"}],"schema:value":[{"@value":1}]},{"@id":"left-inc","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":">><>>"}],"schema:value":[{"@value":0}]},{"@id":"right-con","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":">>>>>"}],"schema:value":[{"@value":1}]},{"@id":"left-neut","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"--<--"}],"schema:value":[{"@value":0}]},{"@id":"right-neut","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"-->--"}],"schema:value":[{"@value":1}]}],"schema:name":[{"@language":"en","@value":"trials"}],"schema:numberOfItems":[{"@value":6}]} +# +# +# blockType = 'test' if 'test' in item['meta']['screen']['@id'].casefold() else 'practice' +# +# blocks = findInput('blocks', inputs) +# blocksIdx = inputs.index(blocks) +# if (blockType == 'test'): +# inputs[blocksIdx] = {"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 1"}],"schema:value":[{"@value":0}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 2"}],"schema:value":[{"@value":1}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 3"}],"schema:value":[{"@value":2}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 4"}],"schema:value":[{"@value":3}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 5"}],"schema:value":[{"@value":4}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 6"}],"schema:value":[{"@value":5}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 7"}],"schema:value":[{"@value":6}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 8"}],"schema:value":[{"@value":7}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 9"}],"schema:value":[{"@value":8}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 10"}],"schema:value":[{"@value":9}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 11"}],"schema:value":[{"@value":10}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 12"}],"schema:value":[{"@value":11}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 13"}],"schema:value":[{"@value":12}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 14"}],"schema:value":[{"@value":13}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 15"}],"schema:value":[{"@value":14}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 16"}],"schema:value":[{"@value":15}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 17"}],"schema:value":[{"@value":16}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 18"}],"schema:value":[{"@value":17}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 19"}],"schema:value":[{"@value":18}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 20"}],"schema:value":[{"@value":19}]}],"schema:name":[{"@language":"en","@value":"blocks"}],"schema:numberOfItems":[{"@value":20}]} +# else: +# inputs[blocksIdx] = {"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 1"}],"schema:value":[{"@value":0}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 2"}],"schema:value":[{"@value":1}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 3"}],"schema:value":[{"@value":2}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 4"}],"schema:value":[{"@value":3}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 5"}],"schema:value":[{"@value":4}]}],"schema:name":[{"@language":"en","@value":"blocks"}],"schema:numberOfItems":[{"@value":5}]} +# +# +# if (findInput('fixationScreen', inputs) is None): +# print('adding fixationScreen') +# inputs.append({"@type":["http://schema.org/Text"],"schema:image":"","schema:name":[{"@language":"en","@value":"fixationScreen"}],"schema:value":[{"@language":"en","@value":"-----"}]}) +# itemChanged = True +# +# if (findInput('fixationDuration', inputs) is None): +# print('adding fixationDuration') +# inputs.append({"@type":["http://schema.org/Number"],"schema:name":[{"@language":"en","@value":"fixationDuration"}],"schema:value":[{"@value":500}]}) +# itemChanged = True +# +# if (findInput('buttons', inputs) is None): +# print('adding buttons') +# inputs.append({"schema:itemListElement":[{"schema:image":"","schema:name":[{"@language":"en","@value":"<"}],"schema:value":[{"@value":0}]},{"schema:image":"","schema:name":[{"@language":"en","@value":">"}],"schema:value":[{"@value":0}]}],"schema:name":[{"@language":"en","@value":"buttons"}]}) +# itemChanged = True +# +# if (findInput('blocks', inputs) is None): +# print('adding blocks') +# inputs.append({"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 1"}],"schema:value":[{"@value":0}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 2"}],"schema:value":[{"@value":1}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 3"}],"schema:value":[{"@value":2}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 4"}],"schema:value":[{"@value":3}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 5"}],"schema:value":[{"@value":4}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 6"}],"schema:value":[{"@value":5}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 7"}],"schema:value":[{"@value":6}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 8"}],"schema:value":[{"@value":7}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 9"}],"schema:value":[{"@value":8}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 10"}],"schema:value":[{"@value":9}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 11"}],"schema:value":[{"@value":10}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 12"}],"schema:value":[{"@value":11}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 13"}],"schema:value":[{"@value":12}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 14"}],"schema:value":[{"@value":13}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 15"}],"schema:value":[{"@value":14}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 16"}],"schema:value":[{"@value":15}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 17"}],"schema:value":[{"@value":16}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 18"}],"schema:value":[{"@value":17}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 19"}],"schema:value":[{"@value":18}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 20"}],"schema:value":[{"@value":19}]}],"schema:name":[{"@language":"en","@value":"blocks"}],"schema:numberOfItems":[{"@value":20}]}) +# itemChanged = True +# +# if (findInput('blockType', inputs) is None): +# value = 'test' if 'test' in item['meta']['screen']['@id'].casefold() else 'practice' +# print('adding blockType', value) +# inputs.append({"@type":["http://schema.org/Text"],"schema:name":[{"@language":"en","@value":"blockType"}],"schema:value":[{"@language":"en","@value":value}]}) +# itemChanged = True +# +# if itemChanged: +# Item().setMetadata(item, item['meta'], validate=False) +# affectedActivityIds.append(activityId) +# +# +# affectedActivityIds = list(set(affectedActivityIds)) +# +# if (len(affectedActivityIds)): +# print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) +# +# # refresh cache for the affected activities +# for activityId in affectedActivityIds: +# print('Refreshing affected activity id=' + str(activityId)) +# activity = Activity().findOne({'_id': activityId}) +# jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) + + + +# old: https://raw.githubusercontent.com/mtg137/Flanker_applet/master/protocols/flanker/flanker_schema +# old: https://raw.githubusercontent.com/mtg137/Flanker_applet/staging/protocols/flanker/flanker_schema +# TODO: file names are renamed. how about responses? diff --git a/girderformindlogger/models/__init__.py b/girderformindlogger/models/__init__.py index dfe988c22..87c6ee8d2 100644 --- a/girderformindlogger/models/__init__.py +++ b/girderformindlogger/models/__init__.py @@ -136,6 +136,9 @@ def pluralize(modelType): def cycleModels(IRIset, modelType=None, meta={}): + # import pandas as pd + from pandas.io.json import json_normalize + from girderformindlogger.constants import REPROLIB_TYPES from girderformindlogger.models.folder import Folder as FolderModel from girderformindlogger.models.item import Item as ItemModel @@ -164,8 +167,9 @@ def cycleModels(IRIset, modelType=None, meta={}): } } - for key in meta: - query['meta.{}'.format(key)] = meta[key] + flattenedMeta = json_normalize(meta).to_dict(orient='records')[0] + for key in flattenedMeta: + query['meta.{}'.format(key)] = flattenedMeta[key] cachedDoc = ItemModel().findOne(query) if modelType == 'screen' else FolderModel().findOne(query) @@ -189,10 +193,13 @@ def smartImport(IRI, user=None, refreshCache=False, modelType=None, meta={}): reprolibCanonize MODELS = MODELS() - mt1 = "screen" if modelType in [ - None, - "external JSON-LD document" - ] else modelType + if modelType in [None, "external JSON-LD document"]: + mt1 = "screen" + if '/' in IRI: + meta['screen'] = {'@id': IRI.split('/')[-1]} + else: + mt1 = modelType + model, modelType = MODELS[mt1]().getFromUrl( IRI, user=user, diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index ba8c9eb90..b85c4d4f2 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -817,6 +817,7 @@ def loadFromSingleFile(document, user, editExisting=False): def importAndCompareModelType(model, url, user, modelType, meta={}, existing=None): import threading from girderformindlogger.utility import firstLower + from functools import reduce if model is None: return(None, None) @@ -870,15 +871,15 @@ def importAndCompareModelType(model, url, user, modelType, meta={}, existing=Non if modelClass.name=='folder': newModel = modelClass.setMetadata( docFolder, - { - modelType: { + reduce(merge, [ + {modelType: { **model, 'schema:url': url, 'url': url - }, - **meta, - 'schema': APPLET_SCHEMA_VERSION - } + }}, + meta, + {'schema': APPLET_SCHEMA_VERSION} + ]) ) elif modelClass.name=='item': item = None @@ -909,15 +910,15 @@ def importAndCompareModelType(model, url, user, modelType, meta={}, existing=Non newModel = modelClass.setMetadata( item, - { - modelType: { + reduce(merge, [ + {modelType: { **model, 'schema:url': url, 'url': url - }, - **meta, - 'schema': APPLET_SCHEMA_VERSION - } + }}, + meta, + {'schema': APPLET_SCHEMA_VERSION} + ]) ) modelClass.update( @@ -937,6 +938,15 @@ def importAndCompareModelType(model, url, user, modelType, meta={}, existing=Non createCache(newModel, formatted, modelType, user) return(formatted, modelType) +def merge(a, b, path=None): + "merges b into a" + if path is None: path = [] + for key in b: + if key in a and isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + else: + a[key] = b[key] + return a def _createContextForStr(s): sp = s.split('/') @@ -1796,7 +1806,7 @@ def formatLdObject( modelClasses = {} - if obj.get('loadedFromSingleFile', False): + if obj.get('loadedFromSingleFile', False) or not reimportFromUrl: activities = ActivityModel().find({'meta.protocolId': obj['_id'], 'meta.activity': { '$exists': True }}) activityIDMapping = {} @@ -1852,7 +1862,7 @@ def formatLdObject( return protocol elif mesoPrefix=='activity': itemIDMapping = {} - if obj.get('loadedFromSingleFile', False): + if obj.get('loadedFromSingleFile', False) or not reimportFromUrl: items = ScreenModel().find({'meta.activityId': obj['_id']}) activity = { @@ -1860,8 +1870,10 @@ def formatLdObject( } for item in items: + if not 'identifier' in item['meta']: + item['meta']['identifier'] = '{}/{}'.format(str(obj['_id']), str(item['_id'])) identifier = item['meta']['identifier'] - itemFormatted = formatLdObject(item, 'screen', user, refreshCache=refreshCache) + itemFormatted = formatLdObject(item, 'screen', user, refreshCache=refreshCache, reimportFromUrl=reimportFromUrl) activity['items'][identifier] = itemFormatted key = '{}/{}'.format(str(item['meta']['activityId']), str(item['_id'])) From 89b00291966eaedb2cf074a8fea1081fe7c0a133 Mon Sep 17 00:00:00 2001 From: karser Date: Fri, 21 Oct 2022 14:35:51 +0300 Subject: [PATCH 03/16] added ability to run against github applets ChildMindInstitute/mindlogger-applet-builder#1066 --- girderformindlogger/utility/jsonld_expander.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index b85c4d4f2..e807f3e59 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -1679,6 +1679,7 @@ def formatLdObject( o, mesoPrefix, refreshCache=refreshCache, + reimportFromUrl=reimportFromUrl, user=user ) for o in obj if o is not None ])) @@ -1741,7 +1742,8 @@ def formatLdObject( protocolObj, 'protocol', user, - refreshCache=refreshCache + refreshCache=refreshCache, + reimportFromUrl=reimportFromUrl ) applet = {} @@ -1812,7 +1814,7 @@ def formatLdObject( activityIDMapping = {} for activity in activities: - formatted = formatLdObject(activity, 'activity', user, refreshCache=refreshCache) + formatted = formatLdObject(activity, 'activity', user, refreshCache=refreshCache, reimportFromUrl=reimportFromUrl) if refreshCache: createCache(activity, formatted, 'activity', user, modelClasses) @@ -1828,7 +1830,7 @@ def formatLdObject( activityFlowIdMapping = {} for activityFlow in activityFlows: - formatted = formatLdObject(activityFlow, 'activityFlow', user, refreshCache=refreshCache) + formatted = formatLdObject(activityFlow, 'activityFlow', user, refreshCache=refreshCache, reimportFromUrl=reimportFromUrl) if refreshCache: createCache(activityFlow, formatted, 'activityFlow', user, modelClasses) @@ -1920,6 +1922,7 @@ def formatLdObject( keepUndefined, dropErrors, refreshCache=True, + reimportFromUrl=reimportFromUrl, responseDates=responseDates ))) import sys, traceback From 42f8988ab35ff54e96d7dc46b38581cb0db21a2d Mon Sep 17 00:00:00 2001 From: karser Date: Tue, 25 Oct 2022 10:47:24 +0300 Subject: [PATCH 04/16] fixes for getting activityId and flanker script --- .../external/convert_applets.py | 19 +++++++++++++++---- girderformindlogger/external/fix_flankers.py | 7 +++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/girderformindlogger/external/convert_applets.py b/girderformindlogger/external/convert_applets.py index 61f012ab1..a362ac53a 100644 --- a/girderformindlogger/external/convert_applets.py +++ b/girderformindlogger/external/convert_applets.py @@ -8,11 +8,12 @@ from girderformindlogger.utility import jsonld_expander from bson.objectid import ObjectId -# '_id': ObjectId('5f0e35523477de8b4a528dd0'), -applets = Applet().find(query={ 'meta.applet': { '$exists': True } }, fields= {"_id": 1}) +# '_id': ObjectId('633fc958b7ee9765ba5447a6') +# github: 'meta.protocol.url': {'$exists': True}, 'meta.applet': {'$exists': True}, 'meta.applet.deleted': {'$exists': False}, 'parentId': ObjectId('5ea689a286d25a5dbb14e82c') +applets = Applet().find(query={'_id': ObjectId('633fc958b7ee9765ba5447a6')}, fields= {"_id": 1}) appletsCount = applets.count() print('total', appletsCount) -skipUntil = None +skipUntil = None # ObjectId('60a398c9acd96cf825f7679d') for index, appletId in enumerate(applets, start=1): if skipUntil == appletId['_id']: skipUntil = None @@ -43,8 +44,18 @@ g = [] activityIRIs = dict.keys(formatted['activities'].copy()) for activityIRI in activityIRIs: + activityLink = formatted['activities'][activityIRI]; + if isinstance(activityLink, ObjectId): + activityId = activityLink + elif isinstance(activityLink, str): + activityId = ObjectId(activityLink) + elif '_id' in activityLink: + activityId = ObjectId(activityLink['_id'].split('/').pop()) + else: + continue + activity = Activity().findOne({ - '_id': ObjectId(formatted['activities'][activityIRI]) + '_id': activityId }) if not activity: diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index e8cdc79ab..ad3fa9b97 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -8,7 +8,7 @@ from bson.objectid import ObjectId -items = Item().find(query={'meta.activityId': ObjectId('628e3d6be50eef3353e63813'), 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) +items = Item().find(query={'meta.activityId': ObjectId('62837282e50eef7782f6c41c'), 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) itemsCount = items.count() print('total', itemsCount) skipUntil = None @@ -24,6 +24,7 @@ affectedActivityIds.append(item['meta']['activityId']) item['meta']['screen']['url'] = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/items/{}'.format(item['meta']['screen']['@id']) + item['meta']['identifier'] = '{}/{}'.format(str(item['meta']['activityId']), str(itemId['_id'])) #after import Item().setMetadata(item, item['meta']) @@ -39,11 +40,13 @@ print('Refreshing affected activity id=' + str(activityId)) activity = Folder().findOne(query={'_id': activityId}) activity['meta']['activity']['url'] = activityUrl + activity['meta']['activity']['_id'] = "activity/{}".format(str(activityId)) #after import + Folder().setMetadata(folder=activity, metadata=activity['meta']) protocolId = activity['meta']['protocolId'] user = User().findOne({'_id': activity['creatorId']}) - res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta={'identifier': activityId, 'protocolId': protocolId}) # meta.activity._id=activity/628e3d6be50eef3353e63813 + # res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta={'identifier': activityId, 'protocolId': protocolId}) # comment out after first run jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) From 8d03f94c3260a7b108fccf6ee08a1d35a0156b36 Mon Sep 17 00:00:00 2001 From: karser Date: Tue, 25 Oct 2022 19:42:21 +0300 Subject: [PATCH 05/16] if object references to empty cache then refresh one --- girderformindlogger/models/applet.py | 3 +++ girderformindlogger/utility/jsonld_expander.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/girderformindlogger/models/applet.py b/girderformindlogger/models/applet.py index a95e3f082..26633047f 100644 --- a/girderformindlogger/models/applet.py +++ b/girderformindlogger/models/applet.py @@ -1943,6 +1943,9 @@ def getNextAppletData(self, activities, nextActivity, bufferSize): activity, 'activity' ) + if not formattedActivity: + print('formattedActivity is empty. ActivityId=' + str(activity['_id'])) + continue buffer['activities'][activityIRI] = formattedActivity['activity'] buffer['items'].update(formattedActivity['items']) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index e807f3e59..73dcbea4a 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -1666,7 +1666,12 @@ def formatLdObject( oc is not None ]): if mesoPrefix == 'item' or mesoPrefix == 'screen' or obj.get('meta', {}).get('schema', '') == APPLET_SCHEMA_VERSION: - return(loadCache(oc)) + cached = loadCache(oc) + if cached is not None: + return cached + else: + refreshCache=True + reimportFromUrl=False else: return {} From 45b0afaec4e4b0b048f7f8dac7bec9976d04bc61 Mon Sep 17 00:00:00 2001 From: karser Date: Fri, 11 Nov 2022 12:54:47 +0200 Subject: [PATCH 06/16] fix activity _id if missing --- girderformindlogger/utility/jsonld_expander.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index 73dcbea4a..dd7c75d9a 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -1868,6 +1868,10 @@ def formatLdObject( return protocol elif mesoPrefix=='activity': + if refreshCache and 'activity' in obj['meta'] and not '_id' in obj['meta']['activity']: + obj['meta']['activity']['_id'] = "activity/{}".format(str(obj['_id'])) + ActivityModel().setMetadata(obj, obj['meta']) + itemIDMapping = {} if obj.get('loadedFromSingleFile', False) or not reimportFromUrl: items = ScreenModel().find({'meta.activityId': obj['_id']}) From 2cf4ec7d6b9707cc20b35173c5c05133702c6fd0 Mon Sep 17 00:00:00 2001 From: karser Date: Tue, 15 Nov 2022 18:58:32 +0200 Subject: [PATCH 07/16] added fix_activity_flow_order script ChildMindInstitute/mindlogger-applet-builder#1066 --- .../external/convert_applets.py | 2 +- .../external/fix_activity_flow_order.py | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 girderformindlogger/external/fix_activity_flow_order.py diff --git a/girderformindlogger/external/convert_applets.py b/girderformindlogger/external/convert_applets.py index a362ac53a..ef59c87bb 100644 --- a/girderformindlogger/external/convert_applets.py +++ b/girderformindlogger/external/convert_applets.py @@ -44,7 +44,7 @@ g = [] activityIRIs = dict.keys(formatted['activities'].copy()) for activityIRI in activityIRIs: - activityLink = formatted['activities'][activityIRI]; + activityLink = formatted['activities'][activityIRI] if isinstance(activityLink, ObjectId): activityId = activityLink elif isinstance(activityLink, str): diff --git a/girderformindlogger/external/fix_activity_flow_order.py b/girderformindlogger/external/fix_activity_flow_order.py new file mode 100644 index 000000000..e0bf8980f --- /dev/null +++ b/girderformindlogger/external/fix_activity_flow_order.py @@ -0,0 +1,47 @@ +# fix broken links in protocol.activityFlowOrder + +from girderformindlogger.models.applet import Applet +from girderformindlogger.models.folder import Folder as FolderModel +from girderformindlogger.utility import jsonld_expander +from bson.objectid import ObjectId + + +def findFlowsStringIds(applet): + flowIds = [] + + protocolId = applet['meta']['protocol'].get('_id').split('/').pop() + docCollection = jsonld_expander.getModelCollection('activityFlow') + activityFlows = FolderModel().find({'meta.protocolId': ObjectId(protocolId), 'parentId': docCollection['_id']}, fields={"_id": 1}) + for af in activityFlows: + flowIds.append(str(af['_id'])) + + return flowIds + + +def main(applets): + appletsCount = applets.count() + print('total', appletsCount) + skipUntil = None # ObjectId('60a398c9acd96cf825f7679d') + for index, appletId in enumerate(applets, start=1): + if skipUntil == appletId['_id']: + skipUntil = None + if skipUntil is not None: + continue + + applet = Applet().findOne(appletId) + protocolId = ObjectId(applet['meta']['protocol'].get('_id').split('/').pop()) + protocol = FolderModel().findOne(query={'_id': protocolId}) + + existingFlowsIds = findFlowsStringIds(applet) + flowOrder = protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] if 'reprolib:terms/activityFlowOrder' in protocol['meta']['protocol'] else [] + filteredFlowOrder = [fo for fo in flowOrder if fo['@id'] in existingFlowsIds] + if flowOrder != filteredFlowOrder: + print('fixing applet id=' + str(applet['_id'])) + protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] = filteredFlowOrder + FolderModel().setMetadata(protocol, protocol['meta']) + jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) + + +if __name__ == '__main__': + applets = Applet().find(query={'_id': ObjectId('634fad045cb700431121d0ad')}, fields={"_id": 1}) + main(applets) From e2864021fc60469c75e7c32bf66c9365bd189a8a Mon Sep 17 00:00:00 2001 From: karser Date: Wed, 30 Nov 2022 13:12:48 +0200 Subject: [PATCH 08/16] Fixed lookup activity by meta. The identifier can be string or object, so passing what was received. --- girderformindlogger/external/fix_flankers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index ad3fa9b97..bc568fdb8 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -43,10 +43,10 @@ activity['meta']['activity']['_id'] = "activity/{}".format(str(activityId)) #after import Folder().setMetadata(folder=activity, metadata=activity['meta']) - protocolId = activity['meta']['protocolId'] user = User().findOne({'_id': activity['creatorId']}) - # res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta={'identifier': activityId, 'protocolId': protocolId}) # comment out after first run + searchCriteria = {'identifier': activity['meta']['identifier'], 'protocolId': activity['meta']['protocolId']} + res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta=searchCriteria) # comment out after first run jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) From 54af7f37bdd1b4d361e72fdf80f2964c2869b225 Mon Sep 17 00:00:00 2001 From: karser Date: Thu, 8 Dec 2022 12:27:22 +0200 Subject: [PATCH 09/16] fix q1 issue in activity versions ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 222 +++++++++---------- 1 file changed, 99 insertions(+), 123 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index bc568fdb8..b14528795 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -1,4 +1,6 @@ -# Add missing properties to flanker items: blocks, buttons, fixationDuration, fixationScreen, blockType +# Update flanker activities from old to new protocol +# Old: https://raw.githubusercontent.com/mtg137/Flanker_applet/master/protocols/flanker/flanker_schema +# New: https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/protocols/flanker/flanker_schema from girderformindlogger.models.item import Item from girderformindlogger.models.activity import Activity @@ -8,136 +10,110 @@ from bson.objectid import ObjectId -items = Item().find(query={'meta.activityId': ObjectId('62837282e50eef7782f6c41c'), 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) -itemsCount = items.count() -print('total', itemsCount) -skipUntil = None -affectedActivityIds = [] -for index, itemId in enumerate(items, start=1): - if skipUntil == itemId['_id']: - skipUntil = None - if skipUntil is not None: - continue - - item = Item().findOne(itemId) - print('processing', item['_id'], index, '/', itemsCount) - affectedActivityIds.append(item['meta']['activityId']) - - item['meta']['screen']['url'] = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/items/{}'.format(item['meta']['screen']['@id']) - item['meta']['identifier'] = '{}/{}'.format(str(item['meta']['activityId']), str(itemId['_id'])) #after import - Item().setMetadata(item, item['meta']) - - -affectedActivityIds = list(set(affectedActivityIds)) - -if (len(affectedActivityIds)): - print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) - -activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema' - -# refresh cache for the affected activities -for activityId in affectedActivityIds: +def prepare_items(activityId): + items = Item().find(query={'meta.activityId': activityId, 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) + itemsCount = items.count() + print('total', itemsCount) + skipUntil = None + affectedActivityIds = [] + for index, itemId in enumerate(items, start=1): + if skipUntil == itemId['_id']: + skipUntil = None + if skipUntil is not None: + continue + + item = Item().findOne(itemId) + print('processing', item['_id'], index, '/', itemsCount) + affectedActivityIds.append(item['meta']['activityId']) + + item['meta']['screen']['url'] = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/items/{}'.format(item['meta']['screen']['@id']) + item['meta']['identifier'] = '{}/{}'.format(str(item['meta']['activityId']), str(itemId['_id'])) #after import + Item().setMetadata(item, item['meta']) + + affectedActivityIds = list(set(affectedActivityIds)) + + if (len(affectedActivityIds)): + print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) + + return affectedActivityIds + + +def findInput(name, inputs): + return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) + + +def fix_q1_issue(activityId): + activityChanged = False + items = Item().find(query={'meta.activityId': activityId, 'meta.screen': {'$exists': True}}) + for item in items: + if not 'reprolib:terms/inputs' in item['meta']['screen']: + continue + inputs = item['meta']['screen']['reprolib:terms/inputs'] + trials = findInput('trials', inputs) + if trials is None or not 'schema:itemListElement' in trials: + continue + print('processing item:', item['_id'], item['meta']['screen']['@id']) + # trialsIdx = inputs.index(trials) + itemChanged = False + for trial in trials['schema:itemListElement']: + if 'q1' == trial['schema:name'][0]['@value']: + trial['schema:name'][0]['@value'] = trial['schema:image'] + trial['schema:image'] = '' + itemChanged = True + activityChanged = True + if itemChanged: + Item().setMetadata(item, item['meta'], validate=False) + return activityChanged + + +def fix_q1_issue_in_versions(activityId): + print('fix_q1_issue_in_versions') + hActivities = Folder().find(query={'meta.originalId': activityId, 'meta.activity': {'$exists': True}}) + hCount = hActivities.count() + if hCount == 0: + return + print('Fixing ' + str(hCount) + ' historical version(s) of the activity id=' + str(activityId)) + for hActivity in hActivities: + print('processing activity: ', hActivity['_id']) + if fix_q1_issue(hActivity['_id']): + print('refreshig activity: ', hActivity['_id']) + jsonld_expander.formatLdObject(hActivity, 'activity', None, refreshCache=True, reimportFromUrl=False) + + +def fix_flankers(activityId, reImport = True): + activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema' print('Refreshing affected activity id=' + str(activityId)) activity = Folder().findOne(query={'_id': activityId}) activity['meta']['activity']['url'] = activityUrl activity['meta']['activity']['_id'] = "activity/{}".format(str(activityId)) #after import + if not 'identifier' in activity['meta']: + activity['meta']['identifier'] = str(activityId) Folder().setMetadata(folder=activity, metadata=activity['meta']) user = User().findOne({'_id': activity['creatorId']}) searchCriteria = {'identifier': activity['meta']['identifier'], 'protocolId': activity['meta']['protocolId']} - res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta=searchCriteria) # comment out after first run - + if reImport: + res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta=searchCriteria) # comment out after first run + # refresh cache for the affected activities jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) -exit(0) - - -# def inputExists(name, inputs): -# return any(i['schema:name'][0]['@value'] == name for i in inputs) -# -# def findInput(name, inputs): -# return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) -# -# items = Item().find(query={'meta.activityId': ObjectId('628e3d6be50eef3353e63813'), 'meta.screen.reprolib:terms/inputType.0.@value': 'visual-stimulus-response'}, fields= {"_id": 1}) -# itemsCount = items.count() -# print('total', itemsCount) -# skipUntil = None -# affectedActivityIds = [] -# for index, itemId in enumerate(items, start=1): -# if skipUntil == itemId['_id']: -# skipUntil = None -# if skipUntil is not None: -# continue -# -# item = Item().findOne(itemId) -# itemChanged = False -# activityId = item['meta']['activityId'] -# print('processing', item['_id'], index, '/', itemsCount) -# inputs = item['meta']['screen']['reprolib:terms/inputs'] -# -# -# itemChanged = True -# -# trials = findInput('trials', inputs) -# trialsIdx = inputs.index(trials) -# inputs[trialsIdx] = {"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"@id":"left-con","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"<<<<<"}],"schema:value":[{"@value":0}]},{"@id":"right-inc","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"<<><<"}],"schema:value":[{"@value":1}]},{"@id":"left-inc","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":">><>>"}],"schema:value":[{"@value":0}]},{"@id":"right-con","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":">>>>>"}],"schema:value":[{"@value":1}]},{"@id":"left-neut","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"--<--"}],"schema:value":[{"@value":0}]},{"@id":"right-neut","@type":["http://schema.org/Property"],"schema:image":"","schema:name":[{"@language":"en","@value":"-->--"}],"schema:value":[{"@value":1}]}],"schema:name":[{"@language":"en","@value":"trials"}],"schema:numberOfItems":[{"@value":6}]} -# -# -# blockType = 'test' if 'test' in item['meta']['screen']['@id'].casefold() else 'practice' -# -# blocks = findInput('blocks', inputs) -# blocksIdx = inputs.index(blocks) -# if (blockType == 'test'): -# inputs[blocksIdx] = {"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 1"}],"schema:value":[{"@value":0}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 2"}],"schema:value":[{"@value":1}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 3"}],"schema:value":[{"@value":2}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 4"}],"schema:value":[{"@value":3}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 5"}],"schema:value":[{"@value":4}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 6"}],"schema:value":[{"@value":5}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 7"}],"schema:value":[{"@value":6}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 8"}],"schema:value":[{"@value":7}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 9"}],"schema:value":[{"@value":8}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 10"}],"schema:value":[{"@value":9}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 11"}],"schema:value":[{"@value":10}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 12"}],"schema:value":[{"@value":11}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 13"}],"schema:value":[{"@value":12}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 14"}],"schema:value":[{"@value":13}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 15"}],"schema:value":[{"@value":14}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 16"}],"schema:value":[{"@value":15}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 17"}],"schema:value":[{"@value":16}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 18"}],"schema:value":[{"@value":17}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 19"}],"schema:value":[{"@value":18}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 20"}],"schema:value":[{"@value":19}]}],"schema:name":[{"@language":"en","@value":"blocks"}],"schema:numberOfItems":[{"@value":20}]} -# else: -# inputs[blocksIdx] = {"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 1"}],"schema:value":[{"@value":0}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 2"}],"schema:value":[{"@value":1}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 3"}],"schema:value":[{"@value":2}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 4"}],"schema:value":[{"@value":3}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 5"}],"schema:value":[{"@value":4}]}],"schema:name":[{"@language":"en","@value":"blocks"}],"schema:numberOfItems":[{"@value":5}]} -# -# -# if (findInput('fixationScreen', inputs) is None): -# print('adding fixationScreen') -# inputs.append({"@type":["http://schema.org/Text"],"schema:image":"","schema:name":[{"@language":"en","@value":"fixationScreen"}],"schema:value":[{"@language":"en","@value":"-----"}]}) -# itemChanged = True -# -# if (findInput('fixationDuration', inputs) is None): -# print('adding fixationDuration') -# inputs.append({"@type":["http://schema.org/Number"],"schema:name":[{"@language":"en","@value":"fixationDuration"}],"schema:value":[{"@value":500}]}) -# itemChanged = True -# -# if (findInput('buttons', inputs) is None): -# print('adding buttons') -# inputs.append({"schema:itemListElement":[{"schema:image":"","schema:name":[{"@language":"en","@value":"<"}],"schema:value":[{"@value":0}]},{"schema:image":"","schema:name":[{"@language":"en","@value":">"}],"schema:value":[{"@value":0}]}],"schema:name":[{"@language":"en","@value":"buttons"}]}) -# itemChanged = True -# -# if (findInput('blocks', inputs) is None): -# print('adding blocks') -# inputs.append({"@type":["http://schema.org/ItemList"],"schema:itemListElement":[{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 1"}],"schema:value":[{"@value":0}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 2"}],"schema:value":[{"@value":1}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 3"}],"schema:value":[{"@value":2}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 4"}],"schema:value":[{"@value":3}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 5"}],"schema:value":[{"@value":4}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 6"}],"schema:value":[{"@value":5}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 7"}],"schema:value":[{"@value":6}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 8"}],"schema:value":[{"@value":7}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 9"}],"schema:value":[{"@value":8}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 10"}],"schema:value":[{"@value":9}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 11"}],"schema:value":[{"@value":10}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 12"}],"schema:value":[{"@value":11}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 13"}],"schema:value":[{"@value":12}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 14"}],"schema:value":[{"@value":13}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 15"}],"schema:value":[{"@value":14}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 16"}],"schema:value":[{"@value":15}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 17"}],"schema:value":[{"@value":16}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 18"}],"schema:value":[{"@value":17}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 19"}],"schema:value":[{"@value":18}]},{"reprolib:terms/order":[{"@list":[{"@id":"left-con"},{"@id":"right-con"},{"@id":"left-inc"},{"@id":"right-inc"},{"@id":"left-neut"},{"@id":"right-neut"}]}],"schema:name":[{"@language":"en","@value":"Block 20"}],"schema:value":[{"@value":19}]}],"schema:name":[{"@language":"en","@value":"blocks"}],"schema:numberOfItems":[{"@value":20}]}) -# itemChanged = True -# -# if (findInput('blockType', inputs) is None): -# value = 'test' if 'test' in item['meta']['screen']['@id'].casefold() else 'practice' -# print('adding blockType', value) -# inputs.append({"@type":["http://schema.org/Text"],"schema:name":[{"@language":"en","@value":"blockType"}],"schema:value":[{"@language":"en","@value":value}]}) -# itemChanged = True -# -# if itemChanged: -# Item().setMetadata(item, item['meta'], validate=False) -# affectedActivityIds.append(activityId) -# -# -# affectedActivityIds = list(set(affectedActivityIds)) -# -# if (len(affectedActivityIds)): -# print('Affected activities:', ','.join('"'+str(activityId)+'"' for activityId in affectedActivityIds)) -# -# # refresh cache for the affected activities -# for activityId in affectedActivityIds: -# print('Refreshing affected activity id=' + str(activityId)) -# activity = Activity().findOne({'_id': activityId}) -# jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) - - - -# old: https://raw.githubusercontent.com/mtg137/Flanker_applet/master/protocols/flanker/flanker_schema -# old: https://raw.githubusercontent.com/mtg137/Flanker_applet/staging/protocols/flanker/flanker_schema -# TODO: file names are renamed. how about responses? + + +def main(activityId): + affectedActivityIds = prepare_items(activityId) + + for activityId in affectedActivityIds: + fix_q1_issue_in_versions(activityId) + + for activityId in affectedActivityIds: + fix_flankers(activityId, True) + + # fix some fields after import + affectedActivityIds = prepare_items(ObjectId('6290ed45e50eef5716db579c')) + for activityId in affectedActivityIds: + fix_flankers(activityId, False) + +if __name__ == '__main__': + activityId = ObjectId('6290ed45e50eef5716db579c') + main(activityId) From 0cab99d3100299d634d5a2c437cab67911fc4176 Mon Sep 17 00:00:00 2001 From: karser Date: Wed, 14 Dec 2022 15:46:59 +0200 Subject: [PATCH 10/16] fix q1 issue in the item cache ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 51 ++++++++++---------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index b14528795..a4047b310 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -6,6 +6,7 @@ from girderformindlogger.models.activity import Activity from girderformindlogger.models.folder import Folder from girderformindlogger.models.user import User +from girderformindlogger.models.cache import Cache from girderformindlogger.utility import jsonld_expander from bson.objectid import ObjectId @@ -14,14 +15,8 @@ def prepare_items(activityId): items = Item().find(query={'meta.activityId': activityId, 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) itemsCount = items.count() print('total', itemsCount) - skipUntil = None affectedActivityIds = [] for index, itemId in enumerate(items, start=1): - if skipUntil == itemId['_id']: - skipUntil = None - if skipUntil is not None: - continue - item = Item().findOne(itemId) print('processing', item['_id'], index, '/', itemsCount) affectedActivityIds.append(item['meta']['activityId']) @@ -41,28 +36,36 @@ def prepare_items(activityId): def findInput(name, inputs): return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) +def fix_q1_issue_in_json(id, model): + if not 'reprolib:terms/inputs' in model: + return False + inputs = model['reprolib:terms/inputs'] + trials = findInput('trials', inputs) + if trials is None or not 'schema:itemListElement' in trials: + return False + print('processing item:', id, model['@id']) + itemChanged = False + for trial in trials['schema:itemListElement']: + if 'q1' == trial['schema:name'][0]['@value']: + trial['schema:name'][0]['@value'] = trial['schema:image'] + trial['schema:image'] = '' + itemChanged = True + + return itemChanged def fix_q1_issue(activityId): activityChanged = False items = Item().find(query={'meta.activityId': activityId, 'meta.screen': {'$exists': True}}) for item in items: - if not 'reprolib:terms/inputs' in item['meta']['screen']: - continue - inputs = item['meta']['screen']['reprolib:terms/inputs'] - trials = findInput('trials', inputs) - if trials is None or not 'schema:itemListElement' in trials: - continue - print('processing item:', item['_id'], item['meta']['screen']['@id']) - # trialsIdx = inputs.index(trials) - itemChanged = False - for trial in trials['schema:itemListElement']: - if 'q1' == trial['schema:name'][0]['@value']: - trial['schema:name'][0]['@value'] = trial['schema:image'] - trial['schema:image'] = '' - itemChanged = True - activityChanged = True - if itemChanged: + if fix_q1_issue_in_json(item['_id'], item['meta']['screen']): Item().setMetadata(item, item['meta'], validate=False) + activityChanged = True + + if 'cached' in item: + cache = Cache().getFromSourceID('item', item['_id']) + if cache is not None and fix_q1_issue_in_json(item['_id'], cache): + Cache().updateCache(item['cached'], 'item', item['_id'], 'screen', cache) + return activityChanged @@ -75,9 +78,7 @@ def fix_q1_issue_in_versions(activityId): print('Fixing ' + str(hCount) + ' historical version(s) of the activity id=' + str(activityId)) for hActivity in hActivities: print('processing activity: ', hActivity['_id']) - if fix_q1_issue(hActivity['_id']): - print('refreshig activity: ', hActivity['_id']) - jsonld_expander.formatLdObject(hActivity, 'activity', None, refreshCache=True, reimportFromUrl=False) + fix_q1_issue(hActivity['_id']) def fix_flankers(activityId, reImport = True): From 3a4c11b33fbe4930ff62a60067ec4c83370c0c05 Mon Sep 17 00:00:00 2001 From: karser Date: Tue, 20 Dec 2022 14:21:59 +0200 Subject: [PATCH 11/16] Added get_activities_for_account ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 34 ++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index a4047b310..cfc34e2f9 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -6,6 +6,8 @@ from girderformindlogger.models.activity import Activity from girderformindlogger.models.folder import Folder from girderformindlogger.models.user import User +from girderformindlogger.models.applet import Applet +from girderformindlogger.models.account_profile import AccountProfile from girderformindlogger.models.cache import Cache from girderformindlogger.utility import jsonld_expander from bson.objectid import ObjectId @@ -115,6 +117,34 @@ def main(activityId): for activityId in affectedActivityIds: fix_flankers(activityId, False) +def get_activities_for_account(email): + print('get_activities_for_account for', email) + user = User().findOne({'email': User().hash(email), 'email_encrypted': True}) + + if user is None: + user = User().findOne({'email': email, 'email_encrypted': {'$ne': True}}) + + if user is None: + raise AccessException('user not found') + + activities = [] + applets = Applet().getAppletsForUser('manager', user, active=True) + for applet in applets: + for activityId in applet['meta']['protocol']['activities']: + activity = Activity().findOne({'_id': activityId}) + if activity is None or not 'url' in activity['meta']['activity']: + continue + url = activity['meta']['activity']['url'] + if 'mtg137/Flanker_applet' in url or 'ChildMindInstitute/mindlogger-flanker-applet' in url: + print('applet', applet['name'], applet['_id'], 'activity', activity['name'], activity['_id']) + activities.append(activity) + print('activities to process', len(activities)) + return activities + + if __name__ == '__main__': - activityId = ObjectId('6290ed45e50eef5716db579c') - main(activityId) + activities = get_activities_for_account('jeligi9407@zneep.com') + for activity in activities: + main(activity['_id']) + # activityId = ObjectId('6290ed45e50eef5716db579c') + # main(activityId) From 96c4d7fc19082ce12a733104010f056a1894b006 Mon Sep 17 00:00:00 2001 From: karser Date: Tue, 20 Dec 2022 15:29:32 +0200 Subject: [PATCH 12/16] find flanker activity by @id ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index cfc34e2f9..7044ee259 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -132,10 +132,9 @@ def get_activities_for_account(email): for applet in applets: for activityId in applet['meta']['protocol']['activities']: activity = Activity().findOne({'_id': activityId}) - if activity is None or not 'url' in activity['meta']['activity']: + if activity is None or not '@id' in activity['meta']['activity']: continue - url = activity['meta']['activity']['url'] - if 'mtg137/Flanker_applet' in url or 'ChildMindInstitute/mindlogger-flanker-applet' in url: + if activity['meta']['activity']['@id'] == 'Flanker_360': print('applet', applet['name'], applet['_id'], 'activity', activity['name'], activity['_id']) activities.append(activity) print('activities to process', len(activities)) From 29dab0418fe357beadb823879ffde747790a8b0c Mon Sep 17 00:00:00 2001 From: karser Date: Fri, 23 Dec 2022 21:49:50 +0200 Subject: [PATCH 13/16] refresh applet cache ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index 7044ee259..20018f479 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -11,7 +11,7 @@ from girderformindlogger.models.cache import Cache from girderformindlogger.utility import jsonld_expander from bson.objectid import ObjectId - +import json def prepare_items(activityId): items = Item().find(query={'meta.activityId': activityId, 'meta.screen.@type.0': 'reprolib:schemas/Field'}, fields= {"_id": 1}) @@ -38,6 +38,7 @@ def prepare_items(activityId): def findInput(name, inputs): return next((i for i in inputs if i['schema:name'][0]['@value'] == name), None) + def fix_q1_issue_in_json(id, model): if not 'reprolib:terms/inputs' in model: return False @@ -55,6 +56,7 @@ def fix_q1_issue_in_json(id, model): return itemChanged + def fix_q1_issue(activityId): activityChanged = False items = Item().find(query={'meta.activityId': activityId, 'meta.screen': {'$exists': True}}) @@ -83,6 +85,12 @@ def fix_q1_issue_in_versions(activityId): fix_q1_issue(hActivity['_id']) +def find_applet_by_activity(activity): + protocolId = activity['meta']['protocolId'] + applet = Folder().findOne(query={'meta.applet': {'$exists': True}, 'meta.applet.deleted': {'$exists': False}, 'meta.protocol._id': "protocol/{}".format(str(protocolId))}) + return applet + + def fix_flankers(activityId, reImport = True): activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema' print('Refreshing affected activity id=' + str(activityId)) @@ -98,9 +106,11 @@ def fix_flankers(activityId, reImport = True): searchCriteria = {'identifier': activity['meta']['identifier'], 'protocolId': activity['meta']['protocolId']} if reImport: res = Activity().getFromUrl(activityUrl, 'activity', user, refreshCache=True, thread=False, meta=searchCriteria) # comment out after first run - # refresh cache for the affected activities + # refresh cache for the affected activity jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) - + applet = find_applet_by_activity(activity) + print('Refreshing affected applet id:', str(applet['_id'])) + jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) def main(activityId): From 0bdc7b9f85dde5ce67654918ffae6aee5a56d931 Mon Sep 17 00:00:00 2001 From: karser Date: Thu, 29 Dec 2022 10:12:55 +0200 Subject: [PATCH 14/16] don't reimport for loadedFromSingleFile ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/external/fix_flankers.py | 13 +++++++++++++ girderformindlogger/utility/jsonld_expander.py | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/girderformindlogger/external/fix_flankers.py b/girderformindlogger/external/fix_flankers.py index 20018f479..0f23ea581 100644 --- a/girderformindlogger/external/fix_flankers.py +++ b/girderformindlogger/external/fix_flankers.py @@ -91,6 +91,14 @@ def find_applet_by_activity(activity): return applet +def fix_editing(applet): + if 'editing' in applet['meta']['applet'] and applet['meta']['applet']['editing']: + applet['meta']['applet']['editing'] = False + Folder().setMetadata(folder=applet, metadata=applet['meta']) + return True + return False + + def fix_flankers(activityId, reImport = True): activityUrl = 'https://raw.githubusercontent.com/ChildMindInstitute/mindlogger-flanker-applet/master/activities/Flanker/Flanker_schema' print('Refreshing affected activity id=' + str(activityId)) @@ -110,6 +118,7 @@ def fix_flankers(activityId, reImport = True): jsonld_expander.formatLdObject(activity, 'activity', None, refreshCache=True, reimportFromUrl=False) applet = find_applet_by_activity(activity) print('Refreshing affected applet id:', str(applet['_id'])) + fix_editing(applet) jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) @@ -157,3 +166,7 @@ def get_activities_for_account(email): main(activity['_id']) # activityId = ObjectId('6290ed45e50eef5716db579c') # main(activityId) + # fix_editing for applet + # applet = Folder().findOne(query={'_id': ObjectId('627d1e2de50eef3d5567ee55')}) + # if fix_editing(applet): + # jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index dd7c75d9a..9d83bcc86 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -1819,7 +1819,7 @@ def formatLdObject( activityIDMapping = {} for activity in activities: - formatted = formatLdObject(activity, 'activity', user, refreshCache=refreshCache, reimportFromUrl=reimportFromUrl) + formatted = formatLdObject(activity, 'activity', user, refreshCache=refreshCache, reimportFromUrl=False) if refreshCache: createCache(activity, formatted, 'activity', user, modelClasses) @@ -1835,7 +1835,7 @@ def formatLdObject( activityFlowIdMapping = {} for activityFlow in activityFlows: - formatted = formatLdObject(activityFlow, 'activityFlow', user, refreshCache=refreshCache, reimportFromUrl=reimportFromUrl) + formatted = formatLdObject(activityFlow, 'activityFlow', user, refreshCache=refreshCache, reimportFromUrl=False) if refreshCache: createCache(activityFlow, formatted, 'activityFlow', user, modelClasses) @@ -1884,7 +1884,7 @@ def formatLdObject( if not 'identifier' in item['meta']: item['meta']['identifier'] = '{}/{}'.format(str(obj['_id']), str(item['_id'])) identifier = item['meta']['identifier'] - itemFormatted = formatLdObject(item, 'screen', user, refreshCache=refreshCache, reimportFromUrl=reimportFromUrl) + itemFormatted = formatLdObject(item, 'screen', user, refreshCache=refreshCache, reimportFromUrl=False) activity['items'][identifier] = itemFormatted key = '{}/{}'.format(str(item['meta']['activityId']), str(item['_id'])) From 2b191d0ad42c30a18af3c15782a219079ecec741 Mon Sep 17 00:00:00 2001 From: karser Date: Thu, 5 Jan 2023 14:01:49 +0200 Subject: [PATCH 15/16] handle case if item history reference is incorrect ChildMindInstitute/mindlogger-applet-builder#910 --- girderformindlogger/api/v1/applet.py | 1 + girderformindlogger/utility/jsonld_expander.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/girderformindlogger/api/v1/applet.py b/girderformindlogger/api/v1/applet.py index fee3e09e4..8424ce471 100644 --- a/girderformindlogger/api/v1/applet.py +++ b/girderformindlogger/api/v1/applet.py @@ -627,6 +627,7 @@ def getProtocolVersions(self, applet, retrieveDate=False): items = list(ItemModel().find({ 'folderId': protocol['meta'].get('contentId', None), + 'version': {'$exists': True}, }, fields=['version', 'created'], sort=[("created", DESCENDING)])) if 'contentId' in protocol['meta'] else [] if retrieveDate: diff --git a/girderformindlogger/utility/jsonld_expander.py b/girderformindlogger/utility/jsonld_expander.py index 9d83bcc86..4f7ceb79e 100644 --- a/girderformindlogger/utility/jsonld_expander.py +++ b/girderformindlogger/utility/jsonld_expander.py @@ -572,12 +572,13 @@ def cacheProtocolContent(protocol, document, user, editExisting=False): cacheModel = CacheModel() - if editExisting and 'baseVersion' in document: - latestItem = ItemModel().findOne({ - 'folderId': contentFolder['_id'], - 'version': document['baseVersion'] - }) + latestItem = ItemModel().findOne({ + 'folderId': contentFolder['_id'], + 'version': document['baseVersion'], + 'content': {'$exists': True} + }) if editExisting and 'baseVersion' in document else None + if latestItem is not None: latestDocument = json_util.loads(latestItem['content']) for key in dict.keys(latestDocument['protocol']['activities']): From 0b879943af1a93114cd9b9221063172cb95a8d9c Mon Sep 17 00:00:00 2001 From: karser Date: Thu, 12 Jan 2023 17:40:02 +0200 Subject: [PATCH 16/16] recover flows where flow.protocolId is incorrect ChildMindInstitute/mindlogger-applet-builder#1066 --- .../external/fix_activity_flow_order.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/girderformindlogger/external/fix_activity_flow_order.py b/girderformindlogger/external/fix_activity_flow_order.py index e0bf8980f..1e7d8f6a9 100644 --- a/girderformindlogger/external/fix_activity_flow_order.py +++ b/girderformindlogger/external/fix_activity_flow_order.py @@ -10,8 +10,7 @@ def findFlowsStringIds(applet): flowIds = [] protocolId = applet['meta']['protocol'].get('_id').split('/').pop() - docCollection = jsonld_expander.getModelCollection('activityFlow') - activityFlows = FolderModel().find({'meta.protocolId': ObjectId(protocolId), 'parentId': docCollection['_id']}, fields={"_id": 1}) + activityFlows = FolderModel().find({'meta.protocolId': ObjectId(protocolId), 'meta.activityFlow': {'$exists': True} }, fields={"_id": 1}) for af in activityFlows: flowIds.append(str(af['_id'])) @@ -33,15 +32,37 @@ def main(applets): protocol = FolderModel().findOne(query={'_id': protocolId}) existingFlowsIds = findFlowsStringIds(applet) + + # find and attach orphan flows + for afId in applet['meta']['protocol']['activityFlows']: + if str(afId) in existingFlowsIds: + continue # exclude that already refer to us + if FolderModel().find(query={'meta.protocol.activityFlows': afId, '_id': {'$ne': appletId['_id']}}).count() > 0: + continue # if no other protocols refer to this flow + flow = FolderModel().findOne(query={'_id': afId}) + if flow is None: + continue + print('attaching orphan flow '+str(afId)+' to applet ' + str(applet['_id'])) + flow['meta']['protocolId'] = protocolId + FolderModel().setMetadata(flow, flow['meta']) + existingFlowsIds.append(str(afId)) + + flowOrder = protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] if 'reprolib:terms/activityFlowOrder' in protocol['meta']['protocol'] else [] + flowOrderIdsMap = [fo['@id'] for fo in flowOrder] + for afId in existingFlowsIds: + if not str(afId) in flowOrderIdsMap: + flowOrder.append({'@id': str(afId)}) + filteredFlowOrder = [fo for fo in flowOrder if fo['@id'] in existingFlowsIds] - if flowOrder != filteredFlowOrder: - print('fixing applet id=' + str(applet['_id'])) + flowOrderChanged = flowOrder != filteredFlowOrder or len(flowOrder) != len(flowOrderIdsMap) + if flowOrderChanged: + print('fixing activityFlowOrder for applet ' + str(applet['_id'])) protocol['meta']['protocol']['reprolib:terms/activityFlowOrder'][0]['@list'] = filteredFlowOrder FolderModel().setMetadata(protocol, protocol['meta']) jsonld_expander.formatLdObject(applet, 'applet', None, refreshCache=True, reimportFromUrl=False) if __name__ == '__main__': - applets = Applet().find(query={'_id': ObjectId('634fad045cb700431121d0ad')}, fields={"_id": 1}) + applets = Applet().find(query={'_id': ObjectId('633e855131f2c2777e5e1bb6')}, fields={"_id": 1}) main(applets)