Miccai/venue configuration

venues/MICCAI/2021/Challenges/python/assign-reviewers.py

@@ -0,0 +1,94 @@
+import argparse
+import re
+import csv
+import openreview
+from tqdm import tqdm
+from openreview import tools
+
+parser = argparse.ArgumentParser()
+parser.add_argument('assignments_file', help='csv file')
+parser.add_argument('--baseurl', help='base url')
+parser.add_argument('--username')
+parser.add_argument('--password')
+args = parser.parse_args()
+
+client = openreview.Client(baseurl=args.baseurl, username=args.username, password=args.password)
+
+conference = openreview.helpers.get_conference(client, '6tlp9iYQvsy')
+
+papers = conference.get_submissions()
+
+revs_by_title = {}
+clincial_revs_by_title = {}
+acs_by_title = {}
+
+with open(args.assignmnents_file) as csv_file:
+    csv_reader = csv.reader(csv_file, delimiter = ',')
+    next(csv_reader, None)
+
+    for row in csv_reader:
+        title = row[0]
+        rev1 = row[2]
+        rev2 = row[3]
+        clincal_rev = row[4]
+        ac = row[5]
+        revs_by_title[title] = [rev1,rev2]
+        clincial_revs_by_title[title] = clincal_rev
+        acs_by_title[title] = ac
+
+def get_readers(num):
+    readers = [
+        conference.get_id(),
+        conference.get_program_chairs_id(),
+        conference.get_area_chairs_id(number = num)
+    ]
+    return readers
+
+for paper in papers:
+    num = paper.number
+    revs = revs_by_title[paper.content['title']]
+    readers = get_readers(num)
+    invitees = ['OpenReview.net/Support']
+    for rev in revs:
+        user, groups = tools.add_assignment(client, num, conference.get_id(), rev,
+                            parent_label = 'Reviewers',
+                            individual_label = 'AnonReviewer',
+                            individual_group_params = {
+                                'readers': readers,
+                                'writers': readers
+                            },
+                            parent_group_params = {
+                                'readers': readers,
+                                'writers': readers
+                            })
+
+        r = re.compile('MICCAI.org/2021/Challenges/Paper.*/AnonReviewer.*')
+        anongroup = list(filter(r.match, groups))[0]
+        invitees.append(anongroup)
+
+    #set only these reviewers as invitees of review invitation
+    rev_invitation = client.get_invitation('MICCAI.org/2021/Challenges/Paper{}/-/Official_Review'.format(num))
+    rev_invitation.invitees = invitees
+
+    clinical_rev = clincial_revs_by_title[paper.content['title']]
+    user, groups = tools.add_assignment(client, num, conference.get_id(), clinical_rev,
+                            parent_label = 'Reviewers',
+                            individual_label = 'AnonReviewer',
+                            individual_group_params = {
+                                'readers': readers,
+                                'writers': readers
+                            },
+                            parent_group_params = {
+                                'readers': readers,
+                                'writers': readers
+                            })
+
+    r = re.compile('MICCAI.org/2021/Challenges/Paper.*/AnonReviewer.*')
+    anongroup = list(filter(r.match, groups))[0]
+
+    #set only this reviewer as invitee of clinical review invitation
+    clinical_rev_invitation = client.get_invitation('MICCAI.org/2021/Challenges/Paper{}/-/Clinical_Review'.format(num))
+    clinical_rev_invitation.invitees = [anongroup, 'OpenReview.net/Support']
+
+    ac = acs_by_title[paper.content['title']]
+    tools.add_assignment(client, num, conference.get_id(), ac, parent_label = 'Area_Chairs', individual_label = 'Area_Chair')

venues/MICCAI/2021/Challenges/python/setup_review_stage.py

@@ -0,0 +1,148 @@
+import argparse
+import re
+import csv
+import openreview
+from tqdm import tqdm
+from openreview import tools
+import datetime
+
+parser = argparse.ArgumentParser()
+parser.add_argument('assignments_file', help='csv file')
+parser.add_argument('--baseurl', help='base url')
+parser.add_argument('--username')
+parser.add_argument('--password')
+args = parser.parse_args()
+
+client = openreview.Client(baseurl=args.baseurl, username=args.username, password=args.password)
+
+conference = openreview.helpers.get_conference(client, '6tlp9iYQvsy')
+
+#set official reviews
+review_stage = conference.set_review_stage(openreview.ReviewStage(due_date = datetime.datetime(2021, 1, 16, 11, 59),release_to_authors=True, 
+additional_fields = {
+    "summary": {
+        "order": 2,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Please provide a summary of this work (max 200000 characters). Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "soundness": {
+        "order": 3,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Is the challenge design concise and sound? (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "quality": {
+        "order": 4,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Is the document in sufficient quality for being uploaded? (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "revisions": {
+        "order": 5,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Provide any revisions essential for the acceptance of the challenge. (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "comments": {
+        "order": 6,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Provide any further comments. (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "rating": {
+        "order": 7,
+        "value-dropdown": [
+            "10: Top 5% of accepted challenges, seminal challenge",
+            "9: Top 15% of accepted challenges, strong challenge",
+            "8: Top 50% of accepted challenges, clear accept",
+            "7: Good challenge, accept",
+            "6: Marginally above acceptance threshold",
+            "5: Marginally below acceptance threshold",
+            "4: Ok but not good enough - rejection",
+            "3: Clear rejection",
+            "2: Strong rejection",
+            "1: Trivial or wrong"
+        ],
+        "required": True
+    },
+    "confidence": {
+        "order": 8,
+        "value-radio": [
+            "5: The reviewer is absolutely certain that the evaluation is correct and very familiar with the relevant literature",
+            "4: The reviewer is confident but not absolutely certain that the evaluation is correct",
+            "3: The reviewer is fairly confident that the evaluation is correct",
+            "2: The reviewer is willing to defend the evaluation, but it is quite likely that the reviewer did not understand central parts of the paper",
+            "1: The reviewer's evaluation is an educated guess"
+        ],
+        "required": True
+    }
+},
+remove_fields = ['review']))
+
+#Set clinical reviews
+clinical_revs = conference.set_review_stage(openreview.ReviewStage(name='Clinical_Review', due_date = datetime.datetime(2021, 1, 16, 11, 59), 
+release_to_authors=True, 
+additional_fields = {
+    "summary": {
+        "order": 2,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Please provide a summary of this work (max 200000 characters). Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "challenge_goals": {
+        "order": 3,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Is the challenge goal clinically relevant? (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "metrics": {
+        "order": 4,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Are the metrics clinically relevant? (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "comments": {
+        "order": 5,
+        "value-regex": "[\\S\\s]{1,200000}",
+        "description": "Provide any further comments. (max 5000 characters) Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq",
+        "required": True,
+        "markdown": True
+    },
+    "rating": {
+        "order": 6,
+        "value-dropdown": [
+            "10: Top 5% of accepted challenges, seminal challenge",
+            "9: Top 15% of accepted challenges, strong challenge",
+            "8: Top 50% of accepted challenges, clear accept",
+            "7: Good challenge, accept",
+            "6: Marginally above acceptance threshold",
+            "5: Marginally below acceptance threshold",
+            "4: Ok but not good enough - rejection",
+            "3: Clear rejection",
+            "2: Strong rejection",
+            "1: Trivial or wrong"
+        ],
+        "required": True
+    },
+    "confidence": {
+        "order": 7,
+        "value-radio": [
+            "5: The reviewer is absolutely certain that the evaluation is correct and very familiar with the relevant literature",
+            "4: The reviewer is confident but not absolutely certain that the evaluation is correct",
+            "3: The reviewer is fairly confident that the evaluation is correct",
+            "2: The reviewer is willing to defend the evaluation, but it is quite likely that the reviewer did not understand central parts of the paper",
+            "1: The reviewer's evaluation is an educated guess"
+        ],
+        "required": True
+    }
+}, 
+remove_fields = ['review']))

venues/MICCAI/2021/Challenges/python/upload-submissions.py

@@ -17,9 +17,6 @@
 
 submission_invitation = client.get_invitation('MICCAI.org/2021/Challenges/-/Submission')
 
-submission_invitation.reply['readers'] = {
-    'values': ['everyone']
-    }
 submission_invitation.reply['signatures'] =  {
     'values': [conference_id]
     }
@@ -39,12 +36,14 @@
         pdf_link = client.put_attachment(args.pdfs_filepath+'/'+row[4], 'MICCAI.org/2021/Challenges/-/Submission', 'pdf')
         authors = [author.strip() for author in row[2].split(',')]
         authorids = [authorid.strip() for authorid in row[3].split(',')]
+        readers = [conference_id]
+        readers.extend(authorids)
 
         miccai_paper = openreview.Note(
             invitation=conference_id+'/-/Submission',
             signatures=[conference_id],
             writers=[conference_id, conference_id+'/Program_Chairs'],
-            readers=['everyone'],
+            readers = readers,
             content={
                 'title': row[0],
                 'authors': authors,