Update backup metadata schema to support all OkPy message types

src/backups/backup_config.json

@@ -5,16 +5,17 @@
         "offset": 0
     },
     "course": {
-        "lab_start": 0,
-        "lab_end": 11,
-        "hw_start": 1,
-        "hw_end": 10,
-        "projects": ["maps", "ants"]
+        "lab_start": 1,
+        "lab_end": 1,
+        "hw_start": 2,
+        "hw_end": 2,
+        "projects": ["maps"]
     },
     "data": {
         "in_roster": "../../data/private/data_c88c_sp25_gradescope_roster.csv",
         "out_roster": "../../data/private/data_c88c_sp25_emails.txt",
         "dump": "../../data/private/data_c88c_sp25_dump.json",
         "database": "../../data/private/data_c88c_sp25_backups.db"
-    }
+    },
+    "deidentify": true
 }

src/backups/db.py

@@ -0,0 +1,105 @@
+DROP_BACKUP_METADATA_TABLE_CMD = "DROP TABLE IF EXISTS backup_metadata"
+
+DROP_OKPY_MESSAGES_TABLE_CMD = "DROP TABLE IF EXISTS okpy_messages"
+
+CREATE_BACKUP_METADATA_TABLE_CMD = """
+CREATE TABLE backup_metadata (
+    backup_id TEXT PRIMARY KEY,
+
+    -- ISO8601 string
+	created TEXT NOT NULL,
+
+    -- okpy endpoint for course (includes semester, e.g. cal/cs88/sp25)
+    course TEXT NOT NULL,
+
+    -- okpy assignment endpoint (not including course endpoint prefix,
+    -- e.g. lab00. to get full endpoint, do {course}/{assignment})
+    assignment TEXT NOT NULL,
+
+    student_email TEXT NOT NULL,
+
+	is_late INTEGER NOT NULL CHECK (is_late = TRUE OR is_late = FALSE),
+
+    -- whether student used --submit flag (educated guess)
+    submitted INTEGER NOT NULL CHECK (submitted = TRUE OR submitted = FALSE),
+
+    -- each backup has one or more kinds of okpy "messages"
+    -- which contain different data about the student's work.
+    -- see okpy_messages table for more information.
+    -- the following columns contain the path to the file
+    -- containing the contents of the okpy message, or NULL if it doesn't exist.
+    autograder_output_location TEXT,
+    grading_location TEXT,
+    file_contents_location TEXT,
+    analytics_location TEXT,
+    scoring_location TEXT,
+    unlock_location TEXT
+);
+"""
+
+CREATE_OKPY_MESSAGES_TABLE_CMD = """
+CREATE TABLE okpy_messages (
+    id INTEGER PRIMARY KEY,
+    type TEXT NOT NULL,
+    description TEXT NOT NULL
+);
+"""
+
+INSERT_BACKUP_METADATA_CMD = """
+INSERT INTO backup_metadata VALUES (
+    :backup_id,
+    :created,
+    :course,
+    :assignment,
+    :student_email,
+
+    :is_late,
+    :submitted,
+
+    :autograder_output_location,
+    :grading_location,
+    :file_contents_location,
+    :analytics_location,
+    :scoring_location,
+    :unlock_location
+);
+"""
+
+INSERT_OKPY_MESSAGES_TABLE_CMD = """
+INSERT INTO okpy_messages VALUES
+(:id, :type, :description)
+"""
+
+OKPY_MESSAGES_VALUES = [
+    {
+        "id": 1,
+        "type": "autograder_output",
+        "description": "OkPy autograder output string",
+    },
+    {
+        "id": 2,
+        "type": "grading",
+        "description": "For each test, a count of how many were locked/passed/failed",
+    },
+    {
+        "id": 3,
+        "type": "file_contents",
+        "description": "Source file names and their contents",
+    },
+    {
+        "id": 4,
+        "type": "analytics",
+        "description": "Count of how many attempts student made on a problem and boolean of whether it was solved",
+    },
+    {
+        "id": 5,
+        "type": "scoring",
+        "description": "Total score for that OkPy run",  # probably only occurs if --score was passed
+    },
+    {
+        "id": 6,
+        "type": "unlock",
+        "description": "Unlocking test output",
+    },
+    # NOTE: there is another okpy message called "email" but that just contains the student's email
+]

src/backups/main.py

@@ -18,7 +18,7 @@
 
 from emails import process_roster
 from request import get_backups_for_all_users_all_assignments
-from storage import setup_db, PREFIX, responses_to_backups, store_all_backups
+from storage import setup_db, PREFIX, responses_to_backups
 
 DEFAULT_CONFIG_FILE = "backup_config.json"
 
@@ -209,6 +209,9 @@ def store(
             help="Name of sqlite database .db file where backups will be stored"
         ),
     ] = None,
+    deidentify: Annotated[
+        bool, typer.Option(help="Whether to deidentify student emails")
+    ] = False,
     config: Annotated[
         str, typer.Option(help="Configuration .json file")
     ] = DEFAULT_CONFIG_FILE,
@@ -238,6 +241,10 @@ def store(
         database = config_dict["data"]["database"]
     assert database.endswith(".db"), "database must be a sqlite .db file"
 
+    deidentify = config_dict.get("deidentify", deidentify)
+    if verbose and deidentify:
+        print("Deidentifying student emails")
+
     # take HTTP response data and persist it in the database
     if timeit:
         start = time()
@@ -253,18 +260,23 @@ def store(
 
     with open(dump, "r") as f:
         emails_to_responses = json.load(f)
-    backups = responses_to_backups(emails_to_responses, course_endpoint)
+
+    num_backups = responses_to_backups(
+        emails_to_responses, course_endpoint, PREFIX, cur, deidentify
+    )
     if verbose:
-        print(f"Processed {len(backups)} backups from {dump}")
+        print(f"Processed {num_backups} backups from {dump}")
 
-    store_all_backups(cur, backups)
-    cur.execute("SELECT COUNT(*) FROM backups_metadata")
+    cur.execute("SELECT COUNT(*) FROM backup_metadata")
     num_rows = cur.fetchone()[0]
+    assert (
+        num_backups == num_rows
+    ), "num_backups should match num_rows in backup_metadata table"
     if verbose:
         print(
-            f"Wrote backup file contents to {storage_dir} and inserted {num_rows} rows into backups_metadata table"
+            f"Wrote backup file contents to {storage_dir} and inserted {num_rows} rows into backup_metadata table"
         )
-        cur.execute("SELECT * FROM backups_metadata LIMIT 10")
+        cur.execute("SELECT * FROM backup_metadata LIMIT 10")
         rows = cur.fetchall()
         print("First 10 rows:")
         for r in rows:
@@ -275,12 +287,6 @@ def store(
         print(f"Finished storing backups in {database} in {end - start} seconds")
 
 
-@app.command()
-def deidentify():
-    """Not implemented yet"""
-    pass
-
-
 @app.command()
 def query():
     """Not implemented yet"""

src/backups/models.py

@@ -0,0 +1,115 @@
+import json
+
+
+class Backup:
+    def __init__(
+        self,
+        backup_id: str,
+        created: str,
+        course: str,
+        assignment: str,
+        student_email: str,
+        is_late: bool,
+        submitted: bool,
+        autograder_output_location: str = None,
+        grading_location: str = None,
+        file_contents_location: str = None,
+        analytics_location: str = None,
+        scoring_location: str = None,
+        unlock_location: str = None,
+    ):
+        self.backup_id = backup_id
+        self.created = created
+        self.course = course
+        self.assignment = assignment
+        self.student_email = student_email
+
+        self.is_late = is_late
+        self.submitted = submitted
+
+        self.autograder_output_location = autograder_output_location
+        self.grading_location = grading_location
+        self.file_contents_location = file_contents_location
+        self.analytics_location = analytics_location
+        self.scoring_location = scoring_location
+        self.unlock_location = unlock_location
+
+
+class OkPyMessage:
+    def __init__(self, contents):
+        self.contents = contents
+
+
+class AutograderOutputMessage(OkPyMessage):
+    @staticmethod
+    def location(directory):
+        return f"{directory}/autograder_output.txt"
+
+    def write(self, directory):
+        with open(AutograderOutputMessage.location(directory), "w") as f:
+            f.write(self.contents)
+
+
+class GradingMessage(OkPyMessage):
+    @staticmethod
+    def location(directory):
+        return f"{directory}/grading.json"
+
+    def write(self, directory):
+        with open(GradingMessage.location(directory), "w") as f:
+            json.dump(self.contents, f, indent=2)
+
+
+class FileContentsMessage(OkPyMessage):
+    @staticmethod
+    def location(directory):
+        # NOTE: a file content message's location is a DIRECTORY rather than a file
+        # since there may be multiple source files in a student's backup
+        return directory
+
+    def write(self, directory):
+        for src_file_name, src_file_contents in self.contents.items():
+            with open(
+                f"{FileContentsMessage.location(directory)}/{src_file_name}", "w"
+            ) as f:
+                f.write(str(src_file_contents))
+
+
+class AnalyticsMessage(OkPyMessage):
+    @staticmethod
+    def location(directory):
+        return f"{directory}/analytics.json"
+
+    def write(self, directory):
+        with open(AnalyticsMessage.location(directory), "w") as f:
+            json.dump(self.contents, f, indent=2)
+
+
+class ScoringMessage(OkPyMessage):
+    @staticmethod
+    def location(directory):
+        return f"{directory}/scoring.json"
+
+    def write(self, directory):
+        with open(ScoringMessage.location(directory), "w") as f:
+            json.dump(self.contents, f, indent=2)
+
+
+class UnlockMessage(OkPyMessage):
+    @staticmethod
+    def location(directory):
+        return f"{directory}/unlock.json"
+
+    def write(self, directory):
+        with open(GradingMessage.location(directory), "w") as f:
+            json.dump(self.contents, f, indent=2)
+
+
+MESSAGE_KIND_TO_CLASS = {
+    "autograder_output": AutograderOutputMessage,
+    "grading": GradingMessage,
+    "file_contents": FileContentsMessage,
+    "analytics": AnalyticsMessage,
+    "scoring": ScoringMessage,
+    "unlock": UnlockMessage,
+}