Remaining dev work before Formative Study 2 session P2 (#116)

* Determine good candidates for study

* Use 61A FA25 when seeding db

* Fix backend bugs

* Implement print statements

* Adjust histogram buckets manually

* Misc bugfixes/todos

* Add TODO about EC problems

* Run rubocop

* ruff check --fix

* Run ruff format on src/notebooks

* Run ruff format on src/backups

* Remove Rails.logger statements

Author: phrdang

src/backups/README.md

@@ -93,17 +93,18 @@ sqlite3 data/private/out/dev/c88c_fa25.db .dump > data/private/out/dev/c88c_fa25
 3. Update the SQL file:
     1. Remove `../../data/private/` prefix from paths. **IMPORTANT:** Make sure you are removing the trailing `/`.
     2. Remove/comment out `CREATE TABLE` statements since that will interfere with the Rails database migrations (Rails will already handle table creation on its own end, so if you have a duplicate `CREATE TABLE` statement Rails will error).
+    3. also remove create index statements (may be created through data track)
 4. **Optional if not done already:** [Generate corresponding Rails model(s)](https://guides.rubyonrails.org/command_line.html#generating-models) **in the `src/snapshots-app` directory** by running the following command. If you are an internal contributor working with the toy data from `data.zip`, skip this step.
 ```sh
 rails generate model <model_name> <column_name:data_type> ...
 ```
 > [!CAUTION]
 > THE FOLLOWING STEP WILL RESET (e.g. delete everything) AND RE-MIGRATE THE RAILS DB. BE CAREFUL!
-5. Run the following command **in the `src/snapshots-app` directory**:
+1. Run the following command **in the `src/snapshots-app` directory**:
 ```sh
 rails db:migrate:reset
 ```
-6. Run the following command **in the root directory** of the repository to execute commands from output `.sql` dump into the Rails app `development.sqlite3` database. Replace `$PATH_TO_SQL_FILE` with the same value from steps 1 and 2:
+1. Run the following command **in the root directory** of the repository to execute commands from output `.sql` dump into the Rails app `development.sqlite3` database. Replace `$PATH_TO_SQL_FILE` with the same value from steps 1 and 2:
 ```sh
 # General command
 sqlite3 src/snapshots-app/storage/development.sqlite3 < $PATH_TO_SQL_FILE

src/backups/configs/dev/dev2.json

@@ -0,0 +1,25 @@
+{
+    "okpy_api": {
+        "course_endpoint": "cal/cs88/fa25",
+        "limit": 150,
+        "offset": 0
+    },
+    "course": {
+        "lab_start": 1,
+        "lab_end": 0,
+        "hw_start": 1,
+        "hw_end": 0,
+        "projects": ["ants"],
+        "assignment_files": {
+            "ants": ["ants.py"]
+        }
+    },
+    "data": {
+        "in_roster": "../../data/private/out/dev2/roster.csv",
+        "out_roster": "../../data/private/out/dev2/emails.txt",
+        "dump": "../../data/private/out/dev2/okpy_api_output.json",
+        "database": "../../data/private/out/dev2/snapshots.db",
+        "lint_json": "../../data/private/out/dev2/lint_errors.json"
+    },
+    "deidentify": true
+}

src/backups/configs/dev/dev3.json

@@ -0,0 +1,25 @@
+{
+    "okpy_api": {
+        "course_endpoint": "cal/cs61a/fa25",
+        "limit": 150,
+        "offset": 0
+    },
+    "course": {
+        "lab_start": 1,
+        "lab_end": 0,
+        "hw_start": 1,
+        "hw_end": 0,
+        "projects": ["ants"],
+        "assignment_files": {
+            "ants": ["ants.py"]
+        }
+    },
+    "data": {
+        "in_roster": "../../data/private/out/dev3/roster.csv",
+        "out_roster": "../../data/private/out/dev3/emails.txt",
+        "dump": "../../data/private/out/dev3/okpy_api_output.json",
+        "database": "../../data/private/out/dev3/snapshots.db",
+        "lint_json": "../../data/private/out/dev3/lint_errors.json"
+    },
+    "deidentify": true
+}

src/backups/email_hash_lookup.py

@@ -0,0 +1,76 @@
+import hashlib
+
+
+def sha256_short(s: str) -> str:
+    """Returns first 8 characters in SHA256 hash of `s`."""
+    return hashlib.sha256(s.encode()).hexdigest()[:8]
+
+
+def find_emails_by_hashes(input_file, output_file, target_hashes):
+    # 1. Create a dictionary to map hashes back to emails
+    # Key: hash string, Value: original email
+    hash_map = {}
+
+    try:
+        with open(input_file, "r") as f:
+            for line in f:
+                email = line.strip()
+                if email:
+                    # Compute hash and store in map
+                    h = sha256_short(email)
+                    hash_map[h] = email
+
+        # 2. Identify which original emails match our target hash list
+        matched_emails = []
+        for h in target_hashes:
+            if h in hash_map:
+                matched_emails.append(hash_map[h])
+            else:
+                print(f"Warning: No email found for hash {h}")
+
+        # 3. Write the results to a new file
+        with open(output_file, "w") as f:
+            for email in matched_emails:
+                f.write(f"{email}\n")
+
+        print(f"Successfully recovered {len(matched_emails)} emails to {output_file}")
+
+    except FileNotFoundError:
+        print(f"Error: The file '{input_file}' or '{output_file}' was not found.")
+
+
+# TODO parameterize as CLI
+if __name__ == "__main__":
+    target_list = [
+        "e3384165",
+        "1faf1492",
+        "0757b4af",
+        "5e0b5dff",
+        "55d9e0b2",
+        "4349b29d",
+        "27f16a00",
+        "1bcf17a8",
+        "bbb281e4",
+        "18e36d10",
+        "d0d1b4b0",
+        "94d2cb91",
+        "09e6bcbc",
+        "08a08a79",
+        "1a3aee97",
+        "3ff28b43",
+        "4972bef4",
+        "a8faf137",
+        "d6797b5b",
+        "fc1888f1",
+        "c2b307c8",
+        "395b6a1a",
+        "f0cd1289",
+        "90cfed97",
+        "41a86dbb",
+    ]
+
+    find_emails_by_hashes(
+        input_file="../../data/private/out/cs61a/fa25/emails.txt",
+        output_file="../../data/private/out/dev3/emails.txt",
+        target_hashes=target_list,
+    )

src/backups/main.py

@@ -298,6 +298,7 @@ def store(
         print(f"Finished storing backups in {database} in {end - start} seconds")
 
 
+# TODO need to rerun lint with new schema for 61a and c88c fa25
 @app.command()
 def lint(
     database: Annotated[