evaluate.py

import os
import requests
from dotenv import load_dotenv

load_dotenv()
EMAIL = os.getenv("EMAIL")

if not EMAIL:
    print("❌ Please set EMAIL in your .env file.")
    exit(1)

# List of job queries
job_queries = [
    "Tax Lawyer",
    "Junior Corporate Lawyer",
    "Radiology",
    "Doctors (MD)",
    "Biology Expert",
    "Anthropology",
    "Mathematics PhD",
    "Quantitative Finance",
    "Bankers",
    "Mechanical Engineers"
]

def normalize(query):
    return query.lower().replace(" ", "_")

print(f"{'Query':30} | {'# Candidates':12} | {'Avg Final Score'}")
print("-" * 60)

for query in job_queries:
    filename = f"results/{normalize(query)}_ids.txt"
    config_path = f"{normalize(query)}.yml"

    if not os.path.exists(filename):
        print(f"{query:30} | MISSING FILE")
        continue

    with open(filename) as f:
        object_ids = [line.strip() for line in f if line.strip()]

    if not object_ids:
        print(f"{query:30} | 0            | 0.00")
        continue

    payload = {
        "config_path": config_path,
        "object_ids": object_ids
    }

    headers = {
        "Content-Type": "application/json",
        "Authorization": EMAIL
    }

    try:
        response = requests.post(
            "https://mercor-dev--search-eng-interview.modal.run/evaluate",
            json=payload,
            headers=headers
        )
        response.raise_for_status()
        result = response.json()

        print(f"{result['config_name']:30} | {result['num_candidates']:12} | {result['average_final_score']:.2f}")

    except Exception as e:
        print(f"{query:30} | ERROR: {str(e)}")