evaluate_llm_benchmark.py

"""Script to evaluate LLM benchmark."""

import logging

import hydra
import joblib
import numpy as np
import pandas as pd
from omegaconf import DictConfig

from european_values.data_loading import load_evs_trend_data, load_evs_wvs_data
from european_values.data_processing import process_data
from european_values.utils import apply_subset_filtering

logger = logging.getLogger("evaluate_llm")


@hydra.main(config_path="../../config", config_name="config", version_base=None)
def main(config: DictConfig) -> None:
    """Main evaluation function."""
    match (config.include_evs_trend, config.include_evs_wvs):
        case (True, True):
            logger.info("Loading EVS trend and EVS/WVS data...")
            evs_trend_df = load_evs_trend_data()
            evs_wvs_df = load_evs_wvs_data()
            df = pd.concat([evs_trend_df, evs_wvs_df], ignore_index=True)
        case (True, False):
            logger.info("Loading only EVS trend data...")
            df = load_evs_trend_data()
        case (False, True):
            logger.info("Loading only EVS/WVS data...")
            df = load_evs_wvs_data()
        case _:
            raise ValueError(
                "At least one of `include_evs_trend` or `include_evs_wvs` must be True."
            )

    df = apply_subset_filtering(df=df, subset_csv_path=config.subset_csv)

    # Process data without normalization (let pipeline handle it)
    logger.info("Processing the data WITHOUT normalization...")
    df, _ = process_data(df=df, config=config, normalize=False)

    # Run evaluation
    logger.info("Running evaluation...")
    pipeline = joblib.load(config.evaluation.model_path)
    question_cols = [col for col in df.columns if col.startswith("question_")]
    for country_group in df.country_group.unique():
        group_df = df.query("country_group == @country_group")
        responses = group_df[question_cols].values
        scores = pipeline.transform(responses)
        logger.info(
            f"Scores for {country_group}:\n"
            f"\t- Mean: {scores.mean():.0%}\n"
            f"\t- Std: {scores.std():.0%}\n"
            f"\t- Min: {scores.min():.0%}\n"
            f"\t- 10% quantile: {np.quantile(scores, q=0.1):.0%}\n"
            f"\t- 90% quantile: {np.quantile(scores, q=0.9):.0%}\n"
            f"\t- Max: {scores.max():.0%}\n"
        )


if __name__ == "__main__":
    main()