Adding prompts as files to promptfoo config

README.md

@@ -30,7 +30,7 @@ For the image scope, the program takes up to two files, depending on the prompt
 |----------------------|-------------------------------------------------------------------|----------|
 | `--submission_type`  | Type of submission (from `arg_options.FileType`)                  | ❌ |
 | `--prompt`           | Pre-defined prompt name or file path to custom prompt file        | ❌ **|
-| `--prompt_text`      | Additional string text prompt that can be fed to model.           | ❌ ** |
+| `--prompt_text`      | Additional string text prompt that can be fed to model or standalone prompt. | ❌ ** |
 | `--scope`            | Processing scope (`image` or `code` or `text`)                    | ✅ |
 | `--submission`       | Submission file path                                              | ✅ |
 | `--question`         | Specific question to evaluate                                     | ❌ |

ai_feedback/__main__.py

@@ -215,7 +215,7 @@ def main() -> int:
         args.submission_type = detect_submission_type(args.submission)
 
     prompt_content = ""
-
+    prompt = {}
     system_instructions = load_system_prompt_content(args.system_prompt)
 
     if args.prompt:
@@ -236,6 +236,7 @@ def main() -> int:
 
     if args.prompt_text:
         prompt_content += args.prompt_text
+        prompt["prompt_content"] = prompt_content
 
     if args.scope == "image":
         prompt = {"prompt_content": prompt_content}

ai_feedback/helpers/constants.py

@@ -2,7 +2,7 @@
 HELP_MESSAGES = {
     "submission_type": "The format of the submission file (e.g., Jupyter notebook, Python script).",
     "prompt": "Pre-defined prompt name (from ai_feedback/data/prompts/user/) or file path to custom prompt file.",
-    "prompt_text": "Additional messages to concatenate to the prompt.",
+    "prompt_text": "Optional standalone prompt or additional text to append to the base prompt.",
     "scope": "The section of the assignment the model should analyze (e.g., code or image).",
     "submission": "The file path for the submission file.",
     "solution": "The file path for the solution file.",

promptfoo/promptfoo_test_runner.py

@@ -34,8 +34,8 @@ def call_api(prompt: str, context: dict, metadata: dict) -> dict:
             options["scope"],
             "--model",
             options["model"],
-            "--prompt",
-            options['prompt'],
+            "--prompt_text",
+            prompt,
             "--llama_mode",
             "server",
             "--output_template",

promptfoo/tests/codellama_tests/codellama_code_tests.yaml

@@ -6,59 +6,59 @@ defaultTest:
     model: codellama:latest
     scope: code
 
-scenarios:
-  - config:
-      - vars: { prompt: code_overall }
-    tests:
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/correct_submission/correct_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/fail_submission/fail_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/incorrect_algo_submission/incorrect_algo_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/style_issues_submission/style_issues_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/correct_submission/correct_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/correctness_1_submission/correctness_1_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/correctness_2_submission/correctness_2_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/efficiency_submission/efficiency_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/style_submission/style_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/correct_submission/correct_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/fail_submission/fail_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/inefficient_submission/inefficient_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/partial_correct_submission/partial_correct_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
+prompts:
+  - file://../../../ai_feedback/data/prompts/user/code_overall.md
+
+tests:
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/correct_submission/correct_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/fail_submission/fail_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/incorrect_algo_submission/incorrect_algo_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/style_issues_submission/style_issues_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/correct_submission/correct_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/correctness_1_submission/correctness_1_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/correctness_2_submission/correctness_2_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/efficiency_submission/efficiency_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/style_submission/style_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/correct_submission/correct_submission.py
+      solution_file:   test_submissions/gac_example/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/fail_submission/fail_submission.py
+      solution_file:   test_submissions/gac_example/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/inefficient_submission/inefficient_submission.py
+      solution_file:   test_submissions/gac_example/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/partial_correct_submission/partial_correct_submission.py
+      solution_file:   test_submissions/gac_example/solution.py

promptfoo/tests/deepseek_r1_tests/deepseek_r1_code_tests.yaml

@@ -6,58 +6,58 @@ defaultTest:
     model: deepSeek-R1:70B
     scope: code
 
-scenarios:
-  - config:
-      - vars: { prompt: code_feedback_r1 }
-    tests:
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/correct_submission/correct_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/fail_submission/fail_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/incorrect_algo_submission/incorrect_algo_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc263_opt_connected/style_issues_submission/style_issues_submission.py
-          solution_file:   test_submissions/csc263_opt_connected/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/correct_submission/correct_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/correctness_1_submission/correctness_1_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/correctness_2_submission/correctness_2_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/efficiency_submission/efficiency_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/csc108/style_submission/style_submission.py
-          solution_file:   test_submissions/csc108/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/correct_submission/correct_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/fail_submission/fail_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/inefficient_submission/inefficient_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
-
-      - vars:
-          submission_file: test_submissions/gac_example/partial_correct_submission/partial_correct_submission.py
-          solution_file:   test_submissions/gac_example/solution.py
+prompts:
+  - file://../../../ai_feedback/data/prompts/user/code_feedback_r1.md
+
+tests:
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/correct_submission/correct_submission.py
+      solution_file: test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/fail_submission/fail_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/incorrect_algo_submission/incorrect_algo_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc263_opt_connected/style_issues_submission/style_issues_submission.py
+      solution_file:   test_submissions/csc263_opt_connected/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/correct_submission/correct_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/correctness_1_submission/correctness_1_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/correctness_2_submission/correctness_2_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/efficiency_submission/efficiency_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/csc108/style_submission/style_submission.py
+      solution_file:   test_submissions/csc108/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/correct_submission/correct_submission.py
+      solution_file:   test_submissions/gac_example/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/fail_submission/fail_submission.py
+      solution_file:   test_submissions/gac_example/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/inefficient_submission/inefficient_submission.py
+      solution_file:   test_submissions/gac_example/solution.py
+
+  - vars:
+      submission_file: test_submissions/gac_example/partial_correct_submission/partial_correct_submission.py
+      solution_file:   test_submissions/gac_example/solution.py

promptfoo/tests/deepseek_r1_tests/deepseek_r1_text_tests.yaml

@@ -6,34 +6,34 @@ defaultTest:
     model: deepSeek-R1:70B
     scope: text
 
-scenarios:
-  - config:
-      - vars: { prompt: text_analyze_r1 }
-    tests:
-      - vars:
-          submission_file: test_submissions/data_collection_ethics_module/average_submission/average_submission.txt
-          solution_file: test_submissions/data_collection_ethics_module/solution.txt
-
-      - vars:
-          submission_file: test_submissions/data_collection_ethics_module/excellent_submission/excellent_submission.txt
-          solution_file: test_submissions/data_collection_ethics_module/solution.txt
-
-      - vars:
-          submission_file: test_submissions/data_collection_ethics_module/off_topic_submission/off_topic_submission.txt
-          solution_file: test_submissions/data_collection_ethics_module/solution.txt
-
-      - vars:
-          submission_file: test_submissions/data_collection_ethics_module/weak_submission/weak_submission.txt
-          solution_file: test_submissions/data_collection_ethics_module/solution.txt
-
-      - vars:
-          submission_file: test_submissions/csc373_eft_optimality_proof/fail_submission/fail_submission.pdf
-          solution_file: test_submissions/csc373_eft_optimality_proof/solution.pdf
-
-      - vars:
-          submission_file: test_submissions/csc373_eft_optimality_proof/incomplete_submission/incomplete_submission.pdf
-          solution_file: test_submissions/csc373_eft_optimality_proof/solution.pdf
-
-      - vars:
-          submission_file: test_submissions/csc373_eft_optimality_proof/induction_submission/induction_submission.pdf
-          solution_file: test_submissions/csc373_eft_optimality_proof/solution.pdf
+prompts:
+  - file://../../../ai_feedback/data/prompts/user/text_analyze_r1.md
+
+tests:
+  - vars:
+      submission_file: test_submissions/data_collection_ethics_module/average_submission/average_submission.txt
+      solution_file: test_submissions/data_collection_ethics_module/solution.txt
+
+  - vars:
+      submission_file: test_submissions/data_collection_ethics_module/excellent_submission/excellent_submission.txt
+      solution_file: test_submissions/data_collection_ethics_module/solution.txt
+
+  - vars:
+      submission_file: test_submissions/data_collection_ethics_module/off_topic_submission/off_topic_submission.txt
+      solution_file: test_submissions/data_collection_ethics_module/solution.txt
+
+  - vars:
+      submission_file: test_submissions/data_collection_ethics_module/weak_submission/weak_submission.txt
+      solution_file: test_submissions/data_collection_ethics_module/solution.txt
+
+  - vars:
+      submission_file: test_submissions/csc373_eft_optimality_proof/fail_submission/fail_submission.pdf
+      solution_file: test_submissions/csc373_eft_optimality_proof/solution.pdf
+
+  - vars:
+      submission_file: test_submissions/csc373_eft_optimality_proof/incomplete_submission/incomplete_submission.pdf
+      solution_file: test_submissions/csc373_eft_optimality_proof/solution.pdf
+
+  - vars:
+      submission_file: test_submissions/csc373_eft_optimality_proof/induction_submission/induction_submission.pdf
+      solution_file: test_submissions/csc373_eft_optimality_proof/solution.pdf