use public prompt sets instead of prompt sets that require auth

Author: rogthefrog

plugins/validation_tests/test_object_creation.py

@@ -1,11 +1,13 @@
 import os
+import re
 
 import pytest
 from flaky import flaky  # type: ignore
 from modelgauge.base_test import PromptResponseTest
 from modelgauge.caching import SqlDictCache
 from modelgauge.config import load_secrets_from_config
 from modelgauge.dependency_helper import FromSourceDependencyHelper
+from modelgauge.external_data import WebData
 from modelgauge.load_plugins import load_plugins
 from modelgauge.locales import EN_US  # see "workaround" below
 from modelgauge.prompt import SUTOptions, TextPrompt
@@ -26,11 +28,21 @@
 _FAKE_SECRETS = fake_all_secrets()
 
 
-@pytest.mark.parametrize("test_name", [key for key, _ in TESTS.items()])
-def test_all_tests_construct_and_record_init(test_name):
-    test = TESTS.make_instance(test_name, secrets=_FAKE_SECRETS)
-    assert hasattr(test, "initialization_record"), "Test is probably missing @modelgauge_test() decorator."
-    assert isinstance(test.initialization_record, InitializationRecord)
+# Some tests require such large downloads / complex processing
+# that we don't want to do that even on expensive_tests.
+# If your Test is timing out, consider adding it here.
+TOO_SLOW = {"real_toxicity_prompts", "bbq"}
+
+
+def ensure_public_dependencies(dependencies):
+    for k, d in dependencies.items():
+        if isinstance(d, WebData):
+            if "practice_prompt" in d.source_url or "heldback_prompt" in d.source_url:
+                new_dependency = WebData(
+                    source_url=re.sub("(practice|heldback|official)_prompt", "demo_prompt", d.source_url)
+                )
+                dependencies[k] = new_dependency
+    return dependencies
 
 
 @pytest.fixture(scope="session")
@@ -39,17 +51,18 @@ def shared_run_dir(tmp_path_factory):
     return tmp_path_factory.mktemp("run_data")
 
 
-# Some tests require such large downloads / complex processing
-# that we don't want to do that even on expensive_tests.
-# If your Test is timing out, consider adding it here.
-TOO_SLOW = {"real_toxicity_prompts", "bbq"}
+@pytest.mark.parametrize("test_name", [key for key, _ in TESTS.items()])
+def test_all_tests_construct_and_record_init(test_name):
+    test = TESTS.make_instance(test_name, secrets=_FAKE_SECRETS)
+    assert hasattr(test, "initialization_record"), "Test is probably missing @modelgauge_test() decorator."
+    assert isinstance(test.initialization_record, InitializationRecord)
 
 
 @expensive_tests
 @pytest.mark.timeout(30)
 @flaky
 @pytest.mark.parametrize("test_name", [key for key, _ in TESTS.items() if key not in TOO_SLOW])
-def test_all_tests_make_test_items(test_name, shared_run_dir):
+def test_all_tests_make_test_items(tmp_path, test_name, shared_run_dir):
     test = TESTS.make_instance(test_name, secrets=_FAKE_SECRETS)
 
     # TODO remove when localized files are handled better
@@ -59,9 +72,11 @@ def test_all_tests_make_test_items(test_name, shared_run_dir):
 
     if isinstance(test, PromptResponseTest):
         test_data_path = os.path.join(shared_run_dir, test.__class__.__name__)
+        dependencies = ensure_public_dependencies(test.get_dependencies())
+
         dependency_helper = FromSourceDependencyHelper(
             test_data_path,
-            test.get_dependencies(),
+            dependencies,
             required_versions={},
         )