.

Author: kritinv

deepeval/openai/evaluate.py

@@ -37,6 +37,7 @@ def add_test_case(
         )
     )
 
+
 ##############################################
 # Evaluation
 ##############################################

deepeval/openai/patch.py

@@ -86,7 +86,7 @@ async def patched_async_openai_method(
             is_observed = len(trace_manager.traces) > 0
 
             if is_observed:
-                @observe(type="llm", model=input_parameters.model)
+                @observe(type="llm", model=input_parameters.model, metrics=metrics)
                 async def llm_generation(*args, **kwargs):
                     response = await orig_method(*args, **kwargs)
                     output_parameters = extract_output_parameters(is_completion_method, response, input_parameters)
@@ -143,7 +143,7 @@ def patched_sync_openai_method(
             is_observed = len(trace_manager.traces) > 0
 
             if is_observed:
-                @observe(type="llm", model=input_parameters.model)
+                @observe(type="llm", model=input_parameters.model, metrics=metrics)
                 def llm_generation(*args, **kwargs):
                     response = orig_method(*args, **kwargs)
                     output_parameters = extract_output_parameters(is_completion_method, response, input_parameters)

tests/integrations/test_async_openai_integration.py

@@ -0,0 +1,67 @@
+from deepeval.metrics import AnswerRelevancyMetric, BiasMetric
+from deepeval.openai import OpenAI
+from deepeval.tracing import observe
+
+client = OpenAI()
+
+##############################################
+# Test end-to-end Evaluation
+##############################################
+
+def test_end_to_end_evaluation():
+    for i in range(5):
+        client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "Hello, how are you?"},
+            ],
+            metrics=[AnswerRelevancyMetric()],
+        )
+
+    for i in range(5):
+        client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {"role": "system", "content": "You are a helpful chatbot."},
+                {"role": "user", "content": "Hello!"},
+            ],
+            metrics=[AnswerRelevancyMetric(), BiasMetric()],
+        )
+
+
+##############################################
+# Test tracing
+##############################################
+
+@observe()
+def llm_app(input: str):
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": "You are a helpful chatbot."},
+            {"role": "user", "content": input},
+        ],
+        metrics=[AnswerRelevancyMetric(), BiasMetric()],
+    )
+    return response.choices[0].message.content
+
+llm_app("hi")
+
+##############################################
+# Test tracing
+##############################################
+
+@observe()
+def llm_app(input: str):
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": "You are a helpful chatbot."},
+            {"role": "user", "content": input},
+        ],
+        metrics=[AnswerRelevancyMetric(), BiasMetric()],
+    )
+    return response.choices[0].message.content
+
+llm_app("hi")

tests/integrations/test_openai_integration.py

@@ -1,24 +1,76 @@
 from deepeval.metrics import AnswerRelevancyMetric, BiasMetric
+from deepeval.tracing import observe
+from deepeval.dataset import Golden
 from deepeval.openai import OpenAI
+from deepeval import evaluate
 
 client = OpenAI()
 
-for i in range(5):
-    client.chat.completions.create(
-        model="gpt-4o",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Hello, how are you?"},
-        ],
-        metrics=[AnswerRelevancyMetric()],
-    )
+##############################################
+# Test end-to-end Evaluation
+##############################################
+
+def test_end_to_end_evaluation():
+    for i in range(5):
+        client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "Hello, how are you?"},
+            ],
+            metrics=[AnswerRelevancyMetric()],
+        )
+
+    for i in range(5):
+        client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {"role": "system", "content": "You are a helpful chatbot."},
+                {"role": "user", "content": "Hello!"},
+            ],
+            metrics=[AnswerRelevancyMetric(), BiasMetric()],
+        )
+
+# test_end_to_end_evaluation()
+
 
-for i in range(5):
-    client.chat.completions.create(
+##############################################
+# Test tracing
+##############################################
+
+@observe()
+def llm_app(input: str):
+    response = client.chat.completions.create(
         model="gpt-4o",
         messages=[
             {"role": "system", "content": "You are a helpful chatbot."},
-            {"role": "user", "content": "Hello!"},
+            {"role": "user", "content": input},
         ],
         metrics=[AnswerRelevancyMetric(), BiasMetric()],
     )
+    return response.choices[0].message.content
+
+def test_tracing():
+    llm_app("hi")
+    llm_app("hello")
+    llm_app("how are you?")
+    llm_app("what is the capital of France?")
+
+# test_tracing()
+
+##############################################
+# Test traceable evaluate
+##############################################
+
+def test_traceable_evaluate():
+    evaluate(
+        observed_callback=llm_app,
+        goldens=[
+            Golden(input="hi"),
+            Golden(input="hello"),
+            Golden(input="how are you?"),
+            Golden(input="what is the capital of France?"),
+        ],
+    )
+
+test_traceable_evaluate()