Merge pull request #37 from agentevals-dev/peterj/addci

add ci

Author: krisztianfekete

.github/workflows/ci.yml

@@ -0,0 +1,56 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv venv
+          uv pip install setuptools
+          uv sync --dev
+
+      - name: Ruff check
+        run: uv run ruff check .
+
+      - name: Ruff format check
+        run: uv run ruff format --check .
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          uv venv --python ${{ matrix.python-version }}
+          uv pip install setuptools
+          uv sync --dev --python ${{ matrix.python-version }}
+
+      - name: Run tests
+        run: uv run pytest -m "not integration and not e2e" --tb=short -q

examples/custom_evaluators/response_quality.py

@@ -35,14 +35,10 @@ def response_quality(input: EvalInput) -> EvalResult:
         if len(inv.final_response.strip()) < min_len:
             score -= 0.3
             issues.append(
-                f"{inv.invocation_id}: response too short "
-                f"({len(inv.final_response.strip())} < {min_len} chars)"
+                f"{inv.invocation_id}: response too short ({len(inv.final_response.strip())} < {min_len} chars)"
             )
 
-        if (
-            inv.user_content
-            and inv.final_response.strip().lower() == inv.user_content.strip().lower()
-        ):
+        if inv.user_content and inv.final_response.strip().lower() == inv.user_content.strip().lower():
             score -= 0.5
             issues.append(f"{inv.invocation_id}: response is just the user input echoed back")
 

examples/dice_agent/agent.py

@@ -22,11 +22,7 @@ def roll_die(sides: int = 6) -> dict:
         return {"error": "Die must have at least 2 sides"}
 
     result = random.randint(1, sides)
-    return {
-        "sides": sides,
-        "result": result,
-        "message": f"Rolled a {sides}-sided die and got {result}"
-    }
+    return {"sides": sides, "result": result, "message": f"Rolled a {sides}-sided die and got {result}"}
 
 
 def check_prime(nums: list[int]) -> dict:
@@ -38,6 +34,7 @@ def check_prime(nums: list[int]) -> dict:
     Returns:
         Dictionary mapping each number to whether it's prime
     """
+
     def is_prime(n: int) -> bool:
         if n < 2:
             return False
@@ -53,16 +50,12 @@ def is_prime(n: int) -> bool:
     results = {num: is_prime(num) for num in nums}
     prime_nums = [n for n, is_p in results.items() if is_p]
 
-    return {
-        "results": results,
-        "prime_count": len(prime_nums),
-        "prime_numbers": prime_nums
-    }
+    return {"results": results, "prime_count": len(prime_nums), "prime_numbers": prime_nums}
 
 
 dice_agent = Agent(
     name="dice_agent",
-    #model="gemini-2.5-flash",
+    # model="gemini-2.5-flash",
     model="gemini-2.5-flash-lite",
     instruction="""You are a helpful assistant that can roll dice and check if numbers are prime.
 

examples/dice_agent/main.py

@@ -82,9 +82,7 @@ async def main():
         user_id = "demo_user"
 
         runner = InMemoryRunner(agent=dice_agent, app_name=app_name)
-        session = await runner.session_service.create_session(
-            app_name=app_name, user_id=user_id
-        )
+        session = await runner.session_service.create_session(app_name=app_name, user_id=user_id)
 
         test_queries = [
             "Hi! Can you help me?",
@@ -95,14 +93,10 @@ async def main():
         for i, query in enumerate(test_queries, 1):
             print(f"\n[{i}/{len(test_queries)}] User: {query}")
 
-            content = types.Content(
-                role="user", parts=[types.Part.from_text(text=query)]
-            )
+            content = types.Content(role="user", parts=[types.Part.from_text(text=query)])
 
             agent_response = ""
-            async for event in runner.run_async(
-                user_id=user_id, session_id=session.id, new_message=content
-            ):
+            async for event in runner.run_async(user_id=user_id, session_id=session.id, new_message=content):
                 if event.content.parts and event.content.parts[0].text:
                     agent_response = event.content.parts[0].text
 
@@ -125,17 +119,11 @@ async def main():
         app_name = "dice_agent_app"
         user_id = "demo_user"
         runner = InMemoryRunner(agent=dice_agent, app_name=app_name)
-        session = await runner.session_service.create_session(
-            app_name=app_name, user_id=user_id
-        )
+        session = await runner.session_service.create_session(app_name=app_name, user_id=user_id)
 
-        content = types.Content(
-            role="user", parts=[types.Part.from_text(text="Roll a 6-sided die")]
-        )
+        content = types.Content(role="user", parts=[types.Part.from_text(text="Roll a 6-sided die")])
 
-        async for event in runner.run_async(
-            user_id=user_id, session_id=session.id, new_message=content
-        ):
+        async for event in runner.run_async(user_id=user_id, session_id=session.id, new_message=content):
             if event.content.parts and event.content.parts[0].text:
                 print(f"Agent: {event.content.parts[0].text}")
 

examples/langchain_agent/agent.py

@@ -17,18 +17,10 @@ def roll_die(sides: int = 6) -> dict:
         Dictionary with sides, result, and message
     """
     if sides < 2:
-        return {
-            "sides": sides,
-            "result": None,
-            "message": "Error: Die must have at least 2 sides"
-        }
+        return {"sides": sides, "result": None, "message": "Error: Die must have at least 2 sides"}
 
     result = random.randint(1, sides)
-    return {
-        "sides": sides,
-        "result": result,
-        "message": f"Rolled a {result} on a {sides}-sided die"
-    }
+    return {"sides": sides, "result": result, "message": f"Rolled a {result} on a {sides}-sided die"}
 
 
 @tool
@@ -41,6 +33,7 @@ def check_prime(nums: list[int]) -> dict:
     Returns:
         Dictionary with results, prime_count, and prime_numbers
     """
+
     def is_prime(n: int) -> bool:
         if n < 2:
             return False
@@ -56,14 +49,10 @@ def is_prime(n: int) -> bool:
     results = {n: is_prime(n) for n in nums}
     prime_numbers = [n for n, is_p in results.items() if is_p]
 
-    return {
-        "results": results,
-        "prime_count": len(prime_numbers),
-        "prime_numbers": prime_numbers
-    }
+    return {"results": results, "prime_count": len(prime_numbers), "prime_numbers": prime_numbers}
 
 
-#def create_dice_agent(model: str = "gpt-3.5-turbo", temperature: float = 0.0):
+# def create_dice_agent(model: str = "gpt-3.5-turbo", temperature: float = 0.0):
 def create_dice_agent(model: str = "gpt-4o-mini", temperature: float = 0.0):
     llm = ChatOpenAI(model=model, temperature=temperature)
     tools = [roll_die, check_prime]

examples/langchain_agent/main.py

@@ -90,9 +90,7 @@ def run_loop_in_background():
     thread = threading.Thread(target=run_loop_in_background, daemon=True)
     thread.start()
 
-    future = asyncio.run_coroutine_threadsafe(
-        processor.connect(eval_set_id=eval_set_id), loop
-    )
+    future = asyncio.run_coroutine_threadsafe(processor.connect(eval_set_id=eval_set_id), loop)
     future.result()
 
     tracer_provider.add_span_processor(processor)
@@ -161,11 +159,7 @@ def main():
                 selected_tool = {t.name: t for t in tools}.get(tool_name)
                 if selected_tool:
                     tool_result = selected_tool.invoke(tool_args)
-                    messages.append(
-                        ToolMessage(
-                            content=str(tool_result), tool_call_id=tool_call["id"]
-                        )
-                    )
+                    messages.append(ToolMessage(content=str(tool_result), tool_call_id=tool_call["id"]))
         else:
             print("     Agent: [Max iterations reached]")
 

examples/sdk_example/async_example.py

@@ -47,18 +47,12 @@ async def main():
         metadata={"model": dice_agent.model},
     ):
         runner = InMemoryRunner(agent=dice_agent, app_name="dice_app")
-        session = await runner.session_service.create_session(
-            app_name="dice_app", user_id="demo_user"
-        )
+        session = await runner.session_service.create_session(app_name="dice_app", user_id="demo_user")
 
         for query in ["Roll a 20-sided die", "Is that number prime?"]:
             print(f"User: {query}")
-            content = types.Content(
-                role="user", parts=[types.Part.from_text(text=query)]
-            )
-            async for event in runner.run_async(
-                user_id="demo_user", session_id=session.id, new_message=content
-            ):
+            content = types.Content(role="user", parts=[types.Part.from_text(text=query)])
+            async for event in runner.run_async(user_id="demo_user", session_id=session.id, new_message=content):
                 if event.content.parts and event.content.parts[0].text:
                     print(f"Agent: {event.content.parts[0].text}")
 

examples/strands_agent/agent.py

@@ -34,6 +34,7 @@ def check_prime(nums: list[int]) -> dict:
     Returns:
         Dictionary with primality results and list of prime numbers
     """
+
     def is_prime(n: int) -> bool:
         if n < 2:
             return False

examples/strands_agent/main.py

@@ -71,9 +71,7 @@ def run_loop_in_background():
     thread = threading.Thread(target=run_loop_in_background, daemon=True)
     thread.start()
 
-    future = asyncio.run_coroutine_threadsafe(
-        processor.connect(eval_set_id=eval_set_id), loop
-    )
+    future = asyncio.run_coroutine_threadsafe(processor.connect(eval_set_id=eval_set_id), loop)
     future.result()
 
     telemetry.tracer_provider.add_span_processor(processor)

examples/zero-code-examples/langchain/run.py

@@ -50,22 +50,17 @@ def main():
 
     os.environ.setdefault(
         "OTEL_RESOURCE_ATTRIBUTES",
-        "agentevals.eval_set_id=langchain_agent_eval,"
-        "agentevals.session_name=langchain-zero-code",
+        "agentevals.eval_set_id=langchain_agent_eval,agentevals.session_name=langchain-zero-code",
     )
 
     resource = Resource.create()
 
     tracer_provider = TracerProvider(resource=resource)
-    tracer_provider.add_span_processor(
-        BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)
-    )
+    tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000))
     trace.set_tracer_provider(tracer_provider)
 
     logger_provider = LoggerProvider(resource=resource)
-    logger_provider.add_log_record_processor(
-        BatchLogRecordProcessor(OTLPLogExporter(), schedule_delay_millis=1000)
-    )
+    logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(), schedule_delay_millis=1000))
     set_logger_provider(logger_provider)
 
     OpenAIInstrumentor().instrument()
@@ -101,11 +96,7 @@ def main():
                 selected_tool = {t.name: t for t in tools}.get(tool_name)
                 if selected_tool:
                     tool_result = selected_tool.invoke(tool_args)
-                    messages.append(
-                        ToolMessage(
-                            content=str(tool_result), tool_call_id=tool_call["id"]
-                        )
-                    )
+                    messages.append(ToolMessage(content=str(tool_result), tool_call_id=tool_call["id"]))
         else:
             print("     Agent: [Max iterations reached]")