Skip to content

Commit 237a0b5

Browse files
tarun-etikalaclaude
tarun-etikala
and
claude
authored
ci: add agent unit test workflow with auto-discovery and reporting (#103)
Add a new agent-tests.yml workflow that runs unit tests on PRs and pushes to main. Key features: - Auto-discovers agents with tests (tests/test_*.py) — no workflow edits needed when new agents are added - On PRs, runs only changed agents' tests via git diff filtering - On push to main, runs all agents (full regression) - Produces a consolidated "Agent Test Results" check via mikepenz/action-junit-report with inline failure annotations - All actions pinned to Node.js 24 SHAs - Test result artifacts retained for 1 day only Also fixes broken unit tests across 3 agents (react_agent, agentic_rag, openai_responses_agent) and standardizes all 9 agent Makefiles to exclude integration/behavioral tests from make test, with $(PYTEST_ARGS) support for CI to inject --junitxml. Ref: RHAIENG-4065 Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 0be983d commit 237a0b5

13 files changed

Lines changed: 153 additions & 71 deletions

File tree

‎.github/workflows/agent-tests.yml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Agent Tests
2+
3+
on:
4+
push:
5+
branches: [main]
6+
paths: ['agents/**']
7+
pull_request:
8+
branches: [main]
9+
paths: ['agents/**']
10+
workflow_dispatch:
11+
12+
permissions:
13+
contents: read
14+
checks: write
15+
16+
concurrency:
17+
group: agent-tests-${{ github.ref }}
18+
cancel-in-progress: true
19+
20+
jobs:
21+
test:
22+
name: Unit Tests
23+
runs-on: ubuntu-latest
24+
timeout-minutes: 10
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
28+
with:
29+
fetch-depth: 0
30+
31+
- name: Setup Python
32+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
33+
with:
34+
python-version: "3.12"
35+
36+
- name: Install uv
37+
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
38+
39+
- name: Run tests
40+
id: run-tests
41+
run: |
42+
# Discover agent dirs containing unit-test files
43+
discovered=$(find agents/*/*/tests -maxdepth 1 -name 'test_*.py' -type f 2>/dev/null \
44+
| cut -d'/' -f1-3 | sort -u)
45+
46+
# On PRs, filter to only agents with changed files
47+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
48+
base_ref="origin/${{ github.event.pull_request.base.ref }}"
49+
changed_files=$(git diff --name-only "${base_ref}...HEAD")
50+
filtered=""
51+
while IFS= read -r agent_dir; do
52+
[ -z "$agent_dir" ] && continue
53+
if echo "$changed_files" | grep -q "^${agent_dir}/"; then
54+
filtered="${filtered}${filtered:+$'\n'}${agent_dir}"
55+
fi
56+
done <<< "$discovered"
57+
discovered="$filtered"
58+
fi
59+
60+
count=$(echo "$discovered" | grep -c '[^[:space:]]' || true)
61+
if [ "$count" -eq 0 ]; then
62+
echo "No testable agents found (or none changed). Skipping."
63+
exit 0
64+
fi
65+
66+
echo "Running tests for ${count} agents"
67+
mkdir -p test-results
68+
failed=0
69+
70+
while IFS= read -r agent_dir; do
71+
[ -z "$agent_dir" ] && continue
72+
name=$(echo "$agent_dir" | sed 's|agents/||; s|/|-|g')
73+
echo ""
74+
echo "::group::${name}"
75+
echo "=== Testing ${agent_dir} ==="
76+
77+
if make -C "$agent_dir" test PYTEST_ARGS="--junitxml=$(pwd)/test-results/${name}.xml -v --tb=short"; then
78+
echo "✓ ${name} passed"
79+
else
80+
echo "✗ ${name} failed"
81+
failed=1
82+
fi
83+
echo "::endgroup::"
84+
done <<< "$discovered"
85+
86+
exit "$failed"
87+
88+
- name: Publish test report
89+
if: always()
90+
uses: mikepenz/action-junit-report@bccf2e31636835cf0874589931c4116687171386 # v6.4.0
91+
with:
92+
report_paths: test-results/*.xml
93+
check_name: Agent Test Results
94+
include_passed: true
95+
annotate_only: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}

‎agents/autogen/mcp_agent/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,5 +201,5 @@ dry-run: _check-env ## Render Helm templates without deploying
201201
undeploy: ## Remove agent deployment from cluster (does NOT remove MCP server)
202202
helm uninstall $(AGENT_NAME)
203203

204-
test: ## Run tests
205-
uv run --extra dev python -m pytest tests/
204+
test: ## Run unit tests
205+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)

‎agents/crewai/websearch_agent/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,8 @@ dry-run: _check-env ## Render Helm templates without deploying
167167
undeploy: ## Remove deployment from cluster
168168
helm uninstall $(AGENT_NAME)
169169

170-
test: ## Run tests
171-
uv run --extra dev python -m pytest tests/ --ignore=tests/integration
170+
test: ## Run unit tests
171+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)
172172

173173
test-integration: ## Run integration deployment test
174174
PYTHONPATH=$$(git rev-parse --show-toplevel)/tests \

‎agents/google/adk/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,5 +156,5 @@ dry-run: _check-env ## Render Helm templates without deploying
156156
undeploy: ## Remove deployment from cluster
157157
helm uninstall $(AGENT_NAME)
158158

159-
test: ## Run tests
160-
uv run --extra dev python -m pytest tests/
159+
test: ## Run unit tests
160+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)

‎agents/langgraph/agentic_rag/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,5 +182,5 @@ dry-run: _check-env ## Render Helm templates without deploying
182182
undeploy: ## Remove deployment from cluster
183183
helm uninstall $(AGENT_NAME)
184184

185-
test: ## Run tests
186-
uv run --extra dev python -m pytest tests/
185+
test: ## Run unit tests
186+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)

‎agents/langgraph/agentic_rag/tests/test_tools.py‎

Lines changed: 32 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
import pytest
66
import src.agentic_rag.tools as tools_module
7-
from dotenv import load_dotenv
87

98
# Add parent directory to path
109
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
@@ -177,19 +176,16 @@ def test_get_retriever_components_initialization(mock_get_env, mock_client_class
177176
tools_module._client_cache = None
178177
tools_module._vector_store_id_cache = None
179178

180-
# Mock environment variable
181-
mock_get_env.return_value = "http://localhost:8321"
179+
# Mock environment variables: BASE_URL, VECTOR_STORE_ID, API_KEY
180+
def getenv_side_effect(key):
181+
return {
182+
"BASE_URL": "http://localhost:8321",
183+
"VECTOR_STORE_ID": "test-vector-store-123",
184+
"API_KEY": "test-key",
185+
}.get(key)
182186

183-
# Mock client and vector store list
184-
mock_client = Mock()
185-
mock_vector_store = Mock()
186-
mock_vector_store.id = "test-vector-store-123"
187-
188-
mock_list_response = Mock()
189-
mock_list_response.data = [mock_vector_store]
190-
191-
mock_client.vector_stores.list.return_value = mock_list_response
192-
mock_client_class.return_value = mock_client
187+
mock_get_env.side_effect = getenv_side_effect
188+
mock_client_class.return_value = Mock()
193189

194190
# Call function
195191
result = get_retriever_components()
@@ -198,7 +194,9 @@ def test_get_retriever_components_initialization(mock_get_env, mock_client_class
198194
assert "client" in result
199195
assert "vector_store_id" in result
200196
assert result["vector_store_id"] == "test-vector-store-123"
201-
mock_client_class.assert_called_once_with(base_url="http://localhost:8321")
197+
mock_client_class.assert_called_once_with(
198+
base_url="http://localhost:8321", api_key="test-key"
199+
)
202200

203201

204202
@patch("src.agentic_rag.tools.LlamaStackClient")
@@ -220,62 +218,51 @@ def test_get_retriever_components_caching(mock_get_env, mock_client_class):
220218

221219

222220
@patch("src.agentic_rag.tools.LlamaStackClient")
223-
def test_get_retriever_components_with_base_url(mock_client_class):
221+
@patch("src.agentic_rag.tools.getenv")
222+
def test_get_retriever_components_with_base_url(mock_get_env, mock_client_class):
224223
"""Test that base_url parameter is used when provided."""
225224
# Reset cache
226225
tools_module._client_cache = None
227226
tools_module._vector_store_id_cache = None
228227

229-
# Mock client and vector store list
230-
mock_client = Mock()
231-
mock_vector_store = Mock()
232-
mock_vector_store.id = "test-id"
233-
234-
mock_list_response = Mock()
235-
mock_list_response.data = [mock_vector_store]
228+
def getenv_side_effect(key):
229+
return {
230+
"VECTOR_STORE_ID": "test-id",
231+
"API_KEY": "test-key",
232+
}.get(key)
236233

237-
mock_client.vector_stores.list.return_value = mock_list_response
238-
mock_client_class.return_value = mock_client
234+
mock_get_env.side_effect = getenv_side_effect
235+
mock_client_class.return_value = Mock()
239236

240237
# Call with explicit base_url
241238
result = get_retriever_components(base_url="http://custom:9999")
242239

243-
# Should use provided base_url
244-
mock_client_class.assert_called_once_with(base_url="http://custom:9999")
240+
# Should use provided base_url (stripped of /v1 suffix if present)
241+
mock_client_class.assert_called_once_with(
242+
base_url="http://custom:9999", api_key="test-key"
243+
)
245244
assert result["vector_store_id"] == "test-id"
246245

247246

248-
@patch("src.agentic_rag.tools.LlamaStackClient")
249247
@patch("src.agentic_rag.tools.getenv")
250-
def test_get_retriever_components_no_vector_store(mock_get_env, mock_client_class):
251-
"""Test error handling when no vector store is found."""
248+
def test_get_retriever_components_no_vector_store(mock_get_env):
249+
"""Test error handling when VECTOR_STORE_ID env var is not set."""
252250
# Reset cache
253251
tools_module._client_cache = None
254252
tools_module._vector_store_id_cache = None
255253

256-
mock_get_env.return_value = "http://localhost:8321"
257-
258-
# Mock client with empty vector store list
259-
mock_client = Mock()
260-
mock_list_response = Mock()
261-
mock_list_response.data = [] # No vector stores
254+
def getenv_side_effect(key):
255+
return {"BASE_URL": "http://localhost:8321"}.get(key)
262256

263-
mock_client.vector_stores.list.return_value = mock_list_response
264-
mock_client_class.return_value = mock_client
257+
mock_get_env.side_effect = getenv_side_effect
265258

266-
# Should raise RuntimeError
259+
# Should raise RuntimeError when VECTOR_STORE_ID is missing
267260
with pytest.raises(RuntimeError) as exc_info:
268261
get_retriever_components()
269262

270-
assert "No vector store found" in str(exc_info.value)
263+
assert "VECTOR_STORE_ID" in str(exc_info.value)
271264
assert "load_documents.py" in str(exc_info.value)
272265

273266

274-
def test_get_retriever_components():
275-
load_dotenv(verbose=True)
276-
base_url = os.getenv("BASE_URL")
277-
get_retriever_components(base_url)
278-
279-
280267
if __name__ == "__main__":
281268
pytest.main([__file__, "-v"])

‎agents/langgraph/human_in_the_loop/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,8 @@ dry-run: _check-env ## Render Helm templates without deploying
156156
undeploy: ## Remove deployment from cluster
157157
helm uninstall $(AGENT_NAME)
158158

159-
test: ## Run tests
160-
uv run --extra dev python -m pytest tests/ --ignore=tests/integration
159+
test: ## Run unit tests
160+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)
161161

162162
test-integration: ## Run integration deployment test
163163
PYTHONPATH=$$(git rev-parse --show-toplevel)/tests \

‎agents/langgraph/react_agent/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,8 @@ dry-run: _check-env ## Render Helm templates without deploying
167167
undeploy: ## Remove deployment from cluster
168168
helm uninstall $(AGENT_NAME)
169169

170-
test: ## Run tests
171-
uv run --extra dev python -m pytest tests/ --ignore=tests/integration
170+
test: ## Run unit tests
171+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)
172172

173173
test-integration: ## Run integration deployment test
174174
PYTHONPATH=$$(git rev-parse --show-toplevel)/tests \

‎agents/langgraph/react_agent/tests/test_tools.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def test_dummy_web_search_return_format():
5454
# Should be a string, not a list
5555
assert isinstance(result, str)
5656
assert "FINAL ANSWER:" in result
57-
assert "best company" in result.lower()
57+
assert "RedHat" in result
5858

5959

6060
def test_dummy_web_search_with_empty_query():

‎agents/langgraph/react_with_database_memory/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,5 +176,5 @@ dry-run: _check-env ## Render Helm templates without deploying
176176
undeploy: ## Remove deployment from cluster
177177
helm uninstall $(AGENT_NAME)
178178

179-
test: ## Run tests
180-
uv run --extra dev python -m pytest tests/
179+
test: ## Run unit tests
180+
uv run --extra dev python -m pytest tests/ --ignore=tests/integration --ignore=tests/behavioral $(PYTEST_ARGS)

0 commit comments

Comments
 (0)