Skip to content

Commit 07c6fdf

Browse files
hoe-jocastler
authored andcommitted
[trlc ai checker] refactor
- update pip requirements - refactor into multiple classes - update api to new schema
1 parent d3a25cc commit 07c6fdf

14 files changed

Lines changed: 1060 additions & 487 deletions

validation/ai_checker/BUILD

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ py_library(
4848
name = "copilot_langchain",
4949
srcs = [
5050
"src/copilot_adapter/__init__.py",
51+
"src/copilot_adapter/_client_manager.py",
52+
"src/copilot_adapter/_errors.py",
53+
"src/copilot_adapter/_message_converter.py",
54+
"src/copilot_adapter/_preflight.py",
55+
"src/copilot_adapter/_tool_converter.py",
5156
"src/copilot_adapter/copilot_langchain.py",
5257
],
5358
imports = ["src"],

validation/ai_checker/ai_checker.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ fi
164164
_COMMON_AI_TEST_ATTRS = {
165165
"model": attr.string(
166166
doc = "AI model name to use for analysis.",
167-
default = "anthropic/claude-sonnet-4-5",
167+
default = "claude-sonnet-4.6",
168168
),
169169
"score_threshold": attr.string(
170170
doc = "Minimum average score required to pass the test (0-10).",

validation/ai_checker/requirements.txt

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -138,13 +138,13 @@ charset-normalizer==3.4.4 \
138138
--hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \
139139
--hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608
140140
# via requests
141-
github-copilot-sdk==0.1.25 \
142-
--hash=sha256:13ef99fa8c709c5f80d820672bf36ee9176bc33f0efce6a2b5cbf6d1bb2369e8 \
143-
--hash=sha256:1a90ee583309ff308fea42f9edec61203645a33ca1d3dc42953628fb8c3eda07 \
144-
--hash=sha256:5249a63d1ac1e4d325c70c9902e81327b0baca53afa46010f52ac3fd3b5a111b \
145-
--hash=sha256:7af33d3afbe09a78dfc9d65a843526e47aba15631e90926c42a21a200fab12da \
146-
--hash=sha256:bc74a3d08ee45313ac02a3f7159c583ec41fc16090ec5f27f88c4b737f03139e \
147-
--hash=sha256:d32c3fc2c393f70923a645a133607da2e562d078b87437f499100d5bb8c1902f
141+
github-copilot-sdk==0.3.0 \
142+
--hash=sha256:7e241d9b00ebf8bb4d10b2d6101c75fcef38de04d144d729e07fa48394270ee1 \
143+
--hash=sha256:93b07c46f60cebbbb003d5bddba22eab886849b1d052b98037b52b6434a5bc07 \
144+
--hash=sha256:b591546d789f9f8243fb59ca71b08cb0bb1dbec818fbef060c3830c6787de2c8 \
145+
--hash=sha256:c5712d57a2c6291b805c79e039c55c48d858034b1a37fc8e1653925403a028e9 \
146+
--hash=sha256:ed8f27989158824c754d7febb473bdf25744a1e6bc07a06f114f7e7deebd2c22 \
147+
--hash=sha256:f4d98a67b8f038885ddd38bd7033d1ac20c3010f04c72ee0fc74ba4984b69ffa
148148
# via -r validation/ai_checker/requirements.txt.in
149149
h11==0.16.0 \
150150
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
@@ -173,10 +173,14 @@ jsonpointer==3.0.0 \
173173
--hash=sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 \
174174
--hash=sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef
175175
# via jsonpatch
176-
langchain-core==1.2.13 \
177-
--hash=sha256:b31823e28d3eff1e237096d0bd3bf80c6f9624eb471a9496dbfbd427779f8d82 \
178-
--hash=sha256:d2773d0d0130a356378db9a858cfeef64c3d64bc03722f1d4d6c40eb46fdf01b
176+
langchain-core==1.4.0 \
177+
--hash=sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f \
178+
--hash=sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c
179179
# via -r validation/ai_checker/requirements.txt.in
180+
langchain-protocol==0.0.15 \
181+
--hash=sha256:461eb794358f83d5e42635a5797799ffec7b4702314e34edf73ac21e75d3ef79 \
182+
--hash=sha256:9ab2d11ee73944754f10e037e717098d3a6796f0e58afa9cadda6154e7655ade
183+
# via langchain-core
180184
langsmith==0.6.3 \
181185
--hash=sha256:33246769c0bb24e2c17e0c34bb21931084437613cd37faf83bd0978a297b826f \
182186
--hash=sha256:44fdf8084165513e6bede9dda715e7b460b1b3f57ac69f2ca3f03afa911233ec
@@ -519,8 +523,8 @@ typing-extensions==4.15.0 \
519523
--hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548
520524
# via
521525
# anyio
522-
# github-copilot-sdk
523526
# langchain-core
527+
# langchain-protocol
524528
# pydantic
525529
# pydantic-core
526530
# typing-inspection

validation/ai_checker/requirements.txt.in

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@ pydantic
55
pyyaml
66

77
# LangChain + GitHub Copilot SDK
8-
github-copilot-sdk>=0.1.23
9-
langchain-core>=1.2.9
8+
github-copilot-sdk>=0.3.0
9+
langchain-core>=1.4.0

validation/ai_checker/src/ai_checker/constants.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717
"""
1818

1919
# Default AI model to use for all analysis operations
20-
DEFAULT_MODEL = "anthropic/claude-sonnet-4-5"
20+
DEFAULT_MODEL = "claude-sonnet-4.6"
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# *******************************************************************************
2+
# Copyright (c) 2026 Contributors to the Eclipse Foundation
3+
#
4+
# See the NOTICE file(s) distributed with this work for additional
5+
# information regarding copyright ownership.
6+
#
7+
# This program and the accompanying materials are made available under the
8+
# terms of the Apache License Version 2.0 which is available at
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# SPDX-License-Identifier: Apache-2.0
12+
# *******************************************************************************
13+
"""Public API for the copilot_adapter package."""
14+
15+
from .copilot_langchain import ChatCopilot
16+
from ._errors import CopilotSetupError
17+
18+
__all__ = ["ChatCopilot", "CopilotSetupError"]
Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
# *******************************************************************************
2+
# Copyright (c) 2026 Contributors to the Eclipse Foundation
3+
#
4+
# See the NOTICE file(s) distributed with this work for additional
5+
# information regarding copyright ownership.
6+
#
7+
# This program and the accompanying materials are made available under the
8+
# terms of the Apache License Version 2.0 which is available at
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# SPDX-License-Identifier: Apache-2.0
12+
# *******************************************************************************
13+
"""Lifecycle management for the Copilot CLI subprocess and SDK client."""
14+
15+
from __future__ import annotations
16+
17+
import logging
18+
from typing import Any, Optional
19+
20+
from copilot import CopilotClient, SubprocessConfig
21+
22+
from ._errors import CopilotSetupError
23+
from ._preflight import (
24+
check_auth_sources,
25+
check_cli_binary,
26+
check_environment,
27+
describe_auth_sources,
28+
resolve_copilot_cli_path,
29+
)
30+
31+
logger = logging.getLogger(__name__)
32+
33+
34+
class CopilotClientManager:
35+
"""Owns the lifecycle of a single CopilotClient / CLI subprocess.
36+
37+
Responsibilities:
38+
- Resolve the CLI binary path (rules_python copy_executables workaround)
39+
- Run pre-flight checks before spawning the process
40+
- Start the subprocess and verify authentication
41+
- Expose the live client for callers
42+
- Shut the process down cleanly on close
43+
44+
This class is intentionally not a Pydantic model — it holds mutable
45+
runtime state that must not be serialised.
46+
"""
47+
48+
def __init__(self, copilot_client_options: dict[str, Any] | None = None) -> None:
49+
self._options: dict[str, Any] = dict(copilot_client_options or {})
50+
self._client: Optional[CopilotClient] = None
51+
self._started: bool = False
52+
53+
# ------------------------------------------------------------------
54+
# Public interface
55+
# ------------------------------------------------------------------
56+
57+
async def ensure_client(self) -> CopilotClient:
58+
"""Return a started, authenticated CopilotClient.
59+
60+
Creates and starts the client on the first call; subsequent calls
61+
return the cached instance immediately.
62+
63+
Pre-flight sequence (runs once, before the CLI is spawned):
64+
1. Resolve the CLI binary path
65+
2. Validate the binary exists and is executable
66+
3. Hard-fail if no auth source is available at all
67+
4. Warn about missing $HOME / HTTPS_PROXY (non-fatal)
68+
5. Start the CLI subprocess
69+
6. Verify authentication via get_auth_status()
70+
71+
Raises:
72+
CopilotSetupError: With a detailed, actionable message for any
73+
failure that prevents the CLI from being used.
74+
"""
75+
if self._client is None:
76+
self._client = self._create_client()
77+
78+
if not self._started:
79+
await self._start_and_verify()
80+
81+
return self._client
82+
83+
async def close(self) -> None:
84+
"""Stop the CLI subprocess if it is running."""
85+
if self._client and self._started:
86+
await self._client.stop()
87+
self._started = False
88+
89+
# ------------------------------------------------------------------
90+
# Internal helpers
91+
# ------------------------------------------------------------------
92+
93+
def _create_client(self) -> CopilotClient:
94+
"""Run pre-flight checks and construct (but do not start) the client."""
95+
opts = dict(self._options)
96+
97+
# --- Resolve CLI binary path ----------------------------------
98+
if "cli_path" not in opts and "cli_url" not in opts:
99+
resolved = resolve_copilot_cli_path()
100+
if resolved:
101+
opts["cli_path"] = resolved
102+
logger.info("Resolved Copilot CLI path: %s", resolved)
103+
else:
104+
logger.warning(
105+
"Could not find copilot_cli (copy_executables target). "
106+
"Falling back to bundled binary — this may fail with "
107+
"PermissionError if the executable bit was stripped."
108+
)
109+
110+
# --- Check binary --------------------------------------------
111+
cli_path = opts.get("cli_path")
112+
if cli_path:
113+
problems = check_cli_binary(cli_path)
114+
if problems:
115+
raise CopilotSetupError(
116+
"Copilot CLI binary check failed:\n"
117+
+ "\n".join(f" - {p}" for p in problems)
118+
)
119+
120+
# --- Hard-fail if no auth source available -------------------
121+
auth_problems = check_auth_sources()
122+
if auth_problems:
123+
raise CopilotSetupError(
124+
"Copilot authentication pre-flight check failed — "
125+
"the CLI process will not be started:\n"
126+
+ "\n".join(f" - {p}" for p in auth_problems)
127+
+ "\n\n"
128+
+ describe_auth_sources()
129+
)
130+
131+
# --- Warn about non-fatal env issues -------------------------
132+
env_problems = check_environment()
133+
if env_problems:
134+
logger.warning(
135+
"Environment issues detected:\n%s\n%s",
136+
"\n".join(f" - {p}" for p in env_problems),
137+
describe_auth_sources(),
138+
)
139+
140+
logger.info("Starting CopilotClient...\n%s", describe_auth_sources())
141+
_subprocess_fields = frozenset(
142+
{
143+
"cli_path",
144+
"cli_args",
145+
"cwd",
146+
"use_stdio",
147+
"port",
148+
"log_level",
149+
"env",
150+
"github_token",
151+
"use_logged_in_user",
152+
"telemetry",
153+
"session_fs",
154+
"session_idle_timeout_seconds",
155+
}
156+
)
157+
subprocess_kwargs = {k: v for k, v in opts.items() if k in _subprocess_fields}
158+
return CopilotClient(SubprocessConfig(**subprocess_kwargs))
159+
160+
async def _start_and_verify(self) -> None:
161+
"""Start the CLI subprocess and verify authentication."""
162+
assert self._client is not None
163+
164+
try:
165+
await self._client.start()
166+
except PermissionError as exc:
167+
raise CopilotSetupError(
168+
f"PermissionError starting Copilot CLI: {exc}\n"
169+
" The CLI binary is not executable. Make sure\n"
170+
" pip.whl_mods / copy_executables is configured in MODULE.bazel\n"
171+
" to create an executable copy of copilot/bin/copilot."
172+
) from exc
173+
except RuntimeError as exc:
174+
if "timeout" in str(exc).lower() or "Timeout" in str(exc):
175+
raise CopilotSetupError(
176+
f"Timeout starting Copilot CLI server: {exc}\n"
177+
" The CLI started but did not become ready in time.\n"
178+
" This usually means the CLI cannot authenticate.\n\n"
179+
+ describe_auth_sources()
180+
+ "\n\n"
181+
" Possible fixes:\n"
182+
" 1. Run 'copilot' in a terminal and sign in interactively.\n"
183+
" 2. Set COPILOT_GITHUB_TOKEN (or GH_TOKEN / GITHUB_TOKEN)\n"
184+
" and pass it via --action_env=COPILOT_GITHUB_TOKEN.\n"
185+
" 3. Ensure HOME is available in the action environment\n"
186+
" (use_default_shell_env = True in the Bazel rule).\n"
187+
" See: https://github.com/github/copilot-sdk/blob/main/docs/auth/index.md"
188+
) from exc
189+
raise
190+
except Exception as exc:
191+
raise CopilotSetupError(
192+
f"Failed to start CopilotClient: {type(exc).__name__}: {exc}\n\n"
193+
+ describe_auth_sources()
194+
) from exc
195+
196+
self._started = True
197+
await self._verify_auth()
198+
199+
async def _verify_auth(self) -> None:
200+
"""Log the result of get_auth_status() as a diagnostic; never hard-fail.
201+
202+
Rationale: get_auth_status() can return isAuthenticated=False even when
203+
the CLI is fully functional — for example:
204+
- The auth state is resolved lazily on the first real request.
205+
- GitHub Enterprise hosts (*.ghe.com) may not be reflected immediately.
206+
- There is a brief window after start() where the status is not yet set.
207+
208+
A false-positive hard-fail here would block valid requests. The actual
209+
LLM call (send_and_wait) will fail with a clear error if auth is truly
210+
broken, so we demote this check to a warning-only diagnostic.
211+
"""
212+
assert self._client is not None
213+
try:
214+
auth_status = await self._client.get_auth_status()
215+
# The SDK uses camelCase on some versions, snake_case on others.
216+
is_auth = getattr(auth_status, "isAuthenticated", None) or getattr(
217+
auth_status, "is_authenticated", None
218+
)
219+
if is_auth:
220+
user = getattr(auth_status, "login", "unknown")
221+
logger.info("Copilot authenticated as: %s", user)
222+
else:
223+
# Log as a warning only — do not raise. The CLI may still work.
224+
logger.warning(
225+
"get_auth_status() reports isAuthenticated=False — "
226+
"continuing anyway; auth may be resolved on first request.\n"
227+
" Auth status: %s\n%s",
228+
auth_status,
229+
describe_auth_sources(),
230+
)
231+
except Exception as exc:
232+
# get_auth_status itself failed — log but do not block.
233+
logger.warning(
234+
"Could not verify auth status (non-fatal): %s: %s",
235+
type(exc).__name__,
236+
exc,
237+
)
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# *******************************************************************************
2+
# Copyright (c) 2026 Contributors to the Eclipse Foundation
3+
#
4+
# See the NOTICE file(s) distributed with this work for additional
5+
# information regarding copyright ownership.
6+
#
7+
# This program and the accompanying materials are made available under the
8+
# terms of the Apache License Version 2.0 which is available at
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# SPDX-License-Identifier: Apache-2.0
12+
# *******************************************************************************
13+
"""Shared error types and constants for the Copilot adapter."""
14+
15+
from __future__ import annotations
16+
17+
# Auth-related environment variables checked by the Copilot CLI (priority order)
18+
AUTH_ENV_VARS: list[str] = [
19+
"COPILOT_GITHUB_TOKEN", # Recommended for explicit Copilot usage
20+
"GH_TOKEN", # GitHub CLI compatible
21+
"GITHUB_TOKEN", # GitHub Actions compatible
22+
]
23+
24+
25+
class CopilotSetupError(RuntimeError):
26+
"""Raised when the Copilot SDK environment is not correctly configured."""

0 commit comments

Comments
 (0)