Skip to content

fix: tagging JSON 파싱 prefix 흡수 (assistant prefill + 슬라이스 fallback) #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gyesswhat merged 1 commit into from
May 28, 2026
Merged

fix: tagging JSON 파싱 prefix 흡수 (assistant prefill + 슬라이스 fallback) #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 48 additions & 8 deletions app/services/tagging/v2_postscore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,16 @@
)


_MAX_TOKENS = 250
"""v1 의 200 보다 +50 — 후보가 5~7개라 output 토큰 약간 증가."""
_MAX_TOKENS = 350
"""5~7개 후보 + JSON 오버헤드 + 도입어/공백 prefix 흡수 마진. 운영 실측 (issue #18)
에서 1건이 250 으로 잘려 ``stop_reason=max_tokens`` 발생 → 안전 마진 추가."""


_ASSISTANT_PREFILL = "{"
"""1차/2차 호출 모두 assistant turn 으로 ``{`` 를 prefill 해 응답 첫 글자를 강제한다.
Haiku 가 ``다음과 같습니다:`` 같은 도입어를 자주 붙여 ``char 0`` JSON 파싱 실패를
유발하던 운영 이슈 (issue #18 — 24h 검증 실패 164회 중 95%) 대응. 응답 텍스트에
``_ASSISTANT_PREFILL + raw`` 로 다시 합쳐 파싱한다."""


_TOKEN_RE = re.compile(r"\{(jobRole|primaryCategory|situationTask|action|result)\}")
Expand Down Expand Up @@ -94,6 +102,24 @@ def _strip_code_fence(text: str) -> str:
return text


def _extract_json_payload(text: str) -> str:
"""LLM 응답에서 JSON object 본문만 추출.

1) 백틱 코드 펜스 (1~3개) 가 둘러쌌으면 벗긴다.
2) 결과에서 첫 ``{`` 부터 마지막 ``}`` 까지 슬라이스해 반환. Haiku 가 응답 앞뒤에
도입어 (``다음과 같습니다:``) · markdown 헤딩 · 공백 같은 prefix/suffix 를 자주
붙여 ``^{`` 만 허용하면 운영 실패율이 과도하게 높다 (issue #18 — 24h 검증 실패
164회 중 95% 가 ``char 0`` JSON 파싱 실패).
3) 어느 단계도 매치 안 되면 원본 그대로 반환 — downstream parser 가 raise.
"""
text = _strip_code_fence(text)
start = text.find("{")
end = text.rfind("}")
if start != -1 and end > start:
return text[start : end + 1]
return text


def _parse_and_validate(raw: str) -> list[str]:
"""후보 검증 — 갯수 :data:`_CANDIDATE_MIN`~:data:`_CANDIDATE_MAX`, 풀 내, 중복 X.

Expand All @@ -102,7 +128,7 @@ def _parse_and_validate(raw: str) -> list[str]:
프롬프트 명세("카테고리 무관, 풀 전체 열려 있다") 와 일치하도록 카테고리
좁힘은 적용하지 않는다.
"""
text = _strip_code_fence(raw)
text = _extract_json_payload(raw)
try:
payload = orjson.loads(text)
except orjson.JSONDecodeError as exc:
Expand Down Expand Up @@ -169,17 +195,25 @@ async def run(req: TaggingRequest, llm: LLMClient, model: str) -> TaggingRespons
workload=WorkloadType.TAGGING,
model=model,
system=system,
messages=[{"role": "user", "content": _USER_TRIGGER}],
messages=[
{"role": "user", "content": _USER_TRIGGER},
{"role": "assistant", "content": _ASSISTANT_PREFILL},
],
max_tokens=_MAX_TOKENS,
)
first_raw = _extract_text(first_msg.content)
first_raw = _ASSISTANT_PREFILL + _extract_text(first_msg.content)

try:
candidates = _parse_and_validate(first_raw)
except TaggingValidationError as first_err:
logger.warning(
"tagging.validation_failed",
extra={**log_ctx, "attempt": 1, "reason": str(first_err)},
extra={
**log_ctx,
"attempt": 1,
"reason": str(first_err),
"raw_head": first_raw[:80],
},
)
retry_msg = await llm.create_message(
workload=WorkloadType.TAGGING,
Expand All @@ -189,16 +223,22 @@ async def run(req: TaggingRequest, llm: LLMClient, model: str) -> TaggingRespons
{"role": "user", "content": _USER_TRIGGER},
{"role": "assistant", "content": first_raw},
{"role": "user", "content": _build_corrective_user_message(str(first_err))},
{"role": "assistant", "content": _ASSISTANT_PREFILL},
],
max_tokens=_MAX_TOKENS,
)
retry_raw = _extract_text(retry_msg.content)
retry_raw = _ASSISTANT_PREFILL + _extract_text(retry_msg.content)
try:
candidates = _parse_and_validate(retry_raw)
except TaggingValidationError as retry_err:
logger.warning(
"tagging.validation_failed",
extra={**log_ctx, "attempt": 2, "reason": str(retry_err)},
extra={
**log_ctx,
"attempt": 2,
"reason": str(retry_err),
"raw_head": retry_raw[:80],
},
)
raise

Expand Down
72 changes: 61 additions & 11 deletions tests/test_tagging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@
LLMUpstreamUnavailableError,
)
from app.services.tagging.exceptions import TaggingValidationError
from app.services.tagging.v2_postscore import (
_extract_json_payload as _v2_extract_json_payload,
)
from app.services.tagging.v2_postscore import (
_parse_and_validate as _v2_parse_and_validate,
)
Expand All @@ -51,9 +54,11 @@

_PII_MARKERS = ("MAGICPII-ST-1234", "MAGICPII-A-5678", "MAGICPII-R-9012")

# v2_postscore happy LLM 응답 — 5개 후보(모두 풀 내·중복 없음). DEVELOPER 가중치로
# 5개 모두 High(3) 동점 → tie-break 입력 순서 top3 = ["#원인분석","#검증및테스트","#반복개선"].
_HAPPY_LLM_RAW = '{"detailTags": ["#원인분석","#검증및테스트","#반복개선","#문제해결","#디버깅"]}'
# v2_postscore happy LLM 응답 — assistant prefill('{') 적용 가정 (issue #18).
# 코드가 prefill + raw 로 합쳐 `{"detailTags": [...]}` JSON 을 완성한다.
# 5개 후보 모두 풀 내·중복 없음. DEVELOPER 가중치로 5개 모두 High(3) 동점 →
# tie-break 입력 순서 top3 = ["#원인분석","#검증및테스트","#반복개선"].
_HAPPY_LLM_RAW = '"detailTags": ["#원인분석","#검증및테스트","#반복개선","#문제해결","#디버깅"]}'
_HAPPY_TOP3 = ["#원인분석", "#검증및테스트", "#반복개선"]


Expand Down Expand Up @@ -137,9 +142,10 @@ def test_v1_tagging_happy_path(client: TestClient, token: str) -> None:
@respx.mock
def test_v1_tagging_corrective_retry_recovers(client: TestClient, token: str) -> None:
# 1차: 풀-외 + 갯수 미달 → 검증 실패. 2차: 풀 내 5개 → top3 자르고 응답.
# mock 응답은 모두 assistant prefill('{') 가정 (issue #18).
route = respx.post(ANTHROPIC_URL).mock(
side_effect=[
Response(200, json=_anthropic_msg('{"detailTags": ["#존재하지않는태그"]}')),
Response(200, json=_anthropic_msg('"detailTags": ["#존재하지않는태그"]}')),
Response(200, json=_anthropic_msg(_HAPPY_LLM_RAW)),
]
)
Expand All @@ -156,8 +162,8 @@ def test_v1_tagging_corrective_retry_recovers(client: TestClient, token: str) ->
def test_v1_tagging_double_validation_failure_returns_422(client: TestClient, token: str) -> None:
route = respx.post(ANTHROPIC_URL).mock(
side_effect=[
Response(200, json=_anthropic_msg('{"detailTags": ["#없는태그A"]}')),
Response(200, json=_anthropic_msg('{"detailTags": ["#없는태그B"]}')),
Response(200, json=_anthropic_msg('"detailTags": ["#없는태그A"]}')),
Response(200, json=_anthropic_msg('"detailTags": ["#없는태그B"]}')),
]
)
r = client.post("/v1/tagging", json=_PAYLOAD, headers={"X-Internal-Token": token})
Expand All @@ -173,9 +179,9 @@ def test_v1_tagging_double_validation_failure_returns_422(client: TestClient, to
@pytest.mark.parametrize(
"bad_response",
[
'{"detailTags": []}',
'{"detailTags": ["#원인분석","#검증및테스트","#반복개선","#문제해결"]}',
'{"detailTags": ["#A","#B","#C","#D","#E","#F","#G","#H"]}',
'"detailTags": []}',
'"detailTags": ["#원인분석","#검증및테스트","#반복개선","#문제해결"]}',
'"detailTags": ["#A","#B","#C","#D","#E","#F","#G","#H"]}',
],
ids=["empty_under_min", "four_tags_under_min", "eight_tags_over_max"],
)
Expand Down Expand Up @@ -266,6 +272,50 @@ def test_v2_postscore_strips_all_backtick_widths(wrapped: str) -> None:
assert _v2_strip_code_fence(wrapped) == '{"detailTags": ["#문제해결"]}'


@pytest.mark.parametrize(
("raw", "expected"),
[
('{"detailTags": ["#문제해결"]}', '{"detailTags": ["#문제해결"]}'),
('다음과 같습니다: {"detailTags": ["#문제해결"]}', '{"detailTags": ["#문제해결"]}'),
(
'# 결과\n{"detailTags": ["#문제해결"]}\n참고: ...',
'{"detailTags": ["#문제해결"]}',
),
(' \n{"detailTags": ["#문제해결"]} ', '{"detailTags": ["#문제해결"]}'),
(
'```json\n{"detailTags": ["#문제해결"]}\n```',
'{"detailTags": ["#문제해결"]}',
),
("브레이스없음", "브레이스없음"),
],
ids=[
"clean",
"korean_prefix",
"markdown_prefix_and_suffix",
"whitespace",
"code_fence",
"no_braces_passthrough",
],
)
def test_v2_postscore_extracts_json_payload_from_messy_responses(raw: str, expected: str) -> None:
"""도입어 / 헤딩 / 공백 / 코드펜스 prefix·suffix 를 흡수해 JSON object 본문만 슬라이스.

issue #18 — Haiku 가 ``다음과 같습니다:`` 같은 도입어를 자주 붙여 ``^{`` 만 허용하면
운영 실패율이 과도하게 높아짐. ``{`` ~ ``}`` 슬라이스 fallback 으로 흡수.
"""
assert _v2_extract_json_payload(raw) == expected


def test_v2_postscore_parse_and_validate_handles_intro_prefix() -> None:
"""도입어가 붙은 응답도 정상 파싱 — _extract_json_payload 가 prefix 흡수."""
raw = (
"다음과 같습니다: "
'{"detailTags": ["#원인분석","#검증및테스트","#반복개선","#문제해결","#디버깅"]}'
)
tags = _v2_parse_and_validate(raw)
assert tags == ["#원인분석", "#검증및테스트", "#반복개선", "#문제해결", "#디버깅"]


def test_v2_postscore_parse_accepts_5_to_7_candidates() -> None:
"""후보 갯수 5~7 범위는 통과."""
five = '{"detailTags": ["#원인분석","#검증및테스트","#반복개선","#문제해결","#디버깅"]}'
Expand Down Expand Up @@ -349,8 +399,8 @@ def test_v1_tagging_does_not_log_star_body_on_validation_failure(
) -> None:
respx.post(ANTHROPIC_URL).mock(
side_effect=[
Response(200, json=_anthropic_msg('{"detailTags": ["#없는1"]}')),
Response(200, json=_anthropic_msg('{"detailTags": ["#없는2"]}')),
Response(200, json=_anthropic_msg('"detailTags": ["#없는1"]}')),
Response(200, json=_anthropic_msg('"detailTags": ["#없는2"]}')),
]
)
caplog.set_level(logging.INFO)
Expand Down
Loading