feat(search): Phase 2Q Diagnose Phase 1B — scaffold + dispatcher

phase-2q-query-rewrite-diagnose.md v6 plan Phase 1 의 fixture 외 잔여.
Phase 1A 446ba82 위 dispatcher + cache + LLM call + API param + eval flag + 21 unit test.
retrieval 합성 (search_with_rewrite) 은 Phase 2 별 commit.

신규:
- app/services/search/query_rewriter.py — LLM_BACKEND_MAP + _resolve + cache + rewrite()
  · slug-based allowlist (no silent fallback), httpx 직접, Priority.FOREGROUND semaphore
  · sampling 박제 (gemma response_format json_object / qwen prompt rule only — Phase 0 inspect 9)
  · manual TTL cache (query_analyzer 패턴 1:1, sha256[:32] NFKC key, LLM_REWRITE_TIMEOUT_MS=15000)
- tests/test_query_rewriter.py — 21 test PASS (resolve / cache key / parser / cache TTL / constants)

수정:
- app/api/search.py — ?rewrite_backend= query param + 400 unknown / 503 unavailable.
  scaffold = call but discard variants (retrieval path 영향 0). Phase 2 에서 합성.
- tests/search_eval/run_eval.py — --rewrite-backend flag + 4 hot spot wire-up.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
hyungi
2026-05-23 22:25:03 +00:00
parent 446ba82c91
commit 3e6866b4ae
4 changed files with 531 additions and 3 deletions
+17 -3
View File
@@ -203,6 +203,7 @@ async def call_search(
snapshot_doc_id_max: int | None = None,
snapshot_chunk_id_max: int | None = None,
reranker_backend: str | None = None,
rewrite_backend: str | None = None,
) -> tuple[list[int], float]:
"""검색 API 호출 → (doc_ids, latency_ms)."""
url = f"{base_url.rstrip('/')}/api/search/"
@@ -222,6 +223,8 @@ async def call_search(
params["snapshot_chunk_id_max"] = snapshot_chunk_id_max
if reranker_backend is not None:
params["reranker_backend"] = reranker_backend
if rewrite_backend is not None:
params["rewrite_backend"] = rewrite_backend
import time
@@ -253,6 +256,7 @@ async def evaluate(
snapshot_doc_id_max: int | None = None,
snapshot_chunk_id_max: int | None = None,
reranker_backend: str | None = None,
rewrite_backend: str | None = None,
) -> list[QueryResult]:
"""전체 쿼리셋 평가."""
results: list[QueryResult] = []
@@ -266,6 +270,7 @@ async def evaluate(
snapshot_doc_id_max=snapshot_doc_id_max,
snapshot_chunk_id_max=snapshot_chunk_id_max,
reranker_backend=reranker_backend,
rewrite_backend=rewrite_backend,
)
results.append(
QueryResult(
@@ -843,6 +848,7 @@ async def call_search_full(
snapshot_doc_id_max: int | None = None,
snapshot_chunk_id_max: int | None = None,
reranker_backend: str | None = None,
rewrite_backend: str | None = None,
) -> tuple[list[dict], float]:
"""call_search와 동일 로직. 단 full result dict 리스트 반환."""
url = f"{base_url.rstrip('/')}/api/search/"
@@ -864,6 +870,8 @@ async def call_search_full(
params["snapshot_chunk_id_max"] = snapshot_chunk_id_max
if reranker_backend is not None:
params["reranker_backend"] = reranker_backend
if rewrite_backend is not None:
params["rewrite_backend"] = rewrite_backend
import time
@@ -1322,6 +1330,12 @@ def main() -> int:
default=None,
help="Phase 2B Diagnose reranker dispatcher slug (baseline | cand_gte_ml_base). 미지정 = production.",
)
parser.add_argument(
"--rewrite-backend",
type=str,
default=None,
help="Phase 2Q Diagnose query rewrite dispatcher slug (baseline | cand_multi_query_macmini | cand_multi_query_macbook). 미지정 = single-query path. Phase 1B scaffold = variants 박제만, retrieval 합성은 Phase 2.",
)
args = parser.parse_args()
@@ -1375,21 +1389,21 @@ def main() -> int:
if args.base_url:
print(f"\n>>> evaluating: {args.base_url}")
results = asyncio.run(
evaluate(queries, args.base_url, args.token, "single", mode=args.mode, fusion=args.fusion, rerank=args.rerank, analyze=args.analyze, embedding_backend=args.embedding_backend, snapshot_doc_id_max=args.snapshot_doc_id_max, snapshot_chunk_id_max=args.snapshot_chunk_id_max, reranker_backend=args.reranker_backend)
evaluate(queries, args.base_url, args.token, "single", mode=args.mode, fusion=args.fusion, rerank=args.rerank, analyze=args.analyze, embedding_backend=args.embedding_backend, snapshot_doc_id_max=args.snapshot_doc_id_max, snapshot_chunk_id_max=args.snapshot_chunk_id_max, reranker_backend=args.reranker_backend, rewrite_backend=args.rewrite_backend)
)
print_summary("single", results, eval_version=args.eval_version)
all_results.extend(results)
else:
print(f"\n>>> baseline: {args.baseline_url}")
baseline_results = asyncio.run(
evaluate(queries, args.baseline_url, args.token, "baseline", mode=args.mode, fusion=args.fusion, rerank=args.rerank, analyze=args.analyze, embedding_backend=args.embedding_backend, snapshot_doc_id_max=args.snapshot_doc_id_max, snapshot_chunk_id_max=args.snapshot_chunk_id_max, reranker_backend=args.reranker_backend)
evaluate(queries, args.baseline_url, args.token, "baseline", mode=args.mode, fusion=args.fusion, rerank=args.rerank, analyze=args.analyze, embedding_backend=args.embedding_backend, snapshot_doc_id_max=args.snapshot_doc_id_max, snapshot_chunk_id_max=args.snapshot_chunk_id_max, reranker_backend=args.reranker_backend, rewrite_backend=args.rewrite_backend)
)
baseline_summary = print_summary("baseline", baseline_results, eval_version=args.eval_version)
print(f"\n>>> candidate: {args.candidate_url}")
candidate_results = asyncio.run(
evaluate(
queries, args.candidate_url, args.token, "candidate", mode=args.mode, fusion=args.fusion, rerank=args.rerank, analyze=args.analyze, embedding_backend=args.embedding_backend, snapshot_doc_id_max=args.snapshot_doc_id_max, snapshot_chunk_id_max=args.snapshot_chunk_id_max, reranker_backend=args.reranker_backend
queries, args.candidate_url, args.token, "candidate", mode=args.mode, fusion=args.fusion, rerank=args.rerank, analyze=args.analyze, embedding_backend=args.embedding_backend, snapshot_doc_id_max=args.snapshot_doc_id_max, snapshot_chunk_id_max=args.snapshot_chunk_id_max, reranker_backend=args.reranker_backend, rewrite_backend=args.rewrite_backend
)
)
candidate_summary = print_summary("candidate", candidate_results, eval_version=args.eval_version)
+212
View File
@@ -0,0 +1,212 @@
"""Phase 2Q Diagnose Phase 1B — query_rewriter scaffold + dispatcher 단위 테스트.
가드레일 (plan v6 §5 + §7 Phase 1):
1. `_resolve_rewrite_backend` — slug resolve, unknown ValueError, baseline → None
2. `_cache_key` — deterministic + NFKC normalize + backend slug 분리
3. `_extract_variants` — valid shape / wrong count / type mismatch / empty / non-list
4. cache set/get/TTL (LRU evict 시뮬레이션)
5. `allowed_slugs` — LLM_BACKEND_MAP keys 1:1
"""
from __future__ import annotations
import asyncio
import logging
import os
import sys
import time
import pytest
# logs/llm_gate.log 가 root 소유 (운영 fastapi daemon write) → pytest 가 hyungi user 로
# import 시 PermissionError. 본 test 한정 FileHandler safe-wrap (다른 test 영향 0).
_orig_file_handler = logging.FileHandler
def _safe_file_handler(filename, *args, **kwargs): # type: ignore
try:
return _orig_file_handler(filename, *args, **kwargs)
except PermissionError:
return logging.NullHandler()
logging.FileHandler = _safe_file_handler # type: ignore[assignment]
# tests/ → 프로젝트 루트 → app/
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
from services.search import query_rewriter
from services.search.query_rewriter import (
EXPECTED_N_VARIANTS,
LLM_BACKEND_MAP,
PROMPT_VERSION,
_cache_key,
_extract_variants,
_resolve_rewrite_backend,
allowed_slugs,
)
# ─── 1. _resolve_rewrite_backend ──────────────────────────
def test_resolve_baseline_returns_none():
assert _resolve_rewrite_backend(None) is None
assert _resolve_rewrite_backend("baseline") is None
def test_resolve_known_slugs():
cfg = _resolve_rewrite_backend("cand_multi_query_macmini")
assert cfg is not None
assert "endpoint" in cfg and "model" in cfg and "sampling" in cfg
assert cfg["model"] == "gemma-4-26b-a4b-it-8bit"
cfg = _resolve_rewrite_backend("cand_multi_query_macbook")
assert cfg is not None
assert cfg["model"] == "mlx-community/Qwen3.6-27B-8bit"
# qwen sampling 에 response_format 없음 (Phase 0 inspect 9 박제)
assert "response_format" not in cfg["sampling"]
def test_resolve_unknown_slug_raises():
with pytest.raises(ValueError, match="unknown_rewrite_backend"):
_resolve_rewrite_backend("cand_bogus")
with pytest.raises(ValueError):
_resolve_rewrite_backend("cand_multi_query_other")
def test_allowed_slugs_matches_map():
assert allowed_slugs() == list(LLM_BACKEND_MAP.keys())
assert "baseline" in allowed_slugs()
assert "cand_multi_query_macmini" in allowed_slugs()
assert "cand_multi_query_macbook" in allowed_slugs()
# ─── 2. _cache_key ────────────────────────────────────────
def test_cache_key_deterministic():
k1 = _cache_key("산업안전보건법 제6장", "cand_multi_query_macmini")
k2 = _cache_key("산업안전보건법 제6장", "cand_multi_query_macmini")
assert k1 == k2
assert len(k1) == 32 # sha256[:32]
def test_cache_key_nfkc_normalize_and_strip_lower():
# whitespace + uppercase → 동일 key
base = _cache_key("ASME Section VIII", "cand_multi_query_macmini")
assert _cache_key(" asme section viii ", "cand_multi_query_macmini") == base
assert _cache_key("ASME SECTION VIII", "cand_multi_query_macmini") == base
def test_cache_key_differs_by_backend_slug():
k_a = _cache_key("query", "cand_multi_query_macmini")
k_b = _cache_key("query", "cand_multi_query_macbook")
assert k_a != k_b
def test_cache_key_includes_prompt_version():
# PROMPT_VERSION 변경 시 cache 분리 — 직접 test 어렵지만 raw 구성 확인
assert PROMPT_VERSION == "v1"
k = _cache_key("query", "cand_multi_query_macmini")
assert len(k) == 32
# ─── 3. _extract_variants ─────────────────────────────────
def test_extract_variants_valid_shape():
raw = '{"variants": ["원본", "한국어 변형", "english"]}'
out = _extract_variants(raw, expected_n=3)
assert out == ["원본", "한국어 변형", "english"]
def test_extract_variants_strips_whitespace():
raw = '{"variants": [" 원본 ", "한국어\\n", " english "]}'
out = _extract_variants(raw, expected_n=3)
assert out == ["원본", "한국어", "english"]
def test_extract_variants_wrong_count_returns_none():
raw = '{"variants": ["only_one"]}'
assert _extract_variants(raw, expected_n=3) is None
raw = '{"variants": ["a", "b", "c", "d"]}'
assert _extract_variants(raw, expected_n=3) is None
def test_extract_variants_missing_key_returns_none():
raw = '{"queries": ["a", "b", "c"]}'
assert _extract_variants(raw, expected_n=3) is None
def test_extract_variants_non_list_returns_none():
raw = '{"variants": "single string"}'
assert _extract_variants(raw, expected_n=3) is None
def test_extract_variants_empty_string_returns_none():
raw = '{"variants": ["a", "", "c"]}'
assert _extract_variants(raw, expected_n=3) is None
def test_extract_variants_non_string_element_returns_none():
raw = '{"variants": ["a", 123, "c"]}'
assert _extract_variants(raw, expected_n=3) is None
def test_extract_variants_invalid_json_returns_none():
raw = "not json at all"
assert _extract_variants(raw, expected_n=3) is None
def test_extract_variants_markdown_fence_fallback():
# parse_json_response 가 ```json fenced 블록 내부 추출 — production parser 재사용 검증
raw = '```json\n{"variants": ["a", "b", "c"]}\n```'
out = _extract_variants(raw, expected_n=3)
assert out == ["a", "b", "c"]
# ─── 4. cache set / get ───────────────────────────────────
@pytest.mark.asyncio
async def test_cache_set_get_roundtrip():
# 격리: 전역 _CACHE 초기화 (다른 테스트와 격리)
query_rewriter._CACHE.clear()
key = _cache_key("__test_unique_key__", "cand_multi_query_macmini")
assert await query_rewriter._get_cached(key) is None
await query_rewriter._set_cached(key, ["v0", "v1", "v2"])
out = await query_rewriter._get_cached(key)
assert out == ["v0", "v1", "v2"]
@pytest.mark.asyncio
async def test_cache_ttl_expiry():
query_rewriter._CACHE.clear()
key = "ttl_test_key"
# manual entry with past expire_at
query_rewriter._CACHE[key] = (time.time() - 1.0, ["a", "b", "c"])
assert await query_rewriter._get_cached(key) is None
# lazy delete verify
assert key not in query_rewriter._CACHE
@pytest.mark.asyncio
async def test_cache_returns_copy_not_reference():
"""_get_cached 반환 list 를 외부에서 수정해도 internal cache 안전."""
query_rewriter._CACHE.clear()
key = "copy_test_key"
await query_rewriter._set_cached(key, ["a", "b", "c"])
out = await query_rewriter._get_cached(key)
out.append("mutated")
out2 = await query_rewriter._get_cached(key)
assert out2 == ["a", "b", "c"]
# ─── 5. constants ─────────────────────────────────────────
def test_constants_match_plan_v6():
assert PROMPT_VERSION == "v1"
assert EXPECTED_N_VARIANTS == 3
assert query_rewriter.LLM_REWRITE_TIMEOUT_MS == 15000
assert query_rewriter.CACHE_TTL == 86400
assert query_rewriter.CACHE_MAXSIZE == 1000