feat(eval): run_eval_ask runner 에 X-Eval-Token/X-Eval-Case-Id 전파 추가

배경: Phase 3.5 fix2 로 서버 /ask 는 X-Source=eval 을 받아들이려면
X-Eval-Token 이 EVAL_RUNNER_TOKEN 와 일치해야 함. runner 에 해당 헤더
주입 경로가 없어 eval 호출이 전부 source='document_server' 로 강등됐음.

변경:
- call_ask / call_analyze: eval_token, eval_case_id 인자 추가. 조건부 헤더 주입
- run_eval: eval_token 파라미터 추가
- CLI: --eval-token 플래그 추가 (env EVAL_RUNNER_TOKEN 자동 fallback)
- main(): --source=eval + --eval-token 미지정 조합에 warning 출력
- eval_case_id 는 item id 자동 전달 → ask_events.eval_case_id join 키로 활용

E.6 재측정의 source='eval' 정확 기록 선결 조건.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Hyungi Ahn
2026-04-17 09:12:24 +09:00
parent c82d52e73f
commit c9f766512d
+26
View File
@@ -98,12 +98,17 @@ class ApiResult:
async def call_ask(
client: httpx.AsyncClient, base_url: str, token: str, source: str,
query: str,
eval_token: str | None = None, eval_case_id: str | None = None,
) -> ApiResult:
url = f"{base_url.rstrip('/')}/api/search/ask"
headers = {
"Authorization": f"Bearer {token}",
"X-Source": source,
}
if eval_token:
headers["X-Eval-Token"] = eval_token
if eval_case_id:
headers["X-Eval-Case-Id"] = eval_case_id
params = {"q": query}
start = time.perf_counter()
try:
@@ -122,12 +127,17 @@ async def call_ask(
async def call_analyze(
client: httpx.AsyncClient, base_url: str, token: str, source: str,
doc_id: int,
eval_token: str | None = None, eval_case_id: str | None = None,
) -> ApiResult:
url = f"{base_url.rstrip('/')}/api/documents/{doc_id}/analyze"
headers = {
"Authorization": f"Bearer {token}",
"X-Source": source,
}
if eval_token:
headers["X-Eval-Token"] = eval_token
if eval_case_id:
headers["X-Eval-Case-Id"] = eval_case_id
start = time.perf_counter()
try:
# analyze 는 서버 내부 timeout 60s (ANALYZE_TIMEOUT_S). client 는 여유 두고 130s.
@@ -307,6 +317,7 @@ def load_items(
async def run_eval(
items: list[dict], base_url: str, token: str, source: str,
output_path: Path, min_interval: float, retries: int, retry_delay: float,
eval_token: str | None = None,
) -> None:
output_path.parent.mkdir(parents=True, exist_ok=True)
total = len(items)
@@ -329,6 +340,7 @@ async def run_eval(
if itype == "ask":
api = await call_with_retry(
call_ask, client, base_url, token, source, item["query"],
eval_token, iid,
retries=retries, retry_delay=retry_delay,
)
row = build_row_ask(item, api)
@@ -345,6 +357,7 @@ async def run_eval(
else:
api = await call_with_retry(
call_analyze, client, base_url, token, source, doc_id,
eval_token, iid,
retries=retries, retry_delay=retry_delay,
)
row = build_row_analyze(item, api)
@@ -390,6 +403,11 @@ def main() -> int:
help="Bearer 토큰 (env DOCSRV_TOKEN)",
)
parser.add_argument("--source", type=str, default="eval", help="X-Source 헤더 값 (default: eval)")
parser.add_argument(
"--eval-token", type=str, default=os.environ.get("EVAL_RUNNER_TOKEN"),
help="X-Eval-Token 헤더 값 (env EVAL_RUNNER_TOKEN 자동 fallback). "
"미지정 시 서버가 eval claim 거부 → source='document_server' 로 강등.",
)
parser.add_argument("--concurrency", type=int, default=1, help="동시 요청 수 (현재 1 고정)")
parser.add_argument("--min-interval", type=float, default=0.3, help="요청 간 최소 간격(초)")
parser.add_argument("--retries", type=int, default=1, help="timeout/5xx 재시도 횟수")
@@ -430,10 +448,18 @@ def main() -> int:
file=sys.stderr,
)
if args.source == "eval" and not args.eval_token:
print(
"WARNING: --source=eval 인데 --eval-token 미지정. "
"서버가 X-Source=eval 을 거부하고 source='document_server' 로 강등합니다.",
file=sys.stderr,
)
asyncio.run(
run_eval(
items, args.base_url, args.token, args.source,
args.output, args.min_interval, args.retries, args.retry_delay,
eval_token=args.eval_token,
)
)
return 0