From c9f766512d1426c0105d4fe09de94e09f6a4a455 Mon Sep 17 00:00:00 2001
From: Hyungi Ahn <hyungiahn@Hyungiui-MacBookPro.local>
Date: Fri, 17 Apr 2026 09:12:24 +0900
Subject: [PATCH] =?UTF-8?q?feat(eval):=20run=5Feval=5Fask=20runner=20?=
 =?UTF-8?q?=EC=97=90=20X-Eval-Token/X-Eval-Case-Id=20=EC=A0=84=ED=8C=8C=20?=
 =?UTF-8?q?=EC=B6=94=EA=B0=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

배경: Phase 3.5 fix2 로 서버 /ask 는 X-Source=eval 을 받아들이려면
X-Eval-Token 이 EVAL_RUNNER_TOKEN 와 일치해야 함. runner 에 해당 헤더
주입 경로가 없어 eval 호출이 전부 source='document_server' 로 강등됐음.

변경:
- call_ask / call_analyze: eval_token, eval_case_id 인자 추가. 조건부 헤더 주입
- run_eval: eval_token 파라미터 추가
- CLI: --eval-token 플래그 추가 (env EVAL_RUNNER_TOKEN 자동 fallback)
- main(): --source=eval + --eval-token 미지정 조합에 warning 출력
- eval_case_id 는 item id 자동 전달 → ask_events.eval_case_id join 키로 활용

E.6 재측정의 source='eval' 정확 기록 선결 조건.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 scripts/run_eval_ask.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/scripts/run_eval_ask.py b/scripts/run_eval_ask.py
index d0b3cd5..76a2484 100755
--- a/scripts/run_eval_ask.py
+++ b/scripts/run_eval_ask.py
@@ -98,12 +98,17 @@ class ApiResult:
 async def call_ask(
     client: httpx.AsyncClient, base_url: str, token: str, source: str,
     query: str,
+    eval_token: str | None = None, eval_case_id: str | None = None,
 ) -> ApiResult:
     url = f"{base_url.rstrip('/')}/api/search/ask"
     headers = {
         "Authorization": f"Bearer {token}",
         "X-Source": source,
     }
+    if eval_token:
+        headers["X-Eval-Token"] = eval_token
+    if eval_case_id:
+        headers["X-Eval-Case-Id"] = eval_case_id
     params = {"q": query}
     start = time.perf_counter()
     try:
@@ -122,12 +127,17 @@ async def call_ask(
 async def call_analyze(
     client: httpx.AsyncClient, base_url: str, token: str, source: str,
     doc_id: int,
+    eval_token: str | None = None, eval_case_id: str | None = None,
 ) -> ApiResult:
     url = f"{base_url.rstrip('/')}/api/documents/{doc_id}/analyze"
     headers = {
         "Authorization": f"Bearer {token}",
         "X-Source": source,
     }
+    if eval_token:
+        headers["X-Eval-Token"] = eval_token
+    if eval_case_id:
+        headers["X-Eval-Case-Id"] = eval_case_id
     start = time.perf_counter()
     try:
         # analyze 는 서버 내부 timeout 60s (ANALYZE_TIMEOUT_S). client 는 여유 두고 130s.
@@ -307,6 +317,7 @@ def load_items(
 async def run_eval(
     items: list[dict], base_url: str, token: str, source: str,
     output_path: Path, min_interval: float, retries: int, retry_delay: float,
+    eval_token: str | None = None,
 ) -> None:
     output_path.parent.mkdir(parents=True, exist_ok=True)
     total = len(items)
@@ -329,6 +340,7 @@ async def run_eval(
                 if itype == "ask":
                     api = await call_with_retry(
                         call_ask, client, base_url, token, source, item["query"],
+                        eval_token, iid,
                         retries=retries, retry_delay=retry_delay,
                     )
                     row = build_row_ask(item, api)
@@ -345,6 +357,7 @@ async def run_eval(
                     else:
                         api = await call_with_retry(
                             call_analyze, client, base_url, token, source, doc_id,
+                            eval_token, iid,
                             retries=retries, retry_delay=retry_delay,
                         )
                         row = build_row_analyze(item, api)
@@ -390,6 +403,11 @@ def main() -> int:
         help="Bearer 토큰 (env DOCSRV_TOKEN)",
     )
     parser.add_argument("--source", type=str, default="eval", help="X-Source 헤더 값 (default: eval)")
+    parser.add_argument(
+        "--eval-token", type=str, default=os.environ.get("EVAL_RUNNER_TOKEN"),
+        help="X-Eval-Token 헤더 값 (env EVAL_RUNNER_TOKEN 자동 fallback). "
+             "미지정 시 서버가 eval claim 거부 → source='document_server' 로 강등.",
+    )
     parser.add_argument("--concurrency", type=int, default=1, help="동시 요청 수 (현재 1 고정)")
     parser.add_argument("--min-interval", type=float, default=0.3, help="요청 간 최소 간격(초)")
     parser.add_argument("--retries", type=int, default=1, help="timeout/5xx 재시도 횟수")
@@ -430,10 +448,18 @@ def main() -> int:
         file=sys.stderr,
     )
 
+    if args.source == "eval" and not args.eval_token:
+        print(
+            "WARNING: --source=eval 인데 --eval-token 미지정. "
+            "서버가 X-Source=eval 을 거부하고 source='document_server' 로 강등합니다.",
+            file=sys.stderr,
+        )
+
     asyncio.run(
         run_eval(
             items, args.base_url, args.token, args.source,
             args.output, args.min_interval, args.retries, args.retry_delay,
+            eval_token=args.eval_token,
         )
     )
     return 0