diff --git a/scripts/run_eval_ask.py b/scripts/run_eval_ask.py index d0b3cd5..76a2484 100755 --- a/scripts/run_eval_ask.py +++ b/scripts/run_eval_ask.py @@ -98,12 +98,17 @@ class ApiResult: async def call_ask( client: httpx.AsyncClient, base_url: str, token: str, source: str, query: str, + eval_token: str | None = None, eval_case_id: str | None = None, ) -> ApiResult: url = f"{base_url.rstrip('/')}/api/search/ask" headers = { "Authorization": f"Bearer {token}", "X-Source": source, } + if eval_token: + headers["X-Eval-Token"] = eval_token + if eval_case_id: + headers["X-Eval-Case-Id"] = eval_case_id params = {"q": query} start = time.perf_counter() try: @@ -122,12 +127,17 @@ async def call_ask( async def call_analyze( client: httpx.AsyncClient, base_url: str, token: str, source: str, doc_id: int, + eval_token: str | None = None, eval_case_id: str | None = None, ) -> ApiResult: url = f"{base_url.rstrip('/')}/api/documents/{doc_id}/analyze" headers = { "Authorization": f"Bearer {token}", "X-Source": source, } + if eval_token: + headers["X-Eval-Token"] = eval_token + if eval_case_id: + headers["X-Eval-Case-Id"] = eval_case_id start = time.perf_counter() try: # analyze 는 서버 내부 timeout 60s (ANALYZE_TIMEOUT_S). client 는 여유 두고 130s. @@ -307,6 +317,7 @@ def load_items( async def run_eval( items: list[dict], base_url: str, token: str, source: str, output_path: Path, min_interval: float, retries: int, retry_delay: float, + eval_token: str | None = None, ) -> None: output_path.parent.mkdir(parents=True, exist_ok=True) total = len(items) @@ -329,6 +340,7 @@ async def run_eval( if itype == "ask": api = await call_with_retry( call_ask, client, base_url, token, source, item["query"], + eval_token, iid, retries=retries, retry_delay=retry_delay, ) row = build_row_ask(item, api) @@ -345,6 +357,7 @@ async def run_eval( else: api = await call_with_retry( call_analyze, client, base_url, token, source, doc_id, + eval_token, iid, retries=retries, retry_delay=retry_delay, ) row = build_row_analyze(item, api) @@ -390,6 +403,11 @@ def main() -> int: help="Bearer 토큰 (env DOCSRV_TOKEN)", ) parser.add_argument("--source", type=str, default="eval", help="X-Source 헤더 값 (default: eval)") + parser.add_argument( + "--eval-token", type=str, default=os.environ.get("EVAL_RUNNER_TOKEN"), + help="X-Eval-Token 헤더 값 (env EVAL_RUNNER_TOKEN 자동 fallback). " + "미지정 시 서버가 eval claim 거부 → source='document_server' 로 강등.", + ) parser.add_argument("--concurrency", type=int, default=1, help="동시 요청 수 (현재 1 고정)") parser.add_argument("--min-interval", type=float, default=0.3, help="요청 간 최소 간격(초)") parser.add_argument("--retries", type=int, default=1, help="timeout/5xx 재시도 횟수") @@ -430,10 +448,18 @@ def main() -> int: file=sys.stderr, ) + if args.source == "eval" and not args.eval_token: + print( + "WARNING: --source=eval 인데 --eval-token 미지정. " + "서버가 X-Source=eval 을 거부하고 source='document_server' 로 강등합니다.", + file=sys.stderr, + ) + asyncio.run( run_eval( items, args.base_url, args.token, args.source, args.output, args.min_interval, args.retries, args.retry_delay, + eval_token=args.eval_token, ) ) return 0