hyungi
98 lines
3.8 KiB
Python
98 lines
3.8 KiB
Python
"""Phase 2A E-4 비교기 — baseline vs 후보 run CSV 들의 per-query 판정.
|
|
|
|
python tests/search_eval/compare_runs.py \
|
|
--baseline baselines/<exact 재측정>.csv \
|
|
--cand qwen06=<...>.csv --cand qwen4=<...>.csv --cand qwen4m=<...>.csv \
|
|
[--epsilon 0.01] [--bootstrap 2000]
|
|
|
|
판정 출력(plan r3 E-4): 전체 graded NDCG 평균 delta · per-query win/loss/tie(|d|<ε=tie)
|
|
· 부트스트랩 95% CI · 카테고리별 평균 · 상위 개선/퇴행 쿼리. failure_expected/error 행 제외.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import random
|
|
import statistics
|
|
from pathlib import Path
|
|
|
|
|
|
def load(path: str) -> dict[str, dict]:
|
|
out = {}
|
|
with Path(path).open(encoding="utf-8") as f:
|
|
for row in csv.DictReader(f):
|
|
if row.get("failure_expected", "").lower() in ("true", "1"):
|
|
continue
|
|
if row.get("error"):
|
|
continue
|
|
try:
|
|
row["_g"] = float(row["graded_ndcg_at_10"])
|
|
except (TypeError, ValueError):
|
|
continue
|
|
out[row["id"]] = row
|
|
return out
|
|
|
|
|
|
def bootstrap_ci(deltas: list[float], n: int, seed: int = 42) -> tuple[float, float]:
|
|
rng = random.Random(seed)
|
|
means = sorted(
|
|
statistics.mean(rng.choices(deltas, k=len(deltas))) for _ in range(n)
|
|
)
|
|
return means[int(0.025 * n)], means[int(0.975 * n)]
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--baseline", required=True)
|
|
p.add_argument("--cand", action="append", required=True, metavar="name=csv")
|
|
p.add_argument("--epsilon", type=float, default=0.01)
|
|
p.add_argument("--bootstrap", type=int, default=2000)
|
|
a = p.parse_args()
|
|
|
|
base = load(a.baseline)
|
|
print(f"baseline: {a.baseline} — scored {len(base)}, "
|
|
f"graded NDCG mean {statistics.mean(r['_g'] for r in base.values()):.4f}")
|
|
|
|
for spec in a.cand:
|
|
name, path = spec.split("=", 1)
|
|
cand = load(path)
|
|
ids = sorted(set(base) & set(cand))
|
|
if len(ids) != len(base):
|
|
print(f" ⚠ {name}: 교집합 {len(ids)} != baseline {len(base)} — 누락 쿼리 확인")
|
|
deltas = [cand[i]["_g"] - base[i]["_g"] for i in ids]
|
|
mean_b = statistics.mean(base[i]["_g"] for i in ids)
|
|
mean_c = statistics.mean(cand[i]["_g"] for i in ids)
|
|
win = sum(1 for d in deltas if d > a.epsilon)
|
|
loss = sum(1 for d in deltas if d < -a.epsilon)
|
|
tie = len(deltas) - win - loss
|
|
lo, hi = bootstrap_ci(deltas, a.bootstrap)
|
|
decided = win + loss
|
|
win_rate = (win / decided * 100) if decided else 0.0
|
|
|
|
print(f"\n== {name} ==")
|
|
print(f" graded NDCG: {mean_b:.4f} → {mean_c:.4f} (delta {mean_c-mean_b:+.4f}, "
|
|
f"bootstrap95% [{lo:+.4f}, {hi:+.4f}])")
|
|
print(f" per-query: win {win} / loss {loss} / tie {tie} (ε={a.epsilon}) — "
|
|
f"win-rate(결정전) {win_rate:.0f}%")
|
|
|
|
cats: dict[str, list[float]] = {}
|
|
for i in ids:
|
|
cats.setdefault(base[i].get("category", "?"), []).append(
|
|
cand[i]["_g"] - base[i]["_g"]
|
|
)
|
|
for c in sorted(cats):
|
|
ds = cats[c]
|
|
cb = statistics.mean(base[i]["_g"] for i in ids if base[i].get("category") == c)
|
|
cc = statistics.mean(cand[i]["_g"] for i in ids if base[i].get("category") == c)
|
|
print(f" {c:<18} {cb:.3f} → {cc:.3f} ({statistics.mean(ds):+.3f}, n={len(ds)})")
|
|
|
|
ranked = sorted(ids, key=lambda i: cand[i]["_g"] - base[i]["_g"])
|
|
worst = [(i, round(cand[i]['_g']-base[i]['_g'],3)) for i in ranked[:3]]
|
|
best = [(i, round(cand[i]['_g']-base[i]['_g'],3)) for i in ranked[-3:][::-1]]
|
|
print(f" 개선 top3 {best} / 퇴행 top3 {worst}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|