hyungi
cd33ded7a8
PR-DocSrv-Hier-PassageRAG-Diagnose-1 c4+c5. 조건부 N=12(retrieval 통제) blind pairwise (hypothesis-blind subagent, 익명 3-file split). 결과 4-way 수렴 = 동등: pairwise prehier4/hier3/tie5(no edge) + axis ±0.08 + objective 동일(halluc36/36) + variance~0(byte-identical 재생성). verbosity artifact 없음(prehier 더 길었으나 승+1). => NO-GO: hier-leaf evidence 무이득. hier leaf = section-outline UI 전용 완전 확정 (UI yes / doc-search NO-GO / passage-RAG NO-GO 3영역 종결). 2026-06-21 freeze input only. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
33 lines
1.7 KiB
Python
33 lines
1.7 KiB
Python
import json, os
|
|
os.chdir(os.path.expanduser("~/Documents/code/hyungi_Document_Server"))
|
|
key = json.load(open("reports/passage_rag_judge_key_2026-05-25.json"))
|
|
verdicts = {
|
|
"pair_01":"B","pair_02":"B","pair_03":"B","pair_04":"tie","pair_05":"B","pair_06":"tie",
|
|
"pair_07":"tie","pair_08":"tie","pair_09":"B","pair_10":"tie","pair_11":"B","pair_12":"B"}
|
|
# also per-axis scores for faithfulness/correctness/completeness avg by variant
|
|
scores = {
|
|
"pair_01":(3,3,3,3,3,3),"pair_02":(2,3,2,3,3,3),"pair_03":(3,3,2,3,3,3),"pair_04":(3,3,3,3,3,3),
|
|
"pair_05":(3,2,2,3,3,3),"pair_06":(3,3,3,3,3,3),"pair_07":(3,3,2,3,3,2),"pair_08":(3,3,3,3,3,3),
|
|
"pair_09":(3,3,3,3,3,3),"pair_10":(3,3,3,3,3,3),"pair_11":(3,3,2,3,3,3),"pair_12":(3,3,2,3,3,3)}
|
|
win = {"prehier":0,"hier_sim_clean":0,"tie":0}
|
|
axis = {"prehier":[0,0,0],"hier_sim_clean":[0,0,0]}; n=0
|
|
print("pair q_id winner_variant")
|
|
for pid,k in key.items():
|
|
v=verdicts[pid]
|
|
wv = "tie" if v=="tie" else k[v]
|
|
win[wv if wv in win else "tie"]+=1
|
|
print(f"{pid} {k['q_id']:10} {('TIE' if v=='tie' else wv)} (A={k['A']},B={k['B']})")
|
|
fa,ca,coa,fb,cb,cob = scores[pid]
|
|
sA = {"f":fa,"c":ca,"co":coa}; sB={"f":fb,"c":cb,"co":cob}
|
|
for slot,sc in ((k['A'],(fa,ca,coa)),(k['B'],(fb,cb,cob))):
|
|
axis[slot][0]+=sc[0]; axis[slot][1]+=sc[1]; axis[slot][2]+=sc[2]
|
|
n+=1
|
|
print(f"\n=== PAIRWISE (N={n}) ===")
|
|
print(f" hier_sim_clean wins: {win['hier_sim_clean']}")
|
|
print(f" prehier wins: {win['prehier']}")
|
|
print(f" tie: {win['tie']}")
|
|
print(f"\n=== AXIS AVG (faithfulness/correctness/completeness, N={n}) ===")
|
|
for v in ["prehier","hier_sim_clean"]:
|
|
f,c,co = axis[v]
|
|
print(f" {v}: faith={f/n:.2f} correct={c/n:.2f} complete={co/n:.2f}")
|