{
  "version": "v0.2-phase2q-eval-dedup",
  "label": "phase_2q_eval_graded_ndcg_dedup_invariant_recovery",
  "date": "2026-05-24",
  "plan": "pr-eval-graded-ndcg-dedup-stormy-tide.md",
  "main_head_pre": "b00d9f5",
  "critical_finding": {
    "summary": "Phase 2Q multi-query 의 실제 net 효과 = 거의 0 (NDCG -0.003 vs baseline). 모든 박제 측정 (0.927/0.876/+0.217) 은 inflation 결과.",
    "root_cause": "_rrf_fuse_variants 의 representative 보존 logic 이 같은 doc_id 의 여러 SearchResult 를 unique 가정 — 실제로는 multi-query path 에서 doc_id 중복 박제. chunk_id dedup (Rerank-Fix) 으로 chunk-level inflation 만 해결, doc-level inflation 잔재.",
    "evidence": "dedup audit = baseline 0/51 정상 vs gemma 42/51 cases with 81 chunks dedup applied",
    "rollout_impact": "Apply opt-in 1주 관찰 결정 재검토 필요. multi-query 의 net 개선 sub-noise level + latency 4x 회귀."
  },
  "snapshot": {
    "doc_id_max": 25180,
    "chunk_id_max": 56526
  },
  "eval_set": {"total_cases": 51, "scored_cases": 46},
  "measurements": {
    "baseline_rewrite_null": {
      "graded_ndcg_at_10": 0.644,
      "graded_recall_at_10_t2": 0.699,
      "graded_recall_at_10_t3": 0.761,
      "latency_p50_ms": 378,
      "latency_p95_ms": 1931,
      "dedup_audit": {
        "cases_with_dedup": "0/51",
        "total_dedup_chunks": 0,
        "status": "✓ 정상 — single-query path 의 retrieval 가 doc unique 박제"
      },
      "by_category": {
        "english_only": {"recall": 0.78, "gndcg": 0.72},
        "exam": {"recall": 0.57, "gndcg": 0.74},
        "korean_only": {"recall": 0.56, "gndcg": 0.57},
        "mixed": {"recall": 0.38, "gndcg": 0.38},
        "standards": {"recall": 0.91, "gndcg": 0.82}
      },
      "csv": "reports/v0_2_phase2q_eval_dedup_baseline_2026-05-24.csv"
    },
    "cand_multi_query_macmini": {
      "graded_ndcg_at_10": 0.641,
      "graded_recall_at_10_t2": 0.716,
      "graded_recall_at_10_t3": 0.728,
      "latency_p50_ms": 1383,
      "latency_p95_ms": 3584,
      "dedup_audit": {
        "cases_with_dedup": "42/51",
        "total_dedup_chunks": 81,
        "status": "⚠️ inflation — _rrf_fuse_variants representative 의 doc_id 중복 박제 (chunk_id dedup 후 잔재)"
      },
      "by_category": {
        "english_only": {"recall": 0.78, "gndcg": 0.74},
        "exam": {"recall": 0.64, "gndcg": 0.67},
        "korean_only": {"recall": 0.57, "gndcg": 0.52},
        "mixed": {"recall": 0.40, "gndcg": 0.39},
        "standards": {"recall": 0.95, "gndcg": 0.87}
      },
      "csv": "reports/v0_2_phase2q_eval_dedup_gemma_2026-05-24.csv"
    },
    "delta_vs_baseline": {
      "overall_ndcg": "-0.003 (사실상 동일, noise level)",
      "recall_t2": "+0.017",
      "recall_t3": "-0.033 (회귀)",
      "english_only": "+0.02",
      "exam": "-0.07",
      "korean_only": "-0.05",
      "mixed": "+0.01",
      "standards": "+0.05",
      "latency_p50": "+1005ms (+266%)",
      "latency_p95": "+1653ms (+86%)"
    }
  },
  "previous_inflated_records": [
    {"source": "Phase 3 commit a41adb6", "ndcg": 0.927, "inflation_source": "chunk_id 중복 (chunks_per_doc cap 만, dedup 0)"},
    {"source": "Rerank-Fix commit b734fc5", "ndcg": 0.876, "inflation_source": "chunk_id dedup 적용 단 doc_id 중복 잔재"},
    {"source": "Category-Analysis commit b00d9f5", "ndcg": 0.876, "note": "Rerank-Fix 측정값 재사용 — 본 정정 후 0.641"}
  ],
  "recommendations": {
    "immediate_user_decision": [
      "Apply rollback 검토 — multi-query 의 실제 net 효과 ≈ 0 + latency 4x 회귀 + LLM endpoint 의존",
      "또는 PR-2Q-Search-Result-Dedup 진입 (real fix) 후 재측정 → 실제 multi-query 효과 측정 후 Apply 결정"
    ],
    "next_pr": "PR-2Q-Search-Result-Dedup — _rrf_fuse_variants representative 가 doc_id unique 가정 invariant 강제. dedup audit 가 42/51 → 0/51 회복 + NDCG 실제 효과 측정 가치"
  },
  "changes_summary": {
    "files_changed": [
      "tests/search_eval/run_eval.py — _dedup_returned_ids helper + count_dedup wrapper + ndcg_at_k/graded_ndcg_at_k 진입 시 dedup + print_summary dedup audit stats + QueryResult.dedup_count + csv schema column",
      "tests/search_eval/test_eval_graded_ndcg_dedup.py — 13 신규 test (dedup helper + invariant + Phase 2Q 실측 case regression)"
    ],
    "test_results": "13 신규 PASS + 38 기존 PASS = 51/51, retrieval path 영향 0"
  }
}
