{
  "version": "v0.2-phase2q",
  "label": "phase_2q_query_rewrite_3_measurement",
  "date": "2026-05-24",
  "snapshot": {
    "doc_id_max": 25180,
    "chunk_id_max": 56526,
    "documents_n": 21365,
    "chunks_n": 30605,
    "source": "v0_2_phase2a_baseline_snapshot_2026-05-23.json (재사용)"
  },
  "eval_set": {
    "total_cases": 51,
    "scored_cases": 46,
    "failure_expected_cases": 5,
    "queries_yaml": "tests/search_eval/queries.yaml"
  },
  "model_config": {
    "embedding": "BAAI/bge-m3 (production)",
    "reranker": "BAAI/bge-reranker-v2-m3 (production)",
    "search_mode": "hybrid",
    "fusion": "rrf_boost (production default)",
    "rerank_enabled": "server_default true",
    "per_variant_k": 16,
    "phase2q_unified_cap": 60,
    "rrf_k": 60,
    "plan": "phase-2q-query-rewrite-diagnose.md v6"
  },
  "candidates": {
    "baseline_rebaseline": {
      "rewrite_backend": null,
      "llm_endpoint": null,
      "overall": {
        "n": 46,
        "graded_ndcg_at_10": 0.659,
        "graded_recall_at_10_t2": 0.695,
        "graded_recall_at_10_t3": 0.761,
        "latency_p50_ms": 478,
        "latency_p95_ms": 1627,
        "failure_precision": "0/5"
      },
      "by_category": {
        "english_only": {"n": 9, "recall": 0.78, "gndcg": 0.78},
        "exam": {"n": 7, "recall": 0.57, "gndcg": 0.74},
        "korean_only": {"n": 9, "recall": 0.55, "gndcg": 0.51},
        "mixed": {"n": 10, "recall": 0.38, "gndcg": 0.39},
        "standards": {"n": 11, "recall": 0.91, "gndcg": 0.87}
      },
      "recurrence_vs_phase2a": "NDCG 0.659 = Phase 2A baseline 0.659. diff 0.000 < 0.005 threshold PASS",
      "csv": "reports/v0_2_phase2q_baseline_rebaseline_2026-05-24.csv"
    },
    "cand_multi_query_macmini": {
      "rewrite_backend": "cand_multi_query_macmini",
      "llm_endpoint": "http://100.76.254.116:8801/v1/chat/completions",
      "llm_model": "gemma-4-26b-a4b-it-8bit",
      "n_variants": 3,
      "cold": {
        "n": 46,
        "graded_ndcg_at_10": 0.927,
        "graded_recall_at_10_t2": 0.687,
        "graded_recall_at_10_t3": 0.728,
        "latency_p50_ms": 2757,
        "latency_p95_ms": 9684,
        "delta_vs_baseline": {
          "ndcg": "+0.268",
          "recall_t2": "-0.008",
          "recall_t3": "-0.033"
        },
        "csv": "reports/v0_2_phase2q_cand_multi_query_macmini_2026-05-24_cold.csv"
      },
      "warm": {
        "graded_ndcg_at_10": 0.927,
        "graded_recall_at_10_t2": 0.687,
        "graded_recall_at_10_t3": 0.728,
        "latency_p50_ms": 998,
        "latency_p95_ms": 2693,
        "cache_hit_speedup": "p50 -1759ms (-64%), p95 -6991ms (-72%)",
        "csv": "reports/v0_2_phase2q_cand_multi_query_macmini_2026-05-24_warm.csv"
      },
      "by_category_cold": {
        "english_only": {"n": 9, "recall": 0.61, "gndcg": 0.77, "delta": "-0.01"},
        "exam": {"n": 7, "recall": 0.62, "gndcg": 1.11, "delta": "+0.37"},
        "korean_only": {"n": 9, "recall": 0.55, "gndcg": 0.71, "delta": "+0.20"},
        "mixed": {"n": 10, "recall": 0.40, "gndcg": 0.57, "delta": "+0.18"},
        "standards": {"n": 11, "recall": 0.91, "gndcg": 1.44, "delta": "+0.57"}
      }
    },
    "cand_multi_query_macbook": {
      "rewrite_backend": "cand_multi_query_macbook",
      "llm_endpoint": "http://100.118.112.84:8810/v1/chat/completions",
      "llm_model": "mlx-community/Qwen3.6-27B-8bit",
      "n_variants": 3,
      "cold": {
        "n": 46,
        "graded_ndcg_at_10": 0.919,
        "graded_recall_at_10_t2": 0.697,
        "graded_recall_at_10_t3": 0.728,
        "latency_p50_ms": 3647,
        "latency_p95_ms": 5202,
        "delta_vs_baseline": {
          "ndcg": "+0.260",
          "recall_t2": "+0.002",
          "recall_t3": "-0.033"
        },
        "csv": "reports/v0_2_phase2q_cand_multi_query_macbook_2026-05-24_cold.csv"
      },
      "warm": {
        "graded_ndcg_at_10": 0.919,
        "graded_recall_at_10_t2": 0.697,
        "graded_recall_at_10_t3": 0.728,
        "latency_p50_ms": 873,
        "latency_p95_ms": 2901,
        "cache_hit_speedup": "p50 -2774ms (-76%), p95 -2301ms (-44%)",
        "csv": "reports/v0_2_phase2q_cand_multi_query_macbook_2026-05-24_warm.csv"
      },
      "by_category_cold": {
        "english_only": {"n": 9, "recall": 0.78, "gndcg": 0.89, "delta": "+0.11"},
        "exam": {"n": 7, "recall": 0.52, "gndcg": 1.04, "delta": "+0.30"},
        "korean_only": {"n": 9, "recall": 0.56, "gndcg": 0.67, "delta": "+0.16"},
        "mixed": {"n": 10, "recall": 0.43, "gndcg": 0.65, "delta": "+0.26"},
        "standards": {"n": 11, "recall": 0.88, "gndcg": 1.31, "delta": "+0.44"}
      }
    }
  },
  "comparison": {
    "macmini_vs_macbook_overall_ndcg_diff": "+0.008 (macmini 우세)",
    "macmini_strengths": ["exam +0.07", "standards +0.13", "korean_only +0.04"],
    "macbook_strengths": ["english_only +0.12", "mixed +0.08", "recall_t2 +0.01"],
    "latency_cold_p50": "macmini 2757ms < macbook 3647ms (-890ms gemma 우세)",
    "latency_warm_p50": "macmini 998ms > macbook 873ms (-125ms qwen 우세, cache hit 시)"
  },
  "incidents": {
    "phase3_first_attempt_catastrophic": {
      "ndcg": 0.033,
      "root_cause": "query_rewriter._call_llm() 가 user 메시지 1개에 prompt template 전체 박음. prompt template 에 {query} placeholder 없음 → _render_prompt no-op. LLM 이 actual query 인식 못 함 → 모든 query 에 동일 default response (`압력용기 설계 기준` 등 마지막 example) 반환.",
      "discovery": "fastapi log [rewrite-variant] 박제에서 query 별 같은 variants 발견.",
      "fix": "_call_llm 을 fixture request_body 형식 (system=prompt template / user=query) 으로 변경. fixture-first invariant 강화.",
      "fix_commit": "TBD (재측정 후)"
    }
  },
  "decisions": {
    "H1_both_net_improve": "Both backends NDCG net improve ≥ +0.03 (macmini +0.268, macbook +0.260)",
    "selected_for_apply": "TBD — 4 factor balance: latency (cold) gemma 우세 / latency (warm) qwen 우세 / category 분포 / cost",
    "apply_pr_recommendation": "PR-2Q-Apply-Query-Rewrite-1 진입 후보. LLM 선택 = 4-factor weighted (latency / category / availability / cost) decision needed.",
    "category_split_observation": "qwen=영어/mixed/recall 강함, gemma=exam/standards/korean 강함. 만약 mixed (0.39 → 0.65) 가 최우선이면 qwen. 만약 exam/standards 가 최우선이면 gemma."
  },
  "follow_ups": {
    "phase4_decision_md": "별 step 으로 작성 — 사용자 검토 후 Apply LLM 선택",
    "fix_commit_required": "variants bug fix 별 commit 필요 (test 추가 + system/user 메시지 분리)",
    "rerank_413_payload_too_large": "fastapi log 에 RRF fallback 다수 관찰 — unified RRF 결과 만으로도 NDCG 0.927 달성. reranker 입력 cap 또는 chunk dedup 별 PR 후보 (Apply 전 결정).",
    "latency_p95_cold": "cand_macmini 9684ms 매우 큼 — production rollout 시 cache prewarm 정책 + 비동기 rewrite 필수",
    "ub2_caffeinate_pid": "PID 37361 (caffeinate -di) 측정 종료 후 사용자 kill 권장"
  }
}
