hyungi_document_server/app/api/search.py

"""하이브리드 검색 API — thin endpoint (Phase 3.1 이후).

실제 검색 파이프라인(retrieval → fusion → rerank → diversity → confidence)
은 `services/search/search_pipeline.py::run_search()` 로 분리되어 있다.
이 파일은 다음만 담당:
  - Pydantic 스키마 (SearchResult / SearchResponse / SearchDebug / DebugCandidate
    / Citation / AskResponse / AskDebug)
  - `/search` endpoint wrapper (run_search 호출 + logger + telemetry + 직렬화)
  - `/ask` endpoint wrapper (Phase 3.3 에서 추가)
"""

import asyncio
import hmac
import time
from datetime import date
from typing import Annotated, Literal

from fastapi import APIRouter, BackgroundTasks, Depends, Header, Query
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession

from core.auth import get_current_user
from core.config import settings
from core.database import get_session
from core.utils import setup_logger
from models.user import User
from services.document_telemetry import sanitize_source
from services.search.classifier_service import ClassifierResult, classify
from services.search.evidence_service import EvidenceItem, extract_evidence
from services.search.fusion_service import DEFAULT_FUSION
from services.search.grounding_check import check as grounding_check
from services.search.refusal_gate import RefusalDecision, decide as refusal_decide
from services.search import query_rewriter
from services.search.retrieval_service import AxisFilter
from services.search.result_decorate import compute_facets, decorate_version_status
from services.search.search_pipeline import PipelineResult, run_search
from services.search.synthesis_service import SynthesisResult, synthesize
from services.search.verifier_service import VerifierResult, verify
from services.prompt_versions import ASK_PROMPT_VERSION, resolve_primary_model
from services.search_telemetry import record_ask_event, record_search_event

# logs/search.log + stdout 동시 출력 (Phase 0.4)
logger = setup_logger("search")

router = APIRouter()


class SearchResult(BaseModel):
    """검색 결과 단일 행.

    Phase 1.2-C: chunk-level vector retrieval 도입으로 chunk 메타 필드 추가.
    text 검색 결과는 chunk_id 등이 None (doc-level).
    vector 검색 결과는 chunk_id 등이 채워짐 (chunk-level).
    """

    id: int  # doc_id (text/vector 공통)
    title: str | None
    ai_domain: str | None
    ai_summary: str | None
    file_format: str
    score: float
    snippet: str | None
    match_reason: str | None = None
    # Phase 1.2-C: chunk 메타 (vector 검색 시 채워짐)
    chunk_id: int | None = None
    chunk_index: int | None = None
    section_title: str | None = None
    # Phase 3.1: reranker raw score 보존 (display score drift 방지).
    # rerank 경로를 탄 chunk에만 채워짐. normalize_display_scores는 이 필드를
    # 건드리지 않는다. Phase 3 evidence fast-path 판단에 사용.
    rerank_score: float | None = None
    # PR-RAG-Time-1: freshness decay 디버그 메타. apply_freshness_decay 가 채움.
    # 비적용 row 도 채워짐(freshness_policy=None). base_score 는 항상 보존.
    freshness_debug: dict | None = None
    # 안전 자료실 C-1: 분류 축 메타 (3 leg SELECT 에서 채움 — additive, ranking 무관).
    # D-1 UI 결과 카드 유형별 렌더 + 해외 법령(B-5) 가동 시 국가 무표지 혼재 차단의 선행 조건.
    material_type: str | None = None
    jurisdiction: str | None = None
    published_date: date | None = None
    # 안전 자료실 C-1 후속: 법령 버전 상태(legal_meta.version_status) — wrapper 1회 decorate.
    # law 결과만 채워짐(legal_meta 위성), 그 외/무매핑 law = None. D-1 버전 뱃지 선행.
    version_status: str | None = None


# ─── Phase 0.4: 디버그 응답 스키마 ─────────────────────────


class DebugCandidate(BaseModel):
    """단계별 후보 (debug=true 응답에서만 노출)."""
    id: int
    rank: int
    score: float
    match_reason: str | None = None


class SearchDebug(BaseModel):
    timing_ms: dict[str, float]
    text_candidates: list[DebugCandidate] | None = None
    vector_candidates: list[DebugCandidate] | None = None
    fused_candidates: list[DebugCandidate] | None = None
    confidence: float
    notes: list[str] = []
    # Phase 1/2 도입 후 채워질 placeholder
    query_analysis: dict | None = None
    reranker_scores: list[DebugCandidate] | None = None


class SearchResponse(BaseModel):
    results: list[SearchResult]
    total: int
    query: str
    mode: str
    debug: SearchDebug | None = None
    # 안전 자료실 C-1 후속: facets=true 일 때만 채워짐(미요청=None, byte 불변).
    # top-K 결과 내 분류 축 분포 라벨 {axis: {label: count}}.
    facets: dict[str, dict[str, int]] | None = None


def _to_debug_candidates(rows: list[SearchResult], n: int = 20) -> list[DebugCandidate]:
    return [
        DebugCandidate(
            id=r.id, rank=i + 1, score=r.score, match_reason=r.match_reason
        )
        for i, r in enumerate(rows[:n])
    ]


def _build_search_debug(pr: PipelineResult) -> SearchDebug:
    """PipelineResult → SearchDebug (기존 search()의 debug 구성 블록 복사)."""
    return SearchDebug(
        timing_ms=pr.timing_ms,
        text_candidates=(
            _to_debug_candidates(pr.text_results)
            if pr.text_results or pr.mode != "vector"
            else None
        ),
        vector_candidates=(
            _to_debug_candidates(pr.vector_results)
            if pr.vector_results or pr.mode in ("vector", "hybrid")
            else None
        ),
        fused_candidates=(
            _to_debug_candidates(pr.results) if pr.mode == "hybrid" else None
        ),
        confidence=pr.confidence_signal,
        notes=pr.notes,
        query_analysis=pr.query_analysis,
    )


@router.get("/", response_model=SearchResponse)
async def search(
    q: str,
    user: Annotated[User, Depends(get_current_user)],
    session: Annotated[AsyncSession, Depends(get_session)],
    background_tasks: BackgroundTasks,
    mode: str = Query("hybrid", pattern="^(fts|trgm|vector|hybrid)$"),
    limit: int = Query(20, ge=1, le=100),
    fusion: str = Query(
        DEFAULT_FUSION,
        pattern="^(legacy|rrf|rrf_boost)$",
        description="hybrid 모드 fusion 전략 (legacy=기존 가중합, rrf=RRF k=60, rrf_boost=RRF+강한신호 boost)",
    ),
    rerank: bool = Query(
        True,
        description="bge-reranker-v2-m3 활성화 (Phase 1.3, hybrid 모드만 동작)",
    ),
    analyze: bool = Query(
        False,
        description="QueryAnalyzer 활성화 (Phase 2.1, LLM 호출). Phase 2.1은 debug 노출만, 검색 경로 영향 X",
    ),
    debug: bool = Query(False, description="단계별 candidates + timing 응답에 포함"),
    embedding_backend: str | None = Query(
        None,
        pattern=r"^(baseline|cand_[a-z0-9_]+)$",
        description="Phase 2A Diagnose dispatcher (R2-2 + R2-B1). slug 만 받음 (raw table name X). baseline|cand_<slug>. 미지정/baseline = production path.",
    ),
    snapshot_doc_id_max: int | None = Query(
        None, ge=1,
        description="Phase 2A snapshot freeze (R2-D + R2-B2). documents.id <= 값 filter. baseline 측정 시에도 동일 filter 적용.",
    ),
    snapshot_chunk_id_max: int | None = Query(
        None, ge=1,
        description="Phase 2A snapshot freeze (R2-D + R2-B2). document_chunks.id <= 값 filter. baseline 측정 시에도 동일 filter 적용.",
    ),
    reranker_backend: str | None = Query(
        None,
        pattern=r"^(baseline|cand_[a-z0-9_]+)$",
        description="Phase 2B Diagnose reranker dispatcher (R2-B1 slug-based). slug 만 받음 (raw endpoint URL X). baseline|cand_<slug>. 미지정/baseline = production reranker.",
    ),
    rewrite_backend: str | None = Query(
        None,
        pattern=r"^(baseline|cand_[a-z0-9_]+)$",
        description=(
            "⚠️ EXPERIMENTAL / DEPRECATED (Phase 2Q closed 2026-05-24 as evaluated experiment). "
            "Result-level dedup 정정 후 net gain marginal (NDCG +0.019, Recall t≥2 +0.030) "
            "vs latency cost 큼 (cold +876%, warm +320%). default production rollout 권고 X. "
            "slug-based, no silent fallback. baseline|cand_multi_query_macmini|cand_multi_query_macbook. "
            "미지정/baseline = single-query path (회귀 0 invariant, 권장 default). "
            "opt-in 실험 reference 만 유지 — docs/phase_2q_apply_opt_in.md 의 closed status 참조."
        ),
    ),
    corpus_variant: str | None = Query(
        None,
        pattern=r"^(prehier|hier_sim_raw|hier_sim_clean)$",
        description=(
            "⚠️ EVAL ONLY (Hier-Replace-Diagnose-1). chunk leg 를 측정 뷰로 교체 — "
            "prehier(legacy baseline) | hier_sim_raw | hier_sim_clean(childless-tiny 제외). "
            "doc-level + fts/trgm 는 documents 테이블 = 변종 무관. 미지정 = production corpus_chunks. "
            "embedding_backend cand 와 동시 사용 불가 (400)."
        ),
    ),
    exact_knn: bool = Query(
        False,
        description=(
            "⚠️ EVAL ONLY (Hier-Replace-Diagnose-1). vector leg 에 SET LOCAL enable_indexscan/"
            "bitmapscan=off → ivfflat 근사 제거(exact seqscan). prehier vs hier_sim 의 index 변수 "
            "분리용. production 검색에는 사용 금지 (latency 큼)."
        ),
    ),
    material_type: str | None = Query(
        None, description="안전 자료실 C-1: 자료유형 필터 CSV (law,paper,incident,...). material_type = ANY"),
    jurisdiction: str | None = Query(
        None, description="안전 자료실 C-1: 관할 필터 (KR/US/EU/JP/GB/INT)"),
    year_from: int | None = Query(None, ge=1900, le=2100, description="published_date 연도 하한 (NULL=created_at fallback)"),
    year_to: int | None = Query(None, ge=1900, le=2100, description="published_date 연도 상한"),
    facets: bool = Query(False, description="안전 자료실 C-1 후속: top-K 결과 분류 축 분포(material_type/jurisdiction/version_status)를 응답 facets 에 집계. 미지정=계산/노출 0"),
):
    """문서 검색 — FTS + ILIKE + 벡터 결합 (Phase 3.1 이후 run_search wrapper)"""
    try:
        axis = AxisFilter(
            material_types=[m.strip() for m in material_type.split(",") if m.strip()]
            if material_type else None,
            jurisdiction=jurisdiction,
            year_from=year_from,
            year_to=year_to,
        )
        pr = await run_search(
            session,
            q,
            mode=mode,  # type: ignore[arg-type]
            limit=limit,
            fusion=fusion,
            rerank=rerank,
            analyze=analyze,
            embedding_backend=embedding_backend,
            snapshot_doc_id_max=snapshot_doc_id_max,
            snapshot_chunk_id_max=snapshot_chunk_id_max,
            reranker_backend=reranker_backend,
            rewrite_backend=rewrite_backend,
            corpus_variant=corpus_variant,
            exact_knn=exact_knn,
            axis=axis,
        )
    except ValueError as e:
        # _resolve_backend / _resolve_reranker / _resolve_rewrite_backend / _resolve_corpus_variant unknown slug → HTTP 400
        msg = str(e)
        if msg.startswith("unknown_corpus_variant") or msg.startswith("corpus_variant_incompatible"):
            return JSONResponse(
                status_code=400,
                content={
                    "error_reason": msg.split(":")[0].split(" ")[0],
                    "corpus_variant_requested": corpus_variant,
                    "allowed": ["prehier", "hier_sim_raw", "hier_sim_clean"],
                    "detail": msg,
                },
            )
        if msg.startswith("unknown_rewrite_backend"):
            return JSONResponse(
                status_code=400,
                content={
                    "error_reason": "unknown_rewrite_backend",
                    "backend_requested": rewrite_backend,
                    "allowed": query_rewriter.allowed_slugs(),
                    "detail": msg,
                },
            )
        if msg.startswith("unknown_reranker_backend"):
            return JSONResponse(
                status_code=400,
                content={
                    "error_reason": "unknown_reranker_backend",
                    "backend_requested": reranker_backend,
                    "allowed": ["baseline", "cand_gte_ml_base"],
                    "detail": msg,
                },
            )
        return JSONResponse(
            status_code=400,
            content={
                "error_reason": "unknown_embedding_backend",
                "backend_requested": embedding_backend,
                "allowed": ["baseline"],
                "detail": msg,
            },
        )
    except RuntimeError as e:
        # query_rewriter.rewrite() 실패 (LLM unavailable / parse fail) → HTTP 503
        msg = str(e)
        if msg.startswith("rewrite_llm_unavailable"):
            return JSONResponse(
                status_code=503,
                content={
                    "error_reason": "rewrite_llm_unavailable",
                    "backend_requested": rewrite_backend,
                    "detail": msg,
                },
            )
        raise

    # 사용자 feedback: 모든 단계 timing은 debug 응답과 별도로 항상 로그로 남긴다
    timing_str = " ".join(f"{k}={v:.0f}" for k, v in pr.timing_ms.items())
    fusion_str = f" fusion={fusion}" if mode == "hybrid" else ""
    analyzer_str = (
        f" analyzer=hit={pr.analyzer_cache_hit}/conf={pr.analyzer_confidence:.2f}/tier={pr.analyzer_tier}"
        if analyze
        else ""
    )
    logger.info(
        "search query=%r mode=%s%s%s results=%d conf=%.2f %s",
        q[:80],
        pr.mode,
        fusion_str,
        analyzer_str,
        len(pr.results),
        pr.confidence_signal,
        timing_str,
    )

    # Phase 0.3: 실패 자동 로깅 (응답 latency에 영향 X — background task)
    # Phase 2.1: analyze=true일 때만 analyzer_confidence 전달 (False는 None → 기존 호환)
    background_tasks.add_task(
        record_search_event,
        q,
        user.id,
        pr.results,
        pr.mode,
        pr.confidence_signal,
        pr.analyzer_confidence if analyze else None,
    )

    debug_obj = _build_search_debug(pr) if debug else None

    # 안전 자료실 C-1 후속 — wrapper decoration (검색 코어 무접촉, ranking 무관)
    await decorate_version_status(session, pr.results)   # 법령 결과에 version_status
    facets_obj = compute_facets(pr.results) if facets else None

    return SearchResponse(
        results=pr.results,
        total=len(pr.results),
        query=q,
        mode=pr.mode,
        debug=debug_obj,
        facets=facets_obj,
    )


# ═══════════════════════════════════════════════════════════
# Phase 3.3: /api/search/ask — Evidence + Grounded Synthesis
# ═══════════════════════════════════════════════════════════


class Citation(BaseModel):
    """answer 본문의 [n] 에 해당하는 근거 단일 행."""

    n: int
    chunk_id: int | None
    doc_id: int
    title: str | None
    section_title: str | None
    span_text: str  # evidence LLM 이 추출한 50~300자
    full_snippet: str  # 원본 800자 (citation 원문 보기 전용)
    relevance: float
    rerank_score: float


class ConfirmedItem(BaseModel):
    """Partial answer 의 개별 aspect 답변."""

    aspect: str
    text: str
    citations: list[int]


class AskDebug(BaseModel):
    """`/ask?debug=true` 응답 확장."""

    timing_ms: dict[str, float]
    search_notes: list[str]
    query_analysis: dict | None = None
    confidence_signal: float
    evidence_candidate_count: int
    evidence_kept_count: int
    evidence_skip_reason: str | None
    synthesis_cache_hit: bool
    synthesis_prompt_preview: str | None = None
    synthesis_raw_preview: str | None = None
    hallucination_flags: list[str] = []
    # Phase 3.5a: per-layer defense 로깅
    defense_layers: dict | None = None


class AskResponse(BaseModel):
    """`/ask` 응답. Phase 3.5a: completeness + aspects 추가."""

    results: list[SearchResult]
    ai_answer: str | None
    citations: list[Citation]
    synthesis_status: Literal[
        "completed", "timeout", "skipped", "no_evidence", "parse_failed", "llm_error",
        # PR-MacBook-RAG-Backend-1: 200 응답에는 등장하지 않음 (해당 status 는 503 분기).
        # Literal 호환성 위해 포함.
        "backend_unavailable",
    ]
    synthesis_ms: float
    confidence: Literal["high", "medium", "low"] | None
    refused: bool
    no_results_reason: str | None
    query: str
    total: int
    # Phase 3.5a
    completeness: Literal["full", "partial", "insufficient"] = "full"
    covered_aspects: list[str] | None = None
    missing_aspects: list[str] | None = None
    confirmed_items: list[ConfirmedItem] | None = None
    # PR-MacBook-RAG-Backend-1: backend dispatcher metadata.
    # backend 미지정 호출은 둘 다 None 으로 유지 (기존 호출자 호환 — Hermes docsrv_ask /
    # voice-memo-bot 응답 형식 변동 0). 명시 opt-in 시만 채워짐.
    backend_requested: str | None = None
    backend_used: str | None = None
    debug: AskDebug | None = None


def _map_no_results_reason(
    pr: PipelineResult,
    evidence: list[EvidenceItem],
    ev_skip: str | None,
    sr: SynthesisResult,
) -> str | None:
    """사용자에게 보여줄 한국어 메시지 매핑.

    Failure mode 표 (plan §Failure Modes) 기반.
    """
    # LLM 자가 refused → 모델이 준 사유 그대로
    if sr.refused and sr.refuse_reason:
        return sr.refuse_reason

    # synthesis 상태 우선
    if sr.status == "no_evidence":
        if not pr.results:
            return "검색 결과가 없습니다."
        return "관련도 높은 근거를 찾지 못했습니다."
    if sr.status == "skipped":
        return "검색 결과가 없습니다."
    if sr.status == "timeout":
        return "답변 생성이 지연되어 생략했습니다. 검색 결과를 확인해 주세요."
    if sr.status == "parse_failed":
        return "답변 형식 오류로 생략했습니다."
    if sr.status == "llm_error":
        return "AI 서버에 일시적 문제가 있습니다."

    # evidence 단계 실패는 fallback 을 탔더라도 notes 용
    if ev_skip == "all_low_rerank":
        return "관련도 높은 근거를 찾지 못했습니다."
    if ev_skip == "empty_retrieval":
        return "검색 결과가 없습니다."

    return None


def _build_citations(
    evidence: list[EvidenceItem], used_citations: list[int]
) -> list[Citation]:
    """answer 본문에 실제로 등장한 n 만 Citation 으로 변환."""
    by_n = {e.n: e for e in evidence}
    out: list[Citation] = []
    for n in used_citations:
        e = by_n.get(n)
        if e is None:
            continue
        out.append(
            Citation(
                n=e.n,
                chunk_id=e.chunk_id,
                doc_id=e.doc_id,
                title=e.title,
                section_title=e.section_title,
                span_text=e.span_text,
                full_snippet=e.full_snippet,
                relevance=e.relevance,
                rerank_score=e.rerank_score,
            )
        )
    return out


def _build_ask_debug(
    pr: PipelineResult,
    evidence: list[EvidenceItem],
    ev_skip: str | None,
    sr: SynthesisResult,
    ev_ms: float,
    synth_ms: float,
    total_ms: float,
) -> AskDebug:
    timing: dict[str, float] = dict(pr.timing_ms)
    timing["evidence_ms"] = ev_ms
    timing["synthesis_ms"] = synth_ms
    timing["ask_total_ms"] = total_ms

    # candidate count 는 rule filter 통과한 수 (recomputable from results)
    # 엄밀히는 evidence_service 내부 숫자인데, evidence 길이 ≈ kept, candidate
    # 는 관측이 어려움 → kept 는 evidence 길이, candidate 는 별도 필드 없음.
    # 단순화: candidate_count = len(evidence) 를 상한 근사로 둠 (debug 전용).
    return AskDebug(
        timing_ms=timing,
        search_notes=pr.notes,
        query_analysis=pr.query_analysis,
        confidence_signal=pr.confidence_signal,
        evidence_candidate_count=len(evidence),
        evidence_kept_count=len(evidence),
        evidence_skip_reason=ev_skip,
        synthesis_cache_hit=sr.cache_hit,
        synthesis_prompt_preview=None,  # 현재 synthesis_service 에서 노출 안 함
        synthesis_raw_preview=sr.raw_preview,
        hallucination_flags=sr.hallucination_flags,
    )


def _detect_synthesis_failure(sr: SynthesisResult) -> str | None:
    """Synthesis 가 유효한 답을 못 냈으면 re_gate 라벨, 아니면 None.

    판정 우선순위 (Phase 3.5 fix3):
      1) sr.refused → LLM self-refuse (status="completed") 또는 mechanical fail 후 refused 전파
         - status=="completed" + refused=True → "synthesis_self_refuse"
         - 그 외 → f"synthesis_failed({status})"
      2) sr.status ∈ {timeout, parse_failed, llm_error} → f"synthesis_failed({status})"
      3) answer 공백 → f"synthesis_failed({status})"
      4) 유효 → None
    """
    if sr.refused:
        if sr.status == "completed":
            return "synthesis_self_refuse"
        return f"synthesis_failed({sr.status})"
    if sr.status in ("timeout", "parse_failed", "llm_error"):
        return f"synthesis_failed({sr.status})"
    if not (sr.answer or "").strip():
        return f"synthesis_failed({sr.status})"
    return None


def _resolve_eval_identity(
    x_source: str | None,
    x_eval_case_id: str | None,
    x_eval_token: str | None,
) -> tuple[str, str | None]:
    """X-Source/X-Eval-Case-Id 신뢰 검증 (Phase 3.5 fix2).

    규칙:
      - 기본값: source='document_server', eval_case_id=None
      - X-Source=eval 또는 X-Eval-Case-Id 가 들어왔다면 eval claim 으로 간주
      - eval claim 은 X-Eval-Token == settings.eval_runner_token 일 때만 수용
        (constant-time compare, env 미설정 시 항상 거부)
      - 거부 시: 헤더 무시 + warning log + source=sanitize(non-eval) / eval_case_id=None
      - 통과 시: source='eval', eval_case_id=x_eval_case_id

    반환: (source, eval_case_id)
    """
    claimed_source = sanitize_source(x_source)
    is_eval_claim = (claimed_source == "eval") or bool(x_eval_case_id)
    if not is_eval_claim:
        # 일반 호출 — eval_case_id 강제 None (source != 'eval' 이면 case_id 의미 없음)
        return claimed_source, None

    # eval claim — token 검증
    expected = settings.eval_runner_token
    presented = x_eval_token or ""
    token_valid = bool(expected) and hmac.compare_digest(presented, expected)
    if not token_valid:
        logger.warning(
            "eval header rejected: source=%s case_id=%s token_present=%s expected_set=%s",
            x_source, x_eval_case_id, bool(x_eval_token), bool(expected),
        )
        # 일반 호출로 강등 — source='eval' 주장은 무시, case_id 도 무시
        # claimed_source 가 'eval' 이면 default 'document_server' 로
        if claimed_source == "eval":
            return "document_server", None
        return claimed_source, None

    # token OK — eval 라벨 수용
    return "eval", x_eval_case_id