Compare commits

..

1 Commits

Author SHA1 Message Date
hyungi 624b9d523d refactor(search): Phase 2A/2B cand reranker 잔재 teardown
- RERANKER_BACKEND_MAP 에서 cand_gte_ml_base 슬러그 제거 (컨테이너·DB 테이블 마이그360·override 이미 종료)
- docker-compose.override.cand.yml / override.rerank-cand.yml 삭제
- search.py allowlist · run_eval.py help 정합
- dispatcher scaffold(_resolve_reranker)는 보존 (후보 재진입 대비)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-18 00:00:42 +00:00
117 changed files with 4922 additions and 4996 deletions
-3
View File
@@ -47,6 +47,3 @@ caddy_data/
*.bak_*
*.pre-*
.pre-*/
# SQLite 로컬 아티팩트 (Django/툴링 잔재)
*.sqlite3
-7
View File
@@ -12,13 +12,6 @@ http://document.hyungi.net {
# 명시 Content-Type match — 기본 match 의 text/* 는 text/event-stream 까지 포함해
# SSE(/api/eid/chat)의 첫 ~512B 를 gzip 버퍼링함. SSE 제외, 기존 압축 대상은 보존.
# (응답 매처는 header <필드> <값> 한 쌍씩 — 여러 줄 = OR. 한 줄 다중 값은 파싱 에러)
# 2026-06-20 보안 헤더 (M: 클릭재킹·MIME 스니핑 방어). HSTS 는 TLS 종단 edge(home-caddy) 소관.
header {
X-Content-Type-Options nosniff
X-Frame-Options SAMEORIGIN
Referrer-Policy strict-origin-when-cross-origin
-Server
}
encode {
gzip
match {
+2 -27
View File
@@ -1,6 +1,5 @@
"""AI 추상화 레이어 — 통합 클라이언트. 기본값은 항상 Qwen3.5."""
import asyncio
import json
import re
from pathlib import Path
@@ -189,25 +188,6 @@ def _load_prompt(name: str) -> str:
CLASSIFY_PROMPT = _load_prompt("classify.txt") if (PROMPTS_DIR / "classify.txt").exists() else ""
# 공유 httpx 클라이언트 — 호출마다 AsyncClient 를 새로 만들던 것(30+ 사이트, 연결풀 재사용 0)을
# 일원화해 keep-alive 재사용. 이벤트루프 바인딩이라 루프 변경(pytest 격리 등) 시 재생성한다.
# close() 는 공유 풀이라 no-op — 프로세스 종료 시 GC.
_shared_http: httpx.AsyncClient | None = None
_shared_http_loop: object | None = None
def _get_shared_http() -> httpx.AsyncClient:
global _shared_http, _shared_http_loop
try:
loop: object | None = asyncio.get_running_loop()
except RuntimeError:
loop = None
if _shared_http is None or _shared_http.is_closed or _shared_http_loop is not loop:
_shared_http = httpx.AsyncClient(timeout=120)
_shared_http_loop = loop
return _shared_http
class AIClient:
"""AI 모델 통합 클라이언트.
@@ -222,7 +202,7 @@ class AIClient:
def __init__(self):
self.ai = settings.ai
self._http = _get_shared_http()
self._http = httpx.AsyncClient(timeout=120)
# ─── 3-tier routing (B-0) ───────────────────────────────────────────────
@@ -366,10 +346,6 @@ class AIClient:
payload["temperature"] = model_config.temperature
if model_config.top_p is not None:
payload["top_p"] = model_config.top_p
if model_config.repetition_penalty is not None:
payload["repetition_penalty"] = model_config.repetition_penalty
if model_config.top_k is not None:
payload["top_k"] = model_config.top_k
response = await self._http.post(
model_config.endpoint,
json=payload,
@@ -380,5 +356,4 @@ class AIClient:
return data["choices"][0]["message"]["content"]
async def close(self):
# 공유 풀(_get_shared_http) 이라 per-use close 안 함 — 연결 재사용. 프로세스 종료 시 GC.
return None
await self._http.aclose()
+5 -327
View File
@@ -28,7 +28,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import ClientDisconnect
from ai.client import AIClient, _load_prompt, parse_json_response
from core.auth import get_current_user, get_egress_class
from core.auth import get_current_user
from core.config import settings
from core.database import async_session, get_session
from core.utils import file_hash
@@ -672,101 +672,16 @@ async def list_duplicates(
)
class ClauseHit(BaseModel):
doc_id: int
doc_title: str
section_title: str | None = None
char_start: int | None = None
chunk_id: int
node_type: str | None = None
class ClauseLookupResponse(BaseModel):
label: str
hits: list[ClauseHit]
# NOTE: '/{doc_id}' (int path param) 라우트보다 먼저 선언해야 '/clause-lookup' 이 doc_id 로
# 잘못 매칭되지 않는다 (FastAPI 선언 순서 매칭). 이동 금지.
@router.get("/clause-lookup", response_model=ClauseLookupResponse)
async def clause_lookup(
label: str,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""절 식별자(예: UG-79)로 크로스-doc 절 위치 조회 — 'UG-79 보여줘' 진입점 (U-1).
절(node_type=clause/clause_split)은 in_corpus=false(검색 비활성)라 의미검색으론 못 찾으므로,
라벨 prefix 정확매칭으로 (doc, char_start) 를 직접 해소해 읽기뷰 점프를 가능케 한다.
대부분 1건; 부록(A-/E-/F-) 등 doc 간 공유 라벨만 다중 반환(에디션 선택). /sections 와 동일하게
document_chunks 직접 조회 — corpus_chunks 우회는 retrieval 아닌 정확지목이므로 의도적 예외.
"""
from sqlalchemy import text as sql_text
lab = (label or "").strip()
if not lab:
return ClauseLookupResponse(label=label, hits=[])
rows = (
await session.execute(
sql_text(
"""
SELECT c.doc_id, d.title AS doc_title, c.section_title, c.char_start, c.node_type,
-- 점프 타깃 = outline(/sections: is_leaf 또는 %_split)에 있는 chunk 여야 딥링크 동작.
-- 자신이 그러면 자신, 아니면(컨테이너 절: 자식 heading 보유·is_leaf=false) 문서순서상
-- 자신 이후 첫 딥링크 가능 chunk(=그 절 내용 시작)로 해소. 그래도 없으면 자신(폴백).
COALESCE(
CASE WHEN c.is_leaf = true OR c.node_type LIKE '%\\_split' ESCAPE '\\' THEN c.id END,
(SELECT ch.id FROM document_chunks ch
WHERE ch.doc_id = c.doc_id AND ch.source_type = 'hier_section'
AND ch.chunk_index >= c.chunk_index
AND (ch.is_leaf = true OR ch.node_type LIKE '%\\_split' ESCAPE '\\')
ORDER BY ch.chunk_index LIMIT 1),
c.id
) AS chunk_id
FROM document_chunks c
JOIN documents d ON d.id = c.doc_id
WHERE c.node_type IN ('clause', 'clause_split')
AND (c.section_title ILIKE :lab_sp OR c.section_title ILIKE :lab_eq)
AND d.deleted_at IS NULL
ORDER BY c.doc_id, c.char_start NULLS LAST
LIMIT 50
"""
).bindparams(lab_sp=lab + " %", lab_eq=lab)
)
).mappings().all()
return ClauseLookupResponse(label=lab, hits=[ClauseHit(**dict(r)) for r in rows])
@router.get("/{doc_id}", response_model=DocumentDetailResponse)
async def get_document(
doc_id: int,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
egress_class: Annotated[str, Depends(get_egress_class)],
):
"""문서 단건 조회. 본문(extracted_text)·canonical markdown 동봉.
cloud egress(갭2): egress=cloud 토큰(예: Claude/MCP)은 search 와 동일한 cloud-eligibility
게이트를 통과한 문서만 열람 가능 — id 직접 fetch 로 비공개/인프라/개인/restricted 문서를
우회 열람하는 경로를 차단한다. 부적격은 404(존재 자체 비노출). local 토큰=게이트 미발동(무회귀).
"""
from sqlalchemy import text as sql_text
from services.search.retrieval_service import cloud_eligible_doc_sql
"""문서 단건 조회. 본문(extracted_text)·canonical markdown 동봉."""
doc = await session.get(Document, doc_id)
if not doc or doc.deleted_at is not None:
raise HTTPException(status_code=404, detail="문서를 찾을 수 없습니다")
if egress_class == "cloud":
eligible = (
await session.execute(
sql_text(
"SELECT 1 FROM documents WHERE id = :doc_id AND deleted_at IS NULL"
+ cloud_eligible_doc_sql("")
).bindparams(doc_id=doc_id)
)
).first()
if eligible is None:
raise HTTPException(status_code=404, detail="문서를 찾을 수 없습니다")
return DocumentDetailResponse.model_validate(doc)
@@ -1048,19 +963,6 @@ async def get_document_image_raw(
DocumentImage.image_key == image_key,
)
)
if img is None:
# clause-KB: 절-문서는 부모 표준 이미지를 공유(md_content=부모 슬라이스) → parent_id 폴백.
from sqlalchemy import text as sql_text
_par = (await session.execute(
sql_text("SELECT parent_id FROM documents WHERE id = :id").bindparams(id=doc_id)
)).scalar()
if _par is not None:
img = await session.scalar(
select(DocumentImage).where(
DocumentImage.document_id == _par,
DocumentImage.image_key == image_key,
)
)
if img is None:
raise HTTPException(status_code=404, detail="이미지를 찾을 수 없습니다")
@@ -1264,10 +1166,8 @@ async def upload_document(
doc.duplicate_of = canonical.id
canonical.duplicate_count = (canonical.duplicate_count or 0) + 1
# document + processing_queue 는 단일 트랜잭션으로 묶어 원자적 정리.
# G2: 첫 stage=presegment (extract 前 번들 PDF 분할, 후보 A 검증완료 2026-06-18).
# 非PDF/단일은 presegment 가 무변 통과 → extract. 번들 PDF 만 N 자식 분할(worker-side gating).
await enqueue_stage(session, doc.id, "presegment")
# document + processing_queue 는 단일 트랜잭션으로 묶어 원자적 정리
await enqueue_stage(session, doc.id, "extract")
await session.commit()
except Exception:
# DB 예외 시 session 은 get_session 컨텍스트 종료로 자동 rollback.
@@ -1310,14 +1210,6 @@ async def update_document(
if val is not None and val not in ("business", "knowledge"):
raise HTTPException(status_code=400, detail="doc_purpose는 business 또는 knowledge만 가능")
# edit_url SSRF 가드 (2026-06-20 M1): 내부/메타데이터 주소 후속 fetch 차단 (news.py 동형 검증)
if update_data.get("edit_url"):
from core.url_validator import validate_feed_url
try:
await asyncio.to_thread(validate_feed_url, update_data["edit_url"])
except Exception as e:
raise HTTPException(status_code=400, detail=f"edit_url 검증 실패: {e}")
for field, value in update_data.items():
setattr(doc, field, value)
doc.updated_at = datetime.now(timezone.utc)
@@ -1598,7 +1490,7 @@ ANALYZE_PROMPT = (
)
ANALYZE_TEXT_LIMIT = 12000 # chars (15000 → 12000, 실측 timeout 빈발)
ANALYZE_TIMEOUT_S = settings.llm_call_timeout_s # 2026-06-20 config 단일소스 (구 60s=빠른 Gemma)
ANALYZE_TIMEOUT_S = 60 # 15,000자 입력 + 4층 출력. 실측 7~45초, safety margin 포함
ANALYZE_CACHE_TTL_S = 1800 # 30분
ANALYZE_CACHE_MAXSIZE = 100
ANALYZE_LAYER_MIN_CHARS = 50 # 이 미만이면 억지 채움으로 보고 제거
@@ -1834,217 +1726,3 @@ async def analyze_document(
error_code=error_code,
source=source,
)
# ─── ASME 절-지식베이스: 유기적 책 네비 (clause-KB, doc_kind='clause' 자식 문서 기반) ───
class ClauseTocItem(BaseModel):
id: int
clause_code: str | None = None
clause_part: str | None = None
clause_order: int | None = None
title: str | None = None
class ClauseBookResponse(BaseModel):
parent_id: int
parent_title: str | None = None
clauses: list[ClauseTocItem]
@router.get("/{doc_id}/clauses", response_model=ClauseBookResponse)
async def get_document_clauses(
doc_id: int,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""부모 표준 doc 의 절-문서 목차(유기적 책 TOC). doc_kind='clause' 자식을 clause_order 순 반환.
절-문서는 in_corpus=false + doc_kind='clause'(검색 제외)라 일반 목록/검색엔 안 뜨지만,
이 책-내 네비는 부모 표준에서 자식 절로 진입하는 전용 경로다(ASME 2025판=한 권의 책).
"""
from sqlalchemy import text as sql_text
parent = await session.get(Document, doc_id)
if not parent or parent.deleted_at is not None:
raise HTTPException(status_code=404, detail="문서를 찾을 수 없습니다")
rows = (
await session.execute(
sql_text(
"""
SELECT id, clause_code, clause_part, clause_order, title
FROM documents
WHERE parent_id = :pid AND doc_kind = 'clause' AND deleted_at IS NULL
ORDER BY clause_order
"""
).bindparams(pid=doc_id)
)
).mappings().all()
return ClauseBookResponse(
parent_id=doc_id,
parent_title=parent.title,
clauses=[ClauseTocItem(**dict(r)) for r in rows],
)
class BacklinkRef(BaseModel):
code: str
doc_id: int | None = None # 해소된 절-문서(같은 부모) — dangling 이면 None
title: str | None = None
anchor: str | None = None
ctx: str | None = None
class BacklinksResponse(BaseModel):
doc_id: int
clause_code: str | None = None
parent_id: int | None = None
prev: ClauseTocItem | None = None
next: ClauseTocItem | None = None
forward: list[BacklinkRef] # 이 절이 참조하는 절들
back: list[BacklinkRef] # 이 절을 참조하는 절들
@router.get("/{doc_id}/backlinks", response_model=BacklinksResponse)
async def get_document_backlinks(
doc_id: int,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""절-문서의 양방향 백링크 + 같은 부모 내 이전/다음 절(유기적 책 흐름)."""
from sqlalchemy import text as sql_text
doc = await session.get(Document, doc_id)
if not doc or doc.deleted_at is not None:
raise HTTPException(status_code=404, detail="문서를 찾을 수 없습니다")
_meta = (await session.execute(sql_text(
"SELECT parent_id, clause_code, clause_order FROM documents WHERE id = :id"
).bindparams(id=doc_id))).mappings().first()
_parent_id = _meta["parent_id"] if _meta else None
_clause_code = _meta["clause_code"] if _meta else None
_clause_order = _meta["clause_order"] if _meta else None
forward = (
await session.execute(
sql_text(
"""
SELECT cl.dst_code AS code, cl.dst_doc_id AS doc_id, cl.anchor, cl.ctx, d.title
FROM clause_links cl
LEFT JOIN documents d ON d.id = cl.dst_doc_id
WHERE cl.src_doc_id = :id
ORDER BY cl.char_off NULLS LAST
LIMIT 300
"""
).bindparams(id=doc_id)
)
).mappings().all()
back = (
await session.execute(
sql_text(
"""
SELECT s.clause_code AS code, cl.src_doc_id AS doc_id, s.title, cl.ctx
FROM clause_links cl
JOIN documents s ON s.id = cl.src_doc_id
WHERE cl.dst_doc_id = :id
ORDER BY s.clause_order NULLS LAST
LIMIT 300
"""
).bindparams(id=doc_id)
)
).mappings().all()
prev = nxt = None
if _parent_id is not None and _clause_order is not None:
prow = (
await session.execute(
sql_text(
"""
SELECT id, clause_code, clause_part, clause_order, title FROM documents
WHERE parent_id = :pid AND doc_kind='clause' AND deleted_at IS NULL
AND clause_order < :ord
ORDER BY clause_order DESC LIMIT 1
"""
).bindparams(pid=_parent_id, ord=_clause_order)
)
).mappings().first()
nrow = (
await session.execute(
sql_text(
"""
SELECT id, clause_code, clause_part, clause_order, title FROM documents
WHERE parent_id = :pid AND doc_kind='clause' AND deleted_at IS NULL
AND clause_order > :ord
ORDER BY clause_order ASC LIMIT 1
"""
).bindparams(pid=_parent_id, ord=_clause_order)
)
).mappings().first()
prev = ClauseTocItem(**dict(prow)) if prow else None
nxt = ClauseTocItem(**dict(nrow)) if nrow else None
return BacklinksResponse(
doc_id=doc_id,
clause_code=_clause_code,
parent_id=_parent_id,
prev=prev,
next=nxt,
forward=[BacklinkRef(**dict(r)) for r in forward],
back=[BacklinkRef(**dict(r)) for r in back],
)
# ─── 관련 문서 (유사도, on-demand pgvector KNN — 저부하·무저장) ───
class RelatedItem(BaseModel):
id: int
title: str | None = None
ai_domain: str | None = None
material_type: str | None = None
year: int | None = None
sim: float | None = None
class RelatedResponse(BaseModel):
doc_id: int
related: list[RelatedItem]
@router.get("/{doc_id}/related", response_model=RelatedResponse)
async def get_related_documents(
doc_id: int,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
limit: int = 8,
same_type: bool = True,
):
"""문서-레벨 임베딩 코사인 최근접 = '관련 문서'. on-demand(저장/배치 없음).
인용그래프가 부적합한 코퍼스(업계 기술기사=인용망 부재)의 대안 연결 레이어.
same_type=true면 같은 material_type 내, false면 전 코퍼스. doc_kind='clause'(절-문서)는 제외.
"""
from sqlalchemy import text as sql_text
lim = max(1, min(limit, 30))
type_clause = "AND d.material_type = src.material_type" if same_type else ""
rows = (
await session.execute(
sql_text(
f"""
WITH src AS (
SELECT embedding, material_type FROM documents WHERE id = :id
)
SELECT d.id, d.title, d.ai_domain, d.material_type, d.facet_year AS year,
round((1 - (d.embedding <=> (SELECT embedding FROM src)))::numeric, 3) AS sim
FROM documents d, src
WHERE d.doc_kind = 'standard' AND d.deleted_at IS NULL
AND d.id <> :id AND d.embedding IS NOT NULL
AND (SELECT embedding FROM src) IS NOT NULL
{type_clause}
ORDER BY d.embedding <=> (SELECT embedding FROM src)
LIMIT :lim
"""
).bindparams(id=doc_id, lim=lim)
)
).mappings().all()
return RelatedResponse(
doc_id=doc_id,
related=[RelatedItem(**{k: r[k] for k in ("id", "title", "ai_domain", "material_type", "year")}, sim=float(r["sim"]) if r["sim"] is not None else None) for r in rows],
)
-255
View File
@@ -1,255 +0,0 @@
"""뷰어 write-back ingest (study-to-viewer P2) — 뷰어 로컬 풀이 세션을 DS 로 흘려 finalize 재생.
흐름(plan study-to-viewer-slice1 P2, r2/r3 불변식):
뷰어 outbox → POST /ingest/study/attempts (Bearer VIEWER_SYNC_TOKEN, study_ingest_enabled gate)
→ pub_id→published.source_id→StudyQuestion 해소(부재 graceful skip) → principal=question.user_id
→ topic 별 그룹(뷰어 subject 퀴즈가 여러 DS topic 걸칠 수 있음) → topic 마다 DS quiz_session
(source='viewer', client_session_uuid) 생성 + attempt(derive_outcome=채점 단일 소스) + 세션 done
→ finalize_session **무수정 재생**(SR/pattern/progress + 4-A/4-B enqueue) → finalized_at 마커
→ 전부 1 트랜잭션(원자) 후 commit.
멱등(r2 P2-2): client_session_uuid 로 기존 세션 있으면 이미 적재된 것 → 캐시 요약 반환(재실행 0).
원자 1-tx 라 'uuid 존재 ⟺ finalize 완료' → at-least-once outbox 재전송에도 SR 이중 advance 없음.
user_id 리터럴 금지(r2): principal = 해소된 질문의 owner(단일, mixed 면 거부).
"""
from __future__ import annotations
import hmac
import logging
from collections import defaultdict
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, Header, HTTPException
from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from core.config import settings
from core.database import async_session
from models.published import Published
from models.study_question import StudyQuestion, StudyQuestionAttempt
from models.study_quiz_session import StudyQuizSession
from services.study.outcome import derive_outcome
from services.study.publish_projection import KIND_QUESTION
from services.study.session_finalize import finalize_session
logger = logging.getLogger(__name__)
router = APIRouter()
def _verify_token(authorization: str | None = Header(default=None)) -> None:
"""뷰어↔DS 발행 채널 Bearer(read 와 동일 토큰, r3 단일토큰 수용). default-deny(미설정=503)."""
if not settings.viewer_sync_token:
raise HTTPException(status_code=503, detail="viewer_sync_token not configured")
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="missing Bearer token")
token = authorization[7:].strip()
if not hmac.compare_digest(token, settings.viewer_sync_token):
raise HTTPException(status_code=403, detail="invalid token")
async def _session() -> AsyncSession:
async with async_session() as s:
yield s
class IngestAttempt(BaseModel):
question_pub_id: str
selected_choice: int | None = None
is_unsure: bool = False
answered_at: str | None = None # 클라(오프라인) ISO 시각 — 미래 스큐 클램프, id 가 타이브레이커
class IngestBody(BaseModel):
client_session_uuid: str
attempts: list[IngestAttempt]
def _already_ingested(rows) -> dict:
"""이미 적재된 세션들의 캐시 요약(멱등 응답). 최초 멱등체크 + 동시경합 흡수 양쪽에서 사용."""
return {
"status": "already_ingested",
"sessions": [
{
"topic_id": s.study_topic_id,
"correct": s.correct_count,
"wrong": s.wrong_count,
"unsure": s.unsure_count,
}
for s in rows
],
}
def _parse_answered_at(s: str | None, now: datetime) -> datetime:
if not s:
return now
try:
dt = datetime.fromisoformat(s.replace("Z", "+00:00"))
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return min(dt, now) # 미래 스큐는 now 로 클램프(클라 시계 오염 방지)
except Exception:
return now
@router.post("/attempts")
async def ingest_attempts(
body: IngestBody,
_auth: None = Depends(_verify_token),
session: AsyncSession = Depends(_session),
):
if not settings.study_ingest_enabled:
raise HTTPException(status_code=503, detail="study_ingest not enabled")
if not body.client_session_uuid or not body.attempts:
raise HTTPException(status_code=400, detail="client_session_uuid 와 attempts 필요")
# 멱등: 이 uuid 로 이미 적재됐나(원자 1-tx 라 존재=완료). 있으면 캐시 요약 반환(재실행 0).
existing = (
await session.execute(
select(StudyQuizSession).where(
StudyQuizSession.client_session_uuid == body.client_session_uuid
)
)
).scalars().all()
if existing:
return _already_ingested(existing)
# pub_id → source_id(내부 질문 id) 해소. deleted tombstone 제외.
pub_ids = list({a.question_pub_id for a in body.attempts})
pub_rows = (
await session.execute(
select(Published.pub_id, Published.source_id).where(
Published.kind == KIND_QUESTION,
Published.pub_id.in_(pub_ids),
Published.deleted.is_(False),
)
)
).all()
src_by_pubid = {r.pub_id: r.source_id for r in pub_rows}
# 질문 fetch(미삭제). principal = owner(단일).
source_ids = list(set(src_by_pubid.values()))
q_rows = (
await session.execute(
select(StudyQuestion).where(
StudyQuestion.id.in_(source_ids), StudyQuestion.deleted_at.is_(None)
)
)
).scalars().all()
q_by_id = {q.id: q for q in q_rows}
owners = {q.user_id for q in q_by_id.values()}
if len(owners) > 1:
raise HTTPException(status_code=400, detail="여러 사용자 소유 질문 혼재 — 단일 principal 위반")
if not owners:
raise HTTPException(status_code=404, detail="해소 가능한 질문 없음")
user_id = owners.pop()
now = datetime.now(timezone.utc)
# topic 별 그룹(해소 실패 attempt 는 graceful skip). 같은 (uuid, topic) 1 세션.
by_topic: dict[int, list[tuple[IngestAttempt, StudyQuestion]]] = defaultdict(list)
skipped: list[str] = []
for a in body.attempts:
src = src_by_pubid.get(a.question_pub_id)
q = q_by_id.get(src) if src is not None else None
if q is None:
skipped.append(a.question_pub_id)
continue
by_topic[q.study_topic_id].append((a, q))
if not by_topic:
raise HTTPException(status_code=404, detail="해소된 attempt 없음")
try:
summaries = []
for topic_id, items in by_topic.items():
qids = [q.id for (_, q) in items]
qs = StudyQuizSession(
user_id=user_id,
study_topic_id=topic_id,
question_ids=qids,
subject_distribution={},
status="done",
cursor=len(qids),
source="viewer",
client_session_uuid=body.client_session_uuid,
finished_at=now,
created_at=now,
updated_at=now,
)
session.add(qs)
await session.flush() # qs.id
c = w = u = 0
for a, q in items:
try:
sel, is_corr, outcome = derive_outcome(a.selected_choice, a.is_unsure, q.correct_choice)
except ValueError:
skipped.append(a.question_pub_id) # 선택 없고 unsure 아님 = 무효 → skip
continue
if outcome == "correct":
c += 1
elif outcome == "wrong":
w += 1
elif outcome == "unsure":
u += 1
session.add(
StudyQuestionAttempt(
user_id=user_id,
study_question_id=q.id,
study_topic_id=topic_id,
selected_choice=sel,
correct_choice=q.correct_choice,
is_correct=is_corr,
outcome=outcome,
quiz_session_id=qs.id,
answered_at=_parse_answered_at(a.answered_at, now),
)
)
qs.correct_count, qs.wrong_count, qs.unsure_count = c, w, u
await session.flush()
# finalize 무수정 재생(progress/SR/pattern + 4-A/4-B enqueue). 그 후 멱등 마커.
summary = await finalize_session(
session, user_id=user_id, study_topic_id=topic_id, quiz_session_id=qs.id
)
qs.finalized_at = now
summaries.append(
{
"topic_id": topic_id,
"quiz_session_id": qs.id,
"correct": summary.correct,
"wrong": summary.wrong,
"unsure": summary.unsure,
"newly_correct": summary.newly_correct,
"relapsed": summary.relapsed,
"recovered": summary.recovered,
}
)
await session.commit()
except IntegrityError:
# 동시 같은 client_session_uuid 경합 — 상대가 먼저 commit → (client_session_uuid,
# study_topic_id) uq(mig376) 위반. 데이터는 안전(원자 1-tx 전체 롤백 → SR 이중 advance
# 없음). 승자 결과로 graceful 수렴(500 대신 already_ingested). uuid 경합이 아닌 진짜
# 무결성 오류면 재조회가 비어 → re-raise 로 표면화.
await session.rollback()
winner = (
await session.execute(
select(StudyQuizSession).where(
StudyQuizSession.client_session_uuid == body.client_session_uuid
)
)
).scalars().all()
if not winner:
raise
logger.info("study_ingest uuid=%s 동시경합 흡수 → already_ingested", body.client_session_uuid)
return _already_ingested(winner)
logger.info(
"study_ingest uuid=%s user=%s sessions=%s skipped=%s",
body.client_session_uuid, user_id, len(summaries), len(skipped),
)
return {"status": "ingested", "skipped": skipped, "sessions": summaries}
+5 -5
View File
@@ -9,7 +9,7 @@ from sqlalchemy import func, select
from sqlalchemy import text as sql_text
from sqlalchemy.ext.asyncio import AsyncSession
from core.auth import get_current_user, require_admin
from core.auth import get_current_user
from core.database import get_session
from core.library import LIBRARY_PREFIX, MAX_DEPTH, normalize_library_path
from models.category import LibraryCategory
@@ -78,7 +78,7 @@ async def list_categories(
@router.post("/categories", response_model=CategoryResponse, status_code=201)
async def create_category(
body: CategoryCreate,
user: Annotated[User, Depends(require_admin)],
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""카테고리 생성 (조상 자동 생성 포함)"""
@@ -133,7 +133,7 @@ async def create_category(
@router.patch("/categories", response_model=CategoryResponse)
async def rename_category(
body: CategoryRename,
user: Annotated[User, Depends(require_admin)],
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""카테고리 이름 변경 (leaf only, path 기반 식별)"""
@@ -214,7 +214,7 @@ async def rename_category(
@router.delete("/categories", status_code=204)
async def delete_category(
path: str = Query(..., description="삭제할 카테고리 경로"),
user: Annotated[User, Depends(require_admin)] = None,
user: Annotated[User, Depends(get_current_user)] = None,
session: Annotated[AsyncSession, Depends(get_session)] = None,
):
"""카테고리 삭제 (leaf only, 문서 없는 경우만)"""
@@ -410,7 +410,7 @@ async def get_facet_values(
@router.post("/facets", response_model=FacetValueResponse, status_code=201)
async def add_facet_value(
body: FacetValueResponse,
user: Annotated[User, Depends(require_admin)],
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""facet 사전에 새 값 추가"""
-254
View File
@@ -1,254 +0,0 @@
"""발행 read API (docsrv-viewer-publish P0-2) — 뷰어가 pull-sync 로 당기는 feed.
published 테이블(발행 워커가 rev 커밋순 gapless 부여)을 rev 커서로 페이지네이션해 반환.
뷰어 = Bearer(settings.viewer_sync_token) 인증, default-deny. read-only(SELECT 만).
GET /published/feed?since={rev}&kind={kind}&limit={n}
rev > since 행을 rev ASC 로 limit 만큼. kind 옵션(study_question|study_explanation|... 후속).
tombstone(deleted=true)도 1급 이벤트로 포함 — 뷰어가 pub_id 로 로컬 삭제(stale 회피).
rev 커서 안전성: 워커가 pg_advisory_xact_lock 단일 라이터로 배치 rev 를 한 트랜잭션에
부여·커밋 → 리더는 rev N 을 N-1 없이 보지 못함(부분가시 0). 뷰어는 next_since 로 반복.
엔벨로프 schema_version = 전송 계약 버전(payload 행별 schema_version 과 별개).
미지원 버전 가시거부는 뷰어 책임(no-silent-fallback) — 여기선 행별 schema_version 그대로 전달.
"""
from __future__ import annotations
import hmac
import logging
import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, Header, HTTPException, Query
from pydantic import BaseModel
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from core.config import settings
from core.database import async_session
from models.published import Published
from models.published import Published
from services.queue_overview import build_overview
logger = logging.getLogger(__name__)
router = APIRouter()
# feed 엔벨로프(전송 계약) 버전 — payload schema_version 과 독립.
FEED_SCHEMA_VERSION = 1
DEFAULT_LIMIT = 200
MAX_LIMIT = 500
def _verify_token(authorization: str | None = Header(default=None)) -> None:
"""뷰어↔DS 발행 채널 Bearer 인증. default-deny(미설정=503). 상수시간 비교(internal_study 정본).
이 토큰은 정답 포함 study payload 를 노출하므로 hmac.compare_digest 로 timing side-channel 차단.
"""
if not settings.viewer_sync_token:
raise HTTPException(status_code=503, detail="viewer_sync_token not configured")
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="missing Bearer token")
token = authorization[7:].strip()
if not hmac.compare_digest(token, settings.viewer_sync_token):
raise HTTPException(status_code=403, detail="invalid token")
async def _session() -> AsyncSession:
async with async_session() as s:
yield s
class FeedItem(BaseModel):
pub_id: str # opaque+stable = 뷰어 dedup키 = progress키
kind: str
source_id: int # DS 내부 소스 행 id (ingest write-back 역해소용, P2)
rev: int
deleted: bool # tombstone — 뷰어 로컬 삭제 트리거
schema_version: int # payload 모양 버전(뷰어 range 수용)
payload: dict # render-ready projection (tombstone 이면 {})
class FeedResponse(BaseModel):
schema_version: int # 엔벨로프(전송 계약) 버전
items: list[FeedItem]
next_since: int # 다음 호출 since (이 배치 max rev; 빈 배치면 입력 since 유지)
has_more: bool # limit 가득 = 더 있을 수 있음(뷰어 반복)
@router.get("/feed", response_model=FeedResponse)
async def published_feed(
since: int = Query(0, ge=0),
kind: str | None = Query(None, max_length=40),
limit: int = Query(DEFAULT_LIMIT, ge=1, le=MAX_LIMIT),
_auth: None = Depends(_verify_token),
session: AsyncSession = Depends(_session),
):
"""rev > since 행을 rev ASC 로 limit 만큼 반환. 뷰어가 next_since 로 incremental pull."""
stmt = select(Published).where(Published.rev > since)
if kind:
stmt = stmt.where(Published.kind == kind)
stmt = stmt.order_by(Published.rev.asc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
items = [
FeedItem(
pub_id=r.pub_id,
kind=r.kind,
source_id=r.source_id,
rev=r.rev,
deleted=r.deleted,
schema_version=r.schema_version,
payload=r.payload if r.payload is not None else {},
)
for r in rows
]
next_since = items[-1].rev if items else since
has_more = len(rows) == limit
logger.info(
"published_feed since=%s kind=%s returned=%s next_since=%s has_more=%s",
since, kind, len(items), next_since, has_more,
)
return FeedResponse(
schema_version=FEED_SCHEMA_VERSION,
items=items,
next_since=next_since,
has_more=has_more,
)
# ── P1-1: 뉴스/다이제스트 발행 read API (docsrv-viewer-publish) ────────────────────
# global_digests(일간 컨테이너) + digest_topics(토픽 N, digest_id FK) -> render-ready
# read-time projection. content-type 파라미터화(plan r2): version 커서=global_digests.id
# (일간 단일 라이터라 gapless 불요·gap 무해) · pub_id=date-as-id(admin-gated feed 라 opacity
# 불필요) · tombstone 없음(다이제스트 미삭제). 엔벨로프는 /feed 와 동일(FeedResponse)=뷰어 재사용.
# scaffold-first: DIGEST_PUBLISH_ENABLED off(기본)=503(명시적 미가동, no-silent).
DIGEST_PAYLOAD_SCHEMA_VERSION = 1
@router.get("/digest", response_model=FeedResponse)
async def published_digest(
since: int = Query(0, ge=0),
limit: int = Query(DEFAULT_LIMIT, ge=1, le=MAX_LIMIT),
_auth: None = Depends(_verify_token),
session: AsyncSession = Depends(_session),
):
"""global_digests.id > since 를 id ASC 로 limit 만큼. 각 digest 에 topics 조인해 render-ready 반환."""
if not settings.digest_publish_enabled:
raise HTTPException(status_code=503, detail="digest publish not enabled (scaffold)")
drows = (await session.execute(
text(
"SELECT id, digest_date, status, total_articles, total_topics, total_countries, created_at "
"FROM global_digests WHERE id > :since ORDER BY id ASC LIMIT :limit"
),
{"since": since, "limit": limit},
)).mappings().all()
if not drows:
return FeedResponse(schema_version=FEED_SCHEMA_VERSION, items=[], next_since=since, has_more=False)
ids = [r["id"] for r in drows]
trows = (await session.execute(
text(
"SELECT digest_id, topic_rank, topic_label, summary, country, article_count, importance_score "
"FROM digest_topics WHERE digest_id = ANY(:ids) ORDER BY digest_id ASC, topic_rank ASC"
),
{"ids": ids},
)).mappings().all()
topics_by_digest: dict[int, list[dict]] = {}
for t in trows:
topics_by_digest.setdefault(t["digest_id"], []).append({
"rank": t["topic_rank"],
"label": t["topic_label"],
"summary": t["summary"],
"country": t["country"],
"article_count": t["article_count"],
"importance": t["importance_score"],
})
items = []
for r in drows:
d_date = r["digest_date"].isoformat() if r["digest_date"] else None
items.append(FeedItem(
pub_id=f"digest:{d_date}",
kind="digest",
source_id=r["id"],
rev=r["id"],
deleted=False,
schema_version=DIGEST_PAYLOAD_SCHEMA_VERSION,
payload={
"digest_date": d_date,
"status": r["status"],
"total_articles": r["total_articles"],
"total_topics": r["total_topics"],
"total_countries": r["total_countries"],
"generated_at": r["created_at"].isoformat() if r["created_at"] else None,
"topics": topics_by_digest.get(r["id"], []),
},
))
next_since = items[-1].rev
has_more = len(drows) == limit
logger.info(
"published_digest since=%s returned=%s next_since=%s has_more=%s",
since, len(items), next_since, has_more,
)
return FeedResponse(
schema_version=FEED_SCHEMA_VERSION,
items=items,
next_since=next_since,
has_more=has_more,
)
# ── P1-2: 가공현황 라이브 스냅샷 API (+P1-4 점검 플래그) ──────────────────────────
# 뷰어 리포트 '문서 가공현황' 섹션용. build_overview(기존 서비스) 재사용 + source_health
# 조인 요약. pull-through(저장 X) — 라이브 수치라 캐시 없음, 소비자(뷰어)가 2~3s timeout 책임
# (plan P1-2). P1-4: maintenance 플래그 동봉 — 소프트락/점검이 워커를 멈춰 수치가 정체로
# 보일 때 뷰어가 '점검·실험 중' 배너로 구분(표면 != 데이터). read-only.
@router.get("/processing-status")
async def published_processing_status(
_auth: None = Depends(_verify_token),
session: AsyncSession = Depends(_session),
):
"""가공현황 스냅샷: queue overview + source_health 요약 + maintenance 플래그."""
overview = await build_overview(session)
sh_rows = (await session.execute(text(
"SELECT ns.name, ns.category, sh.circuit_state, sh.consecutive_failures, sh.empty_streak, "
"sh.last_success_at, sh.last_probe_ok "
"FROM source_health sh JOIN news_sources ns ON ns.id = sh.source_id "
"ORDER BY (sh.circuit_state <> 'closed') DESC, sh.consecutive_failures DESC"
))).mappings().all()
by_state: dict[str, int] = {}
problems: list[dict] = []
for r in sh_rows:
st = r["circuit_state"]
by_state[st] = by_state.get(st, 0) + 1
if st != "closed":
problems.append({
"name": r["name"],
"category": r["category"],
"circuit_state": st,
"consecutive_failures": r["consecutive_failures"],
"empty_streak": r["empty_streak"],
"last_success_at": r["last_success_at"].isoformat() if r["last_success_at"] else None,
"last_probe_ok": r["last_probe_ok"],
})
return {
"schema_version": 1,
"generated_at": datetime.now(timezone.utc).isoformat(),
"overview": overview,
"sources": {
"total": len(sh_rows),
"by_circuit_state": by_state,
"problems": problems,
},
"maintenance": {
"active": settings.maintenance_mode,
"note": settings.maintenance_note,
},
}
+853 -12
View File
@@ -3,28 +3,42 @@
실제 검색 파이프라인(retrieval → fusion → rerank → diversity → confidence)
은 `services/search/search_pipeline.py::run_search()` 로 분리되어 있다.
이 파일은 다음만 담당:
- Pydantic 스키마 (SearchResult / SearchResponse / SearchDebug / DebugCandidate)
- Pydantic 스키마 (SearchResult / SearchResponse / SearchDebug / DebugCandidate
/ Citation / AskResponse / AskDebug)
- `/search` endpoint wrapper (run_search 호출 + logger + telemetry + 직렬화)
- `/ask` endpoint wrapper (Phase 3.3 에서 추가)
"""
import asyncio
import hmac
import time
from datetime import date
from typing import Annotated
from typing import Annotated, Literal
from fastapi import APIRouter, BackgroundTasks, Depends, Query
from fastapi import APIRouter, BackgroundTasks, Depends, Header, Query
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from core.auth import get_current_user, get_egress_class
from core.auth import get_current_user
from core.config import settings
from core.database import get_session
from core.utils import setup_logger
from models.user import User
from services.document_telemetry import sanitize_source
from services.search.classifier_service import ClassifierResult, classify
from services.search.evidence_service import EvidenceItem, extract_evidence
from services.search.fusion_service import DEFAULT_FUSION
from services.search.grounding_check import check as grounding_check
from services.search.refusal_gate import RefusalDecision, decide as refusal_decide
from services.search import query_rewriter
from services.search.retrieval_service import AxisFilter
from services.search.result_decorate import compute_facets, decorate_version_status
from services.search.search_pipeline import PipelineResult, run_search
from services.search_telemetry import record_search_event
from services.search.synthesis_service import SynthesisResult, synthesize
from services.search.verifier_service import VerifierResult, verify
from services.prompt_versions import ASK_PROMPT_VERSION, resolve_primary_model
from services.search_telemetry import record_ask_event, record_search_event
# logs/search.log + stdout 동시 출력 (Phase 0.4)
logger = setup_logger("search")
@@ -139,7 +153,6 @@ def _build_search_debug(pr: PipelineResult) -> SearchDebug:
async def search(
q: str,
user: Annotated[User, Depends(get_current_user)],
egress_class: Annotated[str, Depends(get_egress_class)],
session: Annotated[AsyncSession, Depends(get_session)],
background_tasks: BackgroundTasks,
mode: str = Query("hybrid", pattern="^(fts|trgm|vector|hybrid)$"),
@@ -212,8 +225,6 @@ async def search(
None, description="안전 자료실 C-1: 관할 필터 (KR/US/EU/JP/GB/INT)"),
year_from: int | None = Query(None, ge=1900, le=2100, description="published_date 연도 하한 (NULL=created_at fallback)"),
year_to: int | None = Query(None, ge=1900, le=2100, description="published_date 연도 상한"),
domain_bucket: str | None = Query(None, description="377: domain_bucket 스코프 CSV (Safety,Engineering,Law,Philosophy,Programming,General,News). domain_bucket = ANY"),
exclude_bucket: str | None = Query(None, description="377: domain_bucket 제외 CSV (예: News). 지식질의 시 News 기본제외용"),
facets: bool = Query(False, description="안전 자료실 C-1 후속: top-K 결과 분류 축 분포(material_type/jurisdiction/version_status)를 응답 facets 에 집계. 미지정=계산/노출 0"),
):
"""문서 검색 — FTS + ILIKE + 벡터 결합 (Phase 3.1 이후 run_search wrapper)"""
@@ -224,9 +235,6 @@ async def search(
jurisdiction=jurisdiction,
year_from=year_from,
year_to=year_to,
domain_buckets=[b.strip() for b in domain_bucket.split(",") if b.strip()] if domain_bucket else None,
exclude_buckets=[b.strip() for b in exclude_bucket.split(",") if b.strip()] if exclude_bucket else None,
cloud_egress=(egress_class == "cloud"),
)
pr = await run_search(
session,
@@ -274,7 +282,7 @@ async def search(
content={
"error_reason": "unknown_reranker_backend",
"backend_requested": reranker_backend,
"allowed": ["baseline", "cand_gte_ml_base"],
"allowed": ["baseline"],
"detail": msg,
},
)
@@ -346,3 +354,836 @@ async def search(
debug=debug_obj,
facets=facets_obj,
)
# ═══════════════════════════════════════════════════════════
# Phase 3.3: /api/search/ask — Evidence + Grounded Synthesis
# ═══════════════════════════════════════════════════════════
class Citation(BaseModel):
"""answer 본문의 [n] 에 해당하는 근거 단일 행."""
n: int
chunk_id: int | None
doc_id: int
title: str | None
section_title: str | None
span_text: str # evidence LLM 이 추출한 50~300자
full_snippet: str # 원본 800자 (citation 원문 보기 전용)
relevance: float
rerank_score: float
class ConfirmedItem(BaseModel):
"""Partial answer 의 개별 aspect 답변."""
aspect: str
text: str
citations: list[int]
class AskDebug(BaseModel):
"""`/ask?debug=true` 응답 확장."""
timing_ms: dict[str, float]
search_notes: list[str]
query_analysis: dict | None = None
confidence_signal: float
evidence_candidate_count: int
evidence_kept_count: int
evidence_skip_reason: str | None
synthesis_cache_hit: bool
synthesis_prompt_preview: str | None = None
synthesis_raw_preview: str | None = None
hallucination_flags: list[str] = []
# Phase 3.5a: per-layer defense 로깅
defense_layers: dict | None = None
class AskResponse(BaseModel):
"""`/ask` 응답. Phase 3.5a: completeness + aspects 추가."""
results: list[SearchResult]
ai_answer: str | None
citations: list[Citation]
synthesis_status: Literal[
"completed", "timeout", "skipped", "no_evidence", "parse_failed", "llm_error",
# PR-MacBook-RAG-Backend-1: 200 응답에는 등장하지 않음 (해당 status 는 503 분기).
# Literal 호환성 위해 포함.
"backend_unavailable",
]
synthesis_ms: float
confidence: Literal["high", "medium", "low"] | None
refused: bool
no_results_reason: str | None
query: str
total: int
# Phase 3.5a
completeness: Literal["full", "partial", "insufficient"] = "full"
covered_aspects: list[str] | None = None
missing_aspects: list[str] | None = None
confirmed_items: list[ConfirmedItem] | None = None
# PR-MacBook-RAG-Backend-1: backend dispatcher metadata.
# backend 미지정 호출은 둘 다 None 으로 유지 (기존 호출자 호환 — Hermes docsrv_ask /
# voice-memo-bot 응답 형식 변동 0). 명시 opt-in 시만 채워짐.
backend_requested: str | None = None
backend_used: str | None = None
debug: AskDebug | None = None
def _map_no_results_reason(
pr: PipelineResult,
evidence: list[EvidenceItem],
ev_skip: str | None,
sr: SynthesisResult,
) -> str | None:
"""사용자에게 보여줄 한국어 메시지 매핑.
Failure mode 표 (plan §Failure Modes) 기반.
"""
# LLM 자가 refused → 모델이 준 사유 그대로
if sr.refused and sr.refuse_reason:
return sr.refuse_reason
# synthesis 상태 우선
if sr.status == "no_evidence":
if not pr.results:
return "검색 결과가 없습니다."
return "관련도 높은 근거를 찾지 못했습니다."
if sr.status == "skipped":
return "검색 결과가 없습니다."
if sr.status == "timeout":
return "답변 생성이 지연되어 생략했습니다. 검색 결과를 확인해 주세요."
if sr.status == "parse_failed":
return "답변 형식 오류로 생략했습니다."
if sr.status == "llm_error":
return "AI 서버에 일시적 문제가 있습니다."
# evidence 단계 실패는 fallback 을 탔더라도 notes 용
if ev_skip == "all_low_rerank":
return "관련도 높은 근거를 찾지 못했습니다."
if ev_skip == "empty_retrieval":
return "검색 결과가 없습니다."
return None
def _build_citations(
evidence: list[EvidenceItem], used_citations: list[int]
) -> list[Citation]:
"""answer 본문에 실제로 등장한 n 만 Citation 으로 변환."""
by_n = {e.n: e for e in evidence}
out: list[Citation] = []
for n in used_citations:
e = by_n.get(n)
if e is None:
continue
out.append(
Citation(
n=e.n,
chunk_id=e.chunk_id,
doc_id=e.doc_id,
title=e.title,
section_title=e.section_title,
span_text=e.span_text,
full_snippet=e.full_snippet,
relevance=e.relevance,
rerank_score=e.rerank_score,
)
)
return out
def _build_ask_debug(
pr: PipelineResult,
evidence: list[EvidenceItem],
ev_skip: str | None,
sr: SynthesisResult,
ev_ms: float,
synth_ms: float,
total_ms: float,
) -> AskDebug:
timing: dict[str, float] = dict(pr.timing_ms)
timing["evidence_ms"] = ev_ms
timing["synthesis_ms"] = synth_ms
timing["ask_total_ms"] = total_ms
# candidate count 는 rule filter 통과한 수 (recomputable from results)
# 엄밀히는 evidence_service 내부 숫자인데, evidence 길이 ≈ kept, candidate
# 는 관측이 어려움 → kept 는 evidence 길이, candidate 는 별도 필드 없음.
# 단순화: candidate_count = len(evidence) 를 상한 근사로 둠 (debug 전용).
return AskDebug(
timing_ms=timing,
search_notes=pr.notes,
query_analysis=pr.query_analysis,
confidence_signal=pr.confidence_signal,
evidence_candidate_count=len(evidence),
evidence_kept_count=len(evidence),
evidence_skip_reason=ev_skip,
synthesis_cache_hit=sr.cache_hit,
synthesis_prompt_preview=None, # 현재 synthesis_service 에서 노출 안 함
synthesis_raw_preview=sr.raw_preview,
hallucination_flags=sr.hallucination_flags,
)
def _detect_synthesis_failure(sr: SynthesisResult) -> str | None:
"""Synthesis 가 유효한 답을 못 냈으면 re_gate 라벨, 아니면 None.
판정 우선순위 (Phase 3.5 fix3):
1) sr.refused → LLM self-refuse (status="completed") 또는 mechanical fail 후 refused 전파
- status=="completed" + refused=True → "synthesis_self_refuse"
- 그 외 → f"synthesis_failed({status})"
2) sr.status ∈ {timeout, parse_failed, llm_error} → f"synthesis_failed({status})"
3) answer 공백 → f"synthesis_failed({status})"
4) 유효 → None
"""
if sr.refused:
if sr.status == "completed":
return "synthesis_self_refuse"
return f"synthesis_failed({sr.status})"
if sr.status in ("timeout", "parse_failed", "llm_error"):
return f"synthesis_failed({sr.status})"
if not (sr.answer or "").strip():
return f"synthesis_failed({sr.status})"
return None
def _resolve_eval_identity(
x_source: str | None,
x_eval_case_id: str | None,
x_eval_token: str | None,
) -> tuple[str, str | None]:
"""X-Source/X-Eval-Case-Id 신뢰 검증 (Phase 3.5 fix2).
규칙:
- 기본값: source='document_server', eval_case_id=None
- X-Source=eval 또는 X-Eval-Case-Id 가 들어왔다면 eval claim 으로 간주
- eval claim 은 X-Eval-Token == settings.eval_runner_token 일 때만 수용
(constant-time compare, env 미설정 시 항상 거부)
- 거부 시: 헤더 무시 + warning log + source=sanitize(non-eval) / eval_case_id=None
- 통과 시: source='eval', eval_case_id=x_eval_case_id
반환: (source, eval_case_id)
"""
claimed_source = sanitize_source(x_source)
is_eval_claim = (claimed_source == "eval") or bool(x_eval_case_id)
if not is_eval_claim:
# 일반 호출 — eval_case_id 강제 None (source != 'eval' 이면 case_id 의미 없음)
return claimed_source, None
# eval claim — token 검증
expected = settings.eval_runner_token
presented = x_eval_token or ""
token_valid = bool(expected) and hmac.compare_digest(presented, expected)
if not token_valid:
logger.warning(
"eval header rejected: source=%s case_id=%s token_present=%s expected_set=%s",
x_source, x_eval_case_id, bool(x_eval_token), bool(expected),
)
# 일반 호출로 강등 — source='eval' 주장은 무시, case_id 도 무시
# claimed_source 가 'eval' 이면 default 'document_server' 로
if claimed_source == "eval":
return "document_server", None
return claimed_source, None
# token OK — eval 라벨 수용
return "eval", x_eval_case_id
@router.get("/ask", response_model=AskResponse)
async def ask(
q: str,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
background_tasks: BackgroundTasks,
limit: int = Query(10, ge=1, le=20, description="synthesis 입력 상한"),
debug: bool = Query(False, description="evidence/synthesis 중간 상태 노출"),
backend: Annotated[
str | None,
Query(
pattern="^(qwen-macbook|gemma-macmini|mac-mini-default|claude-cloud|auto)$",
description=(
"PR-2 of DS AI routing policy (2026-05-23) — 명시 backend opt-in via llm-router. "
"미지정 = mac-mini-default (gemma-macmini alias, default). "
"'mac-mini-default' = router 가 tier_b (Mac mini gemma-4-26b). "
"'qwen-macbook' = router 가 named upstream (M5 Max Qwen 3.6 27B). "
"'claude-cloud' = router 가 503 provider_not_configured (활성화 별 PR). "
"'auto' = router 의 rule + LLM triage. "
"backend unavailable 시 503 + error_reason=macbook_unavailable / router_* "
"(자동 fallback 없음 — 다시 호출하거나 backend 인자 제거 후 재시도)."
),
),
] = None,
corpus_variant: str | None = Query(
None,
pattern=r"^(prehier|hier_sim_raw|hier_sim_clean)$",
description=(
"⚠️ EVAL-ONLY (Hier-PassageRAG-Diagnose-1). evidence retrieval 의 chunk leg 를 측정 뷰로 "
"교체 — prehier(legacy) | hier_sim_raw | hier_sim_clean. 운영 UI 미사용. "
"미지정 = production corpus_chunks (기존 /ask 동작 동일)."
),
),
exact_knn: bool = Query(
False,
description=(
"⚠️ EVAL-ONLY (Hier-PassageRAG-Diagnose-1). vector leg exact KNN (ivfflat 근사 제거). "
"passage 변종 공정 비교용. 운영 미사용. 미지정(false) = 기존 /ask 동작 동일."
),
),
x_source: Annotated[str | None, Header(alias="X-Source")] = None,
x_eval_case_id: Annotated[str | None, Header(alias="X-Eval-Case-Id")] = None,
x_eval_token: Annotated[str | None, Header(alias="X-Eval-Token")] = None,
):
"""근거 기반 AI 답변 (Phase 3.5a).
Phase 3.3 기반 + classifier parallel + refusal gate + grounding re-gate.
실패 경로에서도 `results` 는 항상 반환.
Phase 3.5 calibration trust boundary (fix2):
- X-Source / X-Eval-Case-Id 는 X-Eval-Token 이 EVAL_RUNNER_TOKEN 와 일치하는
trusted internal eval runner 에서만 수용된다.
- 일반 client 의 X-Source=eval 시도는 무시되고 source='document_server' 로 강제.
- source != 'eval' 이면 eval_case_id 항상 None.
"""
t_total = time.perf_counter()
defense_log: dict = {} # per-layer flag snapshot
source, eval_case_id = _resolve_eval_identity(x_source, x_eval_case_id, x_eval_token)
# 1. 검색 파이프라인 (corpus_variant/exact_knn = EVAL-ONLY, 미지정 시 기존 동작 동일)
pr = await run_search(
session, q, mode="hybrid", limit=limit,
fusion=DEFAULT_FUSION, rerank=True, analyze=True,
corpus_variant=corpus_variant, exact_knn=exact_knn,
)
# 1.5. ask_includable=false 문서를 evidence 입력에서 제외
# 검색 결과 자체는 유지 (사용자에게 보여줌), evidence만 필터
if pr.results:
from sqlalchemy import select as sa_select
from models.document import Document as DocModel
ask_doc_ids = set()
excluded_ids = {r.id for r in pr.results}
rows = await session.execute(
sa_select(DocModel.id, DocModel.ask_includable).where(
DocModel.id.in_(excluded_ids)
)
)
for doc_id, includable in rows:
if includable is False:
ask_doc_ids.add(doc_id)
evidence_results = [r for r in pr.results if r.id not in ask_doc_ids]
else:
evidence_results = pr.results
# 2. Evidence + Classifier 병렬
t_ev = time.perf_counter()
evidence_task = asyncio.create_task(extract_evidence(q, evidence_results))
# classifier input: top 3 chunks meta + rerank scores
top_chunks = [
{
"title": r.title or "",
"section": r.section_title or "",
"snippet": (r.snippet or "")[:200],
}
for r in pr.results[:3]
]
rerank_scores_top = [
r.rerank_score if r.rerank_score is not None else r.score
for r in pr.results[:3]
]
classifier_task = asyncio.create_task(
classify(q, top_chunks, rerank_scores_top)
)
evidence, ev_skip = await evidence_task
ev_ms = (time.perf_counter() - t_ev) * 1000
# classifier await (timeout 보호 — classifier_service 내부에도 있지만 여기서 이중 보호)
# 2026-05-17: 6s outer wrapper 가 classifier_service.LLM_TIMEOUT_MS (30s) 를 override → 동시 부하 시
# 거의 모든 classifier 호출 timeout → conservative_refuse(no_classifier) 경로. 15s 로 상향 — classifier
# 가 실제 작동하도록 (단, ask 전체 응답 시간 상한 영향: ev_ms + max(classifier_wait, evidence_extract) +
# synth_ms + verifier 누적).
# 2026-05-17 B-3: 15s 도 동시 부하 시 부족 (classifier_service LLM_TIMEOUT_MS 30s 와 misalign).
# 30s 로 align → classifier 동작 안정. ask 응답 latency 상한 ↑ 의도.
try:
classifier_result = await asyncio.wait_for(classifier_task, timeout=30.0)
except asyncio.CancelledError:
raise # 요청 취소는 전파 — broad except 가 삼키지 않게 명시 (R3)
except Exception:
classifier_result = ClassifierResult("timeout", None, [], [], 0.0)
defense_log["classifier"] = {
"status": classifier_result.status,
"verdict": classifier_result.verdict,
"covered_aspects": classifier_result.covered_aspects,
"missing_aspects": classifier_result.missing_aspects,
"elapsed_ms": classifier_result.elapsed_ms,
}
# 3. Refusal gate (multi-signal fusion)
all_rerank_scores = [
e.rerank_score for e in evidence
] if evidence else rerank_scores_top
decision = refusal_decide(all_rerank_scores, classifier_result)
defense_log["score_gate"] = {
"max": max(all_rerank_scores) if all_rerank_scores else 0.0,
"agg_top3": sum(sorted(all_rerank_scores, reverse=True)[:3]),
}
defense_log["refusal"] = {
"refused": decision.refused,
"rule_triggered": decision.rule_triggered,
}
if decision.refused:
total_ms = (time.perf_counter() - t_total) * 1000
no_reason = "관련 근거를 찾지 못했습니다."
if not pr.results:
no_reason = "검색 결과가 없습니다."
logger.info(
"ask REFUSED query=%r rule=%s max_score=%.2f total=%.0f",
q[:80], decision.rule_triggered,
max(all_rerank_scores) if all_rerank_scores else 0.0, total_ms,
)
# telemetry — search + ask_events 두 경로 동시
background_tasks.add_task(
record_search_event, q, user.id, pr.results, "hybrid",
pr.confidence_signal, pr.analyzer_confidence,
)
# input_snapshot (디버깅/재현용)
defense_log["input_snapshot"] = {
"query": q,
"top_chunks_preview": [
{"title": c.get("title", ""), "snippet": c.get("snippet", "")[:100]}
for c in top_chunks[:3]
],
"answer_preview": None,
}
background_tasks.add_task(
record_ask_event,
q, user.id, "insufficient", "skipped", None,
True, classifier_result.verdict,
max(all_rerank_scores) if all_rerank_scores else 0.0,
sum(sorted(all_rerank_scores, reverse=True)[:3]),
[], len(evidence), 0,
defense_log, int(total_ms),
# Phase E.1 측정 필드
answer_length=0,
covered_aspects=classifier_result.covered_aspects or None,
missing_aspects=classifier_result.missing_aspects or None,
model_name=resolve_primary_model(),
prompt_version=ASK_PROMPT_VERSION,
# Phase 3.5 calibration
source=source,
eval_case_id=eval_case_id,
)
debug_obj = None
if debug:
debug_obj = AskDebug(
timing_ms={**pr.timing_ms, "evidence_ms": ev_ms, "ask_total_ms": total_ms},
search_notes=pr.notes,
confidence_signal=pr.confidence_signal,
evidence_candidate_count=len(evidence),
evidence_kept_count=len(evidence),
evidence_skip_reason=ev_skip,
synthesis_cache_hit=False,
hallucination_flags=[],
defense_layers=defense_log,
)
return AskResponse(
results=pr.results,
ai_answer=None,
citations=[],
synthesis_status="skipped",
synthesis_ms=0.0,
confidence=None,
refused=True,
no_results_reason=no_reason,
query=q,
total=len(pr.results),
completeness="insufficient",
covered_aspects=classifier_result.covered_aspects or None,
missing_aspects=classifier_result.missing_aspects or None,
# refusal gate 단계에서는 backend 호출 자체가 일어나지 않음 →
# backend_used = None. backend_requested 는 호출자 의도 표시용.
backend_requested=backend,
backend_used=None,
debug=debug_obj,
)
# 4. Synthesis (backend dispatcher 적용 — PR-MacBook-RAG-Backend-1)
t_synth = time.perf_counter()
sr = await synthesize(q, evidence, debug=debug, backend=backend)
synth_ms = (time.perf_counter() - t_synth) * 1000
# 4.1. backend_unavailable → 503 fail-fast (자동 fallback 금지)
# 명시 opt-in backend (예: qwen-macbook) 가 비가용일 때만 발생. /ask wrapper 는
# 절대 다른 backend 로 재시도하지 않음. 사용자가 backend 인자 제거 또는 wake 후 재시도.
if sr.status == "backend_unavailable":
backend_requested_val = backend or "gemma-macmini"
total_ms = (time.perf_counter() - t_total) * 1000
logger.warning(
"ask backend_unavailable backend=%s query=%r total_ms=%.0f flags=%s",
backend_requested_val, q[:80], total_ms,
",".join(sr.hallucination_flags) if sr.hallucination_flags else "-",
)
# error_reason 명명 — macbook_unavailable 만 정착 (자동 fallback 부재).
error_reason = (
"macbook_unavailable"
if backend_requested_val == "qwen-macbook"
else "backend_unavailable"
)
# telemetry — search 만 기록 (ask_events 는 200 응답 path 전용)
background_tasks.add_task(
record_search_event, q, user.id, pr.results, "hybrid",
pr.confidence_signal, pr.analyzer_confidence,
)
return JSONResponse(
status_code=503,
content={
"error": "backend_unavailable",
"error_reason": error_reason,
"backend_requested": backend_requested_val,
"backend_used": None,
"query": q,
"detail": (
"명시 선택한 backend 가 일시적으로 응답할 수 없습니다. "
"MacBook 깨우거나 backend 인자를 제거하고 (기본 Gemma) 다시 호출하세요."
),
},
)
# 5. Grounding check + Verifier (조건부 병렬) + re-gate (Phase 3.5b)
grounding = grounding_check(q, sr.answer or "", evidence)
# verifier skip: grounding strong 2+ OR retrieval 자체가 망함
grounding_only_strong = [
f for f in grounding.strong_flags if not f.startswith("verifier_")
]
max_rerank = max(all_rerank_scores, default=0.0)
if len(grounding_only_strong) >= 2 or max_rerank < 0.2:
verifier_result = VerifierResult("skipped", [], 0.0)
else:
verifier_task = asyncio.create_task(
verify(q, sr.answer or "", evidence)
)
# 2026-05-17 B-3: 4s outer wait_for 가 verifier_service LLM_TIMEOUT_MS (10s) 를 override
# → classifier 와 동일 패턴 (search.py:522 가 6s→15s swap 했던 case). 10s 로 align.
try:
verifier_result = await asyncio.wait_for(verifier_task, timeout=10.0)
except asyncio.CancelledError:
raise # 요청 취소는 전파 — broad except 가 삼키지 않게 명시 (R3)
except Exception:
verifier_result = VerifierResult("timeout", [], 0.0)
# Verifier contradictions → grounding flags 머지 (prefix 로 구분, severity 3단계)
for c in verifier_result.contradictions:
if c.severity == "strong":
grounding.strong_flags.append(f"verifier_{c.type}:{c.claim[:30]}")
elif c.severity == "medium":
grounding.weak_flags.append(f"verifier_{c.type}_medium:{c.claim[:30]}")
else:
grounding.weak_flags.append(f"verifier_{c.type}:{c.claim[:30]}")
defense_log["evidence"] = {
"skip_reason": ev_skip,
"kept_count": len(evidence),
}
defense_log["grounding"] = {
"strong": grounding.strong_flags,
"weak": grounding.weak_flags,
}
defense_log["verifier"] = {
"status": verifier_result.status,
"contradictions_count": len(verifier_result.contradictions),
"strong_count": sum(1 for c in verifier_result.contradictions if c.severity == "strong"),
"medium_count": sum(1 for c in verifier_result.contradictions if c.severity == "medium"),
"elapsed_ms": verifier_result.elapsed_ms,
}
# ── Re-gate: 7-tier completeness 결정 (Phase 3.5 B2 — Tier 4 신규 삽입, 재번호) ──
# 기존 6-tier (3.5b 4차 리뷰) + Tier 4(g_strong + v_strong_numeric + low_conf → refuse).
# 호환성: defense_layers["re_gate"] 의 string literal 들은 기존 그대로 유지.
# 신규 "refuse(grounding+verifier_numeric)" 만 추가.
completeness: Literal["full", "partial", "insufficient"] = "full"
covered_aspects = classifier_result.covered_aspects or None
missing_aspects = classifier_result.missing_aspects or None
confirmed_items: list[ConfirmedItem] | None = None
# verifier/grounding strong 구분
g_strong = [f for f in grounding.strong_flags if not f.startswith("verifier_")]
v_strong = [f for f in grounding.strong_flags if f.startswith("verifier_")]
v_medium = [f for f in grounding.weak_flags if f.startswith("verifier_") and "_medium:" in f]
has_direct_negation = any("direct_negation" in f for f in v_strong)
# Phase 3.5 B2: verifier strong flags 중 numeric_conflict 만 카운트.
# promote(VERIFIER_NUMERIC_PROMOTE=1) 활성 시 critical numeric_conflict 가 strong 으로 승격되며
# 여기 카운트에 잡힘. promote off 면 항상 0 → Tier 4 활성 안 됨 (기존 동작 유지).
v_strong_numeric = sum(
1 for f in v_strong if f.startswith("verifier_numeric_conflict")
)
# ── Tier 0 (Phase 3.5 fix3): synthesis 자체 실패 처리 ──
# LLM self-refuse, 메커니즘 실패(timeout/parse_failed/llm_error), answer 공백.
# 빈 답에 대해 grounding/verifier flag 가 0건이라 기존 체인이 "else clean" 으로 빠지며
# completeness="full" 초기값이 보존되던 모순을 여기서 일관되게 차단.
# 과거 baseline(v1-400char) 에서 20(self-refuse)+4(timeout) = 24/223 (10.8%) 해당.
tier0_label = _detect_synthesis_failure(sr)
if tier0_label:
completeness = "insufficient"
sr.answer = None
sr.refused = True
sr.confidence = None
defense_log["re_gate"] = tier0_label
elif len(g_strong) >= 2:
# Tier 1: grounding strong 2+ → refuse
completeness = "insufficient"
sr.answer = None
sr.refused = True
sr.confidence = None
defense_log["re_gate"] = "refuse(grounding_2+strong)"
elif g_strong and has_direct_negation:
# Tier 2: grounding strong + verifier direct_negation → refuse
completeness = "insufficient"
sr.answer = None
sr.refused = True
sr.confidence = None
defense_log["re_gate"] = "refuse(grounding+direct_negation)"
elif g_strong and sr.confidence == "low" and max_rerank < 0.25:
# Tier 3: grounding strong 1 + (low confidence AND weak evidence) → refuse
completeness = "insufficient"
sr.answer = None
sr.refused = True
sr.confidence = None
defense_log["re_gate"] = "refuse(grounding+low_conf+weak_ev)"
elif g_strong and v_strong_numeric >= 1 and sr.confidence == "low":
# Tier 4 (B2 신규): grounding strong + verifier numeric_conflict strong + low conf → refuse.
# verifier strong 단독 refuse 금지 원칙 유지 — g_strong 교차 필수.
completeness = "insufficient"
sr.answer = None
sr.refused = True
sr.confidence = None
defense_log["re_gate"] = "refuse(grounding+verifier_numeric)"
elif g_strong or has_direct_negation:
# Tier 5 (기존 4): grounding strong 1 또는 verifier direct_negation 단독 → partial
completeness = "partial"
sr.confidence = "low"
defense_log["re_gate"] = "partial(strong_or_negation)"
elif v_medium:
# Tier 6 (기존 5): verifier medium 누적 → count 기반 confidence 하향
medium_count = len(v_medium)
if medium_count >= 3:
sr.confidence = "low"
defense_log["re_gate"] = f"conf_low(medium_x{medium_count})"
elif medium_count == 2 and sr.confidence == "high":
sr.confidence = "medium"
defense_log["re_gate"] = "conf_cap_medium(medium_x2)"
else:
defense_log["re_gate"] = f"medium_x{medium_count}(no_action)"
elif grounding.weak_flags:
# Tier 7 (기존 6): weak → confidence 한 단계 하향
if sr.confidence == "high":
sr.confidence = "medium"
defense_log["re_gate"] = "conf_lower(weak)"
else:
defense_log["re_gate"] = "clean"
# Confidence cap from refusal gate (classifier 부재 시 conservative)
if decision.confidence_cap and sr.confidence:
conf_rank = {"low": 0, "medium": 1, "high": 2}
if conf_rank.get(sr.confidence, 0) > conf_rank.get(decision.confidence_cap, 2):
sr.confidence = decision.confidence_cap
# Partial 이면 max confidence = medium
if completeness == "partial" and sr.confidence == "high":
sr.confidence = "medium"
sr.hallucination_flags.extend(
[f"strong:{f}" for f in grounding.strong_flags]
+ [f"weak:{f}" for f in grounding.weak_flags]
)
total_ms = (time.perf_counter() - t_total) * 1000
# 6. 응답 구성
citations = _build_citations(evidence, sr.used_citations)
no_reason = _map_no_results_reason(pr, evidence, ev_skip, sr)
if completeness == "insufficient" and not no_reason:
# Tier 0 경로: synthesis self-refuse 는 LLM 이 준 사유가 가장 정확.
if sr.refused and sr.refuse_reason:
no_reason = sr.refuse_reason
else:
no_reason = "답변 검증에서 복수 오류 감지"
logger.info(
"ask query=%r results=%d evidence=%d cite=%d synth=%s conf=%s completeness=%s "
"refused=%s grounding_strong=%d grounding_weak=%d ev_ms=%.0f synth_ms=%.0f total=%.0f",
q[:80], len(pr.results), len(evidence), len(citations),
sr.status, sr.confidence or "-", completeness,
sr.refused, len(grounding.strong_flags), len(grounding.weak_flags),
ev_ms, synth_ms, total_ms,
)
# 7. telemetry — search + ask_events 두 경로 동시
background_tasks.add_task(
record_search_event, q, user.id, pr.results, "hybrid",
pr.confidence_signal, pr.analyzer_confidence,
)
# input_snapshot (디버깅/재현용)
defense_log["input_snapshot"] = {
"query": q,
"top_chunks_preview": [
{"title": (r.title or "")[:50], "snippet": (r.snippet or "")[:100]}
for r in pr.results[:3]
],
"answer_preview": (sr.answer or "")[:200],
}
background_tasks.add_task(
record_ask_event,
q, user.id, completeness, sr.status, sr.confidence,
sr.refused, classifier_result.verdict,
max(all_rerank_scores) if all_rerank_scores else 0.0,
sum(sorted(all_rerank_scores, reverse=True)[:3]),
sr.hallucination_flags, len(evidence), len(citations),
defense_log, int(total_ms),
# Phase E.1 측정 필드
answer_length=len(sr.answer or ""),
covered_aspects=covered_aspects,
missing_aspects=missing_aspects,
model_name=resolve_primary_model(),
prompt_version=ASK_PROMPT_VERSION,
# Phase 3.5 calibration
source=source,
eval_case_id=eval_case_id,
)
debug_obj = None
if debug:
timing = dict(pr.timing_ms)
timing["evidence_ms"] = ev_ms
timing["synthesis_ms"] = synth_ms
timing["ask_total_ms"] = total_ms
debug_obj = AskDebug(
timing_ms=timing,
search_notes=pr.notes,
query_analysis=pr.query_analysis,
confidence_signal=pr.confidence_signal,
evidence_candidate_count=len(evidence),
evidence_kept_count=len(evidence),
evidence_skip_reason=ev_skip,
synthesis_cache_hit=sr.cache_hit,
synthesis_raw_preview=sr.raw_preview,
hallucination_flags=sr.hallucination_flags,
defense_layers=defense_log,
)
# backend_used: synthesize 가 실제 호출한 backend (backend 인자 그대로 신뢰 OK —
# backend_unavailable 은 위 503 분기에서 이미 return 됨).
backend_used_val = backend or "gemma-macmini"
return AskResponse(
results=pr.results,
ai_answer=sr.answer,
citations=citations,
synthesis_status=sr.status,
synthesis_ms=sr.elapsed_ms,
confidence=sr.confidence,
refused=sr.refused,
no_results_reason=no_reason,
query=q,
total=len(pr.results),
completeness=completeness,
covered_aspects=covered_aspects,
missing_aspects=missing_aspects,
confirmed_items=confirmed_items,
backend_requested=backend,
backend_used=backend_used_val,
debug=debug_obj,
)
# ─── PR-DocSrv-Ask-ToolCalling-ReAct-1 ────────────────────────────────────
# /api/search/ask/react — Qwen native tool calling 로 ReAct loop.
# 본 endpoint 는 qwen-macbook only (endpoint 자체가 implicit opt-in).
# MacBook unavailable 시 503 + error_reason=macbook_unavailable. Gemma 자동 fallback X.
# G0-2 counter semantics: max_tool_rounds=2, max LLM calls=3, search exec ≤ 2.
# G0-3 trace exposure: default response 의 debug_trace=None, debug=True 시만 채움.
class AskReactRequest(BaseModel):
query: str
debug: bool = False
class AskReactResponse(BaseModel):
final_answer: str
iterations: int
partial: bool
sources: list[dict]
debug_trace: list[dict] | None = None
@router.post("/ask/react", response_model=AskReactResponse)
async def ask_react(
payload: AskReactRequest,
user: Annotated[User, Depends(get_current_user)],
session: Annotated[AsyncSession, Depends(get_session)],
):
"""ReAct loop endpoint (qwen-macbook only, no fallback).
호출자가 명시 opt-in 한 endpoint. MacBook 가 sleep / unreachable / 5xx 시
HTTP 503 + body `{error_reason: "macbook_unavailable", backend: "qwen-macbook"}`
를 반환한다. Gemma Mac mini 로 자동 fallback 하지 않는다 (정정 4 의 연장).
request body:
- query: str (사용자 원본 질의)
- debug: bool (default false; true 시 응답 `debug_trace` 채움)
response body (성공 200):
- final_answer: str (Qwen 종합문, partial 일 수 있음)
- iterations: int (실제 진행된 tool round 수)
- partial: bool (max_tool_rounds 도달 후 LLM content 비었을 때 true)
- sources: list[dict] (검색에서 모인 evidence 메타, id-기준 dedup)
- debug_trace: list[dict] | null (debug=true 시 round 별 trace)
"""
# 지연 import — 순환 의존성 회피 (react_loop 가 api.search.SearchResult 사용 안 함)
from services.llm.backends import BackendUnavailable, get_backend
from services.search.react_loop import agentic_ask_loop
backend_inst = get_backend("qwen-macbook")
# PR-2 of DS AI routing policy: backend_inst may be RouterBackend (default)
# or QwenMacBookBackend (DS_BACKENDS_VIA_ROUTER=false rollback). Both
# implement generate_with_tools so the ReAct loop is identical.
assert hasattr(backend_inst, "generate_with_tools")
try:
result = await agentic_ask_loop(
session,
payload.query,
backend=backend_inst,
debug=payload.debug,
)
except BackendUnavailable as exc:
logger.warning(
"ask_react backend unavailable backend=%s reason=%s",
exc.backend_name, exc.reason,
)
return JSONResponse(
status_code=503,
content={
"error_reason": "macbook_unavailable",
"backend_requested": "qwen-macbook",
"backend_used": None,
"detail": exc.reason,
},
)
return AskReactResponse(
final_answer=result.final_answer,
iterations=result.iterations,
partial=result.partial,
sources=result.sources,
debug_trace=result.debug_trace,
)
+1 -21
View File
@@ -21,14 +21,12 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from core.auth import get_current_user
from core.config import settings
from core.database import get_session
from models.study_memo_card import StudyMemoCard, StudyMemoCardEvidence, record_card_view
from models.study_memo_card_progress import StudyMemoCardProgress, rate_card
from models.study_question import StudyQuestion
from models.user import User
from services.study.card_normalize import compute_dedup_hash
from services.study.publish_enqueue import enqueue_card_progress_publish, enqueue_card_publish
router = APIRouter()
@@ -250,18 +248,9 @@ async def approve_batch(
StudyMemoCard.needs_review,
)
.values(needs_review=False, flagged_by=None, flagged_at=None)
.returning(StudyMemoCard.id)
)
approved_ids = list(result.scalars().all())
# 방금 검수완료된 카드 발행(같은 tx, flag off 면 no-op). S-2.
if settings.study_publish_enabled and approved_ids:
cards = (
await session.execute(select(StudyMemoCard).where(StudyMemoCard.id.in_(approved_ids)))
).scalars().all()
for c in cards:
await enqueue_card_publish(session, c)
await session.commit()
return {"approved": len(approved_ids)}
return {"approved": result.rowcount or 0}
# ─── 복습(SR) 트랙 ───
@@ -321,9 +310,6 @@ async def rate(
if outcome is None:
raise HTTPException(status_code=422, detail=f"invalid outcome: {body.outcome!r}")
progress = await rate_card(session, card=card, outcome=outcome, now=datetime.now(timezone.utc))
# 카드 SR 상태 발행(같은 tx, flag off=no-op) — ALL row(sentinel/terminal 포함). S-4.
if settings.study_publish_enabled:
await enqueue_card_progress_publish(session, progress)
await session.commit()
return RateResult(
card_id=card.id, outcome=outcome, review_stage=progress.review_stage, due_at=progress.due_at
@@ -406,9 +392,6 @@ async def update_card(
card.flagged_by = None
card.flagged_at = None
# 발행 재투영/tombstone(같은 tx) — 검수완료=발행·검수대기복귀=tombstone(상태 기반). S-2.
if settings.study_publish_enabled:
await enqueue_card_publish(session, card)
try:
await session.commit()
except IntegrityError:
@@ -431,7 +414,4 @@ async def delete_card(
card = await session.get(StudyMemoCard, card_id)
card = _verify_card(card, user)
card.deleted_at = datetime.now(timezone.utc)
# 발행 tombstone(같은 tx) — 삭제는 feed 1급 이벤트. S-2.
if settings.study_publish_enabled:
await enqueue_card_publish(session, card)
await session.commit()
+16 -35
View File
@@ -39,9 +39,6 @@ from services.study.explanation_rag import (
gather_explanation_context,
render_evidence_block,
)
from services.study.publish_enqueue import enqueue_publish, enqueue_question_publish
from services.study.publish_projection import KIND_CARD, KIND_EXPLANATION, KIND_QUESTION
from services.study.outcome import derive_outcome
logger = logging.getLogger(__name__)
router = APIRouter()
@@ -546,9 +543,6 @@ async def create_question_in_topic(
)
session.add(q)
await session.flush()
# 발행 outbox 적재(같은 tx, flag off 면 no-op) — 신규 문항 발행. P0-1b.
if settings.study_publish_enabled:
await enqueue_question_publish(session, q)
await session.commit()
stats = QuestionAttemptStats(attempt_count=0, correct_count=0, wrong_count=0)
@@ -911,16 +905,9 @@ async def update_question(
# 카드는 '구' ai_explanation 에서 추출됐으므로 정정 후 stale 가능 — 즉시 가시화 플래그.
# 최종 stale 정리는 card_extract 워커의 supersede 가 책임(새 버전 추출 시 구버전 retire).
if AI_STALE_TRIGGER & fields_set:
flagged_card_ids = await flag_cards_for_source(session, source_question_id=q.id, reason="source_changed")
# 발행 자격 잃은(검수대기 복귀) 파생 카드 tombstone(같은 tx). S-2.
if settings.study_publish_enabled:
for cid in flagged_card_ids:
await enqueue_publish(session, kind=KIND_CARD, source_id=cid, payload=None, deleted=True)
await flag_cards_for_source(session, source_question_id=q.id, reason="source_changed")
q.updated_at = datetime.now(timezone.utc)
# 발행 재투영(같은 tx) — 문항 갱신 반영. 해설은 ready 일 때만 동봉, stale→tombstone 은 P1-3. P0-1b.
if settings.study_publish_enabled:
await enqueue_question_publish(session, q)
await session.commit()
stats = await _attempt_stats(session, user.id, question_id)
@@ -983,16 +970,7 @@ async def soft_delete_question(
)
# 공부 암기노트: 소스 문제 삭제 시 파생 암기카드를 검토 대기로 마킹(source_deleted).
# study_questions 는 soft-delete 만이라 카드 FK CASCADE 는 미발동 — 이 훅이 실 경로.
flagged_card_ids = await flag_cards_for_source(session, source_question_id=q.id, reason="source_deleted")
# 발행 자격 잃은 파생 카드 tombstone(같은 tx). S-2.
if settings.study_publish_enabled:
for cid in flagged_card_ids:
await enqueue_publish(session, kind=KIND_CARD, source_id=cid, payload=None, deleted=True)
# 발행 tombstone(같은 tx) — 삭제는 feed 1급 이벤트(raw DELETE 금지·워커 경유). 해설 본문 있으면 그 kind 도. P0-1b.
if settings.study_publish_enabled:
await enqueue_publish(session, kind=KIND_QUESTION, source_id=q.id, payload=None, deleted=True)
if q.ai_explanation:
await enqueue_publish(session, kind=KIND_EXPLANATION, source_id=q.id, payload=None, deleted=True)
await flag_cards_for_source(session, source_question_id=q.id, reason="source_deleted")
await session.commit()
@@ -1014,13 +992,19 @@ async def submit_attempt(
q = await session.get(StudyQuestion, question_id)
q = _verify_question_ownership(q, user)
# 채점 단일 소스 — 뷰어 ingest 와 동일 함수(P2). 선택 없고 unsure 아니면 422.
try:
selected, is_correct, outcome = derive_outcome(
body.selected_choice, body.is_unsure, q.correct_choice
if body.is_unsure:
selected = None
is_correct = False
outcome = "unsure"
elif body.selected_choice is None:
raise HTTPException(
status_code=422,
detail="selected_choice (1~4) 또는 is_unsure=true 가 필요합니다",
)
except ValueError as e:
raise HTTPException(status_code=422, detail=str(e))
else:
selected = body.selected_choice
is_correct = selected == q.correct_choice
outcome = "correct" if is_correct else "wrong"
# PR-10: 세션 연동. 기본은 None.
quiz_session: StudyQuizSession | None = None
@@ -1559,8 +1543,8 @@ async def delete_question_image(
# ─── PR-3: AI 풀이 생성 엔드포인트 ───
# 2026-06-20: config 단일소스 (구 하드코딩 30s = 빠른 Gemma 기준).
LLM_TIMEOUT_S = settings.llm_call_timeout_s
# MLX 호출 timeout (초). MLX gate + 26B 추론 평균 ~10s, 안전 마진.
LLM_TIMEOUT_S = 30.0
# 프롬프트 템플릿 lazy load
_PROMPT_PATH = "study_question_explanation.txt"
_prompt_cache: str | None = None
@@ -1729,9 +1713,6 @@ async def generate_ai_explanation(
primary_name = ai_client.ai.primary.model if hasattr(ai_client.ai.primary, "model") else "primary"
q.ai_explanation_model = f"mlx:{primary_name}"
q.updated_at = q.ai_explanation_generated_at
# 발행 재투영(같은 tx) — 실시간 해설 ready → 문항+해설 발행. P0-1b.
if settings.study_publish_enabled:
await enqueue_question_publish(session, q)
await session.commit()
return AIExplanationResponse(
+2 -15
View File
@@ -33,7 +33,6 @@ from ai.client import AIClient, strip_thinking
from eid.ai import EidAIClient
from eid.compose import compose
from core.auth import get_current_user
from core.config import settings
from core.database import get_session
from core.library import LIBRARY_PREFIX, normalize_library_path
from models.document import Document
@@ -47,8 +46,6 @@ from models.eid_study_weakness import EidStudyWeakness
from models.eid_review_set_draft import EidReviewSetDraft
from models.user import User
from services.search.llm_gate import Priority, acquire_mlx_gate
from services.study.publish_enqueue import enqueue_publish, enqueue_topic_publish
from services.study.publish_projection import KIND_TOPIC
from services.study.subject_note_rag import (
SubjectNoteContext,
gather_subject_note_context,
@@ -469,9 +466,6 @@ async def create_study_topic(
session.add(topic)
try:
await session.flush()
# 발행 outbox 적재(같은 tx, flag off 면 no-op) — 신규 주제 발행. S-1.
if settings.study_publish_enabled:
await enqueue_topic_publish(session, topic)
await session.commit()
except IntegrityError:
await session.rollback()
@@ -701,10 +695,6 @@ async def update_study_topic(
topic.focused_at = datetime.now(timezone.utc) if body.focused else None
topic.updated_at = datetime.now(timezone.utc)
# 발행 재투영(같은 tx) — 주제 메타 갱신 반영. payload(name·exam_round_size) 무변경(focused 등)
# 은 워커 (payload_hash, deleted) 디둡이 rev 안 올리고 흡수 = churn 없음. S-1.
if settings.study_publish_enabled:
await enqueue_topic_publish(session, topic)
try:
await session.commit()
except IntegrityError:
@@ -780,9 +770,6 @@ async def delete_study_topic(
)
topic.deleted_at = datetime.now(timezone.utc)
# 발행 tombstone(같은 tx) — 삭제는 feed 1급 이벤트(raw DELETE 금지·워커 경유). S-1.
if settings.study_publish_enabled:
await enqueue_publish(session, kind=KIND_TOPIC, source_id=topic.id, payload=None, deleted=True)
await session.commit()
@@ -1028,7 +1015,7 @@ async def detach_session_from_topic(
# ─── PR-9: 분야 설명 (study_topic_subject_notes) ───
SUBJECT_NOTE_TIMEOUT_S = settings.llm_call_timeout_s
SUBJECT_NOTE_TIMEOUT_S = 30.0
_SUBJECT_NOTE_PROMPT_PATH = "study_subject_note.txt"
_subject_note_prompt_cache: str | None = None
@@ -1255,7 +1242,7 @@ async def generate_subject_note(
# 워커(study_weakness)가 산출한 최신 eid_study_weakness 스냅샷을 '학습 진단 코치'(study overlay)
# 로 번역. 약점/태도 '판정'은 코드 derived(스냅샷) — LLM 은 스냅샷 블록 값만 인용(환각 약점 차단).
# compose("study_diagnosis") = persona+rules+study overlay(+{placeholder}) → 표면이 블록 substitute.
DIAGNOSIS_TIMEOUT_S = settings.llm_call_timeout_s
DIAGNOSIS_TIMEOUT_S = 40.0
class StudyDiagnosisResponse(BaseModel):
+2 -11
View File
@@ -31,11 +31,11 @@ def hash_password(password: str) -> str:
return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
def create_access_token(subject: str, expires_minutes: int | None = None, egress: str = "local") -> str:
def create_access_token(subject: str, expires_minutes: int | None = None) -> str:
minutes = expires_minutes if expires_minutes is not None else ACCESS_TOKEN_EXPIRE_MINUTES
now = datetime.now(timezone.utc)
expire = now + timedelta(minutes=minutes)
payload = {"sub": subject, "exp": expire, "iat": int(now.timestamp()), "type": "access", "egress": egress}
payload = {"sub": subject, "exp": expire, "iat": int(now.timestamp()), "type": "access"}
return jwt.encode(payload, settings.jwt_secret, algorithm=ALGORITHM)
@@ -100,15 +100,6 @@ def verify_totp(code: str, secret: str | None = None) -> bool:
return totp.verify(code)
async def get_egress_class(
credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)],
) -> str:
"""토큰 egress claim -> 'cloud'|'local' (갭2 cloud-egress allowlist). claim 부재=local
(비파괴; 기존 토큰=신뢰/로컬). 쿼리파라미터 아님 -> 호출자가 없음(우회 차단)."""
payload = decode_token(credentials.credentials)
return (payload or {}).get("egress", "local")
async def get_current_user(
credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)],
session: Annotated[AsyncSession, Depends(get_session)],
-36
View File
@@ -30,11 +30,6 @@ class AIModelConfig(BaseModel):
# None = MLX/OpenAI server default. Anthropic branch 는 미적용 (별 plan 범위).
temperature: float | None = None
top_p: float | None = None
# mlx 네이티브 샘플링 — 한국어 장문 코드스위칭(CJK/라틴 누수)·반복루프 억제용.
# Qwen3 권장: top_k=20, repetition_penalty 1.05~1.1. None = 서버 기본값(주입 안 함).
# OpenAI 호환 분기(mlx)만 적용 — Anthropic 분기는 미적용(별 범위).
repetition_penalty: float | None = None
top_k: int | None = None
class DeepSummaryBacklogConfig(BaseModel):
@@ -181,29 +176,16 @@ class Settings(BaseModel):
digest_llm_timeout_s: int = 200
digest_llm_attempts: int = 2
digest_pipeline_hard_cap_s: int = 1800
# 2026-06-20: study/analyze 단일 primary-call 타임아웃 (구 하드코딩 30~60s = 빠른 Gemma 기준,
# Qwen 27B 교체 sweep 누락 → 사용자 대면 504 + 워커 영구 stuck). digest 와 동형 단일소스.
llm_call_timeout_s: int = 200
# PR-MacMini-Derived-Worker-1: study explanation owner = Mac mini
# GPU 측은 false 로 설정 (.env), explanation 분기 skip guard 트리거.
study_explanation_enabled: bool = True
# 공부 암기노트 Phase 1: card_extract 폴러/consumer 게이트. owner 분리 시 false 로.
study_card_extract_enabled: bool = True
# 발행 레이어(docsrv-viewer-publish): publish_outbox 워커 게이트. 저자/4-A enqueue 결선(P0-1b) 후 true.
study_publish_enabled: bool = False
digest_publish_enabled: bool = False # docsrv-viewer-publish P1-1 (뉴스/다이제스트 발행 feed gate)
maintenance_mode: bool = False # P1-4: 점검/실험 중 = 가공현황 배너(표면 != 데이터)
maintenance_note: str = ""
# 뷰어 write-back ingest(study-to-viewer P2) 게이트. /ingest/study/attempts 활성. 기본 false=inert(503).
study_ingest_enabled: bool = False
# internal endpoint Bearer token (Mac mini derived-worker 호출용)
internal_worker_token: str = ""
# 뷰어↔DS 발행 채널 Bearer token (publish read API P0-2 + ingest P2). Mac mini 토큰과 분리(폭발반경 격리).
viewer_sync_token: str = ""
def load_settings() -> Settings:
"""config.yaml + 환경변수에서 설정 로딩"""
@@ -211,13 +193,7 @@ def load_settings() -> Settings:
database_url = os.getenv("DATABASE_URL", "")
study_explanation_enabled = os.getenv("STUDY_EXPLANATION_ENABLED", "true").lower() in ("1", "true", "yes")
study_card_extract_enabled = os.getenv("STUDY_CARD_EXTRACT_ENABLED", "true").lower() in ("1", "true", "yes")
study_publish_enabled = os.getenv("STUDY_PUBLISH_ENABLED", "false").lower() in ("1", "true", "yes")
digest_publish_enabled = os.getenv("DIGEST_PUBLISH_ENABLED", "false").lower() in ("1", "true", "yes")
maintenance_mode = os.getenv("MAINTENANCE_MODE", "false").lower() in ("1", "true", "yes")
maintenance_note = os.getenv("MAINTENANCE_NOTE", "")
study_ingest_enabled = os.getenv("STUDY_INGEST_ENABLED", "false").lower() in ("1", "true", "yes")
internal_worker_token = os.getenv("INTERNAL_WORKER_TOKEN", "")
viewer_sync_token = os.getenv("VIEWER_SYNC_TOKEN", "")
jwt_secret = os.getenv("JWT_SECRET", "")
totp_secret = os.getenv("TOTP_SECRET", "")
eval_runner_token = os.getenv("EVAL_RUNNER_TOKEN", "")
@@ -292,7 +268,6 @@ def load_settings() -> Settings:
digest_llm_timeout_s = 200
digest_llm_attempts = 2
digest_pipeline_hard_cap_s = 1800
llm_call_timeout_s = 200
if config_path.exists() and raw and "pipeline" in raw:
held_raw = (raw.get("pipeline") or {}).get("held_stages") or []
# 스칼라(문자열) 오기입 시 char-split 방지 — 단일 항목 리스트로 수용.
@@ -318,10 +293,6 @@ def load_settings() -> Settings:
digest_pipeline_hard_cap_s = max(60, int(_pl.get("digest_pipeline_hard_cap_s", 1800)))
except (TypeError, ValueError):
digest_pipeline_hard_cap_s = 1800
try:
llm_call_timeout_s = max(1, int(_pl.get("llm_call_timeout_s", 200)))
except (TypeError, ValueError):
llm_call_timeout_s = 200
taxonomy = raw.get("taxonomy", {}) if config_path.exists() and raw else {}
document_types = raw.get("document_types", []) if config_path.exists() and raw else []
@@ -350,19 +321,12 @@ def load_settings() -> Settings:
upload=upload_cfg,
study_explanation_enabled=study_explanation_enabled,
study_card_extract_enabled=study_card_extract_enabled,
study_publish_enabled=study_publish_enabled,
digest_publish_enabled=digest_publish_enabled,
maintenance_mode=maintenance_mode,
maintenance_note=maintenance_note,
study_ingest_enabled=study_ingest_enabled,
internal_worker_token=internal_worker_token,
viewer_sync_token=viewer_sync_token,
pipeline_held_stages=pipeline_held_stages,
mlx_gate_concurrency=mlx_gate_concurrency,
digest_llm_timeout_s=digest_llm_timeout_s,
digest_llm_attempts=digest_llm_attempts,
digest_pipeline_hard_cap_s=digest_pipeline_hard_cap_s,
llm_call_timeout_s=llm_call_timeout_s,
)
+15 -21
View File
@@ -57,12 +57,12 @@ def _parse_migration_files(migrations_dir: Path) -> list[tuple[int, str, Path]]:
def _validate_sql_content(name: str, sql: str) -> None:
"""migration SQL에 BEGIN/COMMIT이 포함되어 있으면 에러 (외부 트랜잭션 깨짐 방지)"""
# 주석(전체 줄 + 인라인 `-- ...`) 제거 후 검사. ★인라인 주석을 안 지우면 설명 주석의
# 'commit/begin' 단어(예 365_scan_jobs 의 `-- commit 시 documents.title 로 전파`)를
# 트랜잭션 제어문으로 false-positive 로 잡아 fresh DB/DR 부트스트랩이 깨진다(verification
# 실측 2026-06). 줄별로 `--` 이후를 잘라 주석 텍스트를 검사에서 제외.
cleaned = [re.sub(r"--.*$", "", line) for line in sql.splitlines()]
stripped = "\n".join(cleaned).upper()
# 주석(-- ...) 라인 제거 후 검사
lines = [
line for line in sql.splitlines()
if not line.strip().startswith("--")
]
stripped = "\n".join(lines).upper()
for keyword in ("BEGIN", "COMMIT", "ROLLBACK"):
# 단어 경계로 매칭 (예: BEGIN_SOMETHING은 제외)
if re.search(rf"\b{keyword}\b", stripped):
@@ -70,13 +70,6 @@ def _validate_sql_content(name: str, sql: str) -> None:
f"migration {name}{keyword} 포함됨 — "
f"migration SQL에는 트랜잭션 제어문을 넣지 마세요"
)
# schema_migrations 수정 금지 (runner 가 스탬프 관리) — 주석 제외(stripped) 검사.
# (구: _run_migrations 의 raw `"schema_migrations" in sql.lower()` 가 주석 미제외라
# 365 의 '-- ... schema_migrations 를 건드리지 않음' 주석을 false-positive 로 잡았음.)
if "SCHEMA_MIGRATIONS" in stripped:
raise RuntimeError(
f"Migration {name} must not modify schema_migrations table"
)
# R1: baseline 스냅샷이 대표하는 마지막 마이그레이션 버전 (이하 버전은 baseline 에 포함).
@@ -174,15 +167,16 @@ async def _run_migrations(conn) -> None:
for version, name, path in pending:
sql = path.read_text(encoding="utf-8")
_validate_sql_content(name, sql) # BEGIN/COMMIT + schema_migrations 검사(주석 제외)
_validate_sql_content(name, sql)
if "schema_migrations" in sql.lower():
raise ValueError(
f"Migration {name} must not modify schema_migrations table"
)
logger.info(f"[migration] {name} 실행 중...")
# raw asyncpg simple 프로토콜로 실행 — baseline 적재(_load_baseline_if_fresh)와 동일.
# ★exec_driver_sql 은 prepared 프로토콜이라 multi-statement 불허("cannot insert multiple
# commands into a prepared statement"). 365_scan_jobs 처럼 테이블+시드+인덱스를 한 파일에
# 담은 마이그(컨벤션상 1-statement 권장이나 이미 prod 적재)도 fresh DB/DR replay 되게
# simple execute 사용. text() :name 콜론-binding 이슈도 동일하게 회피(raw 전달).
raw = await conn.get_raw_connection()
await raw.driver_connection.execute(sql)
# raw driver SQL 사용 — text() 의 :name bind parameter 해석으로
# SQL 주석/literal 에 콜론이 들어가면 InvalidRequestError 발생.
# exec_driver_sql 은 SQL 을 driver(asyncpg) 에 그대로 전달.
await conn.exec_driver_sql(sql)
await conn.execute(
text("INSERT INTO schema_migrations (version, name) VALUES (:v, :n)"),
{"v": version, "n": name},
+1 -4
View File
@@ -2,7 +2,6 @@
import hashlib
import logging
from logging.handlers import RotatingFileHandler
from pathlib import Path
@@ -14,9 +13,7 @@ def setup_logger(name: str, log_dir: str = "logs") -> logging.Logger:
if not logger.handlers:
# 파일 핸들러
fh = RotatingFileHandler(
f"{log_dir}/{name}.log", maxBytes=10 * 1024 * 1024, backupCount=3, encoding="utf-8"
)
fh = logging.FileHandler(f"{log_dir}/{name}.log", encoding="utf-8")
fh.setFormatter(logging.Formatter(
"%(asctime)s [%(levelname)s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
+2 -22
View File
@@ -9,8 +9,6 @@ from sqlalchemy import func, select, text
from api.audio import router as audio_router
from api.internal_study import router as internal_study_router
from api.internal_worker import router as internal_worker_router
from api.published import router as published_router
from api.ingest_study import router as ingest_study_router
from api.auth import router as auth_router
from api.briefing import router as briefing_router
from api.config import router as config_router
@@ -72,7 +70,6 @@ async def lifespan(app: FastAPI):
from workers.study_session_queue_consumer import consume_study_session_queue
from workers.study_memo_card_jobs_consumer import consume_study_memo_card_queue
from workers.study_card_enqueue import run as study_card_enqueue_run
from workers.study_publish_worker import consume_publish_outbox
from workers.study_reminder import run as study_reminder_run
from workers.study_weakness import run as study_weakness_run
from workers.study_question_embed_worker import (
@@ -87,13 +84,6 @@ async def lifespan(app: FastAPI):
# 시작: DB 연결 확인
await init_db()
# 2026-06-20: JWT_SECRET 빈값 fail-loud — credentials.env 미로드/누락 시 빈 키로 전 토큰
# 서명하며 부팅하던 침묵 인증붕괴 차단 (totp_secret 은 per-user 라 미가드).
if not settings.jwt_secret:
raise RuntimeError(
"JWT_SECRET 미설정 — 빈 키 서명 방지. credentials.env / 환경변수 확인."
)
# NAS 마운트 확인 (NFS 미마운트 시 로컬 빈 디렉토리에 쓰는 것 방지)
from pathlib import Path
nas_check = Path(settings.nas_mount_path) / "PKM"
@@ -104,12 +94,7 @@ async def lifespan(app: FastAPI):
)
# APScheduler: 백그라운드 작업
scheduler = AsyncIOScheduler(
timezone="Asia/Seoul",
# 2026-06-20 H4: 기본 misfire_grace_time=1s 는 단일 asyncio 루프가 1초만 혼잡해도
# 1분 컨슈머 틱을 run time missed 로 침묵 스킵(에러·failed row 0). 45s 완화 + coalesce.
job_defaults={"misfire_grace_time": 45, "coalesce": True, "max_instances": 1},
)
scheduler = AsyncIOScheduler(timezone="Asia/Seoul")
# 상시 실행
scheduler.add_job(consume_queue, "interval", minutes=1, id="queue_consumer")
# PR-DocSrv-Markdown-Consumer-Split-1: markdown(marker) 전용 consumer.
@@ -143,9 +128,6 @@ async def lifespan(app: FastAPI):
# 별 테이블/별 consumer 로 기존 study queue 와 격리. settings.study_card_extract_enabled 게이트.
scheduler.add_job(consume_study_memo_card_queue, "interval", minutes=1, id="study_memo_card_consumer")
scheduler.add_job(study_card_enqueue_run, "interval", minutes=1, id="study_card_enqueue")
# 발행 레이어(docsrv-viewer-publish): publish_outbox drain → published rev 부여.
# study_publish_enabled=false(기본) 면 worker 내부 no-op. 단일 라이터(pg_advisory_xact_lock) max_instances=1.
scheduler.add_job(consume_publish_outbox, "interval", minutes=1, id="publish_outbox_consumer", max_instances=1)
# PR-B 레거시 tier 백필 — 30분 주기로 호출되지만 KST 00:00~06:00 시간대만 실제 enqueue.
# safety > law > manual 우선순위로 25건씩. 6720 레거시 → 야간당 ~150건 → 약 45일 소화.
scheduler.add_job(tier_backfill_run, "interval", minutes=30, id="tier_backfill")
@@ -162,7 +144,7 @@ async def lifespan(app: FastAPI):
scheduler.add_job(study_reminder_run, CronTrigger(hour="9,13,19", timezone=KST), id="study_reminder")
# 이드 W3-2: 공부중 토픽 약점 derived 스냅샷 (nightly 04:30 KST, LLM 0). study_diagnosis 표면 source.
scheduler.add_job(study_weakness_run, CronTrigger(hour=4, minute=30, timezone=KST), id="study_weakness")
scheduler.add_job(news_collector_run, CronTrigger(hour="0,6,12,18", timezone=KST), id="news_collector")
scheduler.add_job(news_collector_run, "interval", hours=6, id="news_collector")
# crawl-24x7 A-2 안전망: fulltext 영구 실패(3회 소진) 문서를 RSS 요약 기준으로
# 후속 enqueue (silent skip 누적 방지). 03:40 = dedup_reconcile(03:30) 직후 비충돌 슬롯.
scheduler.add_job(fulltext_reconcile_run, CronTrigger(hour=3, minute=40, timezone=KST), id="fulltext_reconcile")
@@ -238,8 +220,6 @@ app.include_router(briefing_router, prefix="/api/briefing", tags=["briefing"])
app.include_router(audio_router, prefix="/api/audio", tags=["audio"])
app.include_router(internal_study_router, prefix="/internal/study", tags=["internal-study"])
app.include_router(internal_worker_router, prefix="/internal/worker", tags=["internal-worker"])
app.include_router(published_router, prefix="/published", tags=["published"])
app.include_router(ingest_study_router, prefix="/ingest/study", tags=["ingest-study"])
app.include_router(video_router, prefix="/api/video", tags=["video"])
app.include_router(study_sessions_router, prefix="/api/study-sessions", tags=["study-sessions"])
app.include_router(study_topics_router, prefix="/api/study-topics", tags=["study-topics"])
-8
View File
@@ -41,14 +41,6 @@ class Document(Base):
Integer, nullable=False, default=0, server_default="0"
)
# G2 pre-segmentation (migration 362): 번들 PDF → N 자식 분할.
# presegment_role: NULL=일반 단일문서 / 'parent'=번들원본(자체 extract/embed 안 함) /
# 'child'=논리 하위문서(부모 file_path 공유 + bundle_page_start/end 1-based inclusive 범위).
# 부모-자식 관계 자체는 document_lineage(relation_type='segmented_from').
bundle_page_start: Mapped[int | None] = mapped_column(Integer)
bundle_page_end: Mapped[int | None] = mapped_column(Integer)
presegment_role: Mapped[str | None] = mapped_column(Text)
# 2계층: 텍스트 추출
extracted_text: Mapped[str | None] = mapped_column(Text)
extracted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-31
View File
@@ -1,31 +0,0 @@
"""document_lineage 테이블 ORM — 문서 파생 관계 이력 (migration 217).
G2 pre-segmentation relation_type='segmented_from'(번들 자식) 으로 사용 (migration 363).
이력 테이블 FK = ON DELETE RESTRICT (부모 hard delete 차단, soft delete 허용).
"""
from datetime import datetime
from sqlalchemy import BigInteger, ForeignKey, Text, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import TIMESTAMP
from core.database import Base
class DocumentLineage(Base):
__tablename__ = "document_lineage"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
source_document_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("documents.id", ondelete="RESTRICT"), nullable=False
)
derived_document_id: Mapped[int] = mapped_column(
BigInteger, ForeignKey("documents.id", ondelete="RESTRICT"), nullable=False
)
relation_type: Mapped[str] = mapped_column(Text, nullable=False)
# 'metadata' 는 SQLAlchemy 예약속성 → Python 속성명은 meta, DB 컬럼명은 metadata.
meta: Mapped[dict] = mapped_column(
"metadata", JSONB, nullable=False, default=dict, server_default="{}"
)
created_at: Mapped[datetime] = mapped_column(TIMESTAMP(timezone=True), server_default=func.now())
-64
View File
@@ -1,64 +0,0 @@
"""발행 레이어 ORM (docsrv-viewer-publish) — published projection + publish_outbox.
관계(relationship) 없음 = 독립 테이블, configure_mappers 무영향. 마이그 367~372.
published = 뷰어가 read API(P0-2) 당기는 render-ready projection(kind-discriminated).
publish_outbox = 저작/4-A 트랜잭션이 같은 tx에서 INSERT, 발행 워커가 drain 하며 rev 부여.
불변식(plan study-to-viewer-slice1):
pub_id opaque+stable = dedup키 = progress키 / rev = 워커 커밋순 gapless(pg_advisory_lock 단일 라이터)
/ (payload_hash, deleted) 디둡 / 삭제 = tombstone(deleted=true) / schema_version = 엔벨로프 버전.
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import BigInteger, Boolean, DateTime, SmallInteger, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from core.database import Base
class Published(Base):
__tablename__ = "published"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
kind: Mapped[str] = mapped_column(String(40), nullable=False)
source_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
pub_id: Mapped[str] = mapped_column(Text, nullable=False)
payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
payload_hash: Mapped[str] = mapped_column(Text, nullable=False)
schema_version: Mapped[int] = mapped_column(SmallInteger, nullable=False, default=1)
rev: Mapped[int] = mapped_column(BigInteger, nullable=False)
deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=datetime.now, nullable=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=datetime.now, nullable=False
)
# UNIQUE(kind, pub_id)=mig368, UNIQUE(kind, source_id)=mig369, idx(rev)=mig370.
class PublishOutbox(Base):
__tablename__ = "publish_outbox"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
kind: Mapped[str] = mapped_column(String(40), nullable=False)
source_id: Mapped[int] = mapped_column(BigInteger, nullable=False)
payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
payload_hash: Mapped[str] = mapped_column(Text, nullable=False)
schema_version: Mapped[int] = mapped_column(SmallInteger, nullable=False, default=1)
deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=datetime.now, nullable=False
)
processed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# mig378: 행별 격리 재시도/terminal. attempts=savepoint 실패 누적, failed_at=MAX 초과 terminal
# (set 시 워커 select 에서 제외 → head-of-line block 방지).
attempts: Mapped[int] = mapped_column(SmallInteger, nullable=False, default=0)
failed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# 미처리 부분 인덱스 idx(id) WHERE processed_at IS NULL = mig372.
+1 -2
View File
@@ -46,10 +46,9 @@ class ProcessingQueue(Base):
# 'stt' (audio): migration 150 / 'thumbnail' (video): queue_consumer 가 enqueue.
# 'deep_summary' (PR-B B-1): classify_worker 가 에스컬레이션 시 enqueue.
# 'fulltext' (crawl-24x7 A-2): migration 321 — 기사 페이지 fetch 후 본문 승격.
# 'presegment' (G2): migration 364 — extract 前 번들 PDF → N 자식 분할.
# DB enum 변경은 마이그레이션이 처리하므로 create_type=False.
Enum(
"presegment", "extract", "classify", "summarize", "embed", "chunk", "preview",
"extract", "classify", "summarize", "embed", "chunk", "preview",
"stt", "thumbnail", "deep_summary", "markdown", "fulltext",
name="process_stage",
create_type=False,
+8 -32
View File
@@ -25,7 +25,6 @@ from sqlalchemy import (
String,
Text,
func,
select,
text,
update,
)
@@ -100,25 +99,13 @@ async def supersede_old_cards(
*,
source_question_id: int,
keep_generated_at: datetime | None,
) -> list[int]:
) -> int:
"""같은 문제의 '다른 버전' 카드를 deleted_at 마킹(retire).
source_generated_at 카드 적재 '전에' 호출 살아있는 구버전 카드가 dedup PARTIAL
UNIQUE 추출을 막는 것을 방지(정정- stale 잔류 0). 같은 버전은 보존.
Returns: retire 되며 '발행 중이던'(needs_review=False) 카드 id 목록 발행 tombstone
대상(호출측이 enqueue). 검수 됐던(미발행) retire 카드는 tombstone 불요라 제외.
Returns: retire .
"""
# 발행 중이던 retire 대상 선캡처(update 전) — 미발행 카드 스푸리어스 tombstone 회피.
published_retired = (
await session.execute(
select(StudyMemoCard.id).where(
StudyMemoCard.source_question_id == source_question_id,
StudyMemoCard.deleted_at.is_(None),
StudyMemoCard.source_generated_at.is_distinct_from(keep_generated_at),
StudyMemoCard.needs_review.is_(False),
)
)
).scalars().all()
stmt = (
update(StudyMemoCard)
.where(
@@ -128,8 +115,8 @@ async def supersede_old_cards(
)
.values(deleted_at=func.now())
)
await session.execute(stmt)
return list(published_retired)
result = await session.execute(stmt)
return result.rowcount or 0
async def append_card(
@@ -229,24 +216,13 @@ async def flag_cards_for_source(
*,
source_question_id: int,
reason: str,
) -> list[int]:
) -> int:
"""소스 문제 정정/삭제 시 파생 카드를 needs_review=auto 마킹(임시 플래그).
최종 stale 정리는 워커 supersede 책임 이건 사용자 가시화용 즉시 플래그.
reason: 'source_changed' | 'source_deleted'.
Returns: 플래그로 '발행 자격을 잃은'(직전 needs_review=False) 카드 id 목록 발행
tombstone 대상(호출측 enqueue). 이미 검수대기였던(미발행) 카드는 제외.
Returns: 마킹된 .
"""
# 발행 중이던 카드 선캡처(update 전) — 플래그로 needs_review=True 가 되면 발행 자격 상실.
published_ids = (
await session.execute(
select(StudyMemoCard.id).where(
StudyMemoCard.source_question_id == source_question_id,
StudyMemoCard.deleted_at.is_(None),
StudyMemoCard.needs_review.is_(False),
)
)
).scalars().all()
stmt = (
update(StudyMemoCard)
.where(
@@ -255,5 +231,5 @@ async def flag_cards_for_source(
)
.values(needs_review=True, flagged_by=reason, flagged_at=func.now())
)
await session.execute(stmt)
return list(published_ids)
result = await session.execute(stmt)
return result.rowcount or 0
-4
View File
@@ -50,10 +50,6 @@ class StudyQuizSession(Base):
chronic_remaining_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# study-to-viewer P2: 뷰어 ingest 멱등/출처. 라이브 세션=finalized_at·client_session_uuid NULL, source='live'.
finalized_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # 멱등 마커(mig 373)
client_session_uuid: Mapped[str | None] = mapped_column(String(64)) # 뷰어 세션 UUID(mig 374, uq mig376)
source: Mapped[str] = mapped_column(String(20), nullable=False, default="live") # live|viewer(mig 375)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=datetime.now, nullable=False
)
+33
View File
@@ -0,0 +1,33 @@
You are an answerability judge. Given a query and evidence chunks, determine if the evidence can answer the query. Respond ONLY in JSON.
## CALIBRATION (CRITICAL)
- verdict=full: evidence is SUFFICIENT to answer the CORE of the query. Missing minor details does NOT make it insufficient.
- verdict=partial: evidence covers SOME major aspects but CLEARLY MISSES others the user explicitly asked about.
- verdict=insufficient: evidence has NO relevant information for the query, or is completely off-topic.
Example: Query="제6장 주요 내용", Evidence covers 제6장 definition+scope → verdict=full (core is covered).
Example: Query="제6장 처벌 조항", Evidence covers 제6장 definition but NOT 처벌 → verdict=partial.
Example: Query="감귤 출하량", Evidence about 산업안전보건법 → verdict=insufficient.
## Rules
1. Your "verdict" must be based ONLY on whether the CONTENT semantically answers the query. Ignore retrieval scores for this field.
2. "covered_aspects": query aspects that evidence covers. Korean labels for Korean queries.
3. "missing_aspects": query aspects that evidence does NOT cover. Korean labels.
4. Keep aspects concise (2-5 words each), non-overlapping.
## Output Schema
{
"verdict": "full" | "partial" | "insufficient",
"covered_aspects": ["aspect1"],
"missing_aspects": ["aspect2"],
"confidence": "high" | "medium" | "low"
}
## Query
{query}
## Evidence chunks:
{chunks}
## Retrieval scores (for reference only, NOT for verdict):
[{scores}]
+1 -3
View File
@@ -1,5 +1,5 @@
[System]
너는 한국어 문서 태거 + 요약기다. 입력 본문을 읽고 짧은 요약(ai_summary 2~3문장) + TL;DR + 핵심 bullets + tags 생성한다. **여러 문단의 상세 심층요약·entities 는 생성하지 않는다** (깊은 요약은 26B, entity 는 P3b 담당).
너는 한국어 문서 태거 + 짧은 요약기다. 입력 본문을 읽고 TL;DR + 핵심 bullets + tags 생성한다. **상세 문단·entities 는 생성하지 않는다** (깊은 요약은 26B, entity 는 P3b 담당).
subject_description: {subject_description}
@@ -13,7 +13,6 @@ subject_description: {subject_description}
- pii 감지 시 "pii" 추가 + confidence 감점.
요약 규칙:
- **ai_summary**: 2~3문장 문단. 문서의 핵심 내용·목적을 서술 (검색·표시용 요약).
- **TL;DR**: 1문장, 최대 60자.
- **Bullets**: 정확히 5개, 각 30~60자.
- 본문에 없는 정보 추가 금지 (hallucination 금지).
@@ -21,7 +20,6 @@ subject_description: {subject_description}
출력 (JSON only):
{{
"ai_summary": "2~3문장 문단 요약",
"tldr": "1문장 최대 60자",
"bullets": ["...", "...", "...", "...", "..."],
"tags": ["..."],
-41
View File
@@ -1,41 +0,0 @@
You are a document-boundary detector. Output ONLY JSON {is_bundle, segments:[{start_page,end_page,title}]}.
You are given a single PDF that may be a "bundle" — several independent logical documents
concatenated into one file (for example: multiple laws, multiple reports, or multiple papers
scanned together). Your job is to decide whether it is a bundle and, if so, where each logical
document starts and ends.
You receive only a compact sample per page: the page number and the first line / heading of that
page (text may be truncated). Use these heading/first-line signals to detect where a new logical
document begins (a new title page, a new cover, a clearly new document title, a restart of
numbering, etc.). You do NOT receive the full text.
Output rules:
- Respond with STRICT JSON only. No prose, no markdown, no code fence.
- Schema:
{
"is_bundle": true | false,
"segments": [
{"start_page": <int>, "end_page": <int>, "title": "<string or null>"}
]
}
- Page numbers are 1-based and INCLUSIVE. start_page=1 is the first page; end_page equals the last
page of that segment.
- Segments MUST fully cover every page with NO gaps and NO overlaps:
- the first segment MUST start at page 1,
- each next segment MUST start exactly one page after the previous segment's end_page,
- the last segment MUST end at the final page (page_count).
- Order segments by start_page ascending.
- title = a short title for that logical document if you can infer one from its first page,
otherwise null.
If the file is NOT a bundle (it is a single logical document), respond:
{"is_bundle": false, "segments": []}
Be conservative: only report is_bundle=true when the heading signals clearly indicate separate
logical documents. When unsure, return is_bundle=false.
page_count: {page_count}
Per-page samples (one per line, "p{n}: {first line}"):
{page_samples}
+42
View File
@@ -0,0 +1,42 @@
You are a grounding verifier. Given an answer and its evidence sources, check if the answer contradicts or fabricates information. Respond ONLY in JSON.
## Contradiction Types (IMPORTANT — severity depends on type)
- **direct_negation** (CRITICAL): Answer directly contradicts evidence. Examples: evidence "의무" but answer "권고"; evidence "금지" but answer "허용"; negation reversal ("~해야 한다" vs "~할 필요 없다").
- **numeric_conflict**: Answer states a number different from evidence. "50명" in evidence but "100명" in answer. Only flag if the same concept is referenced. severity=critical when the number is the CORE answered quantity (amount/count/rate/date/duration that the query asked for); severity=minor when the number is peripheral (e.g., example/footnote).
- **intent_core_mismatch**: Answer addresses a fundamentally different topic than the query asked about.
- **nuance**: Answer overgeneralizes or adds qualifiers not in evidence (e.g., "모든" when evidence says "일부").
- **unsupported_claim**: Answer makes a factual claim with no basis in any evidence.
## Rules
1. Compare each claim in the answer against the cited evidence. A claim with [n] citation should be checked against evidence [n].
2. NOT a contradiction: Paraphrasing, summarizing, or restating the same fact in different words. Korean formal/informal style (합니다/한다) differences.
3. Numbers must match exactly after normalization (1,000 = 1000). Range values (e.g., "100~200명") satisfy any answer within range.
4. Legal/regulatory terms must preserve original meaning (의무 ≠ 권고, 금지 ≠ 제한, 허용 ≠ 금지).
5. Maximum 5 contradictions (most severe first: direct_negation > numeric_conflict > intent_core_mismatch > nuance > unsupported_claim).
## Output Schema
{
"contradictions": [
{
"type": "direct_negation" | "numeric_conflict" | "intent_core_mismatch" | "nuance" | "unsupported_claim",
"severity": "critical" | "minor",
"claim": "answer 내 해당 구절 (50자 이내)",
"evidence_ref": "대응 근거 내용 (50자 이내, [n] 포함)",
"explanation": "모순 이유 (한국어, 30자 이내)"
}
],
"verdict": "clean" | "minor_issues" | "major_issues"
}
severity mapping:
- direct_negation → "critical"
- numeric_conflict → "critical" if the number is the CORE answered quantity, else "minor"
- All other types → "minor"
If no contradictions: {"contradictions": [], "verdict": "clean"}
## Answer
{answer}
## Evidence
{numbered_evidence}
-2
View File
@@ -42,7 +42,6 @@ _NEWS_WINDOW_SQL = text(f"""
AND d.created_at < :window_end
AND d.embedding IS NOT NULL
AND d.ai_summary IS NOT NULL
AND length(d.ai_summary) > 0
-- 안전 자료실 B-4: licensed_restricted 발행 차단 (digest 동일 공유 술어, 경로 일관성)
AND {restricted_exclude_sql("d")}
""")
@@ -67,7 +66,6 @@ _HISTORICAL_CANDIDATES_SQL = text(f"""
AND d.created_at < :hist_end
AND d.embedding IS NOT NULL
AND d.ai_summary IS NOT NULL
AND length(d.ai_summary) > 0
-- 안전 자료실 B-4: licensed_restricted 발행 차단 (공유 술어)
AND {restricted_exclude_sql("d")}
""")
-1
View File
@@ -42,7 +42,6 @@ _NEWS_WINDOW_SQL = text(f"""
AND d.created_at < :window_end
AND d.embedding IS NOT NULL
AND d.ai_summary IS NOT NULL
AND length(d.ai_summary) > 0
-- 안전 자료실 B-4: licensed_restricted 발행 차단 (모든 경로 공유 술어 = license_filter).
-- news 채널엔 현재 restricted 부재 = 방어적 게이트(미래 유료 news 소스 대비, 경로 누락 방지).
AND {restricted_exclude_sql("d")}
+1 -18
View File
@@ -42,21 +42,6 @@ _ENG = re.compile(
_FENCE = re.compile(r'^\s{0,3}(```|~~~)')
# ASME 절 식별자 (A-1): UG-79 · PG-27.4.1 · UW-11 · UCS-56 · A-69 · PFT-14
# (대문자 1~4 + 하이픈 + 숫자[.숫자]*). _detect_heading 의 ATX 분기에서 node_type='clause' 판정에 사용.
# 한국 법령(제N조)은 _KO_JO 가 별도 처리 — 본 패턴/정제와 무관(무회귀).
_ASME_CLAUSE = re.compile(r'^[A-Z]{1,4}-\d+(?:\.\d+)*\b')
def _clean_label(title: str) -> str:
r"""C-4: marker 가 박는 LaTeX/markdown/페이지번호 아티팩트 제거 — 절번호 패턴 매칭의 전처리 겸 표시 라벨 정제.
실데이터 : '$\textbf{PG-20.1 …} \hspace{0.2cm} \textbf{(25)}$' 'PG-20.1 …' / '(25) **A-69**' 'A-69'.
노이즈 없는 제목(한국 법령·일반 ATX ) inert(무회귀)."""
t = re.sub(r'\\textbf|\\textit|\\mathbf|\\hspace\{[^}]*\}|[${}]|\*\*', '', title)
t = re.sub(r'^\s*\(\d+\)\s*', '', t) # 선두 페이지번호 '(25) '
return re.sub(r'\s{2,}', ' ', t).strip()
def _utf16_units(s: str) -> int:
"""JS 문자열 .length(= UTF-16 code unit 수) 와 동일. astral(BMP 밖)=surrogate pair=2 units.
FE `raw.length` / `out.slice(off)` UTF-16 code unit 단위라 char_start 같은 단위여야 .
@@ -87,9 +72,7 @@ def _detect_heading(line: str) -> tuple[int, str, str] | None:
"""(level, title, node_type) 또는 None. level 은 상대 깊이."""
m = _ATX.match(line)
if m:
title = _clean_label(m.group("title").strip()) # C-4: LaTeX/md/페이지번호 정제(전처리)
nt = "clause" if _ASME_CLAUSE.match(title) else None # A-1: ASME 절 식별자(UG-79 등) → clause
return (len(m.group(1)), title, nt)
return (len(m.group(1)), m.group("title").strip(), None) # node_type 은 후처리에서
for pat, lvl, nt in ((_KO_JANG, 1, "chapter"), (_KO_JEOL, 2, "section"),
(_KO_JO, 3, "clause"), (_ENG, 1, "chapter")):
m = pat.match(line)
+156
View File
@@ -0,0 +1,156 @@
"""Answerability classifier (Phase 3.5a).
Mac mini 26B MLX 기반 (config.yaml ai.models.classifier PR #20 이후 triage/primary/classifier 동일 endpoint). MLX gate 밖 — evidence extraction 과 병렬 실행 (concurrent 안전성 별 검토).
P1 실측 결과: ternary (full/partial/insufficient) 불안정 **binary (sufficient/insufficient)**.
"full" vs "partial" 구분은 grounding_check intent alignment 담당.
Classifier verdict "relevant evidence 가 있나" binary 판단.
covered_aspects / missing_aspects 로깅용으로 유지 (refusal gate 에서 사용 ).
"""
from __future__ import annotations
import asyncio
import time
from dataclasses import dataclass
from typing import Literal
from ai.client import AIClient, _load_prompt, parse_json_response
from core.config import settings
from core.utils import setup_logger
from .llm_gate import Priority, acquire_mlx_gate
logger = setup_logger("classifier")
LLM_TIMEOUT_MS = 30000
CIRCUIT_THRESHOLD = 5
CIRCUIT_RECOVERY_SEC = 60
_failure_count = 0
_circuit_open_until: float | None = None
@dataclass(slots=True)
class ClassifierResult:
status: Literal["ok", "timeout", "error", "circuit_open", "skipped"]
verdict: Literal["sufficient", "insufficient"] | None
covered_aspects: list[str]
missing_aspects: list[str]
elapsed_ms: float
try:
CLASSIFIER_PROMPT = _load_prompt("classifier.txt")
except FileNotFoundError:
CLASSIFIER_PROMPT = ""
logger.warning("classifier.txt not found — classifier will always skip")
def _build_input(
query: str,
top_chunks: list[dict],
rerank_scores: list[float],
) -> str:
"""Y+ input (content + scores with role separation)."""
chunk_block = "\n".join(
f"[{i+1}] title: {c.get('title','')}\n"
f" section: {c.get('section','')}\n"
f" snippet: {c.get('snippet','')}"
for i, c in enumerate(top_chunks[:3])
)
scores_str = ", ".join(f"{s:.2f}" for s in rerank_scores[:3])
return (
CLASSIFIER_PROMPT
.replace("{query}", query)
.replace("{chunks}", chunk_block)
.replace("{scores}", scores_str)
)
async def classify(
query: str,
top_chunks: list[dict],
rerank_scores: list[float],
) -> ClassifierResult:
"""Always-on binary classifier. Parallel with evidence extraction.
Returns:
ClassifierResult with verdict=sufficient|insufficient.
Status "ok" 아니면 verdict=None (caller fallback 처리).
"""
global _failure_count, _circuit_open_until
t_start = time.perf_counter()
# Circuit breaker
if _circuit_open_until and time.time() < _circuit_open_until:
return ClassifierResult("circuit_open", None, [], [], 0.0)
if not CLASSIFIER_PROMPT:
return ClassifierResult("skipped", None, [], [], 0.0)
if not hasattr(settings.ai, "classifier") or settings.ai.classifier is None:
return ClassifierResult("skipped", None, [], [], 0.0)
prompt = _build_input(query, top_chunks, rerank_scores)
client = AIClient()
try:
# 2026-05-17: PR #20 이후 endpoint 가 Mac mini 26B → llm_gate Semaphore(1) 필수.
# Gate 미사용 시 classifier + evidence + synthesis 가 동시에 single-inference
# MLX 에 race → 거의 모두 timeout (실측: 8/10 fixture query). docstring 영구 룰:
# "MLX primary 호출 경로는 예외 없이 gate 획득 필수".
async with acquire_mlx_gate(Priority.FOREGROUND):
async with asyncio.timeout(LLM_TIMEOUT_MS / 1000):
raw = await client._request(settings.ai.classifier, prompt)
_failure_count = 0
except asyncio.TimeoutError:
_failure_count += 1
if _failure_count >= CIRCUIT_THRESHOLD:
_circuit_open_until = time.time() + CIRCUIT_RECOVERY_SEC
logger.error(f"classifier circuit OPEN for {CIRCUIT_RECOVERY_SEC}s")
logger.warning("classifier timeout")
return ClassifierResult(
"timeout", None, [], [],
(time.perf_counter() - t_start) * 1000,
)
except Exception as e:
_failure_count += 1
if _failure_count >= CIRCUIT_THRESHOLD:
_circuit_open_until = time.time() + CIRCUIT_RECOVERY_SEC
logger.error(f"classifier circuit OPEN for {CIRCUIT_RECOVERY_SEC}s")
logger.warning("classifier error: type=%s repr=%r", type(e).__name__, e)
return ClassifierResult(
"error", None, [], [],
(time.perf_counter() - t_start) * 1000,
)
finally:
await client.close()
elapsed_ms = (time.perf_counter() - t_start) * 1000
parsed = parse_json_response(raw)
if not isinstance(parsed, dict):
logger.warning("classifier parse failed raw=%r", (raw or "")[:200])
return ClassifierResult("error", None, [], [], elapsed_ms)
# ternary → binary 매핑
raw_verdict = parsed.get("verdict", "")
if raw_verdict == "insufficient":
verdict: Literal["sufficient", "insufficient"] | None = "insufficient"
elif raw_verdict in ("full", "partial", "sufficient"):
verdict = "sufficient"
else:
verdict = None
covered = parsed.get("covered_aspects") or []
missing = parsed.get("missing_aspects") or []
if not isinstance(covered, list):
covered = []
if not isinstance(missing, list):
missing = []
logger.info(
"classifier ok query=%r verdict=%s (raw=%s) covered=%d missing=%d elapsed_ms=%.0f",
query[:60], verdict, raw_verdict, len(covered), len(missing), elapsed_ms,
)
return ClassifierResult("ok", verdict, covered, missing, elapsed_ms)
+14 -19
View File
@@ -1,6 +1,6 @@
"""Time-aware retrieval freshness decay (PR-RAG-Time-1).
뉴스(source_channel='news') / 재해사례(material_type='incident', KOSHA) 도메인은
뉴스(source_channel='news') / 법령 알림(source_channel='law_monitor') 도메인은
시간이 중요한 문서. 단순 relevance score 만으로는 오래된 문서가 상위에 머물러
검색 품질이 떨어짐. 모듈은 reranker 이후 final score 합성 단계에서
soft multiplier 시간 가중치 적용. 삭제는 없음 ranking demote.
@@ -9,10 +9,9 @@ soft multiplier 로 시간 가중치 적용. 삭제는 없음 — ranking 만 de
- reranker = 의미 관련도, freshness decay = 운영 정책. 단계 분리 유지.
- floor 0.7 (multiplier 0.7 미만으로 떨어짐) 오래되어도 죽지 않음.
- 일반 업로드 / 학습 자료 / KGS Code 원문 / ai_drafted 비적용 (no-op).
- 법령(law) C-1 후속에서 freshness 제외 현행성은 version_status(B-1 버전체인) 처리.
published_date 컬럼이 documents 없음 created_at(수집 시점) 임시 proxy.
news/KOSHA 워커가 수집 즉시 indexing 하므로 created_at published_date.
news/law_monitor 워커가 수집 즉시 indexing 하므로 created_at published_date.
정확도 향상은 후속 PR (worker published_date 메타 채우기) 분리.
"""
@@ -33,10 +32,10 @@ if TYPE_CHECKING:
# ─── Policy ────────────────────────────────────────────────────────
# half-life (일). 90 일: 한 달 ~0.79 / 6개월 ~0.25.
# C-1 후속(2026-06-13): law_365d 폐기 — 법령 현행성은 version_status(B-1 버전체인)가 처리,
# age-decay 는 current 법령을 부당 강등(의도 변경 기록). 재해사례(incident)는 news_90d 흡수.
# 365 일: 1년 ~0.5 / 3년 ~0.13.
HALF_LIFE_DAYS: dict[str, int] = {
"news_90d": 90,
"law_365d": 365,
}
# soft multiplier — final = base * (FLOOR + (1-FLOOR) * decay).
@@ -53,35 +52,32 @@ class _DocMeta:
source_channel: str | None
content_origin: str | None
created_at: datetime | None
material_type: str | None = None
def freshness_policy(meta: _DocMeta | None) -> str | None:
"""문서 메타 → freshness 정책 이름 또는 None (no-op).
적용:
- material_type='incident' (KOSHA 재해사례/사망사고) news_90d (C-1 후속 흡수, 시간 민감)
- source_channel='news' news_90d
- source_channel='news' news_90d
- source_channel='law_monitor' law_365d
비적용 (None 반환):
- meta 자체가 None
- content_origin='ai_drafted' (생성 시점 = 가치 시점, 시간 demote 부적합)
- 법령(source_channel='law_monitor'/material_type='law'): C-1 후속에서 law_365d 폐기.
법령 현행성은 version_status(B-1 버전체인 current/superseded) 처리 age-decay
current 법령을 부당 강등(의도 변경 기록). law 검색 ranking = version_status decorate.
- 모든 source_channel (manual, drive_sync, inbox_route, memo 자연 비적용)
- 모든 source_channel (manual, drive_sync, inbox_route, memo,
Study/Manual/Reference/Academic/Checklist 자연 비적용)
"""
if meta is None:
return None
# 가드 2: content_origin='ai_drafted' 비적용
if meta.content_origin == "ai_drafted":
return None
# 재해사례/사망사고 = 시간 민감 → news 와 동일 90d (source 무관, 업로드 incident 도 포함)
if meta.material_type == "incident":
sc = meta.source_channel
if sc == "news":
return "news_90d"
if meta.source_channel == "news":
return "news_90d"
# 법령 law_365d 폐기 + unknown source_channel → no decay
if sc == "law_monitor":
return "law_365d"
# 가드 6: unknown source_channel → no decay
return None
@@ -133,7 +129,7 @@ async def _fetch_meta(
text(
"""
SELECT id, source_channel::text AS source_channel,
content_origin, material_type, created_at
content_origin, created_at
FROM documents
WHERE id = ANY(:ids)
"""
@@ -145,7 +141,6 @@ async def _fetch_meta(
source_channel=row.source_channel,
content_origin=row.content_origin,
created_at=row.created_at,
material_type=getattr(row, "material_type", None),
)
for row in rows
}
+505
View File
@@ -0,0 +1,505 @@
"""Grounding check — post-synthesis 검증 (Phase 3.5a).
Strong/weak flag 분리:
- **Strong** ( partial 강등 or refuse): fabricated_number, intent_misalignment(important)
- **Weak** ( confidence lower only): uncited_claim, low_overlap, intent_misalignment(generic)
Re-gate 로직 (Phase 3.5a 9라운드 토론 결과):
- strong 1 partial 강등
- strong 2 이상 refuse
- weak confidence "low"
Intent alignment (rule-based):
- query 핵심 명사가 answer 등장하는지 확인
- "처벌" 같은 중요 키워드 누락은 strong
- "주요", "관련" 같은 generic 무시
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
from core.utils import setup_logger
if TYPE_CHECKING:
from .evidence_service import EvidenceItem
logger = setup_logger("grounding")
# "주요", "관련" 등 intent alignment 에서 제외할 generic 단어
GENERIC_TERMS = frozenset({
"주요", "관련", "내용", "정의", "기준", "방법", "설명", "개요",
"대한", "위한", "대해", "무엇", "어떤", "어떻게", "있는",
"하는", "되는", "이런", "그런", "이것", "그것",
})
@dataclass(slots=True)
class GroundingResult:
strong_flags: list[str]
weak_flags: list[str]
_UNIT_CHARS = r'명인개%년월일조항호세건원회'
# "이상/이하/초과/미만" — threshold 표현 (numeric conflict 에서 skip 대상)
_THRESHOLD_SUFFIXES = re.compile(r'이상|이하|초과|미만')
# 약칭/근사치 prefix — 매칭 전 제거 (Phase 3.5 B1).
# ⚠ 최대/최소 는 의도적으로 제외 — 이들은 bound operator 라 의미가 다름 (Phase 3.5 B1 fix3).
# 약/대략/거의/얼추 만 노이즈 prefix 로 strip.
_APPROX_PREFIX_RE = re.compile(r'(약|대략|거의|얼추)\s*')
# 단위 동의어 dict — 추출 직후 정규화 (Phase 3.5 B1)
# 의미가 동일한 단위는 같은 표기로 통일해서 set 비교/range overlap 안정화.
_UNIT_SYNONYMS: dict[str, str] = {
"": "",
"사람": "",
"퍼센트": "%",
"프로": "%",
"KRW": "",
"krw": "",
}
# tolerance(±1%) 허용 단위 — 양적 측정값 (Phase 3.5 B1)
_TOLERANCE_UNITS: frozenset[str] = frozenset({"", "", "%", "", ""})
# tolerance 미적용 단위 — 식별자성 숫자 (연도/조문/횟수)
_EXACT_ONLY_UNITS: frozenset[str] = frozenset({"", "", "", "", "", "", ""})
# 최대/최소 prefix 패턴 — bound operator (Phase 3.5 B1 fix3).
# 매칭된 숫자는 exact pool 에서 제외하고 one-sided range 로 변환.
# 경계값 자체는 clear 대상 아님 (Codex 권장: "최대 100명" + answer "100명" → flag 유지).
_BOUND_PATTERN_RE = re.compile(
rf'(최대|최소)\s*(\d[\d,.]*)\s*([{_UNIT_CHARS}]|인|사람|퍼센트|프로|KRW|krw)'
)
_RANGE_INF = 10**18 # one-sided range 상한 sentinel
def _normalize_unit(unit: str) -> str:
"""단위 동의어 → 대표 표기."""
return _UNIT_SYNONYMS.get(unit, unit)
def _extract_unit(literal: str) -> str | None:
"""리터럴에서 숫자 뒤 단위(한 글자 또는 동의어) 추출 + 정규화."""
# 천단위 콤마 + 옵션 소수 + 한글 단위 한 글자 또는 동의어
m = re.match(rf'[\d,.]+\s*([{_UNIT_CHARS}]|인|사람|퍼센트|프로|KRW|krw)', literal)
if not m:
return None
return _normalize_unit(m.group(1))
def _extract_numeric_corpus(text: str) -> dict:
"""단위별 숫자 + 범위 + bound 통합 추출 (Phase 3.5 B1 fix1+fix3).
Returns:
{
"exact_by_unit": {unit_or_None: set(digits)}, # 평범한 숫자 (bound 제외)
"ranges_by_unit": {unit: [(lo, hi), ...]}, # 양방향(A~B) + 단방향(최대/최소)
}
None 키는 단위 없는 bare 숫자.
`최대 N <unit>` ranges[(0, N-1)] (경계값 자체는 cleared 대상 아님)
`최소 N <unit>` ranges[(N+1, INF)]
"""
cleaned = _APPROX_PREFIX_RE.sub('', text)
exact_by_unit: dict[str | None, set[str]] = {None: set()}
ranges_by_unit: dict[str, list[tuple[int, int]]] = {}
# 1) 최대/최소 — bound. exact pool 에서 제외, one-sided range 로 변환.
bound_spans: list[tuple[int, int]] = [] # 매칭 substring 위치 — 이후 단계에서 skip
for m in _BOUND_PATTERN_RE.finditer(cleaned):
bound_kind = m.group(1)
try:
n = int(m.group(2).replace(',', '').split('.')[0])
except ValueError:
continue
unit = _normalize_unit(m.group(3))
if bound_kind == "최대":
ranges_by_unit.setdefault(unit, []).append((0, max(0, n - 1)))
else: # 최소
ranges_by_unit.setdefault(unit, []).append((n + 1, _RANGE_INF))
bound_spans.append((m.start(), m.end()))
def _in_bound_span(pos: int) -> bool:
return any(s <= pos < e for s, e in bound_spans)
# 2) 천단위 콤마 bare number
for m in re.finditer(r'\d{1,3}(?:,\d{3})+(?:\.\d+)?', cleaned):
if _in_bound_span(m.start()):
continue
exact_by_unit[None].add(m.group().replace(',', ''))
# 3) 단위 있는 숫자 (단위 동의어 포함)
for m in re.finditer(
rf'(\d[\d,.]*)\s*([{_UNIT_CHARS}]|인|사람|퍼센트|프로|KRW|krw)',
cleaned,
):
if _in_bound_span(m.start()):
continue
digits = m.group(1).replace(',', '').split('.')[0]
if not digits:
continue
unit = _normalize_unit(m.group(2))
exact_by_unit.setdefault(unit, set()).add(digits)
# 4) 양방향 범위 표현 (A~B / A 부터 B)
for m in re.finditer(
rf'(\d[\d,.]*)\s*(?:[~\-]|부터)\s*(\d[\d,.]*)\s*([{_UNIT_CHARS}]|인|사람|퍼센트|프로)',
cleaned,
):
if _in_bound_span(m.start()):
continue
try:
lo = int(m.group(1).replace(',', '').split('.')[0])
hi = int(m.group(2).replace(',', '').split('.')[0])
except ValueError:
continue
unit = _normalize_unit(m.group(3))
ranges_by_unit.setdefault(unit, []).append((min(lo, hi), max(lo, hi)))
# 5) bare 2자리+ 단독 숫자
for m in re.finditer(r'\b(\d{2,})\b', cleaned):
if _in_bound_span(m.start()):
continue
exact_by_unit[None].add(m.group())
return {
"exact_by_unit": exact_by_unit,
"ranges_by_unit": ranges_by_unit,
}
def _within_unit_range(
n: int, unit: str | None, ranges_by_unit: dict[str, list[tuple[int, int]]]
) -> bool:
"""unit-matching range 검증.
answer unit None (bare 숫자) 보수적으로 False bare 답변은 range clear 대상 아님.
"""
if unit is None:
return False
return any(lo <= n <= hi for lo, hi in ranges_by_unit.get(unit, []))
def _close_to_unit_pool(
n: int, unit: str | None, exact_by_unit: dict[str | None, set[str]], tol: float
) -> bool:
"""unit-matching tolerance 검증.
answer unit None 이면 False bare 답변은 tolerance 대상 아님.
같은 unit bucket 안의 후보만 비교.
"""
if unit is None:
return False
candidates = exact_by_unit.get(unit, set())
for c in candidates:
try:
cn = int(c)
except ValueError:
continue
if cn == 0:
continue
if abs(n - cn) / cn <= tol:
return True
return False
def _extract_number_literals(text: str) -> set[str]:
"""숫자 + 단위 추출 + normalize (Phase 3.5 B1: 6단계 확장).
1) 약칭 prefix 제거 ("약 100명" "100명")
2) 천단위 콤마 bare number 우선 ("1,000" "1000" set 등록)
3) 한국어 단위 접미사 매칭 (기존)
4) 범위 표현 양쪽 숫자 추출 (separator: ~, -, , 부터)
5) 단위 동의어 정규화 (, 퍼센트%, KRW)
6) bare 2자리+ 추출 (기존)
"""
# 1. 약칭 prefix 제거 (전체 텍스트에서)
cleaned = _APPROX_PREFIX_RE.sub('', text)
# 2. 천단위 콤마 bare number — normalize 된 값을 set 에 선등록
normalized: set[str] = set()
for m in re.finditer(r'\d{1,3}(?:,\d{3})+(?:\.\d+)?', cleaned):
normalized.add(m.group().replace(',', ''))
# 3. 숫자 + 한국어 단위 접미사 (동의어 포함)
raw: set[str] = set(re.findall(
rf'\d[\d,.]*\s*(?:[{_UNIT_CHARS}]|인|사람|퍼센트|프로|KRW|krw)\w{{0,2}}',
cleaned,
))
# 4. 범위 표현 — separator 에 "부터" 추가
for m in re.finditer(
rf'(\d[\d,.]*)\s*(?:[~\-]|부터)\s*(\d[\d,.]*)\s*([{_UNIT_CHARS}]|인|사람|퍼센트|프로)',
cleaned,
):
unit_norm = _normalize_unit(m.group(3))
raw.add(m.group(1) + unit_norm)
raw.add(m.group(2) + unit_norm)
# 5. normalize: 단위 동의어 통일 + 콤마 제거
for r in raw:
# 단위 부분 정규화
m = re.match(r'([\d,.]+)\s*([^\d\s]+)', r)
if m:
digits_part = m.group(1)
unit_part = _normalize_unit(m.group(2))
normalized.add(digits_part + unit_part)
normalized.add(digits_part.replace(',', '') + unit_part)
normalized.add(r.strip())
num_only = re.match(r'[\d,.]+', r)
if num_only:
normalized.add(num_only.group().replace(',', ''))
# 6. 단독 숫자 (2자리+ 만)
for d in re.findall(r'\b(\d{2,})\b', cleaned):
normalized.add(d)
return normalized
def _within_evidence_range(digits: str, raw: str, evidence_text: str) -> bool:
"""evidence 에 'A~B 단위' 가 있고 answer 의 숫자가 그 범위 안이면 True.
범위 단위는 무시 (단위 비교는 호출 단계). digits = 정수 문자열.
"""
try:
n = int(digits)
except ValueError:
return False
cleaned_ev = _APPROX_PREFIX_RE.sub('', evidence_text)
for m in re.finditer(
rf'(\d[\d,.]*)\s*(?:[~\-]|부터)\s*(\d[\d,.]*)\s*[{_UNIT_CHARS}]',
cleaned_ev,
):
try:
lo = int(m.group(1).replace(',', '').split('.')[0])
hi = int(m.group(2).replace(',', '').split('.')[0])
if min(lo, hi) <= n <= max(lo, hi):
return True
except ValueError:
continue
return False
def _close_to_any(n: int, candidates: set[str], tol: float) -> bool:
"""candidates 중 하나라도 (1±tol) 배율 안에 들어오면 True.
n 정수, candidates digits-only 문자열 집합.
"""
for c in candidates:
try:
cn = int(c)
except ValueError:
continue
if cn == 0:
continue
if abs(n - cn) / cn <= tol:
return True
return False
def _extract_content_tokens(text: str) -> set[str]:
"""한국어 2자 이상 명사 + 영어 3자 이상 단어."""
return set(re.findall(r'[가-힣]{2,}|[a-zA-Z]{3,}', text))
def _parse_number_with_unit(literal: str) -> tuple[str, str] | None:
"""숫자 리터럴에서 (digits_only, unit) 분리. 단위 없으면 None."""
m = re.match(rf'([\d,.]+)\s*([{_UNIT_CHARS}])', literal)
if not m:
return None
digits = m.group(1).replace(',', '')
unit = m.group(2)
return (digits, unit)
def _check_evidence_numeric_conflicts(evidence: list["EvidenceItem"]) -> list[str]:
"""evidence 간 숫자 충돌 감지 (Phase 3.5b). evidence >= 2 일 때만 활성.
동일 단위, 다른 숫자 weak flag. "이상/이하/초과/미만" 포함 skip.
bare number 비교 (조항 번호 false positive 방지).
"""
if len(evidence) < 2:
return []
# 각 evidence 에서 단위 있는 숫자 + threshold 여부 추출
# {evidence_idx: [(digits, unit, has_threshold), ...]}
per_evidence: dict[int, list[tuple[str, str, bool]]] = {}
for idx, ev in enumerate(evidence):
nums = re.findall(
rf'\d[\d,.]*\s*[{_UNIT_CHARS}]\w{{0,4}}',
ev.span_text,
)
entries = []
for raw in nums:
parsed = _parse_number_with_unit(raw)
if not parsed:
continue
has_thr = bool(_THRESHOLD_SUFFIXES.search(raw))
entries.append((parsed[0], parsed[1], has_thr))
if entries:
per_evidence[idx] = entries
if len(per_evidence) < 2:
return []
# 단위별로 evidence 간 숫자 비교
# {unit: {digits: [evidence_idx, ...]}}
unit_map: dict[str, dict[str, list[int]]] = {}
for idx, entries in per_evidence.items():
for digits, unit, has_thr in entries:
if has_thr:
continue # threshold 표현은 skip
if unit not in unit_map:
unit_map[unit] = {}
if digits not in unit_map[unit]:
unit_map[unit][digits] = []
if idx not in unit_map[unit][digits]:
unit_map[unit][digits].append(idx)
flags: list[str] = []
for unit, digits_map in unit_map.items():
distinct_values = list(digits_map.keys())
if len(distinct_values) >= 2:
# 가장 많이 등장하는 2개 비교
top2 = sorted(distinct_values, key=lambda d: len(digits_map[d]), reverse=True)[:2]
flags.append(
f"evidence_numeric_conflict:{top2[0]}{unit}_vs_{top2[1]}{unit}"
)
return flags
def check(
query: str,
answer: str,
evidence: list[EvidenceItem],
) -> GroundingResult:
"""답변 vs evidence grounding 검증 + query intent alignment."""
strong: list[str] = []
weak: list[str] = []
if not answer or not evidence:
return GroundingResult([], [])
# ⚠ citation marker [n] 양측 제거 (대칭성 — Phase 3.5 B1)
evidence_text = re.sub(r'\[\d+\]', '', " ".join(e.span_text for e in evidence))
# ── Strong 1: fabricated number (unit-aware 3단계 — Phase 3.5 B1 fix1+fix3) ──
# Codex 지적 반영:
# - fix1: range/tolerance/exact 모두 단위 일치 시에만 clear
# (예: "150원" vs "100~200명" → flag 유지)
# - fix3: 최대/최소 prefix 는 bound 의미 보존
# (예: "최대 100명" + answer "100명" → flag 유지, "최대 100명" + answer "50명" → cleared)
answer_clean = re.sub(r'\[\d+\]', '', answer)
answer_corpus = _extract_numeric_corpus(answer_clean)
evidence_corpus = _extract_numeric_corpus(evidence_text)
ev_exact_by_unit = evidence_corpus["exact_by_unit"]
ev_ranges_by_unit = evidence_corpus["ranges_by_unit"]
# cleared 는 (unit, digits) 쌍 단위로 추적 — 단위 충돌 케이스 방어
cleared_pairs: set[tuple[str | None, str]] = set()
# Pass 1: 각 (unit, digits) 가 evidence 에서 정당화되는지 판정
for unit, digits_set in answer_corpus["exact_by_unit"].items():
for d in digits_set:
# 1) exact match — 같은 unit bucket 내에서만
if d in ev_exact_by_unit.get(unit, set()):
cleared_pairs.add((unit, d))
continue
# bare answer (unit=None) 는 evidence bare bucket 도 보조 매칭
if unit is None and d in ev_exact_by_unit.get(None, set()):
cleared_pairs.add((unit, d))
continue
try:
n = int(d)
except ValueError:
continue
# 2) range — same-unit 만 (bare answer 는 range clear 대상 아님)
if _within_unit_range(n, unit, ev_ranges_by_unit):
cleared_pairs.add((unit, d))
continue
# 3) ±1% tolerance — 단위가 양적(_TOLERANCE_UNITS) + 4자리+ + same-unit
if (
unit in _TOLERANCE_UNITS
and len(d) >= 4
and _close_to_unit_pool(n, unit, ev_exact_by_unit, tol=0.01)
):
cleared_pairs.add((unit, d))
continue
# 식별자성 단위(_EXACT_ONLY_UNITS) 는 tolerance 패스 X.
# Pass 2: cleared 되지 않은 (unit, digits) 를 strong flag.
# 1자리 무시는 unit 이 식별자성(_EXACT_ONLY_UNITS: 년/월/일/조/항/호/회) 이 아닐 때만 적용.
# bare(None) 답변 숫자는 같은 digit 이 다른 unit 에서 cleared 됐으면 skip — 추출 부산물 방어.
# ⚠ 단위 cross-clear (예: "원" cleared → "명" 도 skip) 은 금지: Codex unit-mismatch 케이스가 깨짐.
unit_anchored_cleared: set[str] = {d for (u, d) in cleared_pairs if u is not None}
flagged_keys: set[tuple[str | None, str]] = set()
for unit, digits_set in answer_corpus["exact_by_unit"].items():
for d in digits_set:
if (unit, d) in cleared_pairs or (unit, d) in flagged_keys:
continue
# bare(None) 답변 숫자가 임의의 단위 bucket 에서 cleared 됐으면 duplicate 로 처리.
# 사례: "1,000명" → unit bucket "명" 에 1000 + bare bucket None 에 1000 (comma normalize 부산물).
# 이미 ("명", "1000") 가 cleared 라면 (None, "1000") 도 같은 사실을 가리키므로 skip.
if unit is None and d in unit_anchored_cleared:
continue
if len(d) < 2 and unit not in _EXACT_ONLY_UNITS:
continue
flagged_keys.add((unit, d))
# 사람이 읽기 좋게 "{digits}{unit}" 또는 bare 형태로 표기
label = f"{d}{unit}" if unit else d
strong.append(f"fabricated_number:{label}")
# ── Strong/Weak 2: query-answer intent alignment ──
query_content = _extract_content_tokens(query)
answer_content = _extract_content_tokens(answer)
if query_content:
missing_terms = query_content - answer_content
important_missing = [
t for t in missing_terms
if t not in GENERIC_TERMS and len(t) >= 2
]
if important_missing:
strong.append(
f"intent_misalignment:{','.join(important_missing[:3])}"
)
elif len(missing_terms) > len(query_content) * 0.5:
weak.append(
f"intent_misalignment_generic:"
f"missing({','.join(list(missing_terms)[:5])})"
)
# ── Weak 1: uncited claim ──
sentences = re.split(r'(?<=[.!?。])\s+', answer)
for s in sentences:
if len(s.strip()) > 20 and not re.search(r'\[\d+\]', s):
weak.append(f"uncited_claim:{s[:40]}")
# ── Weak: evidence 간 숫자 충돌 (Phase 3.5b) ──
conflicts = _check_evidence_numeric_conflicts(evidence)
weak.extend(conflicts)
# ── Weak 2: token overlap ──
answer_tokens = _extract_content_tokens(answer)
evidence_tokens = _extract_content_tokens(evidence_text)
if answer_tokens:
overlap = len(answer_tokens & evidence_tokens) / len(answer_tokens)
if overlap < 0.4:
weak.append(f"low_overlap:{overlap:.2f}")
if strong or weak:
logger.info(
"grounding query=%r strong=%d weak=%d flags=%s",
query[:60],
len(strong),
len(weak),
",".join(strong[:3] + weak[:3]),
)
return GroundingResult(strong, weak)
+105
View File
@@ -0,0 +1,105 @@
"""Refusal gate — multi-signal fusion (Phase 3.5a).
Score gate (deterministic) + classifier verdict (semantic, binary) 독립 평가 합성.
Classifier 부재 3-tier conservative fallback.
P1 실측 결과: exaone ternary 불안정 binary (sufficient/insufficient) 축소.
"full" vs "partial" 구분은 grounding check (intent alignment) 담당.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Literal
from core.utils import setup_logger
if TYPE_CHECKING:
from .classifier_service import ClassifierResult
logger = setup_logger("refusal_gate")
# Placeholder thresholds — Phase 3.5b 에서 실측 기반 tuning
# AND 조건이라 false refusal 방어됨 (둘 다 만족해야 refuse)
SCORE_MAX_REFUSE = 0.25
SCORE_AGG_REFUSE = 0.70
# Conservative fallback tiers (classifier 부재 시)
CONSERVATIVE_WEAK = 0.35
CONSERVATIVE_MID = 0.55
@dataclass(slots=True)
class RefusalDecision:
refused: bool
confidence_cap: Literal["high", "medium", "low"] | None # None = no cap
rule_triggered: str | None # 디버깅: 어느 signal 이 결정에 기여?
def decide(
rerank_scores: list[float],
classifier: ClassifierResult | None,
) -> RefusalDecision:
"""Multi-signal fusion. Binary classifier verdict 기반.
Returns:
RefusalDecision. refused=True 이면 synthesis skip.
confidence_cap synthesis 결과의 confidence upper bound 적용.
"""
max_score = max(rerank_scores) if rerank_scores else 0.0
agg_top3 = sum(sorted(rerank_scores, reverse=True)[:3])
score_gate_fails = (
max_score < SCORE_MAX_REFUSE and agg_top3 < SCORE_AGG_REFUSE
)
# ── Classifier 사용 가능 (정상 경로) ──
if classifier and classifier.verdict is not None:
if classifier.verdict == "insufficient":
# Evidence quality override: classifier 가 insufficient 라 해도
# evidence 가 충분히 좋으면 override (토론 8라운드 합의)
# (evidence quality 는 이 함수 밖에서 별도 체크 — caller 에서 처리)
logger.info(
"refusal gate: classifier=insufficient max=%.2f agg=%.2f",
max_score, agg_top3,
)
return RefusalDecision(
refused=True,
confidence_cap=None,
rule_triggered="classifier_insufficient",
)
if score_gate_fails:
logger.info(
"refusal gate: score_low max=%.2f agg=%.2f classifier=%s",
max_score, agg_top3, classifier.verdict,
)
return RefusalDecision(
refused=True,
confidence_cap=None,
rule_triggered="score_low",
)
# Classifier says sufficient → proceed
return RefusalDecision(
refused=False,
confidence_cap=None,
rule_triggered=None,
)
# ── Classifier 부재 → 3-tier conservative ──
if max_score < CONSERVATIVE_WEAK:
return RefusalDecision(
refused=True,
confidence_cap=None,
rule_triggered="conservative_refuse(no_classifier)",
)
if max_score < CONSERVATIVE_MID:
return RefusalDecision(
refused=False,
confidence_cap="low",
rule_triggered="conservative_low(no_classifier)",
)
return RefusalDecision(
refused=False,
confidence_cap="medium",
rule_triggered="conservative_medium(no_classifier)",
)
+4 -5
View File
@@ -44,11 +44,10 @@ RERANK_TIMEOUT = 5.0
# server-side allowlist map. query parameter 가 raw endpoint URL 받지 않음.
RERANKER_BACKEND_MAP: dict[str, dict[str, str] | None] = {
"baseline": None, # production reranker (config.yaml endpoint via AIClient.rerank)
"cand_gte_ml_base": {
"endpoint": "http://rerank-cand-gte-ml-base:80/rerank",
},
# mxbai_large 후보 (deberta-v2 → TEI 1.7 미지원) Phase 2B-Extended 이관
# bge_v2_gemma_2b 후보 (LLM-based reranker, 1_Pooling/config.json 부재) Phase 2B-Extended 이관
# Phase 2B 후보 reranker 전부 NO-GO 종결 (2026-06-18 teardown):
# - cand_gte_ml_base : 컨테이너·DB 테이블(마이그 360)·override.rerank-cand.yml 제거됨
# - mxbai_large (deberta-v2 → TEI 1.7 미지원) / bge_v2_gemma_2b (1_Pooling 부재) 미진입
# dispatcher scaffold(_resolve_reranker)는 향후 후보 재진입 위해 보존.
}
+2 -37
View File
@@ -76,15 +76,10 @@ class AxisFilter:
jurisdiction: str | None = None
year_from: int | None = None
year_to: int | None = None
domain_buckets: list[str] | None = None # 377: domain_bucket = ANY (도메인 스코프)
exclude_buckets: list[str] | None = None # 377: domain_bucket <> ALL (예: News 제외)
cloud_egress: bool = False # 갭2: 클라우드 소비자 cloud-eligibility allowlist 강제(토큰 claim 유래)
def active(self) -> bool:
return bool(self.material_types or self.jurisdiction
or self.year_from is not None or self.year_to is not None
or self.domain_buckets or self.exclude_buckets
or self.cloud_egress)
or self.year_from is not None or self.year_to is not None)
def _axis_sql(alias: str, af: "AxisFilter | None", params: dict) -> str:
@@ -109,22 +104,6 @@ def _axis_sql(alias: str, af: "AxisFilter | None", params: dict) -> str:
if af.year_to is not None:
cl.append(f"COALESCE({p}published_date, {p}created_at::date) <= make_date(:af_yt, 12, 31)")
params["af_yt"] = af.year_to
if af.domain_buckets:
cl.append(f"{p}domain_bucket = ANY(:af_db)")
params["af_db"] = af.domain_buckets
if af.exclude_buckets:
cl.append(f"{p}domain_bucket <> ALL(:af_xdb)")
params["af_xdb"] = af.exclude_buckets
if af.cloud_egress:
# 갭2 클라우드 egress allowlist(default-deny). restricted 는 _license_sql 별도 차단.
cl.append(
f"({p}data_origin = 'external' OR ("
f"{p}data_origin = 'work' "
f"AND {p}domain_bucket IN ('Engineering','Safety','Law') "
f"AND ({p}source_channel IS NULL OR {p}source_channel::text NOT IN ('voice','chat','memo')) "
f"AND {p}category::text IS DISTINCT FROM 'memo' "
f"AND ({p}user_note IS NULL OR {p}user_note = '')))"
)
return " AND " + " AND ".join(cl)
@@ -142,21 +121,7 @@ def _license_sql(alias: str) -> str:
술어 정의 = license_filter.restricted_exclude_sql 공유(digest/briefing/study 풀이와 단일 source).
"""
from services.search.license_filter import restricted_exclude_sql
_p = (alias + ".") if alias else ""
# ASME clause-KB(379): clause docs (doc_kind='clause') = read/nav/backlink only, excluded from retrieval/digest legs.
return " AND " + restricted_exclude_sql(alias) + f" AND {_p}doc_kind = 'standard'"
def cloud_eligible_doc_sql(alias: str = "") -> str:
"""단일 문서가 cloud 소비자(예: Claude/MCP)에게 노출 가능한가 = search retrieval 과
동일한 egress allowlist(갭2) + license 제한(B-4) 결합 술어. fetch_document(cloud)
search byte-동일 게이트를 공유하도록 단일 source([[feedback_structural_integrity_over_path_discipline]]).
cloud_egress·license leg 모두 bind 파라미터 없는 리터럴 술어라 호출측 추가 params 불요.
주의: _license_sql 소유자 단건 다운로드엔 미적용(a안)이지만, cloud 노출은 구매 전자책
verbatim 누출을 막아야 하므로 여기선 항상 적용 = search 동일(local 토큰은 게이트 미발동).
반환 ' AND (egress allowlist) AND (license)' (alias='' = 컬럼 직접 참조). default-deny."""
return _axis_sql(alias, AxisFilter(cloud_egress=True), {}) + _license_sql(alias)
return " AND " + restricted_exclude_sql(alias)
# 2단계 gate (R2-B1) — SQL string interpolation 직전 final allowlist.
+194
View File
@@ -0,0 +1,194 @@
"""Exaone semantic verifier (Phase 3.5b).
답변-근거 의미적 모순(contradiction) 감지. rule-based grounding_check 잡는
미묘한 모순 포착. classifier 동일 패턴: circuit breaker + timeout + fail open.
## Severity 3단계
- strong: direct_negation (완전 모순) re-gate 교차 자격
- medium: numeric_conflict, intent_core_mismatch confidence 하향 (누적 강제 low)
- weak: nuance, unsupported_claim 로깅 + mild confidence 하향
## 핵심 원칙
- **Verifier strong 단독 refuse 금지** grounding strong 교차해야 refuse
- **Timeout 3s** 느리면 없는 낫다 (fail open)
- MLX gate 미사용 (PR #20 이후 Mac mini 26B endpoint — concurrent 안전성 별 검토)
"""
from __future__ import annotations
import asyncio
import os
import time
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Literal
from ai.client import AIClient, _load_prompt, parse_json_response
from core.config import settings
from core.utils import setup_logger
if TYPE_CHECKING:
from .evidence_service import EvidenceItem
logger = setup_logger("verifier")
LLM_TIMEOUT_MS = 10000 # 2026-05-17 B-3: 3s 시 동시 부하 시 verifier 빈발 skip → grounding 약화. Mac mini 26B 가 verifier-style 짧은 LLM call 도 concurrent 호출 시 3s 초과 빈번 — 10s 로 raise
CIRCUIT_THRESHOLD = 5
CIRCUIT_RECOVERY_SEC = 60
_failure_count = 0
_circuit_open_until: float | None = None
# Phase 3.5 B2: numeric_conflict severity promote 실험.
# import time 평가 — env 변경 후 process restart 필수 (docker compose restart fastapi).
# default=0 (off). production 적용은 B3 FP 검증 통과 후만.
_NUMERIC_PROMOTE = os.getenv("VERIFIER_NUMERIC_PROMOTE", "0") == "1"
# severity 매핑 (프롬프트 "critical"/"minor" → 코드 strong/medium/weak)
# Tier 4 (B2): _NUMERIC_PROMOTE=1 일 때 numeric_conflict critical → strong 으로 격상.
# minor 는 medium 유지 (FP 위험 분리).
_SEVERITY_MAP: dict[str, dict[str, Literal["strong", "medium", "weak"]]] = {
"direct_negation": {"critical": "strong", "minor": "strong"},
"numeric_conflict": (
{"critical": "strong", "minor": "medium"} if _NUMERIC_PROMOTE
else {"critical": "medium", "minor": "medium"}
),
"intent_core_mismatch": {"critical": "medium", "minor": "medium"},
"nuance": {"critical": "weak", "minor": "weak"},
"unsupported_claim": {"critical": "weak", "minor": "weak"},
}
@dataclass(slots=True)
class Contradiction:
"""개별 모순 발견."""
type: str # direct_negation / numeric_conflict / intent_core_mismatch / nuance / unsupported_claim
severity: Literal["strong", "medium", "weak"]
claim: str
evidence_ref: str
explanation: str
@dataclass(slots=True)
class VerifierResult:
status: Literal["ok", "timeout", "error", "circuit_open", "skipped"]
contradictions: list[Contradiction]
elapsed_ms: float
try:
VERIFIER_PROMPT = _load_prompt("verifier.txt")
except FileNotFoundError:
VERIFIER_PROMPT = ""
logger.warning("verifier.txt not found — verifier will always skip")
def _build_input(
answer: str,
evidence: list[EvidenceItem],
) -> str:
"""답변 + evidence spans → 프롬프트."""
spans = "\n\n".join(
f"[{e.n}] {(e.title or '').strip()}\n{e.span_text}"
for e in evidence
)
return (
VERIFIER_PROMPT
.replace("{answer}", answer)
.replace("{numbered_evidence}", spans)
)
def _map_severity(ctype: str, raw_severity: str) -> Literal["strong", "medium", "weak"]:
"""type + raw severity → 코드 severity 3단계."""
type_map = _SEVERITY_MAP.get(ctype, {"critical": "weak", "minor": "weak"})
return type_map.get(raw_severity, "weak")
async def verify(
query: str,
answer: str,
evidence: list[EvidenceItem],
) -> VerifierResult:
"""답변-근거 semantic 검증. Parallel with grounding_check.
Returns:
VerifierResult. status "ok" 아니면 contradictions 리스트 (fail open).
"""
global _failure_count, _circuit_open_until
t_start = time.perf_counter()
if _circuit_open_until and time.time() < _circuit_open_until:
return VerifierResult("circuit_open", [], 0.0)
if not VERIFIER_PROMPT:
return VerifierResult("skipped", [], 0.0)
if not hasattr(settings.ai, "verifier") or settings.ai.verifier is None:
return VerifierResult("skipped", [], 0.0)
if not answer or not evidence:
return VerifierResult("skipped", [], 0.0)
prompt = _build_input(answer, evidence)
client = AIClient()
try:
async with asyncio.timeout(LLM_TIMEOUT_MS / 1000):
raw = await client._request(settings.ai.verifier, prompt)
_failure_count = 0
except asyncio.TimeoutError:
_failure_count += 1
if _failure_count >= CIRCUIT_THRESHOLD:
_circuit_open_until = time.time() + CIRCUIT_RECOVERY_SEC
logger.error(f"verifier circuit OPEN for {CIRCUIT_RECOVERY_SEC}s")
logger.warning("verifier timeout")
return VerifierResult(
"timeout", [],
(time.perf_counter() - t_start) * 1000,
)
except Exception as e:
_failure_count += 1
if _failure_count >= CIRCUIT_THRESHOLD:
_circuit_open_until = time.time() + CIRCUIT_RECOVERY_SEC
logger.error(f"verifier circuit OPEN for {CIRCUIT_RECOVERY_SEC}s")
logger.warning(f"verifier error: {e}")
return VerifierResult(
"error", [],
(time.perf_counter() - t_start) * 1000,
)
finally:
await client.close()
elapsed_ms = (time.perf_counter() - t_start) * 1000
parsed = parse_json_response(raw)
if not isinstance(parsed, dict):
logger.warning("verifier parse failed raw=%r", (raw or "")[:200])
return VerifierResult("error", [], elapsed_ms)
# contradiction 파싱
raw_items = parsed.get("contradictions") or []
if not isinstance(raw_items, list):
raw_items = []
results: list[Contradiction] = []
for item in raw_items[:5]:
if not isinstance(item, dict):
continue
ctype = item.get("type", "")
if ctype not in _SEVERITY_MAP:
ctype = "unsupported_claim"
raw_sev = item.get("severity", "minor")
severity = _map_severity(ctype, raw_sev)
claim = str(item.get("claim", ""))[:50]
ev_ref = str(item.get("evidence_ref", ""))[:50]
explanation = str(item.get("explanation", ""))[:30]
results.append(Contradiction(ctype, severity, claim, ev_ref, explanation))
logger.info(
"verifier ok query=%r contradictions=%d strong=%d medium=%d elapsed_ms=%.0f",
query[:60],
len(results),
sum(1 for c in results if c.severity == "strong"),
sum(1 for c in results if c.severity == "medium"),
elapsed_ms,
)
return VerifierResult("ok", results, elapsed_ms)
-25
View File
@@ -1,25 +0,0 @@
"""채점(outcome) 산출 단일 소스 (study-to-viewer P2).
라이브 attempt 엔드포인트(submit_attempt) 뷰어 ingest **동일 함수** 채점
정오 어휘가 (서버)에서 결정(plan r2: ingest raw 신호 selected+unsure 싣고
DS 산출 = '무수정 재생' 실제로 성립시키는 형태). correct_choice 항상 현재 DB .
규칙(라이브 study_questions.py:1008-1020 동일):
is_unsure=True (None, False, 'unsure') # unsure 가 정오 override, selected 폐기
selected None ValueError # 선택 없고 unsure 도 아니면 무효(엔드포인트가 처리)
selected==correct (selected, is_correct, 'correct'|'wrong')
"""
from __future__ import annotations
def derive_outcome(
selected_choice: int | None, is_unsure: bool, correct_choice: int
) -> tuple[int | None, bool, str]:
"""(selected, is_correct, outcome) 반환. skipped 는 여기서 안 나옴(선택 없으면 호출측이 거부/skip)."""
if is_unsure:
return None, False, "unsure"
if selected_choice is None:
raise ValueError("selected_choice (1~4) 또는 is_unsure=true 가 필요합니다")
is_correct = selected_choice == correct_choice
return selected_choice, is_correct, ("correct" if is_correct else "wrong")
-174
View File
@@ -1,174 +0,0 @@
"""발행 outbox enqueue + 초기 백필 (docsrv-viewer-publish).
enqueue_publish: 저작/4-A 트랜잭션이 같은 session(=같은 Postgres tx)에서 호출 caller commit
(P0-1 규율: 콘텐츠 변경과 outbox INSERT 원자성, dual-write 회피). payload/hash 스냅샷.
enqueue_question_publish: 문항 + (ready면)해설을 함께 적재. 저작 쓰기/4-A 완료/백필 공용.
backfill_publish_questions: 기존 active 문항을 bounded 1 outbox 적재(초기 백필, P2-1 bounded page).
멱등 = 발행 워커의 (payload_hash, deleted) 디둡이 no-op 재투영 흡수(중복 enqueue 무해).
주의: 저작 엔드포인트(study_questions create/update)·4-A 워커에서의 enqueue 결선은 P0-1b
(기존 hot 파일 수정이라 increment). 모듈은 호출 라이브러리 + 수동/백필 진입점.
"""
from __future__ import annotations
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models.published import PublishOutbox
from models.study_memo_card import StudyMemoCard
from models.study_memo_card_progress import StudyMemoCardProgress
from models.study_question import StudyQuestion
from models.study_topic import StudyTopic
from services.study.publish_projection import (
KIND_CARD,
KIND_CARD_PROGRESS,
KIND_EXPLANATION,
KIND_QUESTION,
KIND_TOPIC,
SCHEMA_VERSION,
payload_hash,
project_card,
project_card_progress,
project_explanation,
project_question,
project_topic,
)
async def enqueue_publish(
session: AsyncSession,
*,
kind: str,
source_id: int,
payload: dict[str, Any] | None,
deleted: bool = False,
) -> None:
"""outbox 1행 INSERT. caller 가 commit (저자 tx 동봉). deleted=True 면 tombstone(payload={})."""
body: dict[str, Any] = payload if payload is not None else {}
session.add(
PublishOutbox(
kind=kind,
source_id=source_id,
payload=body,
payload_hash=payload_hash(body),
schema_version=SCHEMA_VERSION,
deleted=deleted,
)
)
async def enqueue_question_publish(session: AsyncSession, q: Any) -> None:
"""문항 + (ready면)해설을 outbox 적재. caller commit."""
await enqueue_publish(session, kind=KIND_QUESTION, source_id=q.id, payload=project_question(q))
expl = project_explanation(q)
if expl is not None:
await enqueue_publish(session, kind=KIND_EXPLANATION, source_id=q.id, payload=expl)
async def backfill_publish_questions(session: AsyncSession, *, after_id: int = 0, limit: int = 200) -> tuple[int, int]:
"""active(미삭제) 문항을 id>after_id 부터 bounded 로 outbox 적재.
반환 = (enqueue , 마지막 처리 id). caller ==limit last_id 다음 페이지. caller commit.
"""
rows = (
await session.execute(
select(StudyQuestion)
.where(StudyQuestion.deleted_at.is_(None), StudyQuestion.id > after_id)
.order_by(StudyQuestion.id.asc())
.limit(limit)
)
).scalars().all()
for q in rows:
await enqueue_question_publish(session, q)
return len(rows), (rows[-1].id if rows else after_id)
async def enqueue_topic_publish(session: AsyncSession, topic: Any) -> None:
"""주제 메타를 outbox 적재(S-1). caller commit. 저작 create/update 결선 + 백필 공용."""
await enqueue_publish(session, kind=KIND_TOPIC, source_id=topic.id, payload=project_topic(topic))
async def backfill_publish_topics(session: AsyncSession, *, after_id: int = 0, limit: int = 200) -> tuple[int, int]:
"""active(미삭제) 주제를 id>after_id 부터 bounded 로 outbox 적재(S-1 초기 백필).
반환 = (enqueue , 마지막 처리 id). caller ==limit last_id 다음 페이지. caller commit.
멱등 = 발행 워커의 (payload_hash, deleted) 디둡이 no-op 재투영 흡수(중복 enqueue 무해).
"""
rows = (
await session.execute(
select(StudyTopic)
.where(StudyTopic.deleted_at.is_(None), StudyTopic.id > after_id)
.order_by(StudyTopic.id.asc())
.limit(limit)
)
).scalars().all()
for t in rows:
await enqueue_topic_publish(session, t)
return len(rows), (rows[-1].id if rows else after_id)
async def enqueue_card_publish(session: AsyncSession, card: Any) -> None:
"""카드 상태 기반 발행/tombstone (S-2). caller commit.
검수완료(needs_review=False) & 미삭제 발행 (검수대기 복귀·삭제·retire)
tombstone(feed 1 삭제 이벤트). 발행 자격이 카드 상태에 매여 있어 호출측은 '카드를
건드렸다'만 알면 되고 publish/tombstone 분기는 여기 단일화(경로별 가드 기억 회피).
"""
if card.deleted_at is not None or card.needs_review:
await enqueue_publish(session, kind=KIND_CARD, source_id=card.id, payload=None, deleted=True)
else:
await enqueue_publish(session, kind=KIND_CARD, source_id=card.id, payload=project_card(card))
async def backfill_publish_cards(session: AsyncSession, *, after_id: int = 0, limit: int = 200) -> tuple[int, int]:
"""검수완료(needs_review=False)·미삭제 카드를 id>after_id 부터 bounded 로 outbox 적재(S-2 초기 백필).
반환 = (enqueue , 마지막 처리 id). caller ==limit last_id 다음 페이지. 멱등 = 워커 디둡. caller commit.
"""
rows = (
await session.execute(
select(StudyMemoCard)
.where(
StudyMemoCard.deleted_at.is_(None),
StudyMemoCard.needs_review.is_(False),
StudyMemoCard.id > after_id,
)
.order_by(StudyMemoCard.id.asc())
.limit(limit)
)
).scalars().all()
for c in rows:
await enqueue_card_publish(session, c)
return len(rows), (rows[-1].id if rows else after_id)
async def enqueue_card_progress_publish(session: AsyncSession, progress: Any) -> None:
"""카드 SR progress row 발행(S-4). caller commit. rate_card 결과(ALL row, sentinel/terminal 포함)."""
await enqueue_publish(
session,
kind=KIND_CARD_PROGRESS,
source_id=progress.id,
payload=project_card_progress(progress),
)
async def backfill_publish_card_progress(session: AsyncSession, *, after_id: int = 0, limit: int = 200) -> tuple[int, int]:
"""모든 card progress row 를 id>after_id 부터 bounded 로 outbox 적재(S-4 초기 백필).
필터 없음 = ALL row(due_at NULL sentinel·terminal 포함) due-only 백필은 sentinel 누락.
반환 = (enqueue , 마지막 처리 id). caller ==limit last_id 다음 페이지. 멱등 = 워커 디둡. caller commit.
"""
rows = (
await session.execute(
select(StudyMemoCardProgress)
.where(StudyMemoCardProgress.id > after_id)
.order_by(StudyMemoCardProgress.id.asc())
.limit(limit)
)
).scalars().all()
for p in rows:
await enqueue_card_progress_publish(session, p)
return len(rows), (rows[-1].id if rows else after_id)
-112
View File
@@ -1,112 +0,0 @@
"""발행 projection — 소스 행을 render-ready payload + 안정 해시로 변환 (순수 함수).
뷰어가 보는 '단일 진실' payload 까지 (DS 내부 실험 스키마는 계약 격리).
kind projector. payload_hash = 정렬된 JSON sha256 = (payload_hash, deleted) 디둡 .
주의(plan study-to-viewer-slice1 r2): 과목/시험메타를 per-question payload 인라인
bulk subject rename N행 churn. 정규화(과목= kind subject ref) churn 최적화 후속(P0-1b),
읽기 정합엔 무영향. 지금은 인라인(상관관계 단순)으로 두고 후속 PR 에서 분리.
SCHEMA_VERSION = 엔벨로프 버전. payload 모양 진화 bump + 뷰어 range 수용(P0-2).
"""
from __future__ import annotations
import hashlib
import json
from typing import Any
SCHEMA_VERSION = 1
KIND_QUESTION = "study_question"
KIND_EXPLANATION = "study_explanation"
KIND_TOPIC = "study_topic"
KIND_CARD = "study_card" # ★뷰어 pubstudy.ts 의 KIND_CARD 와 일치 필수(S-3 forward-contract).
KIND_CARD_PROGRESS = "study_card_progress" # 카드 SR 상태 read model (S-4, viewer C-4 소비).
def payload_hash(payload: dict[str, Any]) -> str:
"""정렬 JSON 의 sha256 — (payload_hash, deleted) 디둡 키. 키 순서/공백 비의존."""
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def project_question(q: Any) -> dict[str, Any]:
"""study_question → 발행 payload. 정답 포함(개인 학습툴, plan Q2). 이미지는 ref 만(P0-4, 후속)."""
return {
"topic_id": q.study_topic_id,
"question_text": q.question_text,
"choices": [q.choice_1, q.choice_2, q.choice_3, q.choice_4],
"correct_choice": q.correct_choice,
"subject": q.subject,
"scope": q.scope,
"exam_name": q.exam_name,
"exam_round": q.exam_round,
"exam_question_number": q.exam_question_number,
"explanation": q.explanation, # 수동 해설(있으면). AI 해설은 별 kind.
}
def project_explanation(q: Any) -> dict[str, Any] | None:
"""study_question 의 AI 해설 → 별 발행 kind. ready 일 때만(없으면 None=발행 안 함).
재조우 표시용 선발행. 신규 오답은 4-A 워커가 ~90s ready재발행(P2-3 결선, P0-1b).
"""
if getattr(q, "ai_explanation_status", None) != "ready" or not getattr(q, "ai_explanation", None):
return None
gen = getattr(q, "ai_explanation_generated_at", None)
return {
"question_source_id": q.id,
"explanation_md": q.ai_explanation,
"model": getattr(q, "ai_explanation_model", None),
"generated_at": gen.isoformat() if gen else None,
}
def project_card(c: Any) -> dict[str, Any]:
"""study_memo_card → 발행 payload (S-2). 순수 변환 — 발행 자격(needs_review=false &
미삭제) 판단은 호출측(enqueue_card_publish) 카드 상태로. payload 계약 = 뷰어
pubstudy.ts getCards 동형(format·cue·fact·cloze_text·source_question_id·source_generated_at).
"""
gen = getattr(c, "source_generated_at", None)
return {
"format": c.format,
"cue": c.cue,
"fact": c.fact,
"cloze_text": c.cloze_text,
"source_question_id": c.source_question_id,
"source_generated_at": gen.isoformat() if gen else None,
}
def project_card_progress(p: Any) -> dict[str, Any]:
"""study_memo_card_progress → 발행 payload (S-4) = 카드 SR 상태 read model.
ALL row 발행(due_at NULL sentinel=-on-new · terminal=졸업 포함). due-only 발행하면
sentinel 누락 viewer '미확인' 오분류. SR 계산은 DS(sr_schedule), 여긴 결과만.
card_id = pub_card source_id(=DS card.id) viewer C-4 pub_card LEFT JOIN 하는 .
"""
due = getattr(p, "due_at", None)
rev = getattr(p, "last_reviewed_at", None)
return {
"card_id": p.card_id,
"topic_id": p.study_topic_id,
"last_outcome": p.last_outcome,
"last_reviewed_at": rev.isoformat() if rev else None,
"due_at": due.isoformat() if due else None,
"review_stage": p.review_stage,
}
def project_topic(t: Any) -> dict[str, Any]:
"""study_topic → 발행 payload (S-1, plan study-viewer-port).
topic 메타만 신규 발행 viewer 주제 단위 퀴즈를 만들 최소 정보.
회차 목록은 발행 = viewer pub_content(study_question) exam_name/exam_round
파생(추가 발행 불요, plan S-1 결정). topic_id project_question topic_id(=study_topic_id)
동일 DS 식별자라 viewer 문항주제 상관에 사용(pub_id opaque 상관 아님).
"""
return {
"topic_id": t.id,
"name": t.name,
"exam_round_size": t.exam_round_size,
}
-4
View File
@@ -20,7 +20,6 @@ from models.chunk import DocumentChunk
from models.document import Document
from models.study_question import StudyQuestion
from models.study_topic import StudyTopicDocument
from services.search.license_filter import restricted_exclude_orm
logger = logging.getLogger(__name__)
@@ -114,9 +113,6 @@ async def _gather_document_evidence(
select(Document.id, Document.title, Document.ai_summary).where(
Document.id.in_(doc_ids),
Document.deleted_at.is_(None),
# B-4: licensed_restricted 제외 — explanation_rag 와 동일 술어(a안 U-2①). 누락 시
# 구매 자료 verbatim 이 분야노트 RAG 로 새던 보안 drift(복제 과정 누락).
restricted_exclude_orm(),
)
)
).all()
+13 -26
View File
@@ -40,7 +40,6 @@ from ai.client import (
)
from ai.envelope import EscalationEnvelope
from core.config import settings
from services.search.llm_gate import Priority, acquire_mlx_gate
from core.utils import setup_logger
from models.document import Document
from models.queue import StageDeferred, enqueue_stage
@@ -90,7 +89,6 @@ HARD_ESCALATE_REASONS = {
class TriageOutput(BaseModel):
"""p3a_short_summary (4B) 응답 스키마. 파싱 실패 시 기본값 + escalate=True fallback."""
ai_summary: str = "" # B-1 3→2: triage 가 ai_summary 도 생산 (별 summarize 콜 대체)
tldr: str = ""
bullets: list[str] = Field(default_factory=list)
tags: list[str] = Field(default_factory=list)
@@ -580,7 +578,16 @@ async def process(
"reason": "classify pipeline",
}
# ─── 메타데이터 (classify 완료) — 실제 처리 머신 귀속 (drain=qwen-macbook) ───
# ─── 2. Legacy 요약 (primary 또는 deep) ───
try:
summary = await client.summarize(doc.extracted_text[:50000], cfg=legacy_cfg)
except Exception as exc:
if legacy_cfg is not None and is_deferrable_error(exc):
raise StageDeferred(f"macbook_unavailable:{type(exc).__name__}") from exc
raise
doc.ai_summary = strip_thinking(summary)
# ─── 메타데이터 (legacy 완료) — 실제 처리 머신 귀속 (drain=qwen-macbook) ───
doc.ai_model_version = (legacy_cfg or settings.ai.primary).model
doc.ai_processed_at = datetime.now(timezone.utc)
@@ -590,27 +597,13 @@ async def process(
f"confidence={doc.ai_confidence:.2f}, tags={doc.ai_tags}"
)
# ─── 2+3 통합 (B-1 3→2): tier triage 가 tldr/bullets/tier + ai_summary 생산.
# 기존 별도 summarize 콜 제거 → 본문 prefill 1회 절감 (Mac mini 부하). 실패는 fallback.
# ─── 3. PR-B B-1 — tier triage (4B, 실패는 legacy 결과 보존) ───
try:
await _run_tier_triage(client, doc, session, use_deep=use_deep)
except StageDeferred:
raise # 보류는 실패가 아님 — drain/consumer 가 attempts 미소모 처리
except Exception as exc:
logger.exception(f"[triage] id={document_id} 전체 실패: {exc}")
# ─── ai_summary fallback: triage 가 못 채운 경우만 summarize ───
# (>120K long_context 는 triage 가 LLM skip, 또는 triage 파싱실패). 정상 경로는 미발동.
if not doc.ai_summary:
try:
summary = await client.summarize(doc.extracted_text[:50000], cfg=legacy_cfg)
doc.ai_summary = strip_thinking(summary)
except Exception as exc:
if legacy_cfg is not None and is_deferrable_error(exc):
raise StageDeferred(f"macbook_unavailable:{type(exc).__name__}") from exc
# ai_summary=NULL 로 완료되면 digest/briefing 이 조용히 제외 → ERROR 로 가시화
# (best-effort 강등 자체는 유지, 운영 추적성만 보강).
logger.error(f"[summary-fallback] id={document_id} ai_summary 미생성: {exc}")
logger.exception(f"[triage] id={document_id} 전체 실패 — legacy 유지: {exc}")
finally:
await client.close()
@@ -680,10 +673,7 @@ async def _run_tier_triage(
# 는 아래 generic except 에 먹히지 않게 먼저 전파.
raw_triage = await call_deep_or_defer(client, prompt, cfg=deep_triage_cfg)
else:
# consumer 경로 call_triage 는 PR #20 이후 primary 와 동일 Mac mini endpoint —
# evidence/classifier 처럼 gate 안에서 호출(영구 룰: 같은 endpoint 예외 없이 gate).
async with acquire_mlx_gate(Priority.BACKGROUND):
raw_triage = await client.call_triage(prompt)
raw_triage = await client.call_triage(prompt)
except StageDeferred:
raise # drain 이 attempts 미소모 + 백오프로 처리 (sleep-안전)
except Exception as exc:
@@ -780,9 +770,6 @@ async def _apply_triage_result(
if not parse_error:
doc.ai_tldr = (triage_out.tldr or "").strip() or None
doc.ai_bullets = triage_out.bullets or []
# B-1 3→2: triage 가 ai_summary 도 생산(summarize 콜 대체). 비면 process() 가 fallback.
if triage_out.ai_summary.strip():
doc.ai_summary = triage_out.ai_summary.strip()
# Memo Intake Upgrade PR-2B — event kind hint (4B 가 출력했을 때만)
# 허용 enum 외 값이면 무시 (DB enum 제약). AI worker 는 events row 직접 생성 X.
valid_kinds = {"note", "task", "calendar_event", "activity_log", "reference"}
+7 -16
View File
@@ -140,8 +140,7 @@ async def _download_pdf(url: str, dest: Path) -> int:
if len(resp.content) > _MAX_PDF_BYTES:
raise FeedError(f"PDF 크기 초과 ({len(resp.content)} bytes): {url}")
dest.parent.mkdir(parents=True, exist_ok=True)
# 최대 50MB PDF write 는 동기 blocking — 이벤트루프 점유 회피 to_thread (R5 동형).
await asyncio.to_thread(dest.write_bytes, resp.content)
dest.write_bytes(resp.content)
return len(resp.content)
@@ -191,11 +190,9 @@ async def _ingest_pdf(session, page_slug: str, pdf_url: str) -> bool:
dest = Path(settings.nas_mount_path) / rel_path
size = await _download_pdf(pdf_url, dest)
# 50MB PDF read + sha256 는 동기 blocking(I/O+CPU) — 이벤트루프 점유 회피 to_thread (R5 동형).
file_hash = await asyncio.to_thread(lambda: hashlib.sha256(dest.read_bytes()).hexdigest())
doc = Document(
file_path=rel_path,
file_hash=file_hash,
file_hash=hashlib.sha256(dest.read_bytes()).hexdigest(),
file_format="pdf",
file_size=size,
file_type="immutable",
@@ -377,17 +374,11 @@ async def run(bulk: bool = False, limit: int = 0) -> None:
totals = {"page": 0, "pdf": 0, "skip": 0}
for i, (url, lastmod) in enumerate(todo, 1):
# 2026-06-20 C2: URL 1건 실패가 주간 run 전체를 중단(이후 URL 스킵·watermark 정지)하던 것 차단.
# 각 iteration 은 자체 session(async with) 이라 실패 격리 — 건너뛰고 계속.
try:
async with async_session() as session:
src = await session.get(NewsSource, source_id)
counts = await _ingest_url(session, src, url, lastmod)
_set_watermark(src, lastmod)
await session.commit()
except Exception as e:
logger.error(f"[csb] URL 처리 실패 (건너뜀): {url}{str(e) or repr(e)}")
continue
async with async_session() as session:
src = await session.get(NewsSource, source_id)
counts = await _ingest_url(session, src, url, lastmod)
_set_watermark(src, lastmod)
await session.commit()
for k in totals:
totals[k] += counts[k]
if i % 10 == 0:
+7 -74
View File
@@ -67,45 +67,21 @@ def _postprocess_ocr(text: str) -> str:
return text.strip()
def _extract_pdf_pymupdf(
file_path: Path, start_page: int | None = None, end_page: int | None = None
) -> str:
"""PyMuPDF fallback — 페이지 단위 스트리밍으로 대형 PDF도 저메모리 처리.
G2 (PR-G2-2): start_page/end_page(1-based inclusive) 주어지면 범위만 추출
(번들 자식 doc = 부모 파일 공유 + 자기 page 범위). None = 전체(기존 동작 동일).
"""
def _extract_pdf_pymupdf(file_path: Path) -> str:
"""PyMuPDF fallback — 페이지 단위 스트리밍으로 대형 PDF도 저메모리 처리"""
import fitz
text_parts = []
with fitz.open(str(file_path)) as doc:
if start_page is None and end_page is None:
for page in doc:
text_parts.append(page.get_text())
else:
# 1-based inclusive → 0-based range. 범위는 [0, page_count] 로 클램프(방어).
total = doc.page_count
lo = max(1, start_page or 1) - 1
hi = min(total, end_page or total) # inclusive 끝 (0-based 마지막 인덱스 = hi-1)
for i in range(lo, hi):
text_parts.append(doc.load_page(i).get_text())
for page in doc:
text_parts.append(page.get_text())
return "\n".join(text_parts)
def _get_pdf_page_count(
file_path: Path, start_page: int | None = None, end_page: int | None = None
) -> int:
"""PDF 페이지 수 확인. G2: 범위가 주어지면 그 범위의 페이지 수(자식 doc 밀도 계산용).
None = 전체 페이지 (기존 동작 동일).
"""
def _get_pdf_page_count(file_path: Path) -> int:
"""PDF 페이지 수 확인"""
import fitz
with fitz.open(str(file_path)) as doc:
total = len(doc)
if start_page is None and end_page is None:
return total
lo = max(1, start_page or 1)
hi = min(total, end_page or total)
return max(0, hi - lo + 1)
return len(doc)
async def _call_ocr(file_path: Path, is_image: bool, max_pages: int = 200) -> str | None:
@@ -334,49 +310,6 @@ async def process(document_id: int, session: AsyncSession) -> None:
doc.extracted_at = datetime.now(timezone.utc)
return
# ─── G2 (PR-G2-2): 번들 자식 PDF — 부모 파일 공유 + 자기 page 범위만 추출 ───
# kordoc 서비스는 page-range 파라미터가 없어 전체 파일을 파싱한다(자식엔 부적합) → kordoc
# 우회, PyMuPDF 로 [bundle_page_start, bundle_page_end] 범위만 추출. range OCR 은 본 PR 범위
# 밖(자식은 ToC 존재 = digital text layer 전제 → 대개 OCR 불필요). PyMuPDF 텍스트가 빈약해도
# 그대로 보존하고 사유를 남긴다.
if fmt == "pdf" and doc.bundle_page_start is not None and doc.bundle_page_end is not None:
# 후보 A: 자식 file_path 는 합성값(`{부모}#p{s}-{e}`) → 실파일 = bundle_source_path 로 부모경로
# 복원 + NFC/NFD resolve. (자식 file_path 는 디스크에 없음.)
from workers.presegment_worker import _resolve_path as _resolve_bundle_path
from workers.presegment_worker import bundle_source_path
real_rel = bundle_source_path(doc.file_path)
src = _resolve_bundle_path(str(Path(settings.nas_mount_path) / real_rel))
if src is None:
raise FileNotFoundError(f"번들 원본 파일 없음: {real_rel}")
start, end = doc.bundle_page_start, doc.bundle_page_end
try:
pymupdf_text = _extract_pdf_pymupdf(src, start, end)
page_count = _get_pdf_page_count(src, start, end)
except Exception as e:
logger.error(f"[pymupdf:child] {doc.file_path} pages={start}-{end} 실패: {e}")
raise
meta = doc.extract_meta or {}
meta["presegment_child_range"] = {"start_page": start, "end_page": end}
meta["pymupdf_chars"] = len(pymupdf_text.strip())
should, reason = _should_ocr(pymupdf_text, page_count)
if should:
# range OCR 미지원(후속 PR) — PyMuPDF 결과 유지 + 사유 기록(silent skip 아님).
meta["ocr_skip_reason"] = "presegment_child_range_ocr_unsupported"
meta["ocr_reason"] = reason
logger.warning(
f"[pymupdf:child] {doc.file_path} pages={start}-{end} "
f"OCR 필요({reason})하나 range OCR 미지원 → PyMuPDF 결과 유지"
)
doc.extracted_text = pymupdf_text.replace("\x00", "")
doc.extracted_at = datetime.now(timezone.utc)
doc.extractor_version = PYMUPDF_VERSION if pymupdf_text.strip() else None
doc.extract_meta = meta
logger.info(
f"[pymupdf:child] {doc.file_path} pages={start}-{end} ({len(pymupdf_text)}자)"
)
return
# ─── kordoc 파싱 (HWP/HWPX/PDF) + PyMuPDF fallback + OCR ───
if fmt in KORDOC_FORMATS:
container_path = f"/documents/{doc.file_path}"
+85 -102
View File
@@ -118,18 +118,16 @@ def _route_media(path: Path, expected_category: str | None) -> tuple[str | None,
if expected_category == "library":
# 외부 작성 학습 자료 (KGS Code, 시행규칙 등). 문서 확장자만 수락.
# frontmatter 해석은 classify_worker (옵션 C) 가 담당. file_watcher 는 라우팅만.
# G2: 첫 stage=presegment (후보 A 검증완료). 非PDF/단일 통과, 번들 PDF 만 분할.
if ext in LIBRARY_DOC_EXTS:
return ("library", False, "presegment")
return ("library", False, "extract")
if ext in AUDIO_EXTS or ext in VIDEO_DIRECT_EXTS or ext in VIDEO_QUARANTINE_EXTS:
return (None, False, None) # audio/video 잘못 들어오면 skip
return (None, False, None) # 기타 알 수 없는 확장자 skip
# Inbox: 문서 파이프 (기존). audio/video 확장자가 실수로 여기 들어오면 skip.
# G2: 첫 stage=presegment (후보 A 검증완료). 非PDF/단일 통과, 번들 PDF 만 분할.
if ext in AUDIO_EXTS or ext in VIDEO_DIRECT_EXTS or ext in VIDEO_QUARANTINE_EXTS:
return (None, False, None)
return (None, False, "presegment")
return (None, False, "extract")
# ─── Web/Blog ingest (devonagent 트랙) 헬퍼 ──────────────────────────────────
@@ -228,8 +226,7 @@ async def _ingest_web_file(session, file_path: Path, rel_path: str) -> tuple[int
)
session.add(doc)
await session.flush()
# G2: 첫 stage=presegment (후보 A 검증완료). HTML(非PDF)은 presegment 가 무변 통과 → extract.
await enqueue_stage(session, doc.id, "presegment")
await enqueue_stage(session, doc.id, "extract")
return (1, 0)
@@ -251,118 +248,104 @@ async def watch_inbox():
for extra_path in settings.additional_watch_targets:
targets.append((extra_path, "library"))
# 파일별 독립 세션+commit 으로 격리 — 한 파일 실패(예: rglob↔stat 사이 삭제로 FileNotFoundError,
# flush 오류)가 watch_inbox 전체를 raise·롤백해 그 사이클 등록분을 모두 잃거나, 결정적 poison
# 파일이 매 사이클 같은 지점에서 중단시키는 것을 차단 (news_collector/csb_collector 와 동형).
# ─── Web/ 트랙 (devonagent) — DEVONthink Smart Rule 이 떨군 .html 만 진입 ───
if web_root.exists():
# rglob NFS 디렉토리 walk(blocking stat 다발)를 off-thread 로 수집 (R5).
for file_path in await asyncio.to_thread(lambda: list(web_root.rglob("*.html"))):
if not file_path.is_file() or should_skip(file_path):
continue
rel_path = str(file_path.relative_to(nas_root))
try:
async with async_session() as session:
added, _ = await _ingest_web_file(session, file_path, rel_path)
await session.commit()
async with async_session() as session:
# ─── Web/ 트랙 (devonagent) — DEVONthink Smart Rule 이 떨군 .html 만 진입 ───
if web_root.exists():
# rglob NFS 디렉토리 walk(blocking stat 다발)를 off-thread 로 수집 (R5).
for file_path in await asyncio.to_thread(lambda: list(web_root.rglob("*.html"))):
if not file_path.is_file() or should_skip(file_path):
continue
rel_path = str(file_path.relative_to(nas_root))
added, _ = await _ingest_web_file(session, file_path, rel_path)
new_count += added
except Exception as e:
logger.warning("[Web] 파일 처리 실패 skip path=%s: %s", rel_path, e)
# ─── PKM 트랙 (기존 drive_sync) ─────────────────────────────────────────
for sub, expected_category in targets:
scan_root = pkm_root / sub
if not scan_root.exists():
continue
# ─── PKM 트랙 (기존 drive_sync) ─────────────────────────────────────────
for sub, expected_category in targets:
scan_root = pkm_root / sub
if not scan_root.exists():
continue
# 안전 자료실 A-2/B-4 — 타깃 폴더 기반 (material, jurisdiction, license)
target_mt, target_jur, target_license = _TARGET_AXIS.get(
Path(sub).name, (None, None, None)
)
# NFS 디렉토리 walk(blocking) off-thread 수집 (R5).
for file_path in await asyncio.to_thread(lambda: list(scan_root.rglob("*"))):
if not file_path.is_file() or should_skip(file_path):
continue
category, needs_conversion, next_stage = _route_media(
file_path, expected_category
# 안전 자료실 A-2/B-4 — 타깃 폴더 기반 (material, jurisdiction, license)
target_mt, target_jur, target_license = _TARGET_AXIS.get(
Path(sub).name, (None, None, None)
)
# audio/video 폴더에 엉뚱한 확장자가 들어왔거나 Inbox 에
# audio/video 가 잘못 떨어진 경우 — 이 라운드에서 아예 skip
if category is None and next_stage is None:
continue
# NFS 디렉토리 walk(blocking) off-thread 수집 (R5).
for file_path in await asyncio.to_thread(lambda: list(scan_root.rglob("*"))):
if not file_path.is_file() or should_skip(file_path):
continue
rel_path = str(file_path.relative_to(nas_root))
try:
category, needs_conversion, next_stage = _route_media(
file_path, expected_category
)
# audio/video 폴더에 엉뚱한 확장자가 들어왔거나 Inbox 에
# audio/video 가 잘못 떨어진 경우 — 이 라운드에서 아예 skip
if category is None and next_stage is None:
continue
rel_path = str(file_path.relative_to(nas_root))
# GB 파일 SHA-256 은 이벤트 루프를 점유 → 같은 루프의 모든 1분 주기 consumer
# + FastAPI 요청이 수십초~분 동시 정지. to_thread 오프로드. 스캔 루프가 이미
# 순차라 file_hash 는 한 번에 하나만 실행(직렬화) — 병렬 해싱 X = NFS 2.5GbE
# 대역폭·버퍼 메모리 blowup 방지 (R5). 세션 밖에서 계산(커넥션 미점유).
# 대역폭·버퍼 메모리 blowup 방지 (R5).
fhash = await asyncio.to_thread(file_hash, file_path)
async with async_session() as session:
result = await session.execute(
select(Document).where(Document.file_path == rel_path)
result = await session.execute(
select(Document).where(Document.file_path == rel_path)
)
existing = result.scalar_one_or_none()
if existing is None:
ext = file_path.suffix.lstrip(".").lower() or "unknown"
doc = Document(
file_path=rel_path,
file_hash=fhash,
file_format=ext,
file_size=file_path.stat().st_size,
file_type="immutable",
title=file_path.stem,
source_channel="drive_sync",
category=category,
needs_conversion=needs_conversion,
# 안전 자료실 A-2/B-4 — watch 타깃 매핑 (KGS=law/KR 등, 비대상=NULL)
material_type=target_mt,
jurisdiction=target_jur,
)
existing = result.scalar_one_or_none()
# B-4 — 타깃 폴더 license 주입(restricted 포함, 비대상=미주입). classify 는
# material_type IS NULL 일 때만 제안 + extract_meta 미기록이라 주입 보존.
if target_license:
doc.extract_meta = {"license": dict(target_license)}
session.add(doc)
await session.flush()
if existing is None:
ext = file_path.suffix.lstrip(".").lower() or "unknown"
doc = Document(
file_path=rel_path,
file_hash=fhash,
file_format=ext,
file_size=file_path.stat().st_size,
file_type="immutable",
title=file_path.stem,
source_channel="drive_sync",
category=category,
needs_conversion=needs_conversion,
# 안전 자료실 A-2/B-4 — watch 타깃 매핑 (KGS=law/KR 등, 비대상=NULL)
material_type=target_mt,
jurisdiction=target_jur,
)
# B-4 — 타깃 폴더 license 주입(restricted 포함, 비대상=미주입). classify 는
# material_type IS NULL 일 때만 제안 + extract_meta 미기록이라 주입 보존.
if target_license:
doc.extract_meta = {"license": dict(target_license)}
session.add(doc)
await session.flush()
if next_stage:
await enqueue_stage(session, doc.id, next_stage)
new_count += 1
if next_stage:
await enqueue_stage(session, doc.id, next_stage)
await session.commit()
new_count += 1
elif existing.file_hash != fhash:
existing.file_hash = fhash
existing.file_size = file_path.stat().st_size
# 기존 문서에 category/quarantine flag 가 비어있으면 보정
if existing.category is None and category is not None:
existing.category = category
if needs_conversion and not getattr(existing, "needs_conversion", False):
existing.needs_conversion = True
# B-4 — 축/license 보정(B-4 이전 적재분이 재변경 시): material 미설정 시 주입,
# license 부재 시에만 merge 주입(clobber 회피 — 기존 extract_meta 키 보존).
if existing.material_type is None and target_mt is not None:
existing.material_type = target_mt
existing.jurisdiction = target_jur
if target_license and not (existing.extract_meta or {}).get("license"):
meta = dict(existing.extract_meta or {})
meta["license"] = dict(target_license)
existing.extract_meta = meta
elif existing.file_hash != fhash:
existing.file_hash = fhash
existing.file_size = file_path.stat().st_size
# 기존 문서에 category/quarantine flag 가 비어있으면 보정
if existing.category is None and category is not None:
existing.category = category
if needs_conversion and not getattr(existing, "needs_conversion", False):
existing.needs_conversion = True
# B-4 — 축/license 보정(B-4 이전 적재분이 재변경 시): material 미설정 시 주입,
# license 부재 시에만 merge 주입(clobber 회피 — 기존 extract_meta 키 보존).
if existing.material_type is None and target_mt is not None:
existing.material_type = target_mt
existing.jurisdiction = target_jur
if target_license and not (existing.extract_meta or {}).get("license"):
meta = dict(existing.extract_meta or {})
meta["license"] = dict(target_license)
existing.extract_meta = meta
if next_stage:
await enqueue_stage(session, existing.id, next_stage)
changed_count += 1
if next_stage:
await enqueue_stage(session, existing.id, next_stage)
await session.commit()
changed_count += 1
# else: 무변경 → 쓰기 없음 (세션 자동 닫힘, commit 불요)
except Exception as e:
logger.warning("[PKM] 파일 처리 실패 skip path=%s: %s", rel_path, e)
continue
await session.commit()
if new_count or changed_count:
logger.info(f"[Inbox+§3] 새 파일 {new_count}건, 변경 파일 {changed_count}건 등록")
+367
View File
@@ -0,0 +1,367 @@
"""법령 모니터 워커 — 국가법령정보센터 API 연동
26 법령 모니터링, / 단위 분할 저장, 변경 이력 추적.
매일 07:00 실행 (APScheduler).
"""
import os
import re
from datetime import date, datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import httpx
from sqlalchemy import select
from core.config import settings
from core.database import async_session
from core.utils import create_caldav_todo, file_hash, setup_logger
from models.automation import AutomationState
from models.document import Document
from models.queue import enqueue_stage
logger = setup_logger("law_monitor")
LAW_SEARCH_URL = "https://www.law.go.kr/DRF/lawSearch.do"
LAW_SERVICE_URL = "https://www.law.go.kr/DRF/lawService.do"
# 모니터링 대상 법령 (26개)
MONITORED_LAWS = [
# 산업안전보건 핵심
"산업안전보건법",
"산업안전보건법 시행령",
"산업안전보건법 시행규칙",
"산업안전보건기준에 관한 규칙",
"유해위험작업의 취업 제한에 관한 규칙",
"중대재해 처벌 등에 관한 법률",
"중대재해 처벌 등에 관한 법률 시행령",
# 건설안전
"건설기술 진흥법",
"건설기술 진흥법 시행령",
"건설기술 진흥법 시행규칙",
"시설물의 안전 및 유지관리에 관한 특별법",
# 위험물/화학
"위험물안전관리법",
"위험물안전관리법 시행령",
"위험물안전관리법 시행규칙",
"화학물질관리법",
"화학물질관리법 시행령",
"화학물질의 등록 및 평가 등에 관한 법률",
# 소방/전기/가스
"소방시설 설치 및 관리에 관한 법률",
"소방시설 설치 및 관리에 관한 법률 시행령",
"전기사업법",
"전기안전관리법",
"고압가스 안전관리법",
"고압가스 안전관리법 시행령",
"액화석유가스의 안전관리 및 사업법",
# 근로/환경
"근로기준법",
"환경영향평가법",
]
async def run():
"""법령 변경 모니터링 실행"""
law_oc = os.getenv("LAW_OC", "")
if not law_oc:
logger.warning("LAW_OC 미설정 — 법령 API 승인 대기 중")
return
async with async_session() as session:
state = await session.execute(
select(AutomationState).where(AutomationState.job_name == "law_monitor")
)
state_row = state.scalar_one_or_none()
last_check = state_row.last_check_value if state_row else None
today = datetime.now(timezone.utc).strftime("%Y%m%d")
if last_check == today:
logger.info("오늘 이미 체크 완료")
return
new_count = 0
async with httpx.AsyncClient(timeout=30) as client:
for law_name in MONITORED_LAWS:
try:
count = await _check_law(client, law_oc, law_name, session)
new_count += count
except Exception as e:
logger.error(f"[{law_name}] 체크 실패: {e}")
# 상태 업데이트
if state_row:
state_row.last_check_value = today
state_row.last_run_at = datetime.now(timezone.utc)
else:
session.add(AutomationState(
job_name="law_monitor",
last_check_value=today,
last_run_at=datetime.now(timezone.utc),
))
await session.commit()
logger.info(f"법령 모니터 완료: {new_count}건 신규/변경 감지")
async def _check_law(
client: httpx.AsyncClient,
law_oc: str,
law_name: str,
session,
) -> int:
"""단일 법령 검색 → 변경 감지 → 분할 저장"""
# 법령 검색 (lawSearch.do)
resp = await client.get(
LAW_SEARCH_URL,
params={"OC": law_oc, "target": "law", "type": "XML", "query": law_name},
)
resp.raise_for_status()
root = ET.fromstring(resp.text)
total = root.findtext(".//totalCnt", "0")
if total == "0":
logger.debug(f"[{law_name}] 검색 결과 없음")
return 0
# 정확히 일치하는 법령 찾기
for law_elem in root.findall(".//law"):
found_name = law_elem.findtext("법령명한글", "").strip()
if found_name != law_name:
continue
mst = law_elem.findtext("법령일련번호", "")
proclamation_date = law_elem.findtext("공포일자", "")
revision_type = law_elem.findtext("제개정구분명", "")
if not mst:
continue
# 이미 등록된 법령인지 확인 (같은 법령명 + 공포일자)
existing = await session.execute(
select(Document).where(
Document.title.like(f"{law_name}%"),
Document.source_channel == "law_monitor",
)
)
existing_docs = existing.scalars().all()
# 같은 공포일자 이미 있으면 skip
for doc in existing_docs:
if proclamation_date in (doc.title or ""):
return 0
# 이전 공포일 찾기 (변경 이력용)
prev_date = ""
if existing_docs:
prev_date = max(
(re.search(r'\d{8}', doc.title or "").group() for doc in existing_docs
if re.search(r'\d{8}', doc.title or "")),
default=""
)
# 본문 조회 (lawService.do)
text_resp = await client.get(
LAW_SERVICE_URL,
params={"OC": law_oc, "target": "law", "MST": mst, "type": "XML"},
)
text_resp.raise_for_status()
# 분할 저장
count = await _save_law_split(
session, text_resp.text, law_name, proclamation_date,
revision_type, prev_date,
)
# DB 먼저 커밋 (알림 실패가 저장을 막지 않도록)
await session.commit()
# CalDAV + SMTP 알림 (실패해도 무시)
try:
_send_notifications(law_name, proclamation_date, revision_type)
except Exception as e:
logger.warning(f"[{law_name}] 알림 발송 실패 (무시): {e}")
return count
return 0
async def _save_law_split(
session, xml_text: str, law_name: str, proclamation_date: str,
revision_type: str, prev_date: str,
) -> int:
"""법령 XML → 장(章) 단위 Markdown 분할 저장"""
root = ET.fromstring(xml_text)
# 조문단위에서 장 구분자 찾기 (조문키가 000으로 끝나는 조문)
units = root.findall(".//조문단위")
chapters = [] # [(장제목, [조문들])]
current_chapter = None
current_articles = []
for unit in units:
key = unit.attrib.get("조문키", "")
content = (unit.findtext("조문내용", "") or "").strip()
# 장 구분자: 키가 000으로 끝나고 내용에 "제X장" 포함
if key.endswith("000") and re.search(r"\d+장", content):
# 이전 장/서문 저장
if current_articles:
chapter_name = current_chapter or "서문"
chapters.append((chapter_name, current_articles))
chapter_match = re.search(r"(제\d+장\s*.+)", content)
current_chapter = chapter_match.group(1).strip() if chapter_match else content.strip()
current_articles = []
else:
current_articles.append(unit)
# 마지막 장 저장
if current_articles:
chapter_name = current_chapter or "서문"
chapters.append((chapter_name, current_articles))
# 장 분할 성공
sections = []
if chapters:
for chapter_title, articles in chapters:
md_lines = [f"# {law_name}\n", f"## {chapter_title}\n"]
for article in articles:
title = article.findtext("조문제목", "")
content = article.findtext("조문내용", "")
if title:
md_lines.append(f"\n### {title}\n")
if content:
md_lines.append(content.strip())
section_name = _safe_name(chapter_title)
sections.append((section_name, "\n".join(md_lines)))
else:
# 장 분할 실패 → 전체 1파일
full_md = _law_xml_to_markdown(xml_text, law_name)
sections.append(("전문", full_md))
# 각 섹션 저장
inbox_dir = Path(settings.nas_mount_path) / "PKM" / "Inbox"
inbox_dir.mkdir(parents=True, exist_ok=True)
count = 0
for section_name, content in sections:
filename = f"{law_name}_{proclamation_date}_{section_name}.md"
file_path = inbox_dir / filename
file_path.write_text(content, encoding="utf-8")
rel_path = str(file_path.relative_to(Path(settings.nas_mount_path)))
# 변경 이력 메모
note = ""
if prev_date:
note = (
f"[자동] 법령 개정 감지\n"
f"이전 공포일: {prev_date}\n"
f"현재 공포일: {proclamation_date}\n"
f"개정구분: {revision_type}"
)
# 안전 자료실 A-2 — 공포일 파싱 (law published_date = COALESCE(시행일, 공포일) 계약,
# 본 레거시 워커는 공포일만 보유 — 시행일 기반 버전 체인은 B-1 statute_collector 소관)
_digits = re.sub(r"\D", "", str(proclamation_date or ""))
pub_date = None
if len(_digits) == 8:
try:
pub_date = date(int(_digits[:4]), int(_digits[4:6]), int(_digits[6:8]))
except ValueError:
pub_date = None
doc = Document(
file_path=rel_path,
file_hash=file_hash(file_path),
file_format="md",
file_size=len(content.encode()),
file_type="immutable",
title=f"{law_name} ({proclamation_date}) {section_name}",
source_channel="law_monitor",
data_origin="work",
category="law",
# 안전 자료실 A-2 — ingest 시점 deterministic. 법령 텍스트 = 저작권법 제7조
# 비보호 저작물 (public domain). 본 워커는 휴면(LAW_OC 미설정)이나 코드 경로 유지.
material_type="law",
jurisdiction="KR",
published_date=pub_date,
extract_meta={"license": {"scheme": "public_domain", "redistribute": True,
"attribution": "국가법령정보센터"}},
user_note=note or None,
)
session.add(doc)
await session.flush()
await enqueue_stage(session, doc.id, "extract")
count += 1
logger.info(f"[법령] {law_name} ({proclamation_date}) → {count}개 섹션 저장")
return count
def _xml_section_to_markdown(elem) -> str:
"""XML 섹션(편/장)을 Markdown으로 변환"""
lines = []
for article in elem.iter():
tag = article.tag
text = (article.text or "").strip()
if not text:
continue
if "" in tag:
lines.append(f"\n### {text}\n")
elif "" in tag:
lines.append(f"\n{text}\n")
elif "" in tag:
lines.append(f"- {text}")
elif "" in tag:
lines.append(f" - {text}")
else:
lines.append(text)
return "\n".join(lines)
def _law_xml_to_markdown(xml_text: str, law_name: str) -> str:
"""법령 XML 전체를 Markdown으로 변환"""
root = ET.fromstring(xml_text)
lines = [f"# {law_name}\n"]
for elem in root.iter():
tag = elem.tag
text = (elem.text or "").strip()
if not text:
continue
if "" in tag and "제목" not in tag:
lines.append(f"\n## {text}\n")
elif "" in tag and "제목" not in tag:
lines.append(f"\n## {text}\n")
elif "" in tag:
lines.append(f"\n### {text}\n")
elif "" in tag:
lines.append(f"\n{text}\n")
elif "" in tag:
lines.append(f"- {text}")
elif "" in tag:
lines.append(f" - {text}")
return "\n".join(lines)
def _safe_name(name: str) -> str:
"""파일명 안전 변환"""
return re.sub(r'[^\w가-힣-]', '_', name).strip("_")
def _send_notifications(law_name: str, proclamation_date: str, revision_type: str):
"""CalDAV 할일 알림 (SMTP 발송은 2026-06-10 폐기 — CalDAV 가 단일 알림 채널)"""
caldav_url = os.getenv("CALDAV_URL", "")
caldav_user = os.getenv("CALDAV_USER", "")
caldav_pass = os.getenv("CALDAV_PASS", "")
if caldav_url and caldav_user:
create_caldav_todo(
caldav_url, caldav_user, caldav_pass,
title=f"법령 검토: {law_name}",
description=f"공포일자: {proclamation_date}, 개정구분: {revision_type}",
due_days=7,
)
+11 -65
View File
@@ -39,11 +39,7 @@ from models.queue import ProcessingQueue
logger = logging.getLogger(__name__)
# 마크다운 추출 엔드포인트. compose env `MARKER_ENDPOINT`(base URL)에서 읽는다 —
# 기본=marker(무변), 컷오버=`http://mineru-service:3301` 로 env 플립만으로 전환.
# marker/mineru 가 동일 /convert 계약(file_path·start/end·md+base64 images)이라 워커 무변.
_MARKDOWN_BASE = os.getenv("MARKER_ENDPOINT", "http://marker-service:3300").rstrip("/")
MARKER_ENDPOINT = _MARKDOWN_BASE if _MARKDOWN_BASE.endswith("/convert") else _MARKDOWN_BASE + "/convert"
MARKER_ENDPOINT = "http://marker-service:3300/convert"
MARKER_TIMEOUT = 300 # 큰 PDF 5 분 한도
MAX_PAGES = 200 # 소형 1-shot 경로 /convert max_pages 안전장치
@@ -185,10 +181,7 @@ async def process(document_id: int, session: AsyncSession) -> None:
await _fail(session, document_id, "no file_path")
return
# 후보 A: 자식(bundle cols)은 합성 file_path(`{부모}#p{s}-{e}`) → 실파일 = bundle_source_path
# 로 부모경로 복원. 일반 doc 은 그대로(접미사 없음). marker/mineru 는 실파일 + page 범위로 변환.
from workers.presegment_worker import bundle_source_path
container_path = _to_marker_path(bundle_source_path(doc.file_path))
container_path = _to_marker_path(doc.file_path)
suffix = Path(container_path).suffix.lower()
# ---- (3) office/hwp → md (C-2): PDF 외 지원 포맷은 office_md 하이브리드 변환 ----
@@ -210,21 +203,7 @@ async def process(document_id: int, session: AsyncSession) -> None:
return
# ---- (4) page_count gauge + 분기 (LargeDoc split) ----
# G2 (PR-G2-2): 번들 자식 doc 은 부모 파일 공유 + 자기 page 범위([bundle_page_start, end],
# 1-based inclusive)만 변환해야 한다. page_offset = 절대 시작페이지(부모 파일 기준), page_count =
# 자식 범위의 페이지 수. cols 가 NULL(일반 doc)이면 page_offset=1 + 전체 page_count = 기존 동작 동일.
file_page_count = _get_page_count(container_path)
is_child = doc.bundle_page_start is not None and doc.bundle_page_end is not None
if is_child:
page_offset = doc.bundle_page_start
if file_page_count is not None:
child_end = min(doc.bundle_page_end, file_page_count)
page_count = max(0, child_end - doc.bundle_page_start + 1)
else:
page_count = doc.bundle_page_end - doc.bundle_page_start + 1
else:
page_offset = 1
page_count = file_page_count
page_count = _get_page_count(container_path)
# >MAX_SPLIT_PAGES = 변환 안전상태(manual_review). silently skip 아님.
if page_count is not None and page_count > MAX_SPLIT_PAGES:
@@ -243,35 +222,20 @@ async def process(document_id: int, session: AsyncSession) -> None:
# ---- (6) 변환 분기: 소형 1-shot / 대형(>SPLIT_THRESHOLD) page-range 분할 ----
if page_count is not None and page_count > SPLIT_THRESHOLD_PAGES:
await _process_split(doc, document_id, container_path, page_count, session, page_offset)
await _process_split(doc, document_id, container_path, page_count, session)
else:
await _process_single(doc, document_id, container_path, session, page_count, page_offset)
await _process_single(doc, document_id, container_path, session)
async def _process_single(
doc: Document, document_id: int, container_path: str, session: AsyncSession,
page_count: int | None = None, page_offset: int = 1,
doc: Document, document_id: int, container_path: str, session: AsyncSession
) -> None:
"""소형 PDF(≤ SPLIT_THRESHOLD_PAGES) 통째 1-shot 변환 (Phase 1B/1B.5 기존 경로).
G2 (PR-G2-2): 번들 자식(page_offset>1) [page_offset, page_offset+page_count-1] 범위만
변환하도록 marker start_page/end_page 명시한다. 일반 doc(page_offset=1) 기존과
동일하게 max_pages 보낸다(payload byte-identical).
"""
# 일반 doc = 기존 payload 유지. 자식만 절대 page 범위를 명시(부모 파일 기준 1-based inclusive).
if page_offset > 1 and page_count is not None:
req_json = {
"file_path": container_path,
"start_page": page_offset,
"end_page": page_offset + page_count - 1,
}
else:
req_json = {"file_path": container_path, "max_pages": MAX_PAGES}
"""소형 PDF(≤ SPLIT_THRESHOLD_PAGES) 통째 1-shot 변환 (Phase 1B/1B.5 기존 경로)."""
try:
async with httpx.AsyncClient(timeout=MARKER_TIMEOUT) as client:
resp = await client.post(
MARKER_ENDPOINT,
json=req_json,
json={"file_path": container_path, "max_pages": MAX_PAGES},
)
resp.raise_for_status()
data = resp.json()
@@ -300,11 +264,6 @@ async def _process_single(
f"[marker] transient error id={document_id} kind={type(exc).__name__}: {exc}"
)
raise
except json.JSONDecodeError as exc:
# 200 응답의 truncated/malformed body — 연결 흔들림 등 transient. _fail(non-retryable)
# 로 박지 말고 raise → queue retry (max_attempts 까지). 진짜 손상이면 재시도 후 failed.
logger.warning(f"[marker] malformed json body (200) id={document_id}: {exc}")
raise
except Exception as exc:
logger.exception(f"[marker] unexpected error id={document_id}: {exc}")
await _fail(session, document_id, str(exc)[:1000])
@@ -312,10 +271,6 @@ async def _process_single(
# ---- (7) image persist + md_content rewrite (Phase 1B.5) ----
md_content_raw = data["md_content"]
# 2026-06-20 H1: 빈 추출(스캔/이미지 PDF)을 md_status=success + 빈 md 로 박제 X
# (계약: md_status in {success,partial} => md 非공백). office arm 동형 raise → queue 재시도 후 failed.
if not md_content_raw.strip():
raise ValueError("empty md_content (blank extraction) — success 박제 차단")
images_resp = data.get("images") if MARKDOWN_IMAGE_PERSIST else None
saved_images: list[dict[str, Any]] = []
@@ -554,7 +509,6 @@ async def _process_split(
container_path: str,
page_count: int,
session: AsyncSession,
page_offset: int = 1,
) -> None:
"""대형 PDF page-range 분할 변환.
@@ -565,10 +519,6 @@ async def _process_split(
invariant: page numbering = 1-based inclusive (batch1: 1..BATCH_PAGES, ...).
marker slug(`_page_0_*`) batch 마다 재시작 batch rewrite stitch (충돌 회피).
G2 (PR-G2-2): page_offset = 부모 파일 기준 절대 시작페이지(번들 자식). marker 보내는
page 절대값(page_offset 가산), manifest/기록은 자식 상대값(1-based) 유지 일반 doc
(page_offset=1) abs==rel 이라 기존 동작과 동일.
"""
n_batches = (page_count + BATCH_PAGES - 1) // BATCH_PAGES
succeeded: list[dict[str, Any]] = [] # {start_page, end_page, md}
@@ -580,17 +530,15 @@ async def _process_split(
async with httpx.AsyncClient(timeout=MARKER_TIMEOUT) as client:
for b in range(n_batches):
start_page = b * BATCH_PAGES + 1 # 자식 상대 1-based (manifest/기록용)
start_page = b * BATCH_PAGES + 1
end_page = min((b + 1) * BATCH_PAGES, page_count)
abs_start = start_page + (page_offset - 1) # 부모 파일 절대 page (marker 요청용)
abs_end = end_page + (page_offset - 1)
try:
resp = await client.post(
MARKER_ENDPOINT,
json={
"file_path": container_path,
"start_page": abs_start,
"end_page": abs_end,
"start_page": start_page,
"end_page": end_page,
},
)
resp.raise_for_status()
@@ -662,8 +610,6 @@ async def _process_split(
md_status = "success" if not failed else "partial"
stitched = "\n\n".join(b["md"] for b in succeeded)
if not stitched.strip():
raise ValueError("empty stitched md_content (all batches blank) — success 박제 차단")
md_content = _build_large_md_content(stitched[:LARGE_DOC_MD_CONTENT_HEAD_CHARS], manifest)
quality = _compute_quality(stitched, doc.extracted_text or "", {"page_count": page_count})
+13 -25
View File
@@ -213,25 +213,17 @@ async def _run_locked():
result = await session.execute(
select(NewsSource).where(NewsSource.enabled == True)
)
source_ids = [s.id for s in result.scalars().all()]
sources = result.scalars().all()
if not source_ids:
logger.info("활성화된 뉴스 소스 없음")
return
if not sources:
logger.info("활성화된 뉴스 소스 없음")
return
# 2026-06-20 H3: 소스마다 독립 세션 — 한 소스의 DB 오류가 종단 단일 commit 을 깨뜨려
# 전 소스 insert 를 잃던 것 차단. 실패 시 rollback 후 깨끗한 상태에서 failure 기록.
# (csb_collector 의 per-iteration 세션 패턴과 동형.)
total = 0
for sid in source_ids:
async with async_session() as session:
source = await session.get(NewsSource, sid)
if source is None:
continue
sname = source.name
health = await _get_or_create_health(session, sid)
total = 0
for source in sources:
health = await _get_or_create_health(session, source.id)
if not _should_attempt(health, now):
logger.info(f"[{sname}] circuit {health.circuit_state} — 이번 사이클 skip")
logger.info(f"[{source.name}] circuit {health.circuit_state} — 이번 사이클 skip")
continue
try:
if source.feed_type == "api":
@@ -242,18 +234,14 @@ async def _run_locked():
source.last_fetched_at = datetime.now(timezone.utc)
_record_success(health, count, status == "not_modified", now)
total += count
await session.commit()
except Exception as e:
# str 이 빈 예외(httpx.ConnectError('')) 대비 — health 기록과 동일 규칙
await session.rollback()
logger.error(f"[{sname}] 수집 실패: {str(e) or repr(e)}")
health = await _get_or_create_health(session, sid)
src = await session.get(NewsSource, sid)
if src is not None:
src.last_fetched_at = datetime.now(timezone.utc)
logger.error(f"[{source.name}] 수집 실패: {str(e) or repr(e)}")
source.last_fetched_at = datetime.now(timezone.utc)
_record_failure(health, str(e) or repr(e), now)
await session.commit()
logger.info(f"뉴스 수집 완료: {total}건 신규")
await session.commit()
logger.info(f"뉴스 수집 완료: {total}건 신규")
MAX_RESPONSE_SIZE = 5 * 1024 * 1024 # 5MB
-568
View File
@@ -1,568 +0,0 @@
"""presegment_worker — extract 前 번들 PDF(여러 논리문서 한 파일) → N 자식 분할 (G2 / PR-G2-2).
문서가 presegment stage 진입한다(worker-side gating):
- 非PDF(file_format != pdf · suffix != .pdf) = 즉시 fast-exit enqueue_next_stage extract 흘림.
- PDF = PyMuPDF ToC(level-1) deterministic 분석. '명확한 번들' 자식 분할, 나머지는 단일문서로 extract.
deterministic 경로(PR-G2-2): 판정이 애매하면 보수적으로 분할하지 않고 단일문서로 둔다
(bias to NOT splitting). 분할 = '확실한 번들' :
- page_count >= MIN_BUNDLE_PAGES AND level-1 ToC 항목 >= 2 AND 모든 자식 >= MIN_CHILD_PAGES
AND 단조 증가·비중첩 AND [1, page_count] 범위 커버 AND 2 <= N <= MAX_CHILDREN.
LLM 경계 폴백(PR-G2-3, env PRESEGMENT_LLM_FALLBACK, 기본 OFF scaffold-first): deterministic
'명확한 번들' 만든 대형 PDF(ToC 없음/level-1 없음/게이트 미달) 한해, OFF 오늘과
동일(단일문서)이고 ON 이면 off-card Qwen(맥북, 라우터 :8890, model=qwen-macbook)에게 경계를
제안받는다. compact per-page heading 샘플만 전송(본문 미전송). LLM 출력은 **동일 검증 게이트
(_is_clear_bundle)** 통과 시에만 deterministic 같은 _create_children 경로로 분할
is_bundle=false / 파싱·검증 실패 = 단일문서(오늘과 동일) + presegment_llm_rejected 로깅.
맥북 불가(503/연결/절단) StageDeferred 재시도(백오프, no silent fallback).
분할 후보 A(물리분할 없음, uq_documents_file_path 해소): 자식 file_path = unique 합성값
`{부모경로}#p{start}-{end}` (UNIQUE 제약 통과), 실파일은 `bundle_source_path()` 로 부모 경로 복원.
자식은 bundle_page_start/end(1-based inclusive) 부모 파일의 자기 page 범위만 가리킨다.
부모-자식 관계 정본 = document_lineage(relation_type='segmented_from'). 부모(presegment_role='parent')
파일 홀더라 자체 extract/embed enqueue_next_stage presegmentextract 전이가 'parent'
억제된다(queue_consumer 참조). 자식의 extract 워커가 직접 enqueue. extract_worker/marker_worker
자식 처리 bundle_source_path() 실파일 접근.
멱등: 재실행 같은 부모로 이미 자식이 있으면(document_lineage segmented_from) 재생성하지 않고
수렴( 자식이 extract 활성/완료 상태인지만 보장)한다.
해결 이력 (2026-06-18): 최초 Option A(자식이 부모 file_path 그대로 공유) uq_documents_file_path
UNIQUE 위반(실번들 검증서 발견) 합성 file_path(후보 A) 해소. 인제스트 재활성 = 합성번들 재검증 PASS .
plan: G2 pre-segmentation (PR-G2-2 deterministic ToC segmentation)
"""
import hashlib
import os
import re
import unicodedata
from pathlib import Path
from pydantic import BaseModel, ValidationError
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from ai.client import AIClient, call_deep_or_defer, parse_json_response
from core.config import settings
from core.utils import setup_logger
from models.document import Document
from models.document_lineage import DocumentLineage
from models.queue import enqueue_stage
logger = setup_logger("presegment_worker")
# ─── 임계값 (모듈 상수, env-override 가능, 보수적 = 분할 안 하는 쪽으로 bias) ───
# MIN_BUNDLE_PAGES: 이 미만이면 번들로 보지 않음(단일문서). 짧은 문서의 우연한 level-1 ToC 보호.
MIN_BUNDLE_PAGES = int(os.getenv("PRESEGMENT_MIN_BUNDLE_PAGES", "60"))
# MIN_CHILD_PAGES: 자식 하나라도 이 미만이면 분할 거부(표지/목차만 떼지는 over-split 방지).
MIN_CHILD_PAGES = int(os.getenv("PRESEGMENT_MIN_CHILD_PAGES", "5"))
# MAX_CHILDREN: 자식 수 상한. 초과 = ToC 가 챕터/소제목 수준이라 논리문서 경계가 아님 → 분할 거부.
MAX_CHILDREN = int(os.getenv("PRESEGMENT_MAX_CHILDREN", "50"))
# marker_worker._to_marker_path 와 동일 — NAS 상대경로 → 컨테이너 절대경로 prefix.
CONTAINER_PATH_PREFIX = os.getenv("MARKER_CONTAINER_PATH_PREFIX", "/documents")
# ─── PR-G2-3 LLM 경계 폴백 (scaffold-first, 기본 OFF) ───
# PRESEGMENT_LLM_FALLBACK: 기본 "false". OFF 면 deterministic 경로만(=오늘과 동일 — 애매하면
# 단일문서). ON 이면 deterministic 이 '명확한 번들' 을 못 만든 대형 PDF(page_count >=
# MIN_BUNDLE_PAGES) 에 한해 off-card Qwen(맥북, 라우터 :8890 경유)에게 경계를 제안받아
# **동일 검증 게이트(_is_clear_bundle)** 통과 시에만 deterministic 과 같은 자식 생성 경로로 분할.
# 검증 실패/파싱 실패/is_bundle=false = 단일문서(오늘과 동일) + presegment_llm_rejected 로깅.
PRESEGMENT_LLM_FALLBACK = os.getenv("PRESEGMENT_LLM_FALLBACK", "false").lower() in (
"1", "true", "yes", "on",
)
# LLM 에 보내는 per-page 샘플의 page 당 char 상한 (heading/첫줄만 — 본문 미전송).
PRESEGMENT_LLM_PAGE_CHARS = int(os.getenv("PRESEGMENT_LLM_PAGE_CHARS", "80"))
# 전체 page-sample 블록의 char 상한 (수 KB 가드 — 초과 시 잘라냄, 본문 누출/페이로드 폭발 방지).
PRESEGMENT_LLM_SAMPLE_CHARS = int(os.getenv("PRESEGMENT_LLM_SAMPLE_CHARS", "12000"))
# 경계 폴백 프롬프트 (app/prompts/presegment_boundaries.txt). system 지시 + 1-based inclusive·
# 전범위 커버·무중첩 규칙. {page_count}/{page_samples} 를 str.replace 로 주입.
_PRESEGMENT_PROMPT_PATH = Path(__file__).parent.parent / "prompts" / "presegment_boundaries.txt"
class Segment(BaseModel):
"""LLM 이 제안하는 1-based inclusive page 범위 한 조각."""
start_page: int
end_page: int
title: str | None = None
class SegmentationOutput(BaseModel):
"""presegment_boundaries 응답 스키마. parse_json_response → model_validate."""
is_bundle: bool = False
segments: list[Segment] = []
confidence: float | None = None
def _resolve_path(file_path: str) -> Path | None:
"""NFC(DB) vs NFD(NFS) 한글 경로 차이 흡수. thumbnail_worker._resolve_path 와 동일 패턴."""
candidates = [
file_path,
unicodedata.normalize("NFD", file_path),
unicodedata.normalize("NFC", file_path),
]
for c in candidates:
p = Path(c)
if p.exists():
return p
parent = Path(file_path).parent
if parent.exists():
target = unicodedata.normalize("NFC", Path(file_path).name)
for child in parent.iterdir():
if unicodedata.normalize("NFC", child.name) == target:
return child
return None
def _to_container_path(file_path: str) -> str:
"""file_path 를 컨테이너 내부 절대경로로 변환 (marker_worker._to_marker_path 와 동일)."""
if file_path.startswith("/"):
return file_path
return f"{CONTAINER_PATH_PREFIX}/{file_path}"
# 후보 A: 자식 합성 file_path 패턴 `{부모경로}#p{start}-{end}` (uq_documents_file_path 유일성).
_BUNDLE_SUFFIX_RE = re.compile(r"#p\d+-\d+$")
def bundle_source_path(file_path: str | None) -> str | None:
"""자식 합성 file_path → 부모 실파일 경로 복원. 일반 doc(접미사 없음)은 그대로 반환.
extract_worker/marker_worker 자식 처리 실제 파일 접근에 사용 (자식 file_path
합성값이라 디스크에 없음). 결정적·세션 불필요. lineage 부모-자식 관계의 정본 기록.
"""
if not file_path:
return file_path
return _BUNDLE_SUFFIX_RE.sub("", file_path)
def _is_pdf(doc: Document) -> bool:
"""PDF 판정 — file_format=pdf 또는 .pdf 확장자."""
fmt = (doc.file_format or "").lower()
if fmt == "pdf":
return True
if doc.file_path:
return Path(doc.file_path).suffix.lower() == ".pdf"
return False
def _level1_segments(toc: list, page_count: int) -> list[dict]:
"""get_toc(simple=True) 결과에서 level-1 항목만 골라 자식 후보 segment 리스트 생성.
toc 항목 = [level, title, page] (page 1-based). level==1 채택.
end_page = 다음 level-1 항목의 page - 1, 마지막 = page_count.
동일 page 에서 시작하는 level-1 여럿이면 정렬 인접 항목으로 경계 계산되며,
경우 0-페이지 segment 생겨 후속 검증(MIN_CHILD_PAGES·단조)에서 거부된다.
"""
starts = []
for entry in toc:
# simple=True 는 [level, title, page]. 방어적으로 길이 체크.
if not entry or len(entry) < 3:
continue
level, title, page = entry[0], entry[1], entry[2]
if level != 1:
continue
# ToC page 가 범위 밖(0/음수/page_count 초과)이면 깨진 ToC → 후속 검증에서 거부됨.
starts.append((int(page), (title or "").strip()))
# ToC 가 정렬돼 있지 않을 수 있으므로 page 기준 정렬(원본 순서 보존 위해 안정 정렬).
starts.sort(key=lambda x: x[0])
segments: list[dict] = []
for i, (start_page, title) in enumerate(starts):
if i + 1 < len(starts):
end_page = starts[i + 1][0] - 1
else:
end_page = page_count
segments.append({"start_page": start_page, "end_page": end_page, "title": title})
return segments
def _is_clear_bundle(segments: list[dict], page_count: int) -> tuple[bool, str]:
"""deterministic '명확한 번들' 판정. (clear, reason) 반환.
clear=True reason="" / clear=False reason 거부 사유(로깅용).
모든 조건은 보수적 하나라도 어긋나면 단일문서로 처리(분할 ).
"""
n = len(segments)
if n < 2:
return False, f"too_few_level1_entries(n={n})"
if n > MAX_CHILDREN:
return False, f"too_many_children(n={n}>{MAX_CHILDREN})"
# 첫 segment 가 1페이지에서 시작 + 마지막이 page_count 에서 끝 = 전 범위 커버.
if segments[0]["start_page"] != 1:
return False, f"first_start_not_1(start={segments[0]['start_page']})"
if segments[-1]["end_page"] != page_count:
return False, f"last_end_not_page_count(end={segments[-1]['end_page']},pc={page_count})"
prev_end = 0
for seg in segments:
start, end = seg["start_page"], seg["end_page"]
# 단조 증가 · 비중첩: 각 start 는 직전 end + 1 이어야 빈틈/겹침 없이 [1,pc] 정확 분할.
if start != prev_end + 1:
return False, f"non_contiguous(start={start},prev_end={prev_end})"
if end < start:
return False, f"non_monotonic(start={start},end={end})"
if (end - start + 1) < MIN_CHILD_PAGES:
return False, f"child_too_small(pages={end - start + 1}<{MIN_CHILD_PAGES})"
prev_end = end
if prev_end != page_count:
return False, f"coverage_gap(covered={prev_end},pc={page_count})"
return True, ""
def _child_title(parent: Document, seg: dict) -> str:
"""자식 제목 = 부모 제목 + '' + (segment 제목 또는 page 범위)."""
base = (parent.title or "").strip() or (parent.original_filename or "") or "문서"
seg_title = (seg.get("title") or "").strip()
suffix = seg_title if seg_title else f"p.{seg['start_page']}-{seg['end_page']}"
return f"{base}{suffix}"
def _child_file_hash(parent_hash: str, start: int, end: int) -> str:
"""자식 file_hash = sha256(f"{parent.file_hash}:{start}-{end}"). 결정적 → 재실행 멱등.
부모 file_hash NULL 수는 없으나(NOT NULL) 방어적으로 문자열 처리.
"""
return hashlib.sha256(f"{parent_hash or ''}:{start}-{end}".encode("utf-8")).hexdigest()
async def _ensure_child_extract(session: AsyncSession, child_id: int) -> None:
"""자식이 아직 extract 안 됐으면 extract enqueue (멱등 수렴 경로).
이미 extracted_text 채워졌거나 활성 행이 있으면 enqueue_stage no-op/skip.
"""
child = await session.get(Document, child_id)
if child is None:
return
# 이미 추출 완료면 재enqueue 불필요 (큐 중복은 enqueue_stage 가 막지만 의미상으로도 skip).
if child.extracted_at is not None and child.extracted_text is not None:
return
await enqueue_stage(session, child_id, "extract")
async def _create_children(
doc: Document, segments: list[dict], session: AsyncSession
) -> int:
"""검증된 segments 로 자식 N개 생성 + lineage + extract enqueue + 부모 표식 (멱등).
deterministic '명확한 번들' 경로와 LLM 폴백 경로가 공유하는 단일 자식 생성 경로.
호출 segments 반드시 _is_clear_bundle 검증을 통과해야 한다(여기선 재검증 X).
commit 까지 수행. 반환값 = 실제 생성한 자식 (이미 존재해 수렴만 경우 0).
"""
# ─── 멱등 체크: 이미 자식이 있으면 수렴만 (재생성 금지) ───
existing_children = (
await session.execute(
select(DocumentLineage.derived_document_id).where(
DocumentLineage.source_document_id == doc.id,
DocumentLineage.relation_type == "segmented_from",
)
)
).scalars().all()
if existing_children:
# 부모 표식이 누락된 경우 보정(이전 부분실패 복구).
if doc.presegment_role != "parent":
doc.presegment_role = "parent"
for child_id in existing_children:
await _ensure_child_extract(session, child_id)
await session.commit()
logger.info(
f"[presegment] id={doc.id} children already exist "
f"(n={len(existing_children)}) → converge(ensure extract), no re-create"
)
return 0
# ─── 자식 N개 생성 + lineage + extract enqueue ───
created_ids: list[int] = []
for seg in segments:
start, end = seg["start_page"], seg["end_page"]
child = Document(
# 후보 A: 자식 file_path = unique 합성값 `{부모경로}#p{s}-{e}` (uq_documents_file_path
# 충돌 회피). 실파일은 bundle_source_path() 로 복원(부모 경로). 물리 분할 없음 —
# 자식은 bundle_page_start/end 로 부모 파일을 슬라이스.
file_path=f"{doc.file_path}#p{start}-{end}",
file_hash=_child_file_hash(doc.file_hash, start, end),
file_format=doc.file_format,
file_size=doc.file_size,
file_type=doc.file_type,
import_source=doc.import_source,
original_filename=doc.original_filename,
source_channel=doc.source_channel,
category=doc.category,
data_origin=doc.data_origin,
doc_purpose=doc.doc_purpose,
# 안전 자료실 축은 부모에서 상속(분할이 자료유형/관할을 바꾸지 않음).
material_type=doc.material_type,
jurisdiction=doc.jurisdiction,
title=_child_title(doc, seg),
bundle_page_start=start,
bundle_page_end=end,
presegment_role="child",
)
session.add(child)
await session.flush() # child.id 확보
created_ids.append(child.id)
session.add(
DocumentLineage(
source_document_id=doc.id,
derived_document_id=child.id,
relation_type="segmented_from",
meta={"start_page": start, "end_page": end},
)
)
# 자식 extract 는 워커가 직접 enqueue (부모는 'parent' 라 extract 로 흐르지 않음).
await enqueue_stage(session, child.id, "extract")
# 부모 = 파일 홀더. presegment→extract 전이는 enqueue_next_stage 가 'parent' 면 억제.
doc.presegment_role = "parent"
await session.commit()
logger.info(
f"[presegment] id={doc.id} SPLIT into {len(created_ids)} children "
f"child_ids={created_ids}"
)
return len(created_ids)
def _segments_from_output(out: "SegmentationOutput") -> list[dict]:
"""SegmentationOutput.segments(Pydantic) → _is_clear_bundle / _create_children 가 쓰는 dict 형태."""
return [
{"start_page": s.start_page, "end_page": s.end_page, "title": (s.title or "")}
for s in out.segments
]
def _page_samples(pdf, page_count: int) -> str:
"""LLM 입력용 compact per-page 샘플 — page 당 heading/첫줄만(`p{n}: {firstline}`).
PyMuPDF page.get_text() page 텍스트를 스트리밍하되 page 비공백 줄만,
PRESEGMENT_LLM_PAGE_CHARS 잘라 본문 누출 차단. 전체 블록은 PRESEGMENT_LLM_SAMPLE_CHARS
가드로 상한( KB) 초과 지점에서 중단(앞쪽 페이지 우선 보존).
"""
lines: list[str] = []
total = 0
for i in range(page_count):
try:
text = pdf[i].get_text() or ""
except Exception:
text = ""
first = ""
for ln in text.splitlines():
ln = ln.strip()
if ln:
first = ln
break
first = first[:PRESEGMENT_LLM_PAGE_CHARS]
entry = f"p{i + 1}: {first}"
if total + len(entry) + 1 > PRESEGMENT_LLM_SAMPLE_CHARS:
break
lines.append(entry)
total += len(entry) + 1
return "\n".join(lines)
async def _llm_boundary_fallback(
doc: Document, source: Path, page_count: int, session: AsyncSession
) -> bool:
"""애매 + 대형(ToC-less 등) PDF 에 대해 off-card Qwen 으로 경계 제안 → 검증 → 분할.
반환 True = LLM 경로가 분할을 수행(또는 멱등 수렴)했으므로 호출자는 추가 처리 없이 return.
반환 False = is_bundle=false / 파싱 실패 / 검증 실패 호출자는 단일문서(오늘과 동일) 처리.
맥북 불가(503/연결/절단) call_deep_or_defer StageDeferred raise 재시도(백오프).
silent fallback 금지 deep 슬롯 다른 backend 자동 호출 .
"""
import fitz # PyMuPDF — deterministic 경로와 동일 의존
# per-page 샘플은 파일을 다시 열어 스트리밍(deterministic with 블록과 분리해 그 경로 무회귀).
try:
with fitz.open(str(source)) as pdf:
samples = _page_samples(pdf, page_count)
except Exception as exc:
logger.warning(
f"[presegment] id={doc.id} llm fallback sample 실패 "
f"({type(exc).__name__}: {exc}) → single doc(extract)"
)
return False
try:
template = _PRESEGMENT_PROMPT_PATH.read_text(encoding="utf-8")
except Exception as exc:
logger.warning(
f"[presegment] id={doc.id} prompt 로드 실패 ({type(exc).__name__}: {exc}) "
f"→ single doc(extract)"
)
return False
prompt = template.replace("{page_count}", str(page_count)).replace(
"{page_samples}", samples
)
# off-card 호출 — call_deep_or_defer 가 deep 슬롯(맥북, 라우터 :8890, model=qwen-macbook)
# 으로 라우팅. 맥북 불가는 StageDeferred 로 전파(여기서 잡지 않음 → 큐가 보류/백오프).
# classify_worker 와 동일하게 AIClient() 인스턴스화.
client = AIClient()
try:
raw = await call_deep_or_defer(client, prompt)
finally:
await client.close()
parsed = parse_json_response(raw)
if not parsed:
logger.info(
f"[presegment] presegment_llm_rejected id={doc.id} "
f"reason=parse_failed raw={raw[:160]!r} → single doc(extract)"
)
return False
try:
out = SegmentationOutput.model_validate(parsed)
except (ValidationError, ValueError, TypeError) as exc:
logger.info(
f"[presegment] presegment_llm_rejected id={doc.id} "
f"reason=schema_invalid({type(exc).__name__}) → single doc(extract)"
)
return False
if not out.is_bundle:
logger.info(
f"[presegment] presegment_llm_rejected id={doc.id} "
f"reason=is_bundle_false → single doc(extract)"
)
return False
segments = _segments_from_output(out)
clear, reason = _is_clear_bundle(segments, page_count)
if not clear:
# LLM 출력을 그대로 믿지 않음 — deterministic 과 동일 게이트 미달이면 단일문서.
logger.info(
f"[presegment] presegment_llm_rejected id={doc.id} "
f"reason={reason} n={len(segments)} pages={page_count} → single doc(extract)"
)
return False
n = await _create_children(doc, segments, session)
logger.info(
f"[presegment] id={doc.id} LLM-SPLIT accepted "
f"(pages={page_count} n={len(segments)} created={n} "
f"confidence={out.confidence})"
)
return True
async def process(document_id: int, session: AsyncSession) -> None:
"""presegment stage 워커 진입점. queue_consumer 가 호출.
문서가 진입하며, 非PDF·단일문서는 변경 없이 통과(presegment_role 그대로 NULL) extract 흐른다.
'명확한 번들' PDF 자식 분할 + 부모를 'parent' 표식( 경우 부모는 extract 흐르지 않음).
"""
doc = await session.get(Document, document_id)
if doc is None:
logger.warning(f"[presegment] document {document_id} not found")
return
# ─── (0) 非PDF — fast-exit. presegment_role 그대로 NULL → enqueue_next_stage 가 extract 로 흘림 ───
if not _is_pdf(doc):
logger.info(f"[presegment] id={document_id} non-pdf (fmt={doc.file_format}) → extract")
return
# ─── (0.5) file_path 없음(예: note) — 분할 불가, 단일문서로 통과 ───
if not doc.file_path:
logger.info(f"[presegment] id={document_id} no file_path → extract")
return
# ─── (1) 이미 분할된 자식 자신이 presegment 로 다시 들어온 경우 — 재분할 금지 ───
# (정상 흐름에선 자식은 곧장 extract 로 enqueue 되지만, 재처리 스크립트 등으로 들어올 수 있음.)
if doc.presegment_role in ("child", "parent"):
logger.info(
f"[presegment] id={document_id} already presegment_role={doc.presegment_role} → skip"
)
return
# ─── (2) 파일 열기 + page_count ───
raw = str(Path(settings.nas_mount_path) / doc.file_path)
source = _resolve_path(raw)
if source is None:
# 파일 부재 = extract 가 동일 상황에서 FileNotFoundError 로 처리할 사안.
# presegment 는 분할 불가일 뿐이므로 단일문서로 통과시켜 extract 가 일관되게 처리하게 둔다.
logger.warning(f"[presegment] id={document_id} file not found ({raw}) → extract")
return
import asyncio
import fitz # PyMuPDF — extract_worker/marker_worker 와 동일 의존
def _read_toc(path: str):
# fitz open/get_toc 는 동기 blocking — live 스테이지라 이벤트루프(같은 루프의 1분 consumer +
# FastAPI 요청) 점유 회피 위해 to_thread 오프로드(거대/손상 PDF 파싱 수백 ms~초).
with fitz.open(path) as pdf:
return pdf.page_count, (pdf.get_toc(simple=True) or [])
try:
page_count, toc = await asyncio.to_thread(_read_toc, str(source))
except Exception as exc:
# PDF 손상 등 — 분할 불가. 단일문서로 통과(extract 가 PyMuPDF/OCR 로 재시도하며 가시화).
logger.warning(
f"[presegment] id={document_id} fitz open/toc failed "
f"({type(exc).__name__}: {exc}) → extract"
)
return
# ─── (3) page_count 가 임계 미만 = 단일문서 (대다수 경로) ───
if page_count < MIN_BUNDLE_PAGES:
logger.info(
f"[presegment] id={document_id} single doc "
f"(pages={page_count}<{MIN_BUNDLE_PAGES}) → extract"
)
return
# ─── (4) level-1 ToC → 자식 후보 segment ───
segments = _level1_segments(toc, page_count)
if not segments:
# 큰 PDF 인데 ToC 없음/level-1 없음 = 애매. flag ON 이면 LLM 경계 폴백(PR-G2-3),
# OFF(기본) 이면 오늘과 동일 — 단일문서로 처리하고 사유를 남긴다.
if PRESEGMENT_LLM_FALLBACK:
logger.info(
f"[presegment] presegment_ambiguous id={document_id} "
f"reason=no_level1_toc pages={page_count} → LLM fallback"
)
if await _llm_boundary_fallback(doc, source, page_count, session):
return
# LLM 이 분할하지 않음(is_bundle=false / 검증·파싱 실패) — 단일문서.
return
logger.info(
f"[presegment] presegment_ambiguous id={document_id} "
f"reason=no_level1_toc pages={page_count} → single doc(extract)"
)
return
clear, reason = _is_clear_bundle(segments, page_count)
if not clear:
# 큰 PDF + ToC 는 있으나 '명확한 번들' 기준 미달 = 애매. flag ON 이면 LLM 경계 폴백,
# OFF(기본) 이면 오늘과 동일 — 단일문서(분할 안 함).
if PRESEGMENT_LLM_FALLBACK:
logger.info(
f"[presegment] presegment_ambiguous id={document_id} "
f"reason={reason} pages={page_count} level1={len(segments)} → LLM fallback"
)
if await _llm_boundary_fallback(doc, source, page_count, session):
return
return
logger.info(
f"[presegment] presegment_ambiguous id={document_id} "
f"reason={reason} pages={page_count} level1={len(segments)} → single doc(extract)"
)
return
# ─── (5) 명확한 번들 (deterministic) — 공유 자식 생성 경로 (멱등 수렴 포함) ───
await _create_children(doc, segments, session)
+4 -23
View File
@@ -31,9 +31,9 @@ _hold_logged = False
# embed/chunk 1→10 (2026-06-12 fast-consumer): 건당 <1s 실측 — Phase 0.1 초기 보수값이
# LLM 사이클에 인질로 잡혀 실효 ~580/일 vs 수요 최대 2,700/일 → 적체 원인이었음.
# 10 = TEI/marker 와 GPU 공유 고려한 보수 상향(전용 1분 잡 기준 캡 ~14,400/일).
BATCH_SIZE = {"presegment": 3, "extract": 5, "classify": 3, "summarize": 3, "embed": 10,
"chunk": 10, "preview": 2, "stt": 1, "thumbnail": 3, "deep_summary": 1,
"markdown": 1, "fulltext": 3}
BATCH_SIZE = {"extract": 5, "classify": 3, "summarize": 3, "embed": 10, "chunk": 10,
"preview": 2, "stt": 1, "thumbnail": 3, "deep_summary": 1, "markdown": 1,
"fulltext": 3}
STALE_THRESHOLD_MINUTES = 10
# markdown 대형 split 변환은 한 doc 이 수십 분(5210 ≈ 40분) 동안 processing 상태로 머문다.
# marker_worker 는 queue 행에 heartbeat 를 찍지 않으므로(started_at 고정), main 의 10분
@@ -46,7 +46,7 @@ MARKDOWN_STALE_THRESHOLD_MINUTES = int(os.getenv("MARKDOWN_STALE_MINUTES", "120"
# (reset_stale_items 가 자기 집합만 reset, 교차 시 이중 복구 위험).
# STT 도 장기 작업 가능성이 있으나 본 PR 범위 밖 — main 에 유지(follow-up).
MAIN_QUEUE_STAGES = [
"presegment", "extract", "classify", "summarize",
"extract", "classify", "summarize",
"preview", "stt", "thumbnail", "fulltext",
]
MARKDOWN_QUEUE_STAGES = ["markdown"]
@@ -165,10 +165,6 @@ async def enqueue_next_stage(document_id: int, current_stage: str):
}
next_stages = {
# G2 (PR-G2-2): 전 문서가 presegment → extract. 단, 번들 분할로 'parent' 가 된 문서는
# 파일 홀더라 자체 extract 안 함 — 아래 suppression 으로 이 전이를 건너뛴다(자식 extract 는
# presegment_worker 가 직접 enqueue). 단일/非PDF 문서(role NULL)는 정상적으로 extract 로 흐름.
"presegment": ["extract"],
"extract": ["classify", "preview"],
"classify": ["embed", "chunk", "markdown"],
"stt": ["classify"],
@@ -184,18 +180,6 @@ async def enqueue_next_stage(document_id: int, current_stage: str):
stages = extract_override_by_channel[sc]
else:
stages = next_stages.get(current_stage, [])
elif current_stage == "presegment":
# 번들 분할 parent 는 extract 로 흐르지 않게 억제 (자식이 부모 extract 에 가려지는 것 방지).
# role NULL(단일/非PDF) / 'child' 는 정상 전이. presegment_worker 가 자식 extract 를 직접
# enqueue 하므로 'parent' 만 여기서 no-op.
from models.document import Document
async with async_session() as lookup_session:
doc = await lookup_session.get(Document, document_id)
role = doc.presegment_role if doc else None
if role == "parent":
stages = []
else:
stages = next_stages.get(current_stage, [])
else:
stages = next_stages.get(current_stage, [])
@@ -215,7 +199,6 @@ def _load_workers():
from workers.deep_summary_worker import process as deep_summary_process
from workers.embed_worker import process as embed_process
from workers.extract_worker import process as extract_process
from workers.presegment_worker import process as presegment_process
from workers.preview_worker import process as preview_process
from workers.stt_worker import process as stt_process
from workers.summarize_worker import process as summarize_process
@@ -224,8 +207,6 @@ def _load_workers():
from workers.fulltext_worker import process as fulltext_process
return {
# G2 (PR-G2-2): extract 前 번들 PDF → N 자식 분할 (deterministic ToC). 非PDF/단일은 통과.
"presegment": presegment_process,
"extract": extract_process,
"classify": classify_process,
"summarize": summarize_process,
+2 -8
View File
@@ -25,7 +25,6 @@ import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from ai.client import AIClient, parse_json_response
from core.config import settings
from models.study_question import StudyQuestion
from models.study_question_job import StudyQuestionJob
from services.search.llm_gate import Priority, acquire_mlx_gate
@@ -33,12 +32,11 @@ from services.study.explanation_rag import (
gather_explanation_context,
render_evidence_block,
)
from services.study.publish_enqueue import enqueue_question_publish
logger = logging.getLogger(__name__)
# 2026-06-20: config 단일소스 (구 하드코딩 30s = 빠른 Gemma 기준, Qwen 27B 교체 sweep 누락).
LLM_TIMEOUT_S = settings.llm_call_timeout_s
# PR-3 LLM_TIMEOUT_S 와 동일 안전 마진 (26B 평균 ~10s, gate 직렬화 고려)
LLM_TIMEOUT_S = 30.0
# explanation_md hard cap — 운영 데이터 793/838/866자 사례에서 1200 으로 시작
# (800 은 공식·오답·핵심개념 묶이는 기사시험 풀이에 빡빡함). 1차 운영 후 조정.
@@ -228,10 +226,6 @@ async def run_explanation_job(session: AsyncSession, job: StudyQuestionJob) -> N
question.ai_explanation_model = f"mlx:{primary_name}"
question.updated_at = question.ai_explanation_generated_at
# 발행 재투영(같은 tx, caller commit) — 4-A 해설 ready → 문항+해설 발행. P0-1b.
if settings.study_publish_enabled:
await enqueue_question_publish(session, question)
job.status = "completed"
job.completed_at = now()
return
+3 -10
View File
@@ -24,7 +24,6 @@ import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from ai.client import AIClient, parse_json_response
from core.config import settings
from models.study_memo_card import (
append_card,
append_card_evidence,
@@ -34,8 +33,6 @@ from models.study_memo_card_job import StudyMemoCardJob
from models.study_question import StudyQuestion
from models.user import User # noqa: F401 (mapper 초기화 defensive)
from services.search.llm_gate import Priority, acquire_mlx_gate
from services.study.publish_enqueue import enqueue_publish
from services.study.publish_projection import KIND_CARD
from services.study.explanation_rag import (
gather_explanation_context,
render_evidence_block,
@@ -44,8 +41,8 @@ from services.study.study_memo_card_guards import guard_cards
logger = logging.getLogger("study_memo_card_worker")
# 2026-06-20: config 단일소스 (구 하드코딩 45s = 빠른 Gemma 기준).
CARD_LLM_TIMEOUT_S = settings.llm_call_timeout_s
# 다카드 출력이라 explanation(30s)보다 여유. config primary.timeout(180, soft-lock)은 미변경.
CARD_LLM_TIMEOUT_S = 45.0
SOURCE_KIND_QUESTION = "question"
_ENVELOPE_PROMPT_FILE = "study_card_envelope.txt"
@@ -186,13 +183,9 @@ async def run_card_extract_job(session: AsyncSession, job: StudyMemoCardJob) ->
return
# 5. 성공 — 구버전 카드 retire 후 append (dedup partial unique 충돌 회피).
retired_published_ids = await supersede_old_cards(
await supersede_old_cards(
session, source_question_id=question.id, keep_generated_at=source_version
)
# 발행 중이던 구버전 카드 tombstone(같은 tx) — 재추출 retire 후 viewer stale 잔류 0. S-2.
if settings.study_publish_enabled:
for cid in retired_published_ids:
await enqueue_publish(session, kind=KIND_CARD, source_id=cid, payload=None, deleted=True)
model_name = f"mlx:{primary_name}"
inserted = 0
for g in guarded:
-153
View File
@@ -1,153 +0,0 @@
"""발행 워커 — publish_outbox drain → published 에 rev 부여 (docsrv-viewer-publish).
APScheduler 1(max_instances=1). pg_advisory_xact_lock 단일 라이터 rev 커밋순 gapless
(인플라이트 차단: bigserial seq 폴링이 아니라 outbox id + 단일 라이터 rev 부여).
outbox id(커밋순) 순으로 처리, (kind, source_id) published upsert:
- 기존 행과 (payload_hash, deleted) 동일 no-op(디둡, rev 올림) + processed 마킹
- pub_id 재사용(기존)|신규 uuid, rev = MAX(rev)+1, payload/hash/deleted 갱신
tombstone(deleted=True) 디둡 복합키라 삼켜짐. 배치 단일 트랜잭션.
배치 같은 (kind, source_id) 오면 flush 직전 반영을 다음 select 보게 (최신 ).
study_publish_enabled=False(기본) no-op 저자/4-A enqueue 결선(P0-1b) 전까지 inert.
"""
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from sqlalchemy import func, select, text
from core.config import settings
from core.database import async_session
from core.utils import setup_logger
from models.published import Published, PublishOutbox
logger = setup_logger("study_publish_worker")
BATCH_SIZE = 500
# pg_advisory_xact_lock 전역 단일 라이터 키(발행 워커 전용 임의 상수, 타 advisory 락과 비충돌).
ADVISORY_LOCK_KEY = 838201
# 행별 격리 재시도 상한 — 초과 시 failed_at 스탬프(terminal)로 select 에서 제외.
MAX_OUTBOX_ATTEMPTS = 5
async def consume_publish_outbox() -> None:
"""APScheduler 진입점. 미처리 outbox 를 rev 부여하며 published 로 반영."""
if not settings.study_publish_enabled:
logger.debug("study_publish 비활성 (study_publish_enabled=false)")
return
async with async_session() as session:
try:
# 1) 전역 단일 라이터 락(트랜잭션 스코프 — commit/rollback 시 자동 해제).
await session.execute(
text("SELECT pg_advisory_xact_lock(:k)").bindparams(k=ADVISORY_LOCK_KEY)
)
# 2) 현재 최대 rev.
max_rev = int(
(await session.execute(select(func.coalesce(func.max(Published.rev), 0)))).scalar() or 0
)
# 3) 미처리 outbox 를 커밋순(id)으로. failed_at(terminal) 은 제외 — poison 행이
# head-of-line 을 영구 점유하지 않게 함.
rows = (
await session.execute(
select(PublishOutbox)
.where(
PublishOutbox.processed_at.is_(None),
PublishOutbox.failed_at.is_(None),
)
.order_by(PublishOutbox.id.asc())
.limit(BATCH_SIZE)
)
).scalars().all()
if not rows:
return
now = datetime.now(timezone.utc)
published_count = 0
failed_count = 0
for ob in rows:
try:
# 행 단위 savepoint 격리 — 한 행의 예외가 배치 전체(앞 행 processed_at 포함)를
# 롤백해 poison 행이 다음 사이클에 다시 최저 id 로 선택되는 무한 재선택을 차단.
async with session.begin_nested():
existing = (
await session.execute(
select(Published).where(
Published.kind == ob.kind,
Published.source_id == ob.source_id,
)
)
).scalar_one_or_none()
# (payload_hash, deleted) 디둡 — no-op 재투영은 rev 안 올림.
is_noop = (
existing is not None
and existing.payload_hash == ob.payload_hash
and existing.deleted == ob.deleted
)
if is_noop:
ob.processed_at = now
else:
new_rev = max_rev + 1
if existing is None:
session.add(
Published(
kind=ob.kind,
source_id=ob.source_id,
pub_id=uuid.uuid4().hex,
payload=ob.payload,
payload_hash=ob.payload_hash,
schema_version=ob.schema_version,
rev=new_rev,
deleted=ob.deleted,
created_at=now,
updated_at=now,
)
)
else:
existing.payload = ob.payload
existing.payload_hash = ob.payload_hash
existing.schema_version = ob.schema_version
existing.deleted = ob.deleted
existing.rev = new_rev
existing.updated_at = now
ob.processed_at = now
# 배치 내 동일 (kind, source_id) 후속 행이 직전 반영을 보도록 flush(최신 승).
await session.flush()
except Exception as row_err:
# savepoint 롤백 = 이 행의 쓰기(processed_at 포함) 취소. attempts/failed_at 만
# 바깥 트랜잭션에 누적돼 최종 commit 으로 영속(영구 재선택 방지).
ob.attempts = (ob.attempts or 0) + 1
if ob.attempts >= MAX_OUTBOX_ATTEMPTS:
ob.failed_at = now
failed_count += 1
logger.error(
"publish_outbox_row_terminal id=%s kind=%s source_id=%s attempts=%s: %s",
ob.id, ob.kind, ob.source_id, ob.attempts, row_err,
)
else:
logger.warning(
"publish_outbox_row_retry id=%s kind=%s source_id=%s attempts=%s: %s",
ob.id, ob.kind, ob.source_id, ob.attempts, row_err,
)
continue
else:
# savepoint 커밋 성공 시에만 rev 카운터 전진(실패 행은 rev 미소모 → 드물게 gap,
# 단일 라이터·커밋순 부여라 viewer since-rev 증분 동기 정합엔 무해).
if not is_noop:
max_rev = new_rev
published_count += 1
await session.commit()
logger.info(
"publish_outbox_drained scanned=%s published=%s failed=%s max_rev=%s",
len(rows),
published_count,
failed_count,
max_rev,
)
except Exception as e:
await session.rollback()
logger.exception("publish_outbox_drain_failed: %s", e)
+2 -3
View File
@@ -28,7 +28,6 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from ai.client import AIClient, parse_json_response
from core.config import settings
from models.study_question import StudyQuestion, StudyQuestionAttempt
from models.study_quiz_session import StudyQuizSession
from models.study_quiz_session_analysis import StudyQuizSessionAnalysis
@@ -43,8 +42,8 @@ from services.study.session_summary_rag import gather_session_summary_context
logger = logging.getLogger(__name__)
# 2026-06-20: config 단일소스 (구 하드코딩 30s = 빠른 Gemma 기준).
LLM_TIMEOUT_S = settings.llm_call_timeout_s
# 4-A 와 동일 안전 마진 (26B 평균 ~10s, gate 직렬화 고려)
LLM_TIMEOUT_S = 30.0
# wrong/unsure 5 미만은 분석 의미 X — insufficient_attempts skip
MIN_ATTEMPTS_FOR_ANALYSIS = 5
# 큰 세션 (84건 등) 에서 prompt 과대 + LLM timeout 방어. 가장 최근 attempt 기준 cap.
+1 -6
View File
@@ -91,12 +91,7 @@ async def process(document_id: int, session: AsyncSession, *, use_deep: bool = F
# sleep-안전 불변식: 쓰기는 전체 완주 후에만 — 중간 절단은 StageDeferred 로 빠져
# 이 지점에 도달하지 않는다 (carry 는 로컬 변수, doc 무변경).
final_summary = strip_thinking(summary)
# 2026-06-20 H2: 빈/think-only 요약을 ai_summary 빈문자열로 박제 → completed 마크 → briefing/digest 누출.
# raise → queue 재시도 후 failed(가시화). 기존 raise 계약(not-found·empty-text)과 동형.
if not final_summary.strip():
raise ValueError(f"empty ai_summary after strip (document_id={document_id})")
doc.ai_summary = final_summary
doc.ai_summary = strip_thinking(summary)
doc.ai_model_version = used_cfg.model
doc.ai_processed_at = datetime.now(timezone.utc)
logger.info(
+13 -10
View File
@@ -27,14 +27,20 @@ ai:
context_char_limit: 260000
temperature: 0.3
top_p: 0.9
repetition_penalty: 1.05 # 한국어 장문 반복/코드스위칭(CJK·라틴 누수) 억제 (보수적 시작값)
top_k: 20 # Qwen3 권장
# deep: ★2026-06-29 잠정 보류 (사용자 "맥북 night-drain 의미없어 → 맥미니 일원화").
# 슬롯 제거 → deep_summary 가 primary(맥미니) 경로 복귀 + use_deep/drain 도 맥미니 폴백
# (맥북 라우팅 0). drain-keeper(GPU cron)도 비활성. 맥북 mlx-vlm-server 는 OpenCode 로컬용 보존.
# 복원(night-drain 재개 시): git history 에서 deep 슬롯(qwen-macbook :8890, max_tokens 8192,
# timeout 900, context_char_limit 260000, temp 0.3 / top_p 0.9 / rep 1.05 / top_k 20) 부활 + drain-keeper 재활성.
# deep: 야간 night-drain 전용 — 맥북 M5 Max Qwen3.6-27B-6bit (llm-router :8890 경유,
# model=qwen-macbook alias). 2026-06-11 재도입 (사용자: 자기 전 night-drain 으로 백로그 분담).
# 맥북 불가(503/연결/절단) = StageDeferred 보류 — 맥미니/cloud 강등 없음, attempts 미소모.
# consumer 의 deep_summary 도 슬롯 존재 시 맥북 경유 (잠들어 있으면 30분 백오프 보류 = 무해).
# 슬롯 제거 시 deep_summary 는 primary(맥미니) 경로 복귀.
deep:
endpoint: "http://100.76.254.116:8890/v1/chat/completions"
model: "qwen-macbook"
max_tokens: 8192
timeout: 900
context_char_limit: 260000
temperature: 0.3
top_p: 0.9
# fallback: primary 장애 시 최후 방어선. Claude Sonnet 4 API (소액 한도, 자동 trigger).
# 호출 빈도 낮음 가정 (Mac mini 가 거의 항상 up) → premium 과 budget 공유 OK.
@@ -202,6 +208,3 @@ pipeline:
digest_llm_timeout_s: 300
digest_llm_attempts: 2
digest_pipeline_hard_cap_s: 5400
# 2026-06-20: study/analyze 단일 primary-call 타임아웃 (구 하드코딩 30~60s = 빠른 Gemma 기준).
# Qwen 27B(콜당 ~40~150s)에 맞춰 단일소스화 — 구 30s 즉사 = 사용자 504 + 워커 영구 재시도.
llm_call_timeout_s: 300
+20 -43
View File
@@ -3,13 +3,7 @@ services:
image: pgvector/pgvector:pg16
volumes:
- pgdata:/var/lib/postgresql/data
# ★ 2026-06-29 fresh-DB/DR 부팅 fix: initdb.d 마운트 제거(기존 `./migrations:/docker-entrypoint-initdb.d`).
# 빈 볼륨 첫 기동 시 postgres 엔트리포인트가 migrations/*.sql(001~) 을 psql autocommit 으로 실행해
# 스키마는 만들되 schema_migrations 스탬프는 안 남김(runner 만 생성) → fastapi init_db 가 documents
# 존재로 'fresh' 를 오판해 baseline(_load_baseline_if_fresh) 로드를 건너뛰고, 빈 schema_migrations
# 로 001 부터 재replay → `CREATE TABLE users`(IF NOT EXISTS 없음) 충돌 → 부팅 크래시(DR/신규환경).
# fresh-boot 은 init_db 의 baseline 적재 + migration runner 단일 경로로 일원화(설계 의도). 기존 prod
# 볼륨은 비어있지 않아 init scripts 가 애초에 미발동 → 무영향.
- ./migrations:/docker-entrypoint-initdb.d
environment:
POSTGRES_DB: pkm
POSTGRES_USER: pkm
@@ -22,8 +16,6 @@ services:
timeout: 5s
retries: 5
restart: unless-stopped
# 2026-06-20 tier-0 무장: 글로벌 OOM 시 커널이 postgres(prod DB)를 reap 하지 않도록.
oom_score_adj: -900
kordoc-service:
build: ./services/kordoc
@@ -62,28 +54,24 @@ services:
start_period: 180s
restart: unless-stopped
# MinerU 2.5 VLM PDFmarkdown 추출 — ★ marker-service 대체(컷오버 2026-06-18, A/B 8/8 PASS).
# 단일카드 markdown VRAM ~10GB(marker)→~5.9GB 고정. fastapi 가 MARKER_ENDPOINT 로 호출.
# 동기 do_parse 버그 회피 위해 server.py 는 async aio_do_parse 사용. 포트 3301.
mineru-service:
build: ./services/mineru
mem_limit: 16g # 2026-06-20: VLM 스파이크 봉쇄 (steady ~12GB) — 호스트 30GB 글로벌 OOM 차단
# Phase 1B (2026-05-01): PDFmarkdown 변환. ocr-service 와 별도 컨테이너 (deps 충돌 회피).
marker-service:
build: ./services/marker
ports:
- "127.0.0.1:3301:3301"
- "127.0.0.1:3300:3300"
expose:
- "3301"
- "3300"
environment:
# vlm-engine = 순수 VLM 단일모델. 기본 hybrid-engine 은 다중모델 로드 = OOM(반드시 명시).
- MINERU_BACKEND=vlm-engine
- MINERU_LANG=${MINERU_LANG:-korean}
# 공유 16GB 카드 공존: 절대 VRAM 캡(GB, 공유카드 robust) + vLLM 분율 캡 병용.
- MINERU_VIRTUAL_VRAM_SIZE=${MINERU_VIRTUAL_VRAM_SIZE:-6}
- MINERU_GPU_MEMORY_UTILIZATION=${MINERU_GPU_MEMORY_UTILIZATION:-0.40}
- MINERU_PRELOAD=${MINERU_PRELOAD:-1}
- HF_HOME=/models/huggingface
- TORCH_HOME=/models/torch
# D-1 (crawl-24x7): idle-unload 전환 — 영구 점유(~3.5GB) 해제가 90% 봉투의 전제.
# /ready 는 idle 에서도 200 (fastapi depends_on service_healthy 유지).
# 롤백 = MARKER_PRELOAD=1 + MARKER_IDLE_UNLOAD_MINUTES=0.
- MARKER_PRELOAD=0
- MARKER_IDLE_UNLOAD_MINUTES=${MARKER_IDLE_UNLOAD_MINUTES:-30}
volumes:
- ${NAS_NFS_PATH:-/mnt/nas/Document_Server}:/documents:ro
- mineru_models:/root/.cache
ipc: host # vLLM 공유메모리 — 공식 run 의 --ipc=host 대응.
- marker_models:/models
deploy:
resources:
reservations:
@@ -92,11 +80,11 @@ services:
count: 1
capabilities: [gpu]
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3301/ready"]
test: ["CMD", "curl", "-f", "http://localhost:3300/ready"]
interval: 30s
timeout: 10s
retries: 3
start_period: 900s # VLM 모델 lazy 다운로드(~2.4GB)+엔진 로드 여유.
start_period: 300s
restart: unless-stopped
stt-service:
@@ -182,7 +170,6 @@ services:
fastapi:
build: ./app
oom_score_adj: -900 # 2026-06-20 tier-0 무장 (앱+스케줄러 SPOF 보호)
ports:
- "100.110.63.63:8000:8000"
volumes:
@@ -197,8 +184,7 @@ services:
condition: service_healthy
kordoc-service:
condition: service_healthy
# 마크다운 엔진 = mineru-service (marker-service 제거 2026-06-18, 롤백=git history).
mineru-service:
marker-service:
condition: service_healthy
env_file:
- credentials.env
@@ -206,8 +192,7 @@ services:
- DATABASE_URL=postgresql+asyncpg://pkm:${POSTGRES_PASSWORD}@postgres:5432/pkm
- KORDOC_ENDPOINT=http://kordoc-service:3100
- OCR_ENDPOINT=http://ocr-service:3200
# ★ 컷오버 2026-06-18: marker-service:3300 → mineru-service:3301 (동일 /convert 계약).
- MARKER_ENDPOINT=http://mineru-service:3301
- MARKER_ENDPOINT=http://marker-service:3300
- MARKER_CONTAINER_PATH_PREFIX=/documents
# 2026-05-08 (D9 Track B revised): GPU stt-service 정식 승격, 내부 DNS 사용.
- STT_ENDPOINT=http://stt-service:3300
@@ -216,14 +201,6 @@ services:
# PR-MacMini-Derived-Worker-1
- STUDY_EXPLANATION_ENABLED=${STUDY_EXPLANATION_ENABLED:-true}
- INTERNAL_WORKER_TOKEN=${INTERNAL_WORKER_TOKEN}
# docsrv-viewer-publish: 발행 워커/저작 enqueue 게이트(기본 false=inert) + 뷰어↔DS feed Bearer.
- STUDY_PUBLISH_ENABLED=${STUDY_PUBLISH_ENABLED:-false}
- DIGEST_PUBLISH_ENABLED=${DIGEST_PUBLISH_ENABLED:-false}
- MAINTENANCE_MODE=${MAINTENANCE_MODE:-false}
- MAINTENANCE_NOTE=${MAINTENANCE_NOTE:-}
- VIEWER_SYNC_TOKEN=${VIEWER_SYNC_TOKEN:-}
# study-to-viewer P2: 뷰어 write-back ingest 게이트(기본 false=inert, 검증 후 점등).
- STUDY_INGEST_ENABLED=${STUDY_INGEST_ENABLED:-false}
# Voice Memo PoC v1 — bot 계정 한정 long-expiry access token. default false → 일반 운영 영향 0.
# 활성화: host .env 에 VOICE_MEMO_BOT_TOKEN_ENABLED=true. plan: rosy-launching-otter.md
- VOICE_MEMO_BOT_TOKEN_ENABLED=${VOICE_MEMO_BOT_TOKEN_ENABLED:-false}
@@ -277,7 +254,7 @@ services:
caddy:
image: caddy:2
ports:
- "127.0.0.1:8080:80" # 2026-06-20: LAN 우회 차단 (실 ingress=home-caddy→caddy:80 도커망)
- "8080:80"
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile
- caddy_data:/data
@@ -293,4 +270,4 @@ volumes:
reranker_cache:
ocr_models:
stt_models:
mineru_models:
marker_models:
+1 -1
View File
@@ -1094,7 +1094,7 @@ services:
image: pgvector/pgvector:pg16
volumes:
- pgdata:/var/lib/postgresql/data
# initdb.d 마운트 제거(2026-06-29): fresh-boot 은 fastapi init_db+baseline 단일 경로.
- ./migrations:/docker-entrypoint-initdb.d
environment:
POSTGRES_DB: pkm
POSTGRES_USER: pkm
+2 -2
View File
@@ -71,7 +71,7 @@ GPU 서버의 NFS mount (`/proc/mounts` 실측):
| 컨테이너 | 마운트 | 모드 | 비고 |
|---|---|---|---|
| postgres | `pgdata:/var/lib/postgresql/data` | rw | DB 본체 named volume (initdb.d 마운트는 2026-06-29 제거 — 아래 관찰) |
| postgres | `pgdata:/var/lib/postgresql/data` + `./migrations:/docker-entrypoint-initdb.d` | rw | DB 본체 named volume |
| kordoc-service | `${NAS}/Document_Server:/documents` | **ro** | PDF/HWP parse |
| ocr-service | `${NAS}/Document_Server:/documents` + `ocr_models:/root/.cache` | **ro** + rw | |
| marker-service | `${NAS}/Document_Server:/documents` + `marker_models:/models` | **ro** + rw | PDF→markdown |
@@ -84,7 +84,7 @@ GPU 서버의 NFS mount (`/proc/mounts` 실측):
**관찰**:
- worker 컨테이너 (kordoc/ocr/marker/stt) 는 모두 NAS **read-only** 마운트 → 원본 안전.
- fastapi 만 NAS **rw** → 업로드/preview/extracted_images 쓰기 단일 책임.
- `./migrations` fastapi 의 `/app/migrations` 마운트. migration runner 는 fastapi `init_db()` 단일 경로. (~2026-06-29: postgres `docker-entrypoint-initdb.d` 마운트 제거. 기존엔 "첫 생성 시만 실행 → 효과 X" 로 봤으나, 빈 볼륨 첫 기동 시 postgres 가 migrations/*.sql 을 실제 실행해 스키마는 만들되 schema_migrations 스탬프를 안 남겨 → init_db 의 baseline fresh 판정을 깨고 부팅 크래시 유발. fresh-DB/DR 부팅을 init_db+baseline 단일 경로로 일원화.)
- `./migrations` 이 postgres 의 `docker-entrypoint-initdb.d` fastapi 의 `/app/migrations` 양쪽에 마운트. 단 실제 migration runner 는 fastapi `init_db()` 만 사용 (postgres init scripts 는 첫 생성 시만 실행 → 효과 X, 안전).
## 정책 정리
@@ -0,0 +1,160 @@
<!--
AskAnswerCard.svelte — 검색 결과 페이지 상단 AI 답변 카드 (컴팩트).
/ask 페이지의 AskAnswer.svelte와 달리, 검색 결과를 가리지 않는
보조 영역으로 설계. 출처 목록 클릭이 must-have, 본문 [n] 클릭은 nice-to-have.
-->
<script lang="ts">
import Badge from '$lib/components/ui/Badge.svelte';
import Skeleton from '$lib/components/ui/Skeleton.svelte';
import { Sparkles, X, FileText } from 'lucide-svelte';
import type { AskResponse, Confidence } from '$lib/types/ask';
interface Props {
data: AskResponse | null;
loading: boolean;
error: boolean;
onCitationClick: (docId: number) => void;
onDismiss: () => void;
}
let { data, loading, error, onCitationClick, onDismiss }: Props = $props();
// [n] 파싱 (AskAnswer.svelte에서 가져옴)
type Token =
| { type: 'text'; value: string }
| { type: 'cite'; n: number; raw: string };
function splitAnswer(text: string): Token[] {
return text
.split(/(\[\d+\])/g)
.filter(Boolean)
.map((tok): Token => {
const m = tok.match(/^\[(\d+)\]$/);
return m
? { type: 'cite', n: Number(m[1]), raw: tok }
: { type: 'text', value: tok };
});
}
function confidenceTone(c: Confidence | null): 'success' | 'warning' | 'error' | 'neutral' {
if (c === 'high') return 'success';
if (c === 'medium') return 'warning';
if (c === 'low') return 'error';
return 'neutral';
}
function confidenceLabel(c: Confidence | null): string {
if (c === 'high') return '높음';
if (c === 'medium') return '중간';
if (c === 'low') return '낮음';
return '';
}
let tokens = $derived(data?.ai_answer ? splitAnswer(data.ai_answer) : []);
// 출처 중복 제거 (같은 doc_id)
let uniqueCitations = $derived.by(() => {
if (!data?.citations?.length) return [];
const seen = new Set<number>();
return data.citations.filter((c) => {
if (seen.has(c.doc_id)) return false;
seen.add(c.doc_id);
return true;
});
});
</script>
<div class="bg-surface border border-default rounded-card p-4">
<!-- 헤더 -->
<div class="flex items-center justify-between gap-2 mb-2">
<div class="flex items-center gap-1.5">
<Sparkles size={12} class="text-accent" />
<span class="text-[10px] font-semibold tracking-wider uppercase text-dim">
내 자료 기준 답변
</span>
{#if data?.confidence}
<Badge tone={confidenceTone(data.confidence)} size="sm">
신뢰도 {confidenceLabel(data.confidence)}
</Badge>
{/if}
{#if data?.completeness === 'partial'}
<Badge tone="warning" size="sm">일부 답변</Badge>
{/if}
</div>
<button
type="button"
onclick={onDismiss}
class="p-0.5 rounded text-dim hover:text-text transition-colors"
aria-label="답변 카드 접기"
>
<X size={14} />
</button>
</div>
<!-- 본문 -->
{#if loading}
<div class="space-y-2">
<Skeleton w="w-full" h="h-3" />
<Skeleton w="w-4/5" h="h-3" />
</div>
<p class="mt-3 text-[10px] text-dim flex items-center gap-1.5">
<span class="inline-block w-2.5 h-2.5 rounded-full border-2 border-dim border-t-accent animate-spin"></span>
근거 기반 답변 생성 중…
</p>
{:else if error}
<p class="text-xs text-dim">답변을 가져오지 못했습니다.</p>
{:else if data?.ai_answer}
<!-- 답변 텍스트 -->
<div class="text-sm leading-6 text-text">
{#each tokens as tok}
{#if tok.type === 'cite'}
{@const citation = data?.citations?.find((c) => c.n === tok.n)}
{#if citation}
<button
type="button"
class="inline text-accent font-semibold hover:underline px-0.5"
onclick={() => onCitationClick(citation.doc_id)}
title={citation.title || `문서 #${citation.doc_id}`}
>
{tok.raw}
</button>
{:else}
<span class="text-dim">{tok.raw}</span>
{/if}
{:else}
<span>{tok.value}</span>
{/if}
{/each}
</div>
<!-- partial: 누락 측면 -->
{#if data.completeness === 'partial' && data.missing_aspects?.length}
<p class="mt-2 text-[10px] text-dim">
다루지 못한 부분: {data.missing_aspects.join(', ')}
</p>
{/if}
<!-- 출처 목록 (must-have) -->
{#if uniqueCitations.length > 0}
<div class="mt-3 pt-2 border-t border-default">
<p class="text-[10px] font-medium text-dim mb-1.5">출처</p>
<div class="flex flex-wrap gap-1.5">
{#each uniqueCitations as citation}
<button
type="button"
onclick={() => onCitationClick(citation.doc_id)}
class="inline-flex items-center gap-1 text-[11px] px-2 py-0.5 rounded bg-surface text-text border border-default hover:border-accent hover:text-accent transition-colors"
title={citation.span_text}
>
<FileText size={10} />
<span class="max-w-[200px] truncate">
{citation.title || `문서 #${citation.doc_id}`}
</span>
</button>
{/each}
</div>
</div>
{/if}
{/if}
</div>
@@ -1,45 +0,0 @@
<script>
// 관련 문서 (유사도) — 문서 레벨 임베딩 KNN. 자기완결: docId 받아 /related 조회.
import { onMount } from 'svelte';
import { api } from '$lib/api';
let { documentId } = $props();
let items = $state([]);
let loaded = $state(false);
const KIND = { law: '법령', guide: '지침', paper: '논문', standard: '표준', incident: '사례' };
onMount(async () => {
try {
const r = await api(`/documents/${documentId}/related?limit=6`);
items = r?.related ?? [];
} catch (e) { /* silent */ }
finally { loaded = true; }
});
</script>
{#if items.length}
<div class="rel">
<div class="lab">관련 문서</div>
{#each items as it (it.id)}
<a class="ri" href={`/documents/${it.id}`}>
<span class="rt">{it.title}</span>
<span class="rm">
{#if it.material_type && KIND[it.material_type]}<span class="kind">{KIND[it.material_type]}</span>{/if}
<span class="rs">{Math.round((it.sim ?? 0) * 100)}</span>
</span>
</a>
{/each}
</div>
{/if}
<style>
.rel { background: var(--surface); border: 1px solid var(--border); border-radius: 14px; padding: 13px; }
.lab { font-size: 10.5px; font-weight: 700; color: var(--text-dim); letter-spacing: .4px; margin-bottom: 8px; }
.ri { display: flex; align-items: baseline; gap: 8px; padding: 5px 6px; border-radius: 7px; text-decoration: none; }
.ri:hover { background: var(--surface-hover, #ecf0e8); }
.rt { flex: 1; font-size: 12px; line-height: 1.4; color: var(--text); overflow: hidden; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; }
.rm { flex-shrink: 0; display: flex; align-items: center; gap: 5px; }
.kind { font-size: 9px; font-weight: 700; color: var(--accent-hover, #3d7256); background: #e3efe2; border: 1px solid #cfe3cd; border-radius: 4px; padding: 0 4px; }
.rs { font-size: 10.5px; font-family: ui-monospace, Menlo, monospace; color: var(--faint, #9aa090); }
</style>
+1 -8
View File
@@ -2,7 +2,7 @@
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { api } from '$lib/api';
import { ChevronRight, ChevronDown, FolderOpen, FolderTree, Inbox, Clock, Mail, Scale, StickyNote, GraduationCap, CalendarCheck, MessageCircle, Hash } from 'lucide-svelte';
import { ChevronRight, ChevronDown, FolderOpen, FolderTree, Inbox, Clock, Mail, Scale, StickyNote, GraduationCap, CalendarCheck, MessageCircle } from 'lucide-svelte';
let tree = $state([]);
let loading = $state(true);
@@ -195,13 +195,6 @@
>
<FolderTree size={14} /> 자료실
</a>
<a
href="/clause"
class="w-full flex items-center gap-2 px-3 py-1.5 rounded-md text-sm transition-colors
{$page.url.pathname === '/clause' ? 'bg-accent/15 text-accent' : 'text-dim hover:bg-surface hover:text-text'}"
>
<Hash size={14} /> 절 바로가기
</a>
</div>
<!-- 메모 & Inbox -->
@@ -0,0 +1,228 @@
<!--
AskAnswer.svelte — /ask 페이지 상단 패널.
Answer 본문 + clickable [n] citations + 신뢰도/상태 Badge.
status != completed 또는 refused=true → warning empty state +
no_results_reason + "검색 결과 확인하기" 역링크.
-->
<script lang="ts">
import Badge from '$lib/components/ui/Badge.svelte';
import Button from '$lib/components/ui/Button.svelte';
import EmptyState from '$lib/components/ui/EmptyState.svelte';
import Skeleton from '$lib/components/ui/Skeleton.svelte';
import { AlertTriangle, Sparkles } from 'lucide-svelte';
import type { AskResponse, Confidence, SynthesisStatus } from '$lib/types/ask';
interface Props {
data: AskResponse | null;
loading: boolean;
onCitationClick: (n: number) => void;
}
let { data, loading, onCitationClick }: Props = $props();
type Token =
| { type: 'text'; value: string }
| { type: 'cite'; n: number; raw: string };
function splitAnswer(text: string): Token[] {
return text
.split(/(\[\d+\])/g)
.filter(Boolean)
.map((tok): Token => {
const m = tok.match(/^\[(\d+)\]$/);
return m
? { type: 'cite', n: Number(m[1]), raw: tok }
: { type: 'text', value: tok };
});
}
function confidenceTone(
c: Confidence | null,
): 'success' | 'warning' | 'error' | 'neutral' {
if (c === 'high') return 'success';
if (c === 'medium') return 'warning';
if (c === 'low') return 'error';
return 'neutral';
}
function confidenceLabel(c: Confidence | null): string {
if (c === 'high') return '높음';
if (c === 'medium') return '중간';
if (c === 'low') return '낮음';
return '없음';
}
const STATUS_LABEL: Record<SynthesisStatus, string> = {
completed: '답변 완료',
timeout: '답변 지연',
skipped: '답변 생략',
no_evidence: '근거 없음',
parse_failed: '형식 오류',
llm_error: 'AI 오류',
backend_unavailable: 'Backend 비가용',
};
/**
* backend chip label — `backend_requested` 가 명시 opt-in 인 경우만 표시.
* 미지정 (null/undefined) default 호출은 chip 없음 (시각 noise 회피).
*/
function backendChipLabel(backend: string | null | undefined): string | null {
if (!backend) return null;
if (backend === 'qwen-macbook') return 'Qwen 27B (MacBook)';
if (backend === 'gemma-macmini') return 'Gemma 26B (Mac mini)';
return backend;
}
let tokens = $derived(data?.ai_answer ? splitAnswer(data.ai_answer) : []);
let showFullAnswer = $derived(
!!data && !!data.ai_answer && data.completeness === 'full'
&& data.synthesis_status === 'completed' && !data.refused,
);
let showPartial = $derived(
!!data && data.completeness === 'partial' && !data.refused,
);
let showWarning = $derived(!!data && !showFullAnswer && !showPartial);
</script>
<section class="bg-surface border border-default rounded-card p-5">
<!-- 헤더 -->
<div class="flex items-start justify-between gap-3 mb-4">
<div>
<p class="text-[10px] font-semibold tracking-wider uppercase text-dim flex items-center gap-1.5">
<Sparkles size={12} /> AI Answer
</p>
<h2 class="mt-1 text-base font-semibold text-text">근거 기반 답변</h2>
</div>
{#if data && !loading}
<div class="flex flex-wrap gap-1.5">
<Badge tone={confidenceTone(data.confidence)} size="sm">
신뢰도 {confidenceLabel(data.confidence)}
</Badge>
{#if backendChipLabel(data.backend_requested)}
<span title={`backend_requested=${data.backend_requested} / backend_used=${data.backend_used ?? 'null'}`}>
<Badge tone="neutral" size="sm">
{backendChipLabel(data.backend_requested)}
</Badge>
</span>
{/if}
<Badge tone="neutral" size="sm">
{STATUS_LABEL[data.synthesis_status]}
</Badge>
{#if data.synthesis_ms > 0}
<Badge tone="neutral" size="sm">
{Math.round(data.synthesis_ms)}ms
</Badge>
{/if}
</div>
{/if}
</div>
<!-- 본문 -->
{#if loading}
<div class="space-y-3">
<Skeleton w="w-3/4" h="h-4" />
<Skeleton w="w-full" h="h-4" />
<Skeleton w="w-5/6" h="h-4" />
<p class="mt-4 text-xs text-dim flex items-center gap-2">
<span class="inline-block w-3 h-3 rounded-full border-2 border-dim border-t-accent animate-spin"></span>
근거 기반 답변 생성 중… 약 15초 소요
</p>
</div>
{:else if showFullAnswer && data}
<div class="text-sm leading-7 text-text">
{#each tokens as tok}
{#if tok.type === 'cite'}
<button
type="button"
class="inline-block align-baseline text-accent font-semibold hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent-ring rounded px-0.5"
onclick={() => onCitationClick(tok.n)}
aria-label={`인용 ${tok.n}번 보기`}
>
{tok.raw}
</button>
{:else}
<span>{tok.value}</span>
{/if}
{/each}
</div>
{:else if showPartial && data}
<!-- Phase 3.5a: question-aligned partial structure -->
<div>
<Badge tone="warning" size="sm">일부 답변</Badge>
{#if data.ai_answer}
<div class="mt-3 text-sm leading-7 text-text">
{#each tokens as tok}
{#if tok.type === 'cite'}
<button
type="button"
class="inline-block align-baseline text-accent font-semibold hover:underline rounded px-0.5"
onclick={() => onCitationClick(tok.n)}
>{tok.raw}</button>
{:else}
<span>{tok.value}</span>
{/if}
{/each}
</div>
{:else if data.confirmed_items?.length}
<div class="mt-3">
<h4 class="text-xs font-semibold text-dim uppercase tracking-wider">✓ 답변 가능</h4>
<ul class="mt-2 space-y-2">
{#each data.confirmed_items as item}
<li class="text-sm text-text">
<strong class="text-accent">{item.aspect}:</strong>
<span>{item.text}</span>
{#each item.citations as n}
<button
type="button"
class="text-accent font-semibold hover:underline px-0.5"
onclick={() => onCitationClick(n)}
>[{n}]</button>
{/each}
</li>
{/each}
</ul>
</div>
{/if}
{#if data.missing_aspects?.length}
<div class="mt-4 border-t border-default pt-3">
<h4 class="text-xs font-semibold text-dim uppercase tracking-wider">✗ 답변 불가</h4>
<ul class="mt-2 space-y-1">
{#each data.missing_aspects as aspect}
<li class="text-sm text-dim">{aspect} <span class="text-[10px]">(근거 없음)</span></li>
{/each}
</ul>
</div>
{/if}
<div class="mt-4">
<Button
variant="secondary"
size="sm"
href={`/documents?q=${encodeURIComponent(data.query)}`}
>
검색 결과 확인하기
</Button>
</div>
</div>
{:else if showWarning && data}
<EmptyState
icon={AlertTriangle}
title={data.refused && data.no_results_reason
? data.no_results_reason
: (data.no_results_reason ?? '관련 근거를 찾지 못했습니다.')}
description="검색 결과를 직접 확인해 보세요."
>
<Button
variant="secondary"
size="sm"
href={`/documents?q=${encodeURIComponent(data.query)}`}
>
검색 결과 확인하기
</Button>
</EmptyState>
{/if}
</section>
@@ -0,0 +1,91 @@
<!--
AskEvidence.svelte — /ask 페이지 우측 sticky 패널.
⚠ 영구 룰 (Phase 3.4 plan):
`citation.full_snippet` 은 UI 에 직접 렌더 금지. debug 모드(`?debug=1`)
에서 hover tooltip 으로만 조건부 노출 가능.
이 규칙이 깨지면 backend span-precision UX 가치가 사라진다. 코드 리뷰에서
반드시 reject. span_text 만 본문으로 노출한다.
-->
<script lang="ts">
import Badge from '$lib/components/ui/Badge.svelte';
import EmptyState from '$lib/components/ui/EmptyState.svelte';
import Skeleton from '$lib/components/ui/Skeleton.svelte';
import { BookOpen } from 'lucide-svelte';
import type { AskResponse } from '$lib/types/ask';
interface Props {
data: AskResponse | null;
loading: boolean;
activeCitation: number | null;
registerCitation: (n: number, node: HTMLElement) => { destroy: () => void };
}
let { data, loading, activeCitation, registerCitation }: Props = $props();
let citations = $derived(data?.citations ?? []);
</script>
<section class="bg-surface border border-default rounded-card p-5">
<div class="flex items-start justify-between gap-3 mb-4">
<div>
<p class="text-[10px] font-semibold tracking-wider uppercase text-dim flex items-center gap-1.5">
<BookOpen size={12} /> Evidence Highlights
</p>
<h3 class="mt-1 text-sm font-semibold text-text">인용 근거</h3>
</div>
{#if data && !loading}
<Badge tone="neutral" size="sm">{citations.length}</Badge>
{/if}
</div>
{#if loading}
<div class="space-y-3">
{#each Array(2) as _}
<div class="border border-default rounded-card p-4 space-y-2">
<Skeleton w="w-24" h="h-3" />
<Skeleton w="w-full" h="h-3" />
<Skeleton w="w-5/6" h="h-3" />
<Skeleton w="w-3/4" h="h-3" />
</div>
{/each}
</div>
{:else if citations.length === 0}
<EmptyState title="표시할 근거가 없습니다." class="py-6" />
{:else}
<div class="space-y-3">
{#each citations as citation (citation.n)}
{@const isActive = activeCitation === citation.n}
<article
class="border rounded-card p-4 transition-colors {isActive
? 'border-accent ring-2 ring-accent/20 bg-accent/5'
: 'border-default'}"
use:registerCitation={citation.n}
>
<div class="flex items-start gap-2">
<span class="text-accent font-bold text-sm shrink-0">[{citation.n}]</span>
<div class="flex-1 min-w-0">
<strong class="block text-sm text-text truncate">
{citation.title ?? `문서 ${citation.doc_id}`}
</strong>
{#if citation.section_title}
<p class="mt-0.5 text-xs text-dim truncate">{citation.section_title}</p>
{/if}
</div>
</div>
<!-- ⚠ span_text 만 렌더. full_snippet 금지 -->
<p class="mt-3 text-sm leading-relaxed text-text whitespace-pre-wrap">
{citation.span_text}
</p>
<div class="mt-3 flex gap-2 text-[10px] text-dim">
<span>relevance {citation.relevance.toFixed(2)}</span>
<span>rerank {citation.rerank_score.toFixed(2)}</span>
</div>
</article>
{/each}
</div>
{/if}
</section>
@@ -0,0 +1,78 @@
<!--
AskResults.svelte — /ask 페이지 하단 패널.
검색 결과 리스트. DocumentCard 재사용 X — SearchResult 필드 셋이 달라서
의존성 리스크 회피. inline 간단 카드로 title/score/snippet/section_title 표시.
클릭 시 `/documents/{id}` 로 이동.
-->
<script lang="ts">
import Badge from '$lib/components/ui/Badge.svelte';
import EmptyState from '$lib/components/ui/EmptyState.svelte';
import Skeleton from '$lib/components/ui/Skeleton.svelte';
import { FileText } from 'lucide-svelte';
import type { AskResponse } from '$lib/types/ask';
interface Props {
data: AskResponse | null;
loading: boolean;
}
let { data, loading }: Props = $props();
let results = $derived(data?.results ?? []);
</script>
<section class="bg-surface border border-default rounded-card p-5">
<div class="flex items-start justify-between gap-3 mb-4">
<div>
<p class="text-[10px] font-semibold tracking-wider uppercase text-dim flex items-center gap-1.5">
<FileText size={12} /> Search Results
</p>
<h3 class="mt-1 text-sm font-semibold text-text">검색 결과</h3>
</div>
{#if data && !loading}
<Badge tone="neutral" size="sm">{data.total}</Badge>
{/if}
</div>
{#if loading}
<div class="space-y-3">
{#each Array(5) as _}
<div class="border border-default rounded-card p-4 space-y-2">
<Skeleton w="w-2/3" h="h-4" />
<Skeleton w="w-full" h="h-3" />
<Skeleton w="w-4/5" h="h-3" />
</div>
{/each}
</div>
{:else if results.length === 0}
<EmptyState title="검색 결과가 없습니다." class="py-6" />
{:else}
<div class="space-y-3">
{#each results as result (result.id)}
<a
href={`/documents/${result.id}`}
class="block border border-default rounded-card p-4 hover:border-accent hover:bg-surface-hover transition-colors"
>
<div class="flex items-start justify-between gap-3">
<strong class="text-sm text-text flex-1 min-w-0 truncate">
{result.title ?? `문서 ${result.id}`}
</strong>
<div class="flex gap-1.5 text-[10px] text-dim shrink-0">
<span>score {result.score.toFixed(2)}</span>
{#if result.rerank_score != null}
<span>rerank {result.rerank_score.toFixed(2)}</span>
{/if}
</div>
</div>
{#if result.section_title}
<p class="mt-1 text-xs text-dim truncate">{result.section_title}</p>
{/if}
{#if result.snippet}
<p class="mt-2 text-xs text-dim line-clamp-2">{result.snippet}</p>
{/if}
</a>
{/each}
</div>
{/if}
</section>
+84
View File
@@ -0,0 +1,84 @@
/**
* Phase 3.4: `/api/search/ask` .
*
* Backend Pydantic (`app/api/search.py::AskResponse`) 1:1 .
* .
*/
export type SynthesisStatus =
| 'completed'
| 'timeout'
| 'skipped'
| 'no_evidence'
| 'parse_failed'
| 'llm_error'
| 'backend_unavailable';
export type Confidence = 'high' | 'medium' | 'low';
export interface Citation {
n: number;
chunk_id: number | null;
doc_id: number;
title: string | null;
section_title: string | null;
/** LLM이 추출한 50~300자 핵심 span. UI에서 이것만 노출. */
span_text: string;
/**
* 800 window.
*
* UI . debug hover tooltip
* . full_snippet을 backend span-precision UX
* (plan §Evidence ).
*/
full_snippet: string;
relevance: number;
rerank_score: number;
}
export interface SearchResult {
id: number;
title: string | null;
ai_domain: string | null;
ai_summary: string | null;
file_format: string;
score: number;
snippet: string | null;
match_reason: string | null;
chunk_id: number | null;
chunk_index: number | null;
section_title: string | null;
rerank_score: number | null;
}
export type Completeness = 'full' | 'partial' | 'insufficient';
export interface ConfirmedItem {
aspect: string;
text: string;
citations: number[];
}
export interface AskResponse {
results: SearchResult[];
ai_answer: string | null;
citations: Citation[];
synthesis_status: SynthesisStatus;
synthesis_ms: number;
confidence: Confidence | null;
refused: boolean;
no_results_reason: string | null;
query: string;
total: number;
/** Phase 3.5a */
completeness: Completeness;
covered_aspects: string[] | null;
missing_aspects: string[] | null;
confirmed_items: ConfirmedItem[] | null;
/**
* PR-MacBook-RAG-Backend-1: backend dispatcher metadata.
* backend null ( ). opt-in .
*/
backend_requested?: string | null;
backend_used?: string | null;
}
+61
View File
@@ -0,0 +1,61 @@
/**
* "질문형" ( ).
* true이면 /api/search/ask .
*
* false positive: /ask refused=true .
* false negative: 기존 .
*/
export function isQuestion(q: string): boolean {
const trimmed = q.trim();
if (trimmed.length === 0) return false;
// 1. ?로 끝나면 단일 단어라도 허용 (왜?, 절차?)
if (trimmed.endsWith('?')) return true;
// ? 없으면 단일 단어 / 4자 미만 제외 (키워드 보호)
if (trimmed.length < 4) return false;
if (trimmed.split(/\s+/).length < 2) return false;
// 2. 한국어 질문 어미
const KO_ENDINGS = [
'인가요', '인가', '인지', '있나요', '있나',
'할까요', '할까', '될까요', '될까',
'뭐야', '뭔가', '뭘까', '뭔지', '뭐지', '뭐죠',
'는지', '은지', '일까',
'어때', '어떤가',
'됩니까', '합니까', '입니까', '나요', '까요',
'주세요', '알려줘', '설명해', '비교해',
];
for (const ending of KO_ENDINGS) {
if (trimmed.endsWith(ending)) return true;
}
// 3. 한국어 질문 시작어
const KO_STARTS = [
'어떻게', '왜', '무엇', '무슨', '뭐가', '누가',
'어디', '언제', '몇', '얼마나', '어떤', '어느',
];
for (const start of KO_STARTS) {
if (trimmed.startsWith(start)) return true;
}
// 4. 영어 질문 시작어 (대소문자 무시)
const EN_STARTS = [
'what', 'how', 'why', 'when', 'where', 'who', 'which',
'is', 'are', 'do', 'does', 'can', 'could', 'should', 'would',
'explain', 'describe', 'compare', 'tell me',
];
const lower = trimmed.toLowerCase();
for (const start of EN_STARTS) {
if (lower.startsWith(start + ' ')) return true;
}
// 5. 의미 패턴 (끝 단어)
const SEMANTIC_ENDINGS = ['차이', '비교', '설명', '요약', '정리', '방법', '절차'];
const lastWord = trimmed.split(/\s+/).pop() || '';
for (const pat of SEMANTIC_ENDINGS) {
if (lastWord === pat || lastWord.endsWith(pat)) return true;
}
return false;
}
+305
View File
@@ -0,0 +1,305 @@
<!--
/ask — Phase 3.4 Ask Pipeline Frontend.
URL-driven: `/ask?q=<encoded>[&backend=<alias>]` 가 진입점.
$effect 로 (q, backend) 변화 감지 → `/api/search/ask` 호출 →
3-panel 렌더 (Answer / Evidence / Results).
중복 호출 방지: lastKey (q+backend) 가드.
Backend selector (PR-3 of DS AI routing policy, 2026-05-23,
[[document-server-ai-routing-policy]]) — PR-DocSrv-Web-Ask-Selector-1 확장:
- `auto` (기본, URL param 없음 → router 의 rule + LLM triage)
- `mac-mini-default` (명시, Mac mini gemma-4-26b)
- `qwen-macbook` (명시, M5 Max Qwen3.6-27B; "This device" 토글 on 시만)
- `claude-cloud` (명시, 503 scaffold; VITE_ENABLE_CLOUD_BACKEND_DEV=true 시만)
- "This device" 토글: localStorage `ds_device_self_label = 'macbook-m5-max' | null`.
source IP 의존 0 (PR-0 round 2 발견: caddy 2-hop + X-Forwarded-For 미설정 →
DS 가 보는 source IP = LAN gateway, 신뢰 불가).
- 503 + error_reason ∈ {macbook_unavailable, provider_not_configured, router_*}
시 자동 fallback 금지. UI 가 친절한 메시지 + "Mac mini default 로 재요청" 버튼.
- legacy URL `?backend=default|gemma-macmini` 는 그대로 받아서 mac-mini-default 와 동등 처리.
-->
<script lang="ts">
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { api, type ApiError } from '$lib/api';
import { addToast } from '$lib/stores/toast';
import EmptyState from '$lib/components/ui/EmptyState.svelte';
import Button from '$lib/components/ui/Button.svelte';
import AskAnswer from '$lib/components/ask/AskAnswer.svelte';
import AskEvidence from '$lib/components/ask/AskEvidence.svelte';
import AskResults from '$lib/components/ask/AskResults.svelte';
import { Sparkles, Search, AlertCircle } from 'lucide-svelte';
import type { AskResponse } from '$lib/types/ask';
type BackendChoice = 'auto' | 'mac-mini-default' | 'qwen-macbook' | 'claude-cloud';
function parseBackend(raw: string | null): BackendChoice {
if (raw === 'qwen-macbook') return 'qwen-macbook';
if (raw === 'mac-mini-default') return 'mac-mini-default';
if (raw === 'claude-cloud') return 'claude-cloud';
// legacy aliases (?backend=default | gemma-macmini) → mac-mini-default 와 동등
if (raw === 'default' || raw === 'gemma-macmini') return 'mac-mini-default';
return 'auto';
}
// Build-time feature flag — Claude Cloud UI 노출 여부 (default false).
// VITE_ENABLE_CLOUD_BACKEND_DEV=true npm run build 시만 활성. 운영 토글 X
// (build-time 한계). DS runtime feature flag API migrate 는 후속 PR.
const CLOUD_DEV_ENABLED = import.meta.env.VITE_ENABLE_CLOUD_BACKEND_DEV === 'true';
const DEVICE_LABEL_KEY = 'ds_device_self_label';
const DEVICE_LABEL_M5_MAX = 'macbook-m5-max';
// ── state ───────────────────────────────────────────
let queryInput = $state('');
let selectedBackend = $state<BackendChoice>('auto');
let data = $state<AskResponse | null>(null);
let loading = $state(false);
let backendUnavailable = $state(false);
let backendUnavailableMessage = $state('');
// "I am on MacBook M5 Max" 토글. mount 시 localStorage 에서 복원.
let isMacBookM5Max = $state(false);
$effect(() => {
if (typeof window === 'undefined') return;
const stored = window.localStorage.getItem(DEVICE_LABEL_KEY);
isMacBookM5Max = stored === DEVICE_LABEL_M5_MAX;
});
function toggleMacBookM5Max() {
isMacBookM5Max = !isMacBookM5Max;
if (typeof window === 'undefined') return;
if (isMacBookM5Max) {
window.localStorage.setItem(DEVICE_LABEL_KEY, DEVICE_LABEL_M5_MAX);
} else {
window.localStorage.removeItem(DEVICE_LABEL_KEY);
// 토글 off 시 qwen-macbook 선택돼 있었으면 auto 로 복귀 (선택권 박탈 X 명시 신호).
if (selectedBackend === 'qwen-macbook') {
selectedBackend = 'auto';
}
}
}
// 중복 호출 방지 가드 (hydration + reactive trigger 이중 발동 방지)
let lastKey = '';
// citation scroll 연동: Answer 가 [n] 클릭 → Evidence 카드로 이동 + highlight
const citationNodes = new Map<number, HTMLElement>();
let activeCitation = $state<number | null>(null);
function registerCitation(n: number, node: HTMLElement) {
citationNodes.set(n, node);
return {
destroy() {
citationNodes.delete(n);
},
};
}
function scrollToCitation(n: number) {
activeCitation = n;
const node = citationNodes.get(n);
node?.scrollIntoView({ behavior: 'smooth', block: 'center' });
}
// ── URL 빌더: backend !== 'auto' 일 때만 param 추가 ─────
function buildAskUrl(q: string, backend: BackendChoice): string {
const params = new URLSearchParams({ q });
if (backend !== 'auto') params.set('backend', backend);
return `/ask?${params.toString()}`;
}
// ── submit (URL-driven, back 자동) ──────────────────
function submit() {
const q = queryInput.trim();
if (!q) return;
goto(buildAskUrl(q, selectedBackend));
}
function handleKeydown(e: KeyboardEvent) {
if (e.key === 'Enter' && !e.isComposing) {
e.preventDefault();
submit();
}
}
// 503 후 "Mac mini default 로 재요청" — auto 로 reset (param 명시 제거).
function retryWithMacMiniDefault() {
const q = $page.url.searchParams.get('q') ?? queryInput.trim();
if (!q) return;
goto(`/ask?q=${encodeURIComponent(q)}`);
}
// PR-3 of routing policy — error_reason → 친절 메시지 매핑.
// silent fallback 금지 ([[feedback_no_silent_fallback_explicit_opt_in]]):
// 모든 503 case 는 명시 표시, 다른 backend 자동 재호출 X.
function friendlyErrorMessage(reason: string | undefined, detail: string): string {
switch (reason) {
case 'macbook_unavailable':
return 'MacBook M5 Max 가 응답하지 않습니다. 깨우거나 (launchctl start) Mac mini default 로 다시 요청하세요.';
case 'provider_not_configured':
return 'Claude Cloud 백엔드는 아직 구성되지 않았습니다 (2026-06-15 이후 별 PR 활성화 예정).';
default:
if (reason && reason.startsWith('router_')) {
return `라우터 호출 실패 (${reason}). Mac mini default 로 다시 요청하거나 잠시 후 재시도하세요.`;
}
if (reason && reason.startsWith('upstream_')) {
return `Upstream backend 가 응답하지 않습니다 (${reason}). 잠시 후 재시도하세요.`;
}
return detail || '답변 생성 실패';
}
}
// ── API 호출 ───────────────────────────────────────
async function runAsk(q: string, backend: BackendChoice) {
loading = true;
activeCitation = null;
backendUnavailable = false;
backendUnavailableMessage = '';
const path = backend !== 'auto'
? `/search/ask?q=${encodeURIComponent(q)}&backend=${encodeURIComponent(backend)}&limit=10`
: `/search/ask?q=${encodeURIComponent(q)}&limit=10`;
try {
data = await api<AskResponse>(path);
} catch (err) {
const apiErr = err as ApiError;
if (apiErr.status === 503) {
backendUnavailable = true;
backendUnavailableMessage = friendlyErrorMessage(apiErr.errorReason, apiErr.detail);
data = null;
} else {
addToast('error', apiErr.detail || '답변 생성 실패');
data = null;
}
} finally {
loading = false;
}
}
// ── URL → runAsk (중복 가드) ────────────────────────
$effect(() => {
const q = $page.url.searchParams.get('q') ?? '';
const backend = parseBackend($page.url.searchParams.get('backend'));
queryInput = q;
selectedBackend = backend;
if (!q) {
data = null;
loading = false;
backendUnavailable = false;
lastKey = '';
return;
}
const key = `${q}|${backend}`;
if (key === lastKey) return;
lastKey = key;
runAsk(q, backend);
});
</script>
<svelte:head>
<title>질문 - PKM</title>
</svelte:head>
<div class="h-full overflow-auto">
<!-- 상단 검색바 (sticky) -->
<div class="sticky top-0 z-10 bg-bg/80 backdrop-blur border-b border-default px-4 py-3">
<div class="flex flex-wrap items-center gap-2 max-w-[1680px] mx-auto">
<div class="relative flex-1 min-w-0">
<Search
size={14}
class="absolute left-3 top-1/2 -translate-y-1/2 text-dim pointer-events-none"
/>
<input
data-search-input
type="text"
bind:value={queryInput}
onkeydown={handleKeydown}
placeholder="질문을 입력하세요 (/ 키로 포커스)"
class="w-full pl-9 pr-3 py-2 bg-surface border border-default rounded-lg text-text text-sm focus:border-accent outline-none"
/>
</div>
<label
class="flex items-center gap-1.5 text-xs text-dim cursor-pointer select-none"
title="이 디바이스가 MacBook M5 Max 인 경우 체크 — This device (qwen-macbook) 옵션 활성화됩니다. localStorage 저장."
>
<input
type="checkbox"
checked={isMacBookM5Max}
onchange={toggleMacBookM5Max}
class="accent-accent"
/>
<span>This is M5 Max</span>
</label>
<select
bind:value={selectedBackend}
title="Backend 선택 — silent fallback 0 정책 (선택한 backend 만 시도, 실패 시 503)."
class="py-2 px-2 bg-surface border border-default rounded-lg text-text text-xs focus:border-accent outline-none min-w-0 max-w-[42vw] truncate"
>
<option value="auto">Auto (router)</option>
<option value="mac-mini-default">Mac mini (default)</option>
<option
value="qwen-macbook"
disabled={!isMacBookM5Max}
title={isMacBookM5Max
? 'MacBook M5 Max Qwen3.6-27B (직접 호출)'
: 'Check "This is M5 Max" toggle to enable'}
>
{isMacBookM5Max ? 'This device (Qwen MacBook)' : 'This device (unavailable)'}
</option>
<option
value="claude-cloud"
disabled={!CLOUD_DEV_ENABLED}
title={CLOUD_DEV_ENABLED
? 'Claude Cloud (dev mode — returns 503 until activation PR)'
: 'Cloud backend not configured yet'}
>
Claude Cloud {CLOUD_DEV_ENABLED ? '(dev)' : '(unavailable)'}
</option>
</select>
</div>
</div>
<!-- 본문 -->
<div class="max-w-[1680px] mx-auto p-4">
{#if backendUnavailable}
<div class="py-16">
<EmptyState
icon={AlertCircle}
title="Backend 가 응답하지 않습니다"
description={backendUnavailableMessage}
>
<Button variant="primary" size="sm" onclick={retryWithMacMiniDefault}>
Mac mini (default) 로 재요청
</Button>
</EmptyState>
</div>
{:else if !queryInput && !loading && !data}
<div class="py-16">
<EmptyState
icon={Sparkles}
title="근거 기반 답변을 받아보세요"
description="질문을 입력하면 문서에서 근거를 찾아 인용 기반 답변을 생성합니다."
/>
</div>
{:else}
<div class="grid gap-4 lg:grid-cols-[1.2fr_0.9fr] items-start">
<!-- 좌: Answer + Results -->
<div class="flex flex-col gap-4 min-w-0">
<AskAnswer {data} {loading} onCitationClick={scrollToCitation} />
<AskResults {data} {loading} />
</div>
<!-- 우: Evidence (lg 이상 sticky) -->
<div class="lg:sticky lg:top-20 lg:self-start min-w-0">
<AskEvidence
{data}
{loading}
{activeCitation}
{registerCitation}
/>
</div>
</div>
{/if}
</div>
</div>
-209
View File
@@ -1,209 +0,0 @@
<script>
// ASME/법령 절-KB — 코드북·공부 리더 (r2). parent 표준/법령을 한 권의 책처럼.
// 좌 인덱스(Part/章→절/조) · 중 본문(MarkdownDoc=공식·표·이미지) · breadcrumb·이전다음·양방향 백링크.
import { onMount, tick } from 'svelte';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { api } from '$lib/api';
import MarkdownDoc from '$lib/components/MarkdownDoc.svelte';
let parentId = $state(null);
let parentTitle = $state('');
let clauses = $state([]);
let selectedId = $state(null);
let clauseDoc = $state(null);
let links = $state(null);
let expanded = $state({});
let loading = $state(false);
let q = $state('');
let parts = $derived.by(() => {
const out = [], idx = {};
for (const c of clauses) {
const p = c.clause_part || '·';
if (!(p in idx)) { idx[p] = out.length; out.push({ part: p, items: [] }); }
out[idx[p]].items.push(c);
}
return out;
});
let visibleParts = $derived.by(() => {
const term = q.trim().toLowerCase();
if (!term) return parts;
return parts.map(g => ({ part: g.part, items: g.items.filter(c =>
(c.clause_code || '').toLowerCase().includes(term) || (c.title || '').toLowerCase().includes(term)) }))
.filter(g => g.items.length);
});
let selMeta = $derived(clauses.find((c) => c.id === selectedId) || null);
const strip = (t, c) => (t || '').replace(c || '', '').replace(/^[(\s)]+|[(\s)]+$/g, '').trim();
async function loadBook() {
const r = await api(`/documents/${parentId}/clauses`);
parentTitle = r?.parent_title ?? '';
clauses = r?.clauses ?? [];
const e = {};
for (const c of clauses) e[c.clause_part || '·'] = true;
expanded = e;
}
async function loadClause(id) {
if (!id) return;
loading = true;
selectedId = id;
try {
const [d, l] = await Promise.all([api(`/documents/${id}`), api(`/documents/${id}/backlinks`)]);
clauseDoc = d; links = l;
const sel = clauses.find((c) => c.id === id);
if (sel) expanded = { ...expanded, [sel.clause_part || '·']: true };
goto(`/book/${parentId}?c=${id}`, { replaceState: true, keepFocus: true, noScroll: true });
await tick(); window.scrollTo({ top: 0 });
} finally { loading = false; }
}
onMount(async () => {
parentId = Number($page.params.id);
await loadBook();
const c = Number($page.url.searchParams.get('c'));
await loadClause(c && clauses.find((x) => x.id === c) ? c : clauses[0]?.id);
});
</script>
<div class="book">
<!-- top bar -->
<div class="bar">
<span class="brand">절-KB</span>
<span class="crumbs">{parentTitle} {#if selMeta}<b class="sep"></b> {selMeta.clause_part} <b class="sep"></b> <b>{links?.clause_code ?? selMeta.clause_code}</b>{/if}</span>
<div class="search"><input placeholder="절·조 번호 또는 키워드" bind:value={q} /></div>
<div class="tools"><span class="tool on">읽기</span><span class="tool">형광펜</span><span class="tool">노트</span><span class="tool">암기카드</span></div>
</div>
<div class="main">
<!-- left index -->
<aside class="idx">
<a class="btitle" href={`/documents/${parentId}`}>{parentTitle || '표준'}</a>
<div class="bmeta">{clauses.length} · 한 권의 책처럼 탐색</div>
{#each visibleParts as g (g.part)}
<div class="parttab" role="button" tabindex="0" onclick={() => (expanded = { ...expanded, [g.part]: !expanded[g.part] })}>
<span class="bar2"></span><span class="pname">{g.part}</span><span class="ct">{g.items.length}</span>
</div>
{#if expanded[g.part] || q.trim()}
{#each g.items as c (c.id)}
<div class="ci" class:on={c.id === selectedId} role="button" tabindex="0" onclick={() => loadClause(c.id)}>
<span class="no">{c.clause_code}</span><span class="tt">{strip(c.title, c.clause_code)}</span>
</div>
{/each}
{/if}
{/each}
</aside>
<!-- reader -->
<section class="read">
<div class="col">
{#if clauseDoc}
<div class="studybar">
<button class="sbtn" title="형광펜"></button>
<button class="sbtn" title="노트"></button>
<button class="sbtn" title="암기카드 추가"></button>
</div>
<div class="kicker"><span class="pth">{selMeta?.clause_part}</span></div>
<div class="h-no">{links?.clause_code ?? selMeta?.clause_code}</div>
<h1 class="h-title">{strip(clauseDoc.title, links?.clause_code ?? '')}</h1>
<div class="flow">
<button class="fl" disabled={!links?.prev} onclick={() => loadClause(links?.prev?.id)}>{links?.prev?.clause_code ?? ''}</button>
<button class="fl next" disabled={!links?.next} onclick={() => loadClause(links?.next?.id)}>{links?.next?.clause_code ?? ''}</button>
</div>
{#key clauseDoc.id}
<div class="docbody">
<MarkdownDoc documentId={clauseDoc.id} mdContent={clauseDoc.md_content ?? clauseDoc.extracted_text} mdStatus={null} class="prose prose-base max-w-none" />
</div>
{/key}
{#if links && (links.forward.length || links.back.length)}
<section class="conn">
{#if links.forward.length}
<div><h4>이 절이 참조 <span>{links.forward.length}</span></h4>
<div class="chiprow">{#each links.forward as f}
{#if f.doc_id}<button class="ref" onclick={() => loadClause(f.doc_id)}>{f.code}</button>
{:else}<span class="ref dg" title="외부/미분해">{f.code}</span>{/if}
{/each}</div></div>
{/if}
{#if links.back.length}
<div><h4>이 절을 참조 <span>{links.back.length}</span></h4>
<div class="chiprow">{#each links.back as b}<button class="ref" onclick={() => loadClause(b.doc_id)}>{b.code}</button>{/each}</div></div>
{/if}
</section>
{/if}
<div class="pager">
<button class="pg" disabled={!links?.prev} onclick={() => loadClause(links?.prev?.id)}>
<div class="d">← 이전</div><div class="t"><span class="pno">{links?.prev?.clause_code ?? '—'}</span> {strip(links?.prev?.title, links?.prev?.clause_code)}</div></button>
<button class="pg next" disabled={!links?.next} onclick={() => loadClause(links?.next?.id)}>
<div class="d">다음 →</div><div class="t"><span class="pno">{links?.next?.clause_code ?? '—'}</span> {strip(links?.next?.title, links?.next?.clause_code)}</div></button>
</div>
{:else}
<p class="empty">{loading ? '불러오는 중…' : '왼쪽에서 절을 선택하세요'}</p>
{/if}
</div>
</section>
</div>
</div>
<style>
:global(body) { background: var(--bg); }
.book { --paper:#fbfcf9; --serif:"Iowan Old Style","Palatino Linotype","Noto Serif KR",Georgia,serif;
display:flex; flex-direction:column; min-height:100vh; }
.bar { display:flex; align-items:center; gap:14px; height:50px; padding:0 18px; background:var(--paper); border-bottom:1px solid var(--border); }
.brand { font-weight:700; font-size:13.5px; color:var(--text); }
.crumbs { color:var(--text-dim); font-size:12.5px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; max-width:46%; }
.crumbs b { color:var(--text); font-weight:600; } .crumbs .sep { color:#c8d6c0; margin:0 5px; }
.search { margin-left:auto; }
.search input { width:280px; background:var(--surface); border:1px solid var(--border); border-radius:9px; padding:7px 12px; font-size:13px; color:var(--text); outline:none; }
.search input:focus { border-color:var(--accent); }
.tools { display:flex; gap:2px; }
.tool { font-size:12px; color:var(--text-dim); padding:6px 10px; border-radius:8px; border:1px solid transparent; cursor:pointer; }
.tool:hover { background:var(--surface); } .tool.on { background:#ecf0e8; border-color:var(--border); color:var(--accent-hover); font-weight:600; }
.main { display:flex; align-items:flex-start; flex:1; }
.idx { width:264px; flex-shrink:0; align-self:stretch; border-right:1px solid var(--border);
background:linear-gradient(180deg,#f6f8f3,#f1f4ec); padding:16px 10px 30px 16px; position:sticky; top:0; max-height:100vh; overflow:auto; }
.btitle { display:block; font-family:var(--serif); font-size:15.5px; font-weight:600; color:var(--text); text-decoration:none; line-height:1.32; }
.btitle:hover { text-decoration:underline; }
.bmeta { font-size:11px; color:#9aa090; margin:3px 0 14px; }
.parttab { display:flex; align-items:center; gap:8px; margin:11px 0 4px; padding:3px 4px; border-radius:6px; cursor:pointer;
font-size:11px; font-weight:700; letter-spacing:.5px; color:var(--text-dim); text-transform:uppercase; }
.parttab:hover { background:#fff; } .parttab .bar2 { width:3px; height:12px; border-radius:2px; background:var(--domain-engineering); }
.parttab .pname { flex:1; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } .parttab .ct { color:#9aa090; font-weight:600; letter-spacing:0; }
.ci { display:flex; gap:9px; align-items:baseline; padding:4px 9px; border-radius:7px; cursor:pointer; line-height:1.4; }
.ci .no { font-family:ui-monospace,Menlo,monospace; font-size:11px; color:var(--accent); font-weight:600; min-width:52px; white-space:nowrap; }
.ci .tt { font-size:12.5px; color:var(--text-dim); overflow:hidden; text-overflow:ellipsis; }
.ci:hover { background:#fff; }
.ci.on { background:#fff; box-shadow:inset 3px 0 0 var(--accent), 0 1px 2px rgba(35,41,31,.05); }
.ci.on .no { color:var(--accent-hover); font-weight:700; } .ci.on .tt { color:var(--text); font-weight:600; }
.read { flex:1; min-width:0; padding:34px 40px 80px; }
.col { max-width:680px; margin:0 auto; position:relative; }
.studybar { position:absolute; right:-30px; top:4px; display:flex; flex-direction:column; gap:6px; }
.sbtn { width:34px; height:34px; border-radius:9px; border:1px solid var(--border); background:var(--paper); color:var(--text-dim); font-size:13px; cursor:pointer; }
.sbtn:hover { background:var(--surface); color:var(--accent-hover); }
.kicker { margin-bottom:5px; } .kicker .pth { font-size:11.5px; color:#9aa090; font-weight:600; letter-spacing:.3px; }
.h-no { font-family:ui-monospace,Menlo,monospace; font-size:13px; color:var(--accent); font-weight:700; letter-spacing:.5px; }
.h-title { font-family:var(--serif); font-size:26px; line-height:1.24; font-weight:600; margin:2px 0 14px; letter-spacing:-.2px; color:var(--text); }
.flow { display:flex; justify-content:space-between; gap:8px; margin-bottom:18px; }
.flow .fl { font-size:11.5px; color:var(--text-dim); background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:5px 11px; cursor:pointer; }
.flow .fl:hover:not(:disabled) { background:#ecf0e8; } .flow .fl:disabled { opacity:.35; cursor:default; }
.docbody { font-size:15.5px; }
.docbody :global(.prose) { color:#2a3024; line-height:1.78; }
.docbody :global(.prose h1), .docbody :global(.prose h2), .docbody :global(.prose h3) { font-family:var(--serif); }
.docbody :global(a) { color:var(--accent-hover); }
.conn { margin-top:34px; padding-top:18px; border-top:1px solid var(--border); display:grid; grid-template-columns:1fr 1fr; gap:22px; }
.conn h4 { font-size:11px; font-weight:700; color:var(--text-dim); letter-spacing:.4px; margin:0 0 9px; } .conn h4 span { color:#9aa090; font-weight:500; }
.chiprow { display:flex; flex-wrap:wrap; gap:5px; }
.ref { font-family:ui-monospace,Menlo,monospace; font-size:11.5px; font-weight:600; color:var(--accent-hover); background:#eef4ec; border:1px solid #d9e6d8; border-radius:6px; padding:2px 8px; cursor:pointer; }
.ref:hover { background:#e2efe0; } .ref.dg { color:#9aa090; background:var(--surface); border-color:var(--border); cursor:default; }
.pager { display:flex; gap:10px; margin-top:30px; }
.pg { flex:1; text-align:left; border:1px solid var(--border); border-radius:11px; padding:11px 14px; background:var(--paper); cursor:pointer; }
.pg.next { text-align:right; } .pg:hover:not(:disabled) { border-color:#cfd7c6; background:#fff; } .pg:disabled { opacity:.4; cursor:default; }
.pg .d { font-size:10.5px; color:#9aa090; } .pg .t { font-size:12.5px; color:var(--text-dim); font-weight:600; margin-top:1px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
.pg .pno { font-family:ui-monospace,Menlo,monospace; color:var(--accent); }
.empty { color:#9aa090; text-align:center; padding:80px 0; }
@media(max-width:820px){ .idx{display:none} .read{padding:24px 18px} .conn{grid-template-columns:1fr} .studybar{display:none} .crumbs{max-width:30%} .search input{width:150px} }
</style>
-73
View File
@@ -1,73 +0,0 @@
<script>
// 절(clause) 바로가기 — ASME 절 식별자(예: UG-79)로 크로스-doc 위치를 조회해 읽기뷰로 이동 (U-1).
// 절은 in_corpus=false(의미검색 비활성)라 일반 검색으론 안 잡히므로 라벨 정확지목 전용 진입점.
import { api } from '$lib/api';
import { goto } from '$app/navigation';
let label = $state('');
let hits = $state([]);
let loading = $state(false);
let searched = $state(false);
let error = $state('');
async function lookup() {
const q = label.trim();
if (!q) return;
loading = true;
error = '';
try {
const res = await api(`/documents/clause-lookup?label=${encodeURIComponent(q)}`);
hits = res?.hits ?? [];
searched = true;
} catch (e) {
error = '조회에 실패했습니다.';
hits = [];
} finally {
loading = false;
}
}
</script>
<div class="mx-auto max-w-3xl px-6 py-10">
<h1 class="mb-1 text-2xl font-bold text-base">절 바로가기</h1>
<p class="mb-6 text-sm text-dim">
ASME 절 식별자(예: <code class="text-accent">UG-79</code>, <code class="text-accent">PG-5</code>)로 문서·위치를 찾아 이동합니다.
</p>
<form onsubmit={(e) => { e.preventDefault(); lookup(); }} class="mb-6 flex gap-2">
<input
bind:value={label}
placeholder="절 식별자 (UG-79, PG-5.6, A-1 …)"
autocomplete="off"
class="flex-1 rounded-lg border border-default bg-surface px-4 py-2.5 text-base outline-none focus:border-accent"
/>
<button
type="submit"
disabled={loading || !label.trim()}
class="rounded-lg bg-accent px-5 py-2.5 font-medium text-white hover:bg-accent-hover disabled:opacity-50"
>
{loading ? '조회 중…' : '찾기'}
</button>
</form>
{#if error}
<p class="text-sm text-accent">{error}</p>
{:else if searched && hits.length === 0}
<p class="text-sm text-dim">'{label}' 에 해당하는 절을 찾지 못했습니다. (절은 분해된 코드 문서에만 존재합니다)</p>
{:else if hits.length > 0}
<div class="space-y-2">
{#if hits.length > 1}
<p class="text-xs text-dim">{hits.length}개 문서에 존재 — 에디션/부록을 선택하세요.</p>
{/if}
{#each hits as hit (hit.chunk_id)}
<button
onclick={() => goto(`/documents/${hit.doc_id}?section=${hit.chunk_id}`)}
class="block w-full rounded-lg border border-default bg-surface px-4 py-3 text-left transition hover:border-accent hover:bg-surface-hover"
>
<div class="font-medium text-base">{hit.section_title}</div>
<div class="mt-0.5 text-xs text-dim">{hit.doc_title}</div>
</button>
{/each}
</div>
{/if}
</div>
+49 -1
View File
@@ -8,7 +8,7 @@
import { goto } from '$app/navigation';
import { api } from '$lib/api';
import { addToast } from '$lib/stores/toast';
import { X, Plus, Trash2, Tag, FolderTree, ArrowUpDown } from 'lucide-svelte';
import { X, Plus, Trash2, Tag, FolderTree, Sparkles, ArrowUpDown } from 'lucide-svelte';
import MarkdownStatusBadge from '$lib/components/MarkdownStatusBadge.svelte';
import { isMdStatusVisible } from '$lib/utils/mdStatus';
import UploadDropzone from '$lib/components/UploadDropzone.svelte';
@@ -22,7 +22,9 @@
import { useIsXl } from '$lib/composables/useMedia.svelte';
import { useListKeyboardNav } from '$lib/composables/useListKeyboardNav.svelte';
import { pLimit } from '$lib/utils/pLimit';
import { isQuestion } from '$lib/utils/isQuestion';
import { domainBgClass, domainLabel } from '$lib/utils/domainSlug';
import AskAnswerCard from '$lib/components/AskAnswerCard.svelte';
const FORMATS = ['pdf', 'hwp', 'hwpx', 'md', 'docx', 'xlsx', 'png', 'jpg'];
@@ -46,6 +48,30 @@
let searchResults = $state(null);
let selectedDoc = $state(null);
// 이드 답변 상태 (질문형 검색)
let askData = $state(null);
let askLoading = $state(false);
let askError = $state(false);
let askDismissed = $state(false);
async function askSearch(q) {
askLoading = true; askError = false; askData = null;
try {
askData = await api(`/search/ask?q=${encodeURIComponent(q)}&limit=10`);
} catch {
askError = true; askData = null;
} finally {
askLoading = false;
}
}
let showAskCard = $derived(
!askDismissed && (
askLoading ||
(askData && !askData.refused && askData.ai_answer && askData.synthesis_status === 'completed')
)
);
// 인스펙터(우측) 토글 — xl+ inline, < xl Drawer.
const isXl = useIsXl();
let inspectorOpen = $state(
@@ -119,6 +145,7 @@
selectedDoc = null;
selectedIds = new Set();
kbIndex = 0;
askData = null; askLoading = false; askError = false; askDismissed = false;
if (ui.isDrawerOpen('meta')) ui.closeDrawer();
if (urlQ) doSearch(urlQ, urlMode);
else loadDocuments();
@@ -164,6 +191,7 @@
async function doSearch(q, mode) {
loading = true;
if (isQuestion(q)) askSearch(q);
try {
const data = await api(`/search/?q=${encodeURIComponent(q)}&mode=${mode}&limit=50`);
searchResults = data.results;
@@ -378,6 +406,13 @@
<option value="trgm">부분매칭</option>
<option value="vector">의미검색</option>
</select>
{#if searchQuery.trim()}
<a
href={`/ask?q=${encodeURIComponent(searchQuery.trim())}`}
class="flex items-center px-2 py-1.5 rounded-lg border border-default text-dim hover:text-accent hover:border-accent transition-colors"
title="이 쿼리로 AI 답변"
><Sparkles size={14} /></a>
{/if}
</div>
<!-- 필터 칩 row -->
@@ -448,6 +483,19 @@
{/if}
</div>
<!-- AI 답변 (질문형 검색) — 목록 상단 고정, 아래로 목록 스크롤 -->
{#if showAskCard}
<div class="px-3 py-2 shrink-0 border-b border-default max-h-[55vh] overflow-y-auto">
<AskAnswerCard
data={askData}
loading={askLoading}
error={askError}
onCitationClick={(docId) => goto(`/documents/${docId}`)}
onDismiss={() => { askDismissed = true; }}
/>
</div>
{/if}
<!-- 선택 toolbar -->
{#if selectionCount > 0}
<div class="flex flex-wrap items-center gap-2 px-3 py-2 shrink-0 bg-accent/10 border-y border-accent/30">
@@ -16,7 +16,6 @@
import Skeleton from '$lib/components/ui/Skeleton.svelte';
import HandwriteCanvas from '$lib/components/HandwriteCanvas.svelte';
import MarkdownDoc from '$lib/components/MarkdownDoc.svelte';
import RelatedDocs from '$lib/components/RelatedDocs.svelte';
import { renderDocMarkdown } from '$lib/utils/docMarkdown';
import MarkdownStatusBadge from '$lib/components/MarkdownStatusBadge.svelte';
import NoteEditor from '$lib/components/editors/NoteEditor.svelte';
@@ -131,9 +130,6 @@
let manageOpen = $state(false);
// 기본 선택 = 첫 본문 Part 의 첫 절(front-matter TOC 가 아니라 실제 내용으로 진입, front-matter 접힘 유지).
let defaultSelId = $derived.by(() => {
// 딥링크 진입: ?section=<chunk_id> 가 outline 에 있으면 그 절로 (/clause 절 바로가기 → 해당 절 표시).
const deep = Number($page.url.searchParams.get('section'));
if (deep && outline.some((it) => it.section.chunk_id === deep)) return deep;
if (treeGroups) {
const body = treeGroups.find((g) => !g.isFrontMatter);
if (body && body.items.length) return body.items[0].section.chunk_id;
@@ -322,7 +318,6 @@
<!-- ════ 우 슬림 레일 (시안 카드 스타일) ════ -->
{#snippet rail()}
<div style="display:flex;flex-direction:column;gap:11px;font-size:14px;">
<RelatedDocs documentId={doc.id} />
{#if doc.ai_tldr || doc.ai_summary}
<div style="background:#f4f7f1;border:1px solid #dde3d6;border-radius:14px;padding:13px;">
<div style="font-size:10.5px;font-weight:700;color:#697061;letter-spacing:.4px;margin-bottom:7px;">TL;DR</div>
@@ -1,10 +0,0 @@
-- 362: G2 pre-segmentation — 번들 PDF(여러 논리문서 한 파일) → N 자식 문서 분할.
-- 자식 doc 의 원본 내 page 범위(1-based inclusive) + 분할 역할 표식.
-- 부모-자식 관계 자체는 document_lineage(relation_type='segmented_from', migration 363).
-- presegment_role: NULL=일반 단일문서(대다수) / 'parent'=번들원본(자체 extract/embed 안 함) /
-- 'child'=논리 하위문서(부모 file_path 공유 + bundle_page_start/end 범위로 슬라이스).
-- 단일 ALTER(다중 절) = 1 statement (asyncpg 멀티스테이트먼트 제약 준수).
ALTER TABLE documents
ADD COLUMN IF NOT EXISTS bundle_page_start INTEGER,
ADD COLUMN IF NOT EXISTS bundle_page_end INTEGER,
ADD COLUMN IF NOT EXISTS presegment_role TEXT;
@@ -1,8 +0,0 @@
-- 363: G2 — document_lineage.relation_type 에 'segmented_from'(번들 → 자식) 추가.
-- 217 의 column-level CHECK(PG 자동명 document_lineage_relation_type_check, 배포 DB 실측 확인)
-- 를 교체. DROP + ADD 를 단일 ALTER 의 두 절로 = 1 statement.
-- 멱등: DROP ... IF EXISTS 라 재실행 안전(이미 교체됐으면 새 제약 DROP 후 동일 재생성).
ALTER TABLE document_lineage
DROP CONSTRAINT IF EXISTS document_lineage_relation_type_check,
ADD CONSTRAINT document_lineage_relation_type_check
CHECK (relation_type IN ('cited','summarized_from','generated_from','revised_from','segmented_from'));
@@ -1,5 +0,0 @@
-- 364: G2 — process_stage 큐 스테이지 enum 에 'presegment' 추가 (extract 前 번들 분할 단계).
-- PG16: ALTER TYPE ADD VALUE 는 트랜잭션 내 실행 가능(값 추가만, 同 트랜잭션 내 사용은 안 함 —
-- 사용은 후속 마이그/런타임). IF NOT EXISTS = 재실행 멱등.
-- (이 한 줄 단독 파일 — 1 statement.)
ALTER TYPE process_stage ADD VALUE IF NOT EXISTS 'presegment';
-56
View File
@@ -1,56 +0,0 @@
-- 스캔 기능: 잡 모델 + 배치 + 에이전트 생존 (plan: scan-feature-build r3)
-- 웹(fastapi)=intent/명령, 호스트 스캔 에이전트=결과. 싱글톤 스캐너 직렬화.
-- 주: 러너 규약상 이 파일은 schema_migrations 를 건드리지 않음(스탬프는 외부). BEGIN/COMMIT 없음.
-- 순서: 테이블 먼저 → 시드 → 인덱스 (인덱스 실패가 테이블 생성 막지 않게).
-- 잡: 한 스캔 세션 = 한 논리 문서 (배치 N개 → 합본 1 PDF → Inbox)
CREATE TABLE IF NOT EXISTS scan_jobs (
id BIGSERIAL PRIMARY KEY,
title TEXT NOT NULL, -- 사람 입력 제목 (commit 시 documents.title 로 전파)
settings JSONB NOT NULL DEFAULT '{}'::jsonb, -- mode/resolution/source(ADF Duplex) 등 스캔 프로파일
status TEXT NOT NULL DEFAULT 'draft', -- draft|queued|ready|scanning|assembling|preview|committing|committed|failed|canceled
batch_count INTEGER NOT NULL DEFAULT 0, -- 스캔 완료 배치 수
page_count INTEGER, -- 최종 합본 페이지 수 (assembling 후)
last_activity_at TIMESTAMPTZ, -- ready 휴지 벽시계 idle 타임아웃 기준 (방치 데드락 방지)
last_progress_at TIMESTAMPTZ, -- 잡 진행 갱신 (에이전트 생존과 분리)
staging_path TEXT, -- 호스트 로컬 잡 스테이징 디렉토리
nas_staging_path TEXT, -- NAS .scan-staging 합본 경로 (B안 미리보기/commit 소스)
inbox_path TEXT, -- 최종 PKM/Inbox 경로 (commit 후)
file_hash CHAR(64), -- 합본 sha256 = 정체성/멱등 커밋 키 (commit 시 채움)
doc_id BIGINT REFERENCES documents(id) ON DELETE SET NULL, -- commit 후 연결 (title 전파)
error TEXT, -- failed 사유 (no-silent)
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- 배치: 스캔 1회(ADF 한 묶음) 단위. batch_seq = 결합 순서(글롭 정렬 아님).
CREATE TABLE IF NOT EXISTS scan_job_batches (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT REFERENCES scan_jobs(id) ON DELETE CASCADE NOT NULL,
batch_seq INTEGER NOT NULL, -- 1-based 결합 순서
staging_path TEXT, -- 이 배치 PDF (호스트 로컬)
page_count INTEGER,
status TEXT NOT NULL DEFAULT 'scanned', -- scanned | discarded (잼 폐기 후 재스캔)
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (job_id, batch_seq)
);
-- 에이전트 생존: 싱글톤 1행. 잡 진행(last_progress_at)과 분리 — queued 잡 stale 오탐 방지.
CREATE TABLE IF NOT EXISTS scan_agent_status (
id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1), -- 단일 행 강제
last_heartbeat TIMESTAMPTZ,
agent_version TEXT,
current_job_id BIGINT REFERENCES scan_jobs(id) ON DELETE SET NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO scan_agent_status (id) VALUES (1) ON CONFLICT (id) DO NOTHING; -- 시드 1행
-- 활성 잡 락: 스캐너 싱글톤 → in-progress 잡은 전체에서 1개만(나머지 queued).
-- 상수 TRUE 에 unique + in-progress 필터 = 그 상태 행 최대 1개 강제.
CREATE UNIQUE INDEX IF NOT EXISTS uq_scan_jobs_single_active
ON scan_jobs ((TRUE))
WHERE status IN ('ready','scanning','assembling','preview','committing');
CREATE INDEX IF NOT EXISTS idx_scan_jobs_queued ON scan_jobs (created_at) WHERE status = 'queued';
CREATE INDEX IF NOT EXISTS idx_scan_jobs_file_hash ON scan_jobs (file_hash) WHERE file_hash IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_scan_job_batches_job ON scan_job_batches (job_id, batch_seq);
-4
View File
@@ -1,4 +0,0 @@
-- 스캔 잡 명령 채널 (이중 라이터: API=intent/명령, 에이전트=result) — plan scan-feature-build r3
-- API/수동이 pending_command 설정 → 에이전트가 조건부 claim(WHERE pending_command=X AND status=기대값) → 실행 → 결과 status write.
ALTER TABLE scan_jobs ADD COLUMN IF NOT EXISTS pending_command TEXT; -- scan_batch | finish | commit | cancel
ALTER TABLE scan_jobs ADD COLUMN IF NOT EXISTS command_requested_at TIMESTAMPTZ; -- 명령 요청 시각(staleness/디버그)
-21
View File
@@ -1,21 +0,0 @@
-- 367_published.sql
-- 발행 레이어(docsrv-viewer-publish) projection 테이블. 뷰어가 read API로 당겨 자기 SQLite로 복제.
-- kind-discriminated 단일 테이블(study_question | study_explanation | ... 후속 news/document).
-- pub_id = opaque+stable(워커가 (kind,source_id)당 1회 부여, republish=rev bump에도 불변) = 뷰어 dedup키=progress키.
-- source_id = 내부 소스 행 id (pub_id→내부 역매핑, ingest write-back 해소용).
-- rev = 발행 워커 커밋순 gapless 커서(pg_advisory_lock 단일 라이터). 뷰어 feed = WHERE rev>since.
-- payload_hash = sha256(정렬 JSON). (payload_hash, deleted) 디둡 — no-op 재투영 억제, tombstone 보존.
-- deleted = tombstone(삭제/만료도 feed 1급 이벤트). schema_version = 엔벨로프 버전(미지원 가시거부).
CREATE TABLE IF NOT EXISTS published (
id BIGSERIAL PRIMARY KEY,
kind VARCHAR(40) NOT NULL,
source_id BIGINT NOT NULL,
pub_id TEXT NOT NULL,
payload JSONB NOT NULL,
payload_hash TEXT NOT NULL,
schema_version SMALLINT NOT NULL DEFAULT 1,
rev BIGINT NOT NULL,
deleted BOOLEAN NOT NULL DEFAULT false,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
@@ -1,3 +0,0 @@
-- 368_published_kind_pubid_uq.sql
-- pub_id 는 kind 내 유일(뷰어 dedup/progress 키 무결성, pub_id→내부 역해소 유일성 보장).
CREATE UNIQUE INDEX IF NOT EXISTS published_kind_pubid_uq ON published (kind, pub_id);
@@ -1,3 +0,0 @@
-- 369_published_kind_source_uq.sql
-- (kind, source_id) 당 발행 행 1개 — 발행 워커 upsert 타깃 + pub_id 재사용(같은 source=같은 pub_id) 키.
CREATE UNIQUE INDEX IF NOT EXISTS published_kind_source_uq ON published (kind, source_id);
-3
View File
@@ -1,3 +0,0 @@
-- 370_published_rev_idx.sql
-- 뷰어 pull-sync feed: SELECT ... WHERE rev > :since ORDER BY rev LIMIT :page (P0-2).
CREATE INDEX IF NOT EXISTS published_rev_idx ON published (rev);
-15
View File
@@ -1,15 +0,0 @@
-- 371_publish_outbox.sql
-- transactional outbox — 저작/4-A 트랜잭션이 같은 tx에서 여기 INSERT(P0-1 규율),
-- 단일 발행 워커가 id(커밋순) 순으로 drain 하며 published 에 rev 부여(소스 updated_at 폴링 금지=갭 재발).
-- processed_at = 워커 drain 시 스탬프(NULL=미처리). payload/hash 는 enqueue 시점 스냅샷.
CREATE TABLE IF NOT EXISTS publish_outbox (
id BIGSERIAL PRIMARY KEY,
kind VARCHAR(40) NOT NULL,
source_id BIGINT NOT NULL,
payload JSONB NOT NULL,
payload_hash TEXT NOT NULL,
schema_version SMALLINT NOT NULL DEFAULT 1,
deleted BOOLEAN NOT NULL DEFAULT false,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
processed_at TIMESTAMPTZ
);
@@ -1,3 +0,0 @@
-- 372_publish_outbox_unprocessed_idx.sql
-- 워커 drain 쿼리: WHERE processed_at IS NULL ORDER BY id (커밋순). 부분 인덱스로 미처리분만.
CREATE INDEX IF NOT EXISTS publish_outbox_unprocessed_idx ON publish_outbox (id) WHERE processed_at IS NULL;
@@ -1,4 +0,0 @@
-- 373_quiz_session_finalized_at.sql
-- 발행 ingest(study-to-viewer P2) finalize 멱등 마커. finalize 성공 후 스탬프 →
-- 같은 세션 재전송(at-least-once outbox) 시 SR 이중 advance 차단. 라이브 세션은 NULL 유지(무영향).
ALTER TABLE study_quiz_sessions ADD COLUMN IF NOT EXISTS finalized_at TIMESTAMPTZ;
@@ -1,3 +0,0 @@
-- 374_quiz_session_client_uuid.sql
-- 뷰어 로컬 세션 UUID. ingest 가 (uuid, topic) 로 DS 세션 find-or-create = 멱등 키. 라이브=NULL.
ALTER TABLE study_quiz_sessions ADD COLUMN IF NOT EXISTS client_session_uuid TEXT;
-3
View File
@@ -1,3 +0,0 @@
-- 375_quiz_session_source.sql
-- 세션 출처 구분(live | viewer). 감사/필터용. 기존 행=live.
ALTER TABLE study_quiz_sessions ADD COLUMN IF NOT EXISTS source VARCHAR(20) NOT NULL DEFAULT 'live';
@@ -1,3 +0,0 @@
-- 376_quiz_session_client_uuid_uq.sql
-- (client_session_uuid, study_topic_id) 유일 — 뷰어 1세션이 topic 별 1 DS세션. partial(uuid 있는 viewer 행만).
CREATE UNIQUE INDEX IF NOT EXISTS study_quiz_sessions_client_uuid_topic_uq ON study_quiz_sessions (client_session_uuid, study_topic_id) WHERE client_session_uuid IS NOT NULL;
-23
View File
@@ -1,23 +0,0 @@
-- 377_domain_bucket.sql
-- ai_domain(반자유 AI 분류, 드리프트 존재)을 검색 스코프용 7버킷으로 결정적 롤업.
-- 축: ai_domain(routing/해석 축)의 coarsening — category(UI축) 아님 (feedback_category_vs_ai_domain_axis 준수).
-- 버킷: News / Safety / Law / Engineering / General / Philosophy / Programming.
-- STORED generated → 신규/재분류 문서도 ai_domain 붙으면 자동 버킷. ai_domain 원본 보존(하위 검색 유지).
-- 롤백: ALTER TABLE documents DROP COLUMN domain_bucket;
ALTER TABLE documents ADD COLUMN IF NOT EXISTS domain_bucket text
GENERATED ALWAYS AS (
CASE
WHEN ai_domain LIKE 'News%' THEN 'News'
WHEN ai_domain = '법령' OR ai_domain LIKE 'Industrial_Safety/Legislation%' THEN 'Law'
WHEN ai_domain = 'Safety' OR ai_domain LIKE 'Safety/%'
OR ai_domain LIKE 'Industrial_Safety%'
OR ai_domain = 'Knowledge/Industrial_Safety' THEN 'Safety'
WHEN ai_domain LIKE 'Engineering%' OR ai_domain = 'Knowledge/Engineering' THEN 'Engineering'
WHEN ai_domain LIKE 'Philosophy%' THEN 'Philosophy'
WHEN ai_domain LIKE 'Programming%' THEN 'Programming'
ELSE 'General'
END
) STORED;
CREATE INDEX IF NOT EXISTS documents_domain_bucket_idx
ON documents (domain_bucket) WHERE deleted_at IS NULL;
@@ -1,9 +0,0 @@
-- 378_publish_outbox_attempts_failed.sql
-- (번호: 멀티세션 중 prod 가 377_domain_bucket 을 선점 → 378 로 리넘버.)
-- publish_outbox poison row head-of-line block 차단. 발행 워커가 행별 savepoint 격리 후
-- 예외 시 attempts++ 하고 MAX 초과 시 failed_at 스탬프(terminal) → 그 행을 select 에서 제외해
-- 후속 발행이 막히지 않게 함. 기존 미처리 행은 attempts=0 / failed_at=NULL 로 정상 재처리.
-- (단일 ALTER = 1 statement = asyncpg prepared 호환.)
ALTER TABLE publish_outbox
ADD COLUMN IF NOT EXISTS attempts SMALLINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS failed_at TIMESTAMPTZ;
-37
View File
@@ -1,37 +0,0 @@
-- 379_asme_clause_kb.sql
-- ASME 절-지식베이스: 절 = 개별 documents 행(parent_id) + 절↔절 백링크 + 태깅 (additive, idempotent)
-- 검색 무접촉: 절 doc 은 embedding NULL(벡터 제외) + doc_kind='clause'(retrieval doc-leg 필터로 제외).
ALTER TABLE documents
ADD COLUMN IF NOT EXISTS parent_id bigint REFERENCES documents(id),
ADD COLUMN IF NOT EXISTS doc_kind text NOT NULL DEFAULT 'standard',
ADD COLUMN IF NOT EXISTS clause_code text,
ADD COLUMN IF NOT EXISTS clause_part text,
ADD COLUMN IF NOT EXISTS clause_order int;
CREATE INDEX IF NOT EXISTS idx_documents_parent_id ON documents(parent_id) WHERE parent_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_documents_doc_kind ON documents(doc_kind);
CREATE INDEX IF NOT EXISTS idx_documents_clause_code ON documents(clause_code) WHERE clause_code IS NOT NULL;
-- 절↔절 백링크 (dangling 허용: dst_doc_id nullable)
CREATE TABLE IF NOT EXISTS clause_links (
id bigserial PRIMARY KEY,
src_doc_id bigint NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
dst_code text NOT NULL,
dst_doc_id bigint REFERENCES documents(id) ON DELETE SET NULL,
anchor text,
ctx text,
char_off int
);
CREATE INDEX IF NOT EXISTS idx_clause_links_src ON clause_links(src_doc_id);
CREATE INDEX IF NOT EXISTS idx_clause_links_dst ON clause_links(dst_doc_id) WHERE dst_doc_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_clause_links_dstcode ON clause_links(dst_code);
-- 태깅 (Part 자동 + 주제)
CREATE TABLE IF NOT EXISTS document_tags (
doc_id bigint NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
tag text NOT NULL,
tag_kind text NOT NULL DEFAULT 'topic',
PRIMARY KEY (doc_id, tag)
);
CREATE INDEX IF NOT EXISTS idx_document_tags_tag ON document_tags(tag);
-53
View File
@@ -1,53 +0,0 @@
#!/usr/bin/env python3
"""ASME clause-KB backlinks: resolve clause-id mentions in each clause doc -> clause_links.
dst resolved to the clause doc of the same parent (top-level code); sub-code mention -> anchor;
unresolved (cross-standard / material spec not split) -> dangling (dst_doc_id NULL).
Idempotent per parent. Usage: python3 asme_backlinks_persist.py <parent_id> [--commit]
"""
import asyncio, os, re, sys
MENTION_RE = re.compile(r'(?<![A-Za-z0-9])([A-Z]{1,4}-\d+(?:\.\d+)*[A-Za-z]?)(?![A-Za-z0-9])')
def top(code): return re.match(r'^[A-Z]{1,4}-\d+', code).group(0)
async def main():
parent = int(sys.argv[1]); commit = '--commit' in sys.argv
import asyncpg
conn = await asyncpg.connect(os.environ['DATABASE_URL'].replace('+asyncpg', ''))
docs = await conn.fetch("SELECT id, clause_code, md_content FROM documents "
"WHERE parent_id=$1 AND doc_kind='clause' ORDER BY clause_order", parent)
code2id = {d['clause_code']: d['id'] for d in docs}
edges = [] # (src_id, dst_code, dst_doc_id, anchor, ctx, char_off)
resolved = dangling = 0
for d in docs:
body = d['md_content']; src_top = d['clause_code']
seen = set()
for m in MENTION_RE.finditer(body):
code = m.group(1); t = top(code)
if t == src_top: continue # self-reference
if (d['id'], code) in seen: continue # dedup per (src,dst_code)
seen.add((d['id'], code))
dst_id = code2id.get(t) # resolve to same-parent clause doc
anchor = code.lower().replace('.', '-') if code != t else None
off = m.start()
ctx = re.sub(r'\s+', ' ', body[max(0, off-50):off+50]).strip()
edges.append((d['id'], code, dst_id, anchor, ctx, off))
if dst_id: resolved += 1
else: dangling += 1
print(f"parent={parent} clause_docs={len(docs)} edges={len(edges)} resolved={resolved} dangling={dangling}")
# top referenced clauses
from collections import Counter
tgt = Counter(top(e[1]) for e in edges if e[2])
print("most-referenced:", tgt.most_common(8))
if not commit:
print("DRY-RUN. pass --commit to persist."); await conn.close(); return
async with conn.transaction():
ids = [d['id'] for d in docs]
await conn.execute("DELETE FROM clause_links WHERE src_doc_id = ANY($1::bigint[])", ids)
await conn.executemany(
"INSERT INTO clause_links(src_doc_id,dst_code,dst_doc_id,anchor,ctx,char_off) "
"VALUES ($1,$2,$3,$4,$5,$6)", edges)
n = await conn.fetchval("SELECT count(*) FROM clause_links WHERE src_doc_id = ANY($1::bigint[])", ids)
print(f"COMMITTED: {n} clause_links for parent {parent}")
await conn.close()
asyncio.run(main())
-118
View File
@@ -1,118 +0,0 @@
#!/usr/bin/env python3
"""ASME clause-KB persist (v2: over-CAP pagination). Split a parent standard into per-clause
documents (A-granularity); over-CAP clause bodies are paginated into readable page-docs.
Idempotent per parent. doc_kind='clause', embedding NULL (search-excluded), parent_id=<parent>.
Usage: python3 asme_clause_persist.py <parent_id> [--commit]
"""
import asyncio, os, re, sys, hashlib, statistics
CAP = 12000; PAGE_TOK = 11000
EN, KO = 0.217, 0.529
LINE_RE = re.compile(r'^([ \t#>*]{0,8})([A-Z]{2,4}-\d+(?:\.\d+)*[A-Za-z]?)(.*)$')
MENTION_RE = re.compile(r'(?<![A-Za-z0-9])([A-Z]{1,4}-\d+(?:\.\d+)*[A-Za-z]?)(?![A-Za-z0-9])')
EXACT_TOP = re.compile(r'^[A-Z]{2,4}-\d+$')
TITLE_AFTER = re.compile(r'^[\s.]*[A-Z(]')
REF_LEAD = re.compile(r'^[\s.]*(and|or|to|of|in|on|the|as|is|are|shall|through|per|see|with|'
r'for|by|that|which|such|또는|및|등|의|은|는|에|을|를|과|와)\b', re.I)
def tok(s):
ko = sum(1 for c in s if '' <= c <= ''); return int((len(s)-ko)*EN + ko*KO)
def clean_title(rest):
t = re.sub(r'<sup>ð</sup>\s*\**\d*\**\s*<sup>Þ</sup>', '', rest)
t = re.sub(r'ð\**\d*\**Þ', '', t)
t = t.replace('**', '').replace('#', '')
return re.sub(r'\s+', ' ', t).strip(' *:—-')
def is_header(markup, rest):
if '#' in markup or '*' in markup: return True
rs = rest.strip()
if rs == '': return True
if REF_LEAD.match(rest): return False
if rs[0] in ',;.)': return False
if '' <= rs[0] <= '': return False
if rs[0].islower(): return False
return bool(TITLE_AFTER.match(rs))
def paginate(body):
"""split an over-CAP body into <=MAX_PAGES line-aligned pages of ~PAGE_TOK tokens."""
pages, cur, ct = [], [], 0
for ln in body.split('\n'):
lt = tok(ln) + 1
if ct + lt > PAGE_TOK and cur:
pages.append('\n'.join(cur)); cur, ct = [ln], lt
else:
cur.append(ln); ct += lt
if cur: pages.append('\n'.join(cur))
return pages
def build_clauses(text):
lines = text.split('\n'); off = []; a = 0
for ln in lines: off.append(a); a += len(ln) + 1
bounds = []; seen = set()
for i, ln in enumerate(lines):
m = LINE_RE.match(ln)
if not m: continue
markup, code, rest = m.group(1), m.group(2), m.group(3)
if not EXACT_TOP.match(code): continue
if not is_header(markup, rest): continue
if code in seen: continue
seen.add(code); bounds.append((off[i], code, clean_title(rest)))
raw = []
for idx, (start, code, title) in enumerate(bounds):
end = bounds[idx+1][0] if idx+1 < len(bounds) else len(text)
body = text[start:end]
part = re.match(r'^[A-Z]{2,4}', code).group(0)
links = sorted(set(re.match(r'^[A-Z]{1,4}-\d+', mm).group(0)
for mm in MENTION_RE.findall(body)) - {code})
raw.append(dict(code=code, part=part, title=(code + (' ' + title if title else '')),
body=body, tok=tok(body), links=links))
# expand over-CAP into pages; assign running clause_order
final, order = [], 0
for c in raw:
if c['tok'] <= CAP:
final.append({**c, 'order': order}); order += 1; continue
pages = paginate(c['body'])
for pi, pb in enumerate(pages):
code = c['code'] if pi == 0 else f"{c['code']}·p{pi+1}"
title = c['title'] if pi == 0 else f"{c['title']} (페이지 {pi+1}/{len(pages)})"
final.append(dict(code=code, part=c['part'], order=order, title=title,
body=pb, tok=tok(pb), links=c['links'] if pi == 0 else []))
order += 1
return final
async def main():
parent = int(sys.argv[1]); commit = '--commit' in sys.argv
import asyncpg
conn = await asyncpg.connect(os.environ['DATABASE_URL'].replace('+asyncpg', ''))
row = await conn.fetchrow("SELECT md_content, ai_domain, data_origin FROM documents WHERE id=$1", parent)
if not row: print(f"parent {parent} not found"); return
clauses = build_clauses(row['md_content'])
toks = [c['tok'] for c in clauses]
over = [c for c in clauses if c['tok'] > CAP]
print(f"parent={parent} clause_docs={len(clauses)} median_tok={int(statistics.median(toks))} "
f"max_tok={max(toks)} over_cap_remaining={len(over)}")
if over: print("still over-CAP:", [f"{c['code']}:{c['tok']}t" for c in over])
if not commit:
print("DRY-RUN. pass --commit to persist."); await conn.close(); return
async with conn.transaction():
deld = await conn.execute("DELETE FROM documents WHERE parent_id=$1 AND doc_kind='clause'", parent)
print("deleted prior:", deld)
for c in clauses:
fh = hashlib.sha256(f"{parent}:{c['code']}:{c['body']}".encode()).hexdigest()
cid = await conn.fetchval("""
INSERT INTO documents
(file_format, file_hash, title, md_content, parent_id, doc_kind,
clause_code, clause_part, clause_order, ai_domain, data_origin,
md_status, review_status, conversion_status, preview_status)
VALUES ('md',$1,$2,$3,$4,'clause',$5,$6,$7,$8,$9,'success','approved','none','none')
RETURNING id
""", fh, c['title'], c['body'], parent, c['code'], c['part'], c['order'],
row['ai_domain'], row['data_origin'] or 'external')
await conn.execute("INSERT INTO document_tags(doc_id,tag,tag_kind) VALUES ($1,$2,'part') "
"ON CONFLICT DO NOTHING", cid, c['part'])
n = await conn.fetchval("SELECT count(*) FROM documents WHERE parent_id=$1 AND doc_kind='clause'", parent)
print(f"COMMITTED: {n} clause docs for parent {parent}")
await conn.close()
asyncio.run(main())
+1 -1
View File
@@ -289,7 +289,7 @@ async def run(topic_id: int, exam_round: str, apply: bool, abort_threshold: int)
host="postgres",
port=5432,
user="pkm",
password=os.environ["POSTGRES_PASSWORD"],
password="uW38friypljVS0X2ULoMnw",
database="pkm",
)
try:
-74
View File
@@ -1,74 +0,0 @@
"""S-4 초기 백필 — 모든 study_memo_card_progress row 를 발행 outbox 에 적재.
ALL row(필터 없음) due_at NULL sentinel(-on-new)·terminal(졸업) 포함. due-only 백필은
sentinel 누락 viewer 미확인 오분류. 멱등(워커 (payload_hash, deleted) 디둡). flag on 워커 drain.
실행 (GPU 서버):
docker exec hyungi_document_server-fastapi-1 python /app/scripts/backfill_publish_card_progress.py
docker exec hyungi_document_server-fastapi-1 python /app/scripts/backfill_publish_card_progress.py --dry-run
"""
import argparse
import asyncio
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
# standalone-model-registry-fix: app(라우터 경유 전 모델 import)과 달리 script 는 부분 모델만
# import → SQLAlchemy mapper string 관계(StudyTopic.sessions->StudySession 등) 해소 실패.
# 전 모델 모듈 import 로 레지스트리 완성(전부 컨테이너서 import 가능 = app 이 기동 시 로드).
import importlib as _il, pkgutil as _pu
import models as _mp
for _m in _pu.iter_modules(_mp.__path__):
_il.import_module("models." + _m.name)
from sqlalchemy import func, select
from core.config import settings
from core.database import async_session
from models.study_memo_card_progress import StudyMemoCardProgress
from services.study.publish_enqueue import backfill_publish_card_progress
# 페이지 배치 크기 — after_id 루프로 전량 처리(bounded tx).
PAGE = 5000
async def run(dry_run: bool) -> None:
async with async_session() as session:
total = (
await session.execute(
select(func.count()).select_from(StudyMemoCardProgress)
)
).scalar() or 0
print(f"[info] study_publish_enabled={settings.study_publish_enabled} "
f"(False 면 적재는 되나 워커가 drain 안 함)")
print(f"[info] card progress row {total}건 (ALL row 발행)")
if dry_run:
print("[dry-run] 적재 안 함. 실제 실행은 --dry-run 제거.")
return
total = 0
after = 0
while True:
async with async_session() as session:
n, after = await backfill_publish_card_progress(session, after_id=after, limit=PAGE)
await session.commit()
total += n
if n < PAGE:
break
print(f"\n[ok] outbox 적재 {total}건 — 발행 워커가 drain(flag on 시) 하며 rev 부여.")
def main() -> None:
parser = argparse.ArgumentParser(description="S-4 pub_card_progress 초기 백필")
parser.add_argument("--dry-run", action="store_true", default=False)
args = parser.parse_args()
asyncio.run(run(args.dry_run))
if __name__ == "__main__":
main()
-79
View File
@@ -1,79 +0,0 @@
"""S-2 초기 백필 — 검수완료(needs_review=False)·미삭제 study_memo_cards 를 발행 outbox 에 적재.
publish_outbox 에만 적재(멱등: 워커 (payload_hash, deleted) 디둡). study_publish_enabled=True
발행 워커가 drain published(kind=study_card) rev 부여 viewer pull-sync.
실행 (GPU 서버):
docker exec hyungi_document_server-fastapi-1 python /app/scripts/backfill_publish_cards.py
docker exec hyungi_document_server-fastapi-1 python /app/scripts/backfill_publish_cards.py --dry-run
"""
import argparse
import asyncio
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
# standalone-model-registry-fix: app(라우터 경유 전 모델 import)과 달리 script 는 부분 모델만
# import → SQLAlchemy mapper string 관계(StudyTopic.sessions->StudySession 등) 해소 실패.
# 전 모델 모듈 import 로 레지스트리 완성(전부 컨테이너서 import 가능 = app 이 기동 시 로드).
import importlib as _il, pkgutil as _pu
import models as _mp
for _m in _pu.iter_modules(_mp.__path__):
_il.import_module("models." + _m.name)
from sqlalchemy import func, select
from core.config import settings
from core.database import async_session
from models.study_memo_card import StudyMemoCard
from services.study.publish_enqueue import backfill_publish_cards
# 페이지 배치 크기 — after_id 루프로 전량 처리(bounded tx). 워커는 BATCH_SIZE 로 drain.
PAGE = 5000
async def run(dry_run: bool) -> None:
async with async_session() as session:
active = (
await session.execute(
select(func.count())
.select_from(StudyMemoCard)
.where(
StudyMemoCard.deleted_at.is_(None),
StudyMemoCard.needs_review.is_(False),
)
)
).scalar() or 0
print(f"[info] study_publish_enabled={settings.study_publish_enabled} "
f"(False 면 적재는 되나 워커가 drain 안 함)")
print(f"[info] 검수완료·미삭제 카드 {active}")
if dry_run:
print("[dry-run] 적재 안 함. 실제 실행은 --dry-run 제거.")
return
total = 0
after = 0
while True:
async with async_session() as session:
n, after = await backfill_publish_cards(session, after_id=after, limit=PAGE)
await session.commit()
total += n
if n < PAGE:
break
print(f"\n[ok] outbox 적재 {total}건 — 발행 워커가 drain(flag on 시) 하며 rev 부여.")
def main() -> None:
parser = argparse.ArgumentParser(description="S-2 pub_card 초기 백필")
parser.add_argument("--dry-run", action="store_true", default=False)
args = parser.parse_args()
asyncio.run(run(args.dry_run))
if __name__ == "__main__":
main()
-82
View File
@@ -1,82 +0,0 @@
"""S-1 초기 백필 — 기존 active study_topics 를 발행 outbox 에 1회 적재.
publish_outbox 에만 적재한다(멱등: 발행 워커의 (payload_hash, deleted) 디둡이
중복 enqueue no-op 으로 흡수). study_publish_enabled=True 발행 워커가
1 주기로 drain published rev 부여 viewer pull-sync.
주제 수는 개인 학습툴이라 소량 bounded page 사실상 1페이지지만 PAGE 도달
overflow 가드로 페이징 누락을 경보(silent truncation 금지).
실행 (GPU 서버):
docker exec hyungi_document_server-fastapi-1 python /app/scripts/backfill_publish_topics.py
# dry-run(적재 없이 카운트만):
docker exec hyungi_document_server-fastapi-1 python /app/scripts/backfill_publish_topics.py --dry-run
"""
import argparse
import asyncio
import os
import sys
# fastapi 컨테이너 WORKDIR=/app — `from models...` import 가능하게 path 추가.
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
# standalone-model-registry-fix: app(라우터 경유 전 모델 import)과 달리 script 는 부분 모델만
# import → SQLAlchemy mapper string 관계(StudyTopic.sessions->StudySession 등) 해소 실패.
# 전 모델 모듈 import 로 레지스트리 완성(전부 컨테이너서 import 가능 = app 이 기동 시 로드).
import importlib as _il, pkgutil as _pu
import models as _mp
for _m in _pu.iter_modules(_mp.__path__):
_il.import_module("models." + _m.name)
from sqlalchemy import func, select
from core.config import settings
from core.database import async_session
from models.study_topic import StudyTopic
from services.study.publish_enqueue import backfill_publish_topics
# 페이지 배치 크기 — after_id 루프로 전량 처리(bounded tx).
PAGE = 5000
async def run(dry_run: bool) -> None:
async with async_session() as session:
active = (
await session.execute(
select(func.count())
.select_from(StudyTopic)
.where(StudyTopic.deleted_at.is_(None))
)
).scalar() or 0
print(f"[info] study_publish_enabled={settings.study_publish_enabled} "
f"(False 면 적재는 되나 워커가 drain 안 함)")
print(f"[info] active 주제 {active}")
if dry_run:
print("[dry-run] 적재 안 함. 실제 실행은 --dry-run 제거.")
return
total = 0
after = 0
while True:
async with async_session() as session:
n, after = await backfill_publish_topics(session, after_id=after, limit=PAGE)
await session.commit()
total += n
if n < PAGE:
break
print(f"\n[ok] outbox 적재 {total}건 — 발행 워커가 drain(flag on 시) 하며 rev 부여.")
def main() -> None:
parser = argparse.ArgumentParser(description="S-1 pub_topics 초기 백필")
parser.add_argument("--dry-run", action="store_true", default=False)
args = parser.parse_args()
asyncio.run(run(args.dry_run))
if __name__ == "__main__":
main()
+1 -1
View File
@@ -106,7 +106,7 @@ async def main() -> None:
"SELECT count(*) FROM pg_indexes WHERE indexname='uq_attempt_session_question'"))).scalar()
mx = (await s.execute(text("SELECT max(version) FROM schema_migrations"))).scalar()
print(f"SCHEMA OK — max_migration={mx} documents={docs} purge_col={purge} cand_qwen={cand} attempt_uq={uq}")
assert docs and purge == 1 and cand == 0 and uq == 1 and mx == 378, "FAIL: 기대 스키마 상태 불일치"
assert docs and purge == 1 and cand == 0 and uq == 1 and mx == 361, "FAIL: 기대 스키마 상태 불일치"
# ── 5) /health 직접 호출 ──────────────────────────────────────────────
health = await main.health_check()
-100
View File
@@ -1,100 +0,0 @@
#!/usr/bin/env python3
"""기술지침(KOSHA guide) 절-KB persist: 번호섹션(# 1. 목적 / ## 4.1) 단위 분해 + 제본.
ASME/법령과 동일 clause-KB 모델(doc_kind='clause', parent_id=지침, 검색제외, /book 리더 공용).
Usage: python3 guide_clause_persist.py <id|all> [--commit]
"""
import asyncio, os, re, sys, hashlib, statistics
CAP = 12000; PAGE_TOK = 11000
EN, KO = 0.217, 0.529
# 번호섹션 헤더: '# 1. 목 적', '## 4.1 누출...' (번호 1~3자리=연도(4자리) 배제)
ART_RE = re.compile(r'^#{1,6}\s*(\d{1,3}(?:\.\d{1,3})*)\.?\s+(\S.*)$')
TOP_RE = re.compile(r'^\d{1,3}$')
# 외부 표준/법규 참조(대부분 dangling): ASME B16.5 · KS B 1501 · 규칙 제N조
EXT_RE = re.compile(r'(ASME\s+[A-Z][0-9.]+|KS\s+[A-Z]\s*[0-9]+|ISO\s+[0-9]+|제\d+조)')
def tok(s):
ko = sum(1 for c in s if '' <= c <= ''); return int((len(s)-ko)*EN + ko*KO)
def build_sections(text):
lines = text.split('\n'); off = []; a = 0
for ln in lines: off.append(a); a += len(ln) + 1
bounds = []; seen = set()
for i, ln in enumerate(lines):
m = ART_RE.match(ln)
if not m: continue
code, name = m.group(1), m.group(2).strip()
if not TOP_RE.match(code): continue # top-level 번호섹션만 경계
if code in seen: continue
if len(name) < 1: continue
seen.add(code); bounds.append((off[i], code, name))
out = []
for idx, (start, code, name) in enumerate(bounds):
end = bounds[idx+1][0] if idx+1 < len(bounds) else len(text)
body = text[start:end].strip()
ext = sorted(set(EXT_RE.findall(body)))[:8]
out.append(dict(code=code, part='본문', order=0, title=f"{code}. {name}"[:120],
body=body, tok=tok(body), links=[], ext=ext))
# over-CAP 페이지네이션 + 순번
final, order = [], 0
for c in out:
if c['tok'] <= CAP:
final.append({**c, 'order': order}); order += 1; continue
pages, cur, ct = [], [], 0
for ln in c['body'].split('\n'):
lt = tok(ln)+1
if ct+lt > PAGE_TOK and cur: pages.append('\n'.join(cur)); cur=[ln]; ct=lt
else: cur.append(ln); ct+=lt
if cur: pages.append('\n'.join(cur))
for pi, pb in enumerate(pages):
final.append(dict(code=c['code'] if pi==0 else f"{c['code']}·p{pi+1}", part='본문',
order=order, title=c['title'] if pi==0 else f"{c['title']} (p{pi+1})",
body=pb, tok=tok(pb), links=[], ext=[]))
order += 1
return final
async def process_one(conn, gid, commit, verbose=True):
row = await conn.fetchrow("SELECT title, md_content, ai_domain, data_origin FROM documents WHERE id=$1", gid)
if not row: return ('notfound', 0)
if not row['md_content']: return ('nullmd', 0)
secs = build_sections(row['md_content'])
if len(secs) < 2: return ('few', len(secs)) # 섹션 2 미만 = 번호구조 아님
toks = [c['tok'] for c in secs]
if verbose:
print(f"guide={gid} «{(row['title'] or '')[:40]}» 섹션={len(secs)} median={int(statistics.median(toks))} max={max(toks)}")
print(" 샘플:", [c['title'][:26] for c in secs[:7]])
if not commit: return ('dry', len(secs))
async with conn.transaction():
await conn.execute("DELETE FROM clause_links WHERE src_doc_id IN (SELECT id FROM documents WHERE parent_id=$1 AND doc_kind='clause')", gid)
await conn.execute("DELETE FROM documents WHERE parent_id=$1 AND doc_kind='clause'", gid)
for c in secs:
fh = hashlib.sha256(f"{gid}:{c['code']}:{c['body']}".encode()).hexdigest()
cid = await conn.fetchval("""
INSERT INTO documents (file_format,file_hash,title,md_content,parent_id,doc_kind,
clause_code,clause_part,clause_order,ai_domain,data_origin,
md_status,review_status,conversion_status,preview_status)
VALUES ('md',$1,$2,$3,$4,'clause',$5,$6,$7,$8,$9,'success','approved','none','none') RETURNING id
""", fh, c['title'], c['body'], gid, c['code'], c['part'], c['order'], row['ai_domain'], row['data_origin'] or 'external')
await conn.execute("INSERT INTO document_tags(doc_id,tag,tag_kind) VALUES ($1,'기술지침','kind') ON CONFLICT DO NOTHING", cid)
n = await conn.fetchval("SELECT count(*) FROM documents WHERE parent_id=$1 AND doc_kind='clause'", gid)
print(f" COMMITTED: {n} 섹션 for guide {gid}")
return ('committed', len(secs))
async def main():
import asyncpg
arg = sys.argv[1]; commit = '--commit' in sys.argv
conn = await asyncpg.connect(os.environ['DATABASE_URL'].replace('+asyncpg', ''))
if arg == 'all':
gs = await conn.fetch("SELECT id FROM documents WHERE material_type='guide' AND doc_kind='standard' "
"AND deleted_at IS NULL AND md_content IS NOT NULL ORDER BY id")
agg = {}; tot = 0
for i, r in enumerate(gs):
st, n = await process_one(conn, r['id'], commit, verbose=False)
agg[st] = agg.get(st, 0)+1; tot += n if st in ('dry','committed') else 0
if commit and (i+1) % 40 == 0: print(f"{i+1}/{len(gs)} (누적섹션 {tot})")
print(f"BATCH {'COMMIT' if commit else 'DRY'} guides={len(gs)} status={agg} 총섹션={tot}")
else:
await process_one(conn, int(arg), commit, verbose=True)
await conn.close()
asyncio.run(main())
-146
View File
@@ -1,146 +0,0 @@
#!/usr/bin/env python3
"""법령 조-KB persist: 법령을 조(條) 단위 개별 문서로 분해 + 조↔조 백링크 + 장(章) 태그.
ASME clause-KB와 동일 모델(doc_kind='clause', parent_id=법령, embedding NULL, 검색제외).
법령 추출 노이즈( ### 메타 반복) 트림. Usage: python3 law_clause_persist.py <law_id> [--commit]
"""
import asyncio, os, re, sys, hashlib, statistics
CAP = 12000; PAGE_TOK = 11000
EN, KO = 0.217, 0.529
# 조 헤더: '### 제3조의2(가스안전관리...) 본문'
ART_RE = re.compile(r'^#{0,6}\s*(제\d+조(?:의\d+)?)\s*\(([^)]*)\)\s*(.*)$')
CHAP_RE = re.compile(r'^#{1,6}\s*(제\d+장(?:의\d+)?)\s*(.*)$') # 장 = part
# 같은-법 조 멘션(백링크)
MENTION_RE = re.compile(r'\d+조(?:의\d+)?')
# 타법 참조: 「법명」 ... 제N조
EXTLAW_RE = re.compile(r'「([^」]+)」')
def tok(s):
ko = sum(1 for c in s if '' <= c <= ''); return int((len(s)-ko)*EN + ko*KO)
def art_code(c): return c # '제3조의2'
def build_articles(text):
lines = text.split('\n'); off = []; a = 0
for ln in lines: off.append(a); a += len(ln) + 1
arts = [] # (line_idx, code, name, part)
cur_part = None
for i, ln in enumerate(lines):
ch = CHAP_RE.match(ln)
if ch and not ART_RE.match(ln):
cur_part = (ch.group(1) + (' ' + ch.group(2).strip() if ch.group(2).strip() else '')).strip()
continue
m = ART_RE.match(ln)
if m:
arts.append((i, m.group(1), m.group(2).strip(), cur_part))
# 본문 슬라이스 + 다음 조 앞 메타 노이즈 트림
out = []
for idx, (li, code, name, part) in enumerate(arts):
end_li = arts[idx+1][0] if idx+1 < len(arts) else len(lines)
body_lines = lines[li:end_li]
# 트림: 끝에서부터 '### {짧은 메타}' (조번호/조문/날짜/제목, [개정] 제N조 아님) 제거
while len(body_lines) > 1:
last = body_lines[-1].strip()
if last == '':
body_lines.pop(); continue
mh = re.match(r'^#{1,6}\s+(.*)$', last)
if mh:
c = mh.group(1).strip()
if not c.startswith('[') and not c.startswith('') and (
c in ('조문', 'N') or re.fullmatch(r'\d+', c) or re.fullmatch(r'\d{8}', c) or len(c) <= 30):
body_lines.pop(); continue
break
body = '\n'.join(body_lines).strip()
links = sorted(set(MENTION_RE.findall(body)) - {code})
ext = sorted(set(EXTLAW_RE.findall(body)))[:6]
out.append(dict(code=code, part=part or '본칙', order=0,
title=f"{code}({name})" if name else code,
body=body, tok=tok(body), links=links, ext=ext))
# 페이지네이션(over-CAP) + 순번
final, order = [], 0
for c in out:
if c['tok'] <= CAP:
final.append({**c, 'order': order}); order += 1; continue
# 11K 토큰 라인 단위 분할
pages, cur, ct = [], [], 0
for ln in c['body'].split('\n'):
lt = tok(ln)+1
if ct+lt > PAGE_TOK and cur: pages.append('\n'.join(cur)); cur=[ln]; ct=lt
else: cur.append(ln); ct+=lt
if cur: pages.append('\n'.join(cur))
for pi, pb in enumerate(pages):
final.append(dict(code=c['code'] if pi==0 else f"{c['code']}·p{pi+1}", part=c['part'],
order=order, title=c['title'] if pi==0 else f"{c['title']} (p{pi+1}/{len(pages)})",
body=pb, tok=tok(pb), links=c['links'] if pi==0 else [], ext=[]))
order += 1
return final
async def process_one(conn, law, commit, verbose=True):
row = await conn.fetchrow("SELECT title, coalesce(md_content, extracted_text) AS md_content, ai_domain, data_origin FROM documents WHERE id=$1", law)
if not row: return ('notfound', 0, 0)
if not row['md_content']: return ('nullmd', 0, 0)
arts = build_articles(row['md_content'])
if not arts: return ('noart', 0, 0)
toks = [c['tok'] for c in arts]
nlink = sum(len(c['links']) for c in arts)
if verbose:
parts = {}
for c in arts: parts[c['part']] = parts.get(c['part'], 0)+1
print(f"law={law} «{(row['title'] or '')[:34]}» 조문={len(arts)} median={int(statistics.median(toks))} "
f"max={max(toks)} 장={len(parts)} 백링크={nlink}")
print(" 샘플:", [c['title'][:22] for c in arts[:6]])
if not commit:
return ('dry', len(arts), nlink)
async with conn.transaction():
await conn.execute(
"DELETE FROM clause_links WHERE src_doc_id IN (SELECT id FROM documents WHERE parent_id=$1 AND doc_kind='clause')", law)
await conn.execute("DELETE FROM documents WHERE parent_id=$1 AND doc_kind='clause'", law)
code2id = {}
for c in arts:
fh = hashlib.sha256(f"{law}:{c['code']}:{c['body']}".encode()).hexdigest()
cid = await conn.fetchval("""
INSERT INTO documents (file_format,file_hash,title,md_content,parent_id,doc_kind,
clause_code,clause_part,clause_order,ai_domain,data_origin,
md_status,review_status,conversion_status,preview_status)
VALUES ('md',$1,$2,$3,$4,'clause',$5,$6,$7,$8,$9,'success','approved','none','none') RETURNING id
""", fh, c['title'], c['body'], law, c['code'], c['part'], c['order'],
row['ai_domain'], row['data_origin'] or 'external')
code2id[c['code']] = cid
await conn.execute("INSERT INTO document_tags(doc_id,tag,tag_kind) VALUES ($1,$2,'chapter') ON CONFLICT DO NOTHING", cid, c['part'])
# 조↔조 백링크 (같은 법 내부; 타법 참조는 dangling)
edges = []
for c in arts:
src = code2id[c['code']]
for dst in c['links']:
edges.append((src, dst, code2id.get(dst), None, None, None))
if edges:
await conn.executemany(
"INSERT INTO clause_links(src_doc_id,dst_code,dst_doc_id,anchor,ctx,char_off) VALUES ($1,$2,$3,$4,$5,$6)", edges)
n = await conn.fetchval("SELECT count(*) FROM documents WHERE parent_id=$1 AND doc_kind='clause'", law)
print(f" COMMITTED: {n} 조문 + {len(edges)} 백링크 for law {law}")
return ('committed', n, len(edges))
async def main():
import asyncpg
arg = sys.argv[1]; commit = '--commit' in sys.argv
conn = await asyncpg.connect(os.environ['DATABASE_URL'].replace('+asyncpg', ''))
if arg == 'all':
laws = await conn.fetch("SELECT lm.document_id AS id FROM legal_meta lm "
"JOIN documents d ON d.id=lm.document_id "
"WHERE lm.law_doc_kind='primary' AND lm.version_status='current' "
"AND coalesce(d.md_content, d.extracted_text) IS NOT NULL "
"ORDER BY lm.document_id")
agg = {}; tot_art = tot_link = 0; zero = []
for i, r in enumerate(laws):
st, na, nl = await process_one(conn, r['id'], commit, verbose=False)
agg[st] = agg.get(st, 0) + 1
tot_art += na; tot_link += nl
if st == 'noart': zero.append(r['id'])
if commit and (i + 1) % 30 == 0: print(f"{i+1}/{len(laws)} (누적 조 {tot_art})")
print(f"BATCH {'COMMIT' if commit else 'DRY'} laws={len(laws)} status={agg} 총조문={tot_art} 총백링크={tot_link}")
if zero: print(f" 0-조(추출구조 이질) {len(zero)}건: {zero[:20]}")
else:
await process_one(conn, int(arg), commit, verbose=True)
await conn.close()
asyncio.run(main())
-51
View File
@@ -1,51 +0,0 @@
#!/usr/bin/env python3
"""논문 인용그래프 가능성 측정(read-only) — 본문 DOI로 코퍼스내 인용 엣지 추정.
own_doi = 헤더( 2500) DOI / cited = References 이후(또는 전체) DOI. owner 엣지.
"""
import asyncio, os, re, sys
DOI_RE = re.compile(r'10\.\d{4,9}/[^\s"<>)\]\},;]+')
REF_RE = re.compile(r'(references|참고문헌|bibliography|reference\s*list)', re.I)
def norm(d): return d.rstrip('.').lower()
async def main():
import asyncpg
conn = await asyncpg.connect(os.environ['DATABASE_URL'].replace('+asyncpg', ''))
rows = await conn.fetch("SELECT id, title, coalesce(md_content, extracted_text) AS txt FROM documents "
"WHERE material_type='paper' AND doc_kind='standard' AND deleted_at IS NULL "
"AND coalesce(md_content, extracted_text) IS NOT NULL")
owner = {} # doi -> paper id (헤더 DOI = 그 논문 소유)
cited = {} # paper id -> set(cited doi)
n_own = n_refsec = 0
for r in rows:
txt = r['txt']
head = txt[:2500]
hdois = [norm(d) for d in DOI_RE.findall(head)]
if hdois:
owner.setdefault(hdois[0], r['id']); n_own += 1
m = REF_RE.search(txt)
body = txt[m.start():] if m else ''
if m: n_refsec += 1
cds = set(norm(d) for d in DOI_RE.findall(body))
if cds: cited[r['id']] = cds
# 엣지: paper -> owner(cited doi)
edges = []
for pid, cds in cited.items():
for d in cds:
o = owner.get(d)
if o and o != pid: edges.append((pid, o, d))
cited_papers = set(e[0] for e in edges)
target_papers = set(e[1] for e in edges)
print(f"papers={len(rows)} 헤더DOI보유={n_own} References보유={n_refsec} owner_map={len(owner)}")
print(f"인용엣지(코퍼스내)={len(edges)} 인용하는논문={len(cited_papers)} 피인용논문={len(target_papers)}")
# 피인용 top
from collections import Counter
top = Counter(e[1] for e in edges).most_common(6)
if top:
idmap = {r['id']: r['title'] for r in rows}
print("피인용 top:")
for pid, c in top: print(f" {c}회 ← {(idmap.get(pid) or '')[:48]}")
await conn.close()
asyncio.run(main())
-30
View File
@@ -1,30 +0,0 @@
#!/usr/bin/env python3
"""OpenAlex 보강 타당성 테스트 — 소수 논문 제목으로 매칭/메타 확인 (외부 API)."""
import asyncio, os, re
async def main():
import asyncpg, httpx
conn = await asyncpg.connect(os.environ['DATABASE_URL'].replace('+asyncpg', ''))
rows = await conn.fetch("SELECT id, title FROM documents WHERE material_type='paper' "
"AND doc_kind='standard' AND deleted_at IS NULL AND title IS NOT NULL "
"AND length(title) > 15 ORDER BY id LIMIT 6")
async with httpx.AsyncClient(timeout=20) as client:
for r in rows:
title = re.sub(r'\s+', ' ', r['title']).strip()
try:
resp = await client.get("https://api.openalex.org/works",
params={"search": title[:200], "per_page": 1, "mailto": "hyun49196@gmail.com"})
js = resp.json()
res = (js.get("results") or [])
if not res:
print(f"[{r['id']}] NO MATCH | {title[:50]}"); continue
w = res[0]
oid = (w.get("id") or "").split("/")[-1]
print(f"[{r['id']}] {title[:46]}")
print(f" → OA {oid} | {(w.get('title') or '')[:46]} | {w.get('publication_year')} | "
f"cited_by={w.get('cited_by_count')} | refs={len(w.get('referenced_works') or [])} | doi={w.get('doi')}")
except Exception as e:
print(f"[{r['id']}] ERROR {type(e).__name__}: {e}")
await conn.close()
asyncio.run(main())

Some files were not shown because too many files have changed in this diff Show More