feat(chunk): Phase 0.1 chunk 인덱싱 — ORM/worker/migration 정리

GPU 서버에 untracked로만 존재하던 Phase 0.1 코드를 정식 commit:
- app/models/chunk.py — DocumentChunk ORM (country/source/domain 메타 포함)
- app/workers/chunk_worker.py — 6가지 chunking 전략 (legal/news/markdown/email/long_pdf/default)
- migrations/014_document_chunks.sql — pgvector + FTS + trigram 인덱스
- app/models/queue.py — ProcessingQueue enum에 'chunk' stage 추가
- app/workers/queue_consumer.py — chunk stage 등록, classify→[embed,chunk] 자동 연결

Phase 1 reranker 통합 작업의 전제 조건. document_chunks 테이블 기반 retrieval에 사용.
This commit is contained in:
Hyungi Ahn
2026-04-07 13:26:37 +09:00
parent a2941487fe
commit 378fbc7845
5 changed files with 442 additions and 3 deletions

View File

@@ -11,7 +11,7 @@ from models.queue import ProcessingQueue
logger = setup_logger("queue_consumer")
# stage별 배치 크기
BATCH_SIZE = {"extract": 5, "classify": 3, "summarize": 3, "embed": 1, "preview": 2}
BATCH_SIZE = {"extract": 5, "classify": 3, "summarize": 3, "embed": 1, "chunk": 1, "preview": 2}
STALE_THRESHOLD_MINUTES = 10
@@ -34,7 +34,7 @@ async def reset_stale_items():
async def enqueue_next_stage(document_id: int, current_stage: str):
"""현재 stage 완료 후 다음 stage를 pending으로 등록"""
next_stages = {"extract": ["classify", "preview"], "classify": ["embed"]}
next_stages = {"extract": ["classify", "preview"], "classify": ["embed", "chunk"]}
stages = next_stages.get(current_stage, [])
if not stages:
return
@@ -62,6 +62,7 @@ async def enqueue_next_stage(document_id: int, current_stage: str):
async def consume_queue():
"""큐에서 pending 항목을 가져와 stage별 워커 실행"""
from workers.classify_worker import process as classify_process
from workers.chunk_worker import process as chunk_process
from workers.embed_worker import process as embed_process
from workers.extract_worker import process as extract_process
from workers.preview_worker import process as preview_process
@@ -72,6 +73,7 @@ async def consume_queue():
"classify": classify_process,
"summarize": summarize_process,
"embed": embed_process,
"chunk": chunk_process,
"preview": preview_process,
}