feat(chunk): Phase 0.1 chunk 인덱싱 — ORM/worker/migration 정리
GPU 서버에 untracked로만 존재하던 Phase 0.1 코드를 정식 commit: - app/models/chunk.py — DocumentChunk ORM (country/source/domain 메타 포함) - app/workers/chunk_worker.py — 6가지 chunking 전략 (legal/news/markdown/email/long_pdf/default) - migrations/014_document_chunks.sql — pgvector + FTS + trigram 인덱스 - app/models/queue.py — ProcessingQueue enum에 'chunk' stage 추가 - app/workers/queue_consumer.py — chunk stage 등록, classify→[embed,chunk] 자동 연결 Phase 1 reranker 통합 작업의 전제 조건. document_chunks 테이블 기반 retrieval에 사용.
This commit is contained in:
@@ -11,7 +11,7 @@ from models.queue import ProcessingQueue
|
||||
logger = setup_logger("queue_consumer")
|
||||
|
||||
# stage별 배치 크기
|
||||
BATCH_SIZE = {"extract": 5, "classify": 3, "summarize": 3, "embed": 1, "preview": 2}
|
||||
BATCH_SIZE = {"extract": 5, "classify": 3, "summarize": 3, "embed": 1, "chunk": 1, "preview": 2}
|
||||
STALE_THRESHOLD_MINUTES = 10
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ async def reset_stale_items():
|
||||
|
||||
async def enqueue_next_stage(document_id: int, current_stage: str):
|
||||
"""현재 stage 완료 후 다음 stage를 pending으로 등록"""
|
||||
next_stages = {"extract": ["classify", "preview"], "classify": ["embed"]}
|
||||
next_stages = {"extract": ["classify", "preview"], "classify": ["embed", "chunk"]}
|
||||
stages = next_stages.get(current_stage, [])
|
||||
if not stages:
|
||||
return
|
||||
@@ -62,6 +62,7 @@ async def enqueue_next_stage(document_id: int, current_stage: str):
|
||||
async def consume_queue():
|
||||
"""큐에서 pending 항목을 가져와 stage별 워커 실행"""
|
||||
from workers.classify_worker import process as classify_process
|
||||
from workers.chunk_worker import process as chunk_process
|
||||
from workers.embed_worker import process as embed_process
|
||||
from workers.extract_worker import process as extract_process
|
||||
from workers.preview_worker import process as preview_process
|
||||
@@ -72,6 +73,7 @@ async def consume_queue():
|
||||
"classify": classify_process,
|
||||
"summarize": summarize_process,
|
||||
"embed": embed_process,
|
||||
"chunk": chunk_process,
|
||||
"preview": preview_process,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user