hyungi/ai-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: document pipeline (embedding->Korean translation->HTML). Add /pipeline/ingest endpoint

Browse Source
This commit is contained in:
hyungi
2025-08-13 08:45:01 +09:00
parent b430a27215
commit a280304adc
3 changed files with 106 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@@ -213,6 +213,22 @@ curl -s -X POST http://localhost:26000/paperless/hook \
해당 훅은 문서 도착을 통지받는 용도로 제공됩니다. 실제 본문 텍스트는 Paperless API로 조회해 `/index/upsert`로 추가하세요. 해당 훅은 문서 도착을 통지받는 용도로 제공됩니다. 실제 본문 텍스트는 Paperless API로 조회해 `/index/upsert`로 추가하세요.
### Paperless 배치 동기화(`/paperless/sync`) ### Paperless 배치 동기화(`/paperless/sync`)
### 문서 파이프라인(`/pipeline/ingest`)
첨부 문서(텍스트가 준비된 상태: OCR/추출 선행) → 벡터 임베딩 → 한국어 번역 → HTML 생성까지 한 번에 처리합니다.
```bash
curl -s -X POST http://localhost:26000/pipeline/ingest \
-H 'Content-Type: application/json' -H 'X-API-Key: <키>' \
-d '{
"doc_id": "doc-2025-08-13-001",
"text": "(여기에 문서 텍스트)",
"generate_html": true
}'
```
응답에 `html_path`가 포함됩니다. 한국어 번역본이 `outputs/html/<doc_id>.html`로 생성되고, 번역문은 인덱스에 추가되어 RAG로 검색됩니다.
Paperless에서 다수 문서를 일괄 인덱싱합니다. Paperless에서 다수 문서를 일괄 인덱싱합니다.

14
server/main.py
View File

@@ -11,6 +11,7 @@ from .index_store import JsonlIndex
from .security import require_api_key from .security import require_api_key
from .paperless_client import PaperlessClient from .paperless_client import PaperlessClient
from .utils import chunk_text from .utils import chunk_text
from .pipeline import DocumentPipeline
app = FastAPI(title="Local AI Server", version="0.2.1") app = FastAPI(title="Local AI Server", version="0.2.1")
@@ -28,6 +29,7 @@ app.add_middleware(
) )
ollama = OllamaClient(settings.ollama_host) ollama = OllamaClient(settings.ollama_host)
index = JsonlIndex(settings.index_path) index = JsonlIndex(settings.index_path)
pipeline = DocumentPipeline(ollama, settings.embedding_model, settings.boost_model)
class ChatRequest(BaseModel): class ChatRequest(BaseModel):
@@ -55,6 +57,12 @@ class UpsertRequest(BaseModel):
batch: int = 16 batch: int = 16
class PipelineIngestRequest(BaseModel):
doc_id: str
text: str
generate_html: bool = True
@app.get("/health") @app.get("/health")
def health() -> Dict[str, Any]: def health() -> Dict[str, Any]:
return { return {
@@ -152,6 +160,12 @@ def index_reload() -> Dict[str, Any]:
return {"total": total} return {"total": total}
@app.post("/pipeline/ingest")
def pipeline_ingest(req: PipelineIngestRequest, _: None = Depends(require_api_key)) -> Dict[str, Any]:
result = pipeline.process(doc_id=req.doc_id, text=req.text, index=index, generate_html=req.generate_html)
return {"status": "ok", "doc_id": result.doc_id, "added": result.added_chunks, "chunks": result.chunks, "html_path": result.html_path}
# Paperless webhook placeholder (to be wired with user-provided details) # Paperless webhook placeholder (to be wired with user-provided details)
class PaperlessHook(BaseModel): class PaperlessHook(BaseModel):
document_id: int document_id: int

76
server/pipeline.py Normal file
View File

@@ -0,0 +1,76 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import List, Dict, Any
from .utils import chunk_text
from .ollama_client import OllamaClient
from .index_store import IndexRow
@dataclass
class PipelineResult:
doc_id: str
html_path: str | None
added_chunks: int
chunks: int
class DocumentPipeline:
def __init__(self, ollama: OllamaClient, embedding_model: str, boost_model: str, output_dir: str = "outputs") -> None:
self.ollama = ollama
self.embedding_model = embedding_model
self.boost_model = boost_model
self.output_dir = Path(output_dir)
(self.output_dir / "html").mkdir(parents=True, exist_ok=True)
def translate_to_korean(self, parts: List[str]) -> List[str]:
translated: List[str] = []
sys_prompt = (
"당신은 전문 번역가입니다. 입력 텍스트를 자연스러운 한국어로 충실히 번역하세요. "
"의미를 임의로 축약하거나 추가하지 마세요. 코드/수식/표기는 가능한 유지하세요."
)
for p in parts:
messages = [
{"role": "system", "content": sys_prompt},
{"role": "user", "content": f"아래 텍스트를 한국어로 번역하세요:\n\n{p}"},
]
resp = self.ollama.chat(self.boost_model, messages, stream=False, options={"temperature": 0.2, "num_ctx": 32768})
content = resp.get("message", {}).get("content") or resp.get("response", "")
translated.append(content.strip())
return translated
def build_html(self, doc_id: str, title: str, ko_text: str) -> str:
html_path = self.output_dir / "html" / f"{doc_id}.html"
html = f"""
<!doctype html>
<html lang=\"ko\">\n<head>\n<meta charset=\"utf-8\"/>\n<title>{title}</title>\n<style>
body{{font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Noto Sans KR', 'Apple SD Gothic Neo', Arial, sans-serif; line-height:1.6; margin:24px;}}
article{{max-width: 900px; margin: auto;}}
h1{{font-size: 1.6rem; margin-bottom: 1rem;}}
.chunk{{white-space: pre-wrap; margin: 1rem 0;}}
</style>\n</head>\n<body>\n<article>\n<h1>{title}</h1>\n"""
for idx, para in enumerate(ko_text.split("\n\n")):
if para.strip():
html += f"<div class=\"chunk\" id=\"c{idx}\">{para}</div>\n"
html += "</article>\n</body>\n</html>\n"
html_path.write_text(html, encoding="utf-8")
return str(html_path)
def process(self, *, doc_id: str, text: str, index, generate_html: bool = True) -> PipelineResult:
parts = chunk_text(text, max_chars=1200, overlap=200)
translated = self.translate_to_korean(parts)
to_append: List[IndexRow] = []
for i, t in enumerate(translated):
vec = self.ollama.embeddings(self.embedding_model, t)
to_append.append(IndexRow(id=f"pipeline:{doc_id}:{i}", text=t, vector=vec, source=f"pipeline/{doc_id}"))
added = index.append(to_append) if to_append else 0
html_path: str | None = None
if generate_html:
html_path = self.build_html(doc_id, title=f"문서 {doc_id} (한국어 번역본)", ko_text="\n\n".join(translated))
return PipelineResult(doc_id=doc_id, html_path=html_path, added_chunks=added, chunks=len(translated))