feat: AI 서비스 MLX 듀얼 백엔드 및 모델 최적화
- MLX(맥미니 27B) 우선 → Ollama(조립컴 9B) fallback 구조 - pydantic-settings 기반 config 전환 - health check에 MLX 상태 추가 - 텍스트 모델 qwen3:8b → qwen3.5:9b-q8_0 변경 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import httpx
|
||||
from config import settings
|
||||
|
||||
@@ -6,29 +7,55 @@ class OllamaClient:
|
||||
def __init__(self):
|
||||
self.base_url = settings.OLLAMA_BASE_URL
|
||||
self.timeout = httpx.Timeout(float(settings.OLLAMA_TIMEOUT), connect=10.0)
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||
return self._client
|
||||
|
||||
async def close(self):
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def generate_embedding(self, text: str) -> list[float]:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/embeddings",
|
||||
json={"model": settings.OLLAMA_EMBED_MODEL, "prompt": text},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["embedding"]
|
||||
client = await self._get_client()
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/embeddings",
|
||||
json={"model": settings.OLLAMA_EMBED_MODEL, "prompt": text},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["embedding"]
|
||||
|
||||
async def batch_embeddings(self, texts: list[str]) -> list[list[float]]:
|
||||
results = []
|
||||
for text in texts:
|
||||
emb = await self.generate_embedding(text)
|
||||
results.append(emb)
|
||||
return results
|
||||
async def batch_embeddings(self, texts: list[str], concurrency: int = 5) -> list[list[float]]:
|
||||
semaphore = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def _embed(text: str) -> list[float]:
|
||||
async with semaphore:
|
||||
return await self.generate_embedding(text)
|
||||
|
||||
return await asyncio.gather(*[_embed(t) for t in texts])
|
||||
|
||||
async def generate_text(self, prompt: str, system: str = None) -> str:
|
||||
messages = []
|
||||
if system:
|
||||
messages.append({"role": "system", "content": system})
|
||||
messages.append({"role": "user", "content": prompt})
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
client = await self._get_client()
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{settings.MLX_BASE_URL}/chat/completions",
|
||||
json={
|
||||
"model": settings.MLX_TEXT_MODEL,
|
||||
"messages": messages,
|
||||
"max_tokens": 2048,
|
||||
"temperature": 0.3,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()["choices"][0]["message"]["content"]
|
||||
except Exception:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/chat",
|
||||
json={
|
||||
@@ -42,16 +69,21 @@ class OllamaClient:
|
||||
return response.json()["message"]["content"]
|
||||
|
||||
async def check_health(self) -> dict:
|
||||
result = {}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(5.0)) as client:
|
||||
response = await client.get(f"{self.base_url}/api/tags")
|
||||
models = response.json().get("models", [])
|
||||
return {
|
||||
"status": "connected",
|
||||
"models": [m["name"] for m in models],
|
||||
}
|
||||
client = await self._get_client()
|
||||
response = await client.get(f"{self.base_url}/api/tags")
|
||||
models = response.json().get("models", [])
|
||||
result["ollama"] = {"status": "connected", "models": [m["name"] for m in models]}
|
||||
except Exception:
|
||||
return {"status": "disconnected"}
|
||||
result["ollama"] = {"status": "disconnected"}
|
||||
try:
|
||||
client = await self._get_client()
|
||||
response = await client.get(f"{settings.MLX_BASE_URL}/health")
|
||||
result["mlx"] = {"status": "connected", "model": settings.MLX_TEXT_MODEL}
|
||||
except Exception:
|
||||
result["mlx"] = {"status": "disconnected"}
|
||||
return result
|
||||
|
||||
|
||||
ollama_client = OllamaClient()
|
||||
|
||||
Reference in New Issue
Block a user