feat: 맥미니 MLX 연동 — OpenAI-compat 프록시 + 모델 배치 정정

- proxy_openai.py 추가: MLX 서버 SSE 패스스루
- chat.py: openai-compat 백엔드 타입 라우팅 추가
- backends.json: GPU=embed(bge-m3)만, 맥미니MLX=채팅(qwen3.5:35b-a3b)
- LAN IP(192.168.1.122) 사용 (같은 서브넷, Tailscale 불필요)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Hyungi Ahn
2026-03-31 15:09:21 +09:00
parent 3794afff95
commit 7b28252d4f
3 changed files with 113 additions and 4 deletions

View File

@@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel
from middleware.rate_limit import check_backend_rate_limit
from services import proxy_ollama
from services import proxy_ollama, proxy_openai
from services.registry import registry
router = APIRouter(prefix="/v1", tags=["chat"])
@@ -79,7 +79,24 @@ async def chat_completions(body: ChatRequest, request: Request):
)
return JSONResponse(content=result)
# Placeholder for other backend types
if backend.type == "openai-compat":
if body.stream:
return StreamingResponse(
proxy_openai.stream_chat(
backend.url, body.model, messages, **kwargs
),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
},
)
else:
result = await proxy_openai.complete_chat(
backend.url, body.model, messages, **kwargs
)
return JSONResponse(content=result)
raise HTTPException(
status_code=501,
detail={

View File

@@ -0,0 +1,83 @@
"""OpenAI-compatible proxy (MLX server, vLLM, etc.) — SSE passthrough."""
from __future__ import annotations
import json
import logging
from collections.abc import AsyncGenerator
import httpx
logger = logging.getLogger(__name__)
async def stream_chat(
base_url: str,
model: str,
messages: list[dict],
**kwargs,
) -> AsyncGenerator[str, None]:
"""Proxy OpenAI-compatible chat streaming. SSE passthrough with model field override."""
payload = {
"model": model,
"messages": messages,
"stream": True,
**{k: v for k, v in kwargs.items() if v is not None},
}
async with httpx.AsyncClient(timeout=120.0) as client:
async with client.stream(
"POST",
f"{base_url}/v1/chat/completions",
json=payload,
) as resp:
if resp.status_code != 200:
body = await resp.aread()
error_msg = body.decode("utf-8", errors="replace")
yield _error_event(f"Backend error ({resp.status_code}): {error_msg}")
return
async for line in resp.aiter_lines():
if not line.strip():
continue
# Pass through SSE lines as-is (already in OpenAI format)
if line.startswith("data: "):
yield f"{line}\n\n"
elif line == "data: [DONE]":
yield "data: [DONE]\n\n"
async def complete_chat(
base_url: str,
model: str,
messages: list[dict],
**kwargs,
) -> dict:
"""Non-streaming OpenAI-compatible chat."""
payload = {
"model": model,
"messages": messages,
"stream": False,
**{k: v for k, v in kwargs.items() if v is not None},
}
async with httpx.AsyncClient(timeout=120.0) as client:
resp = await client.post(f"{base_url}/v1/chat/completions", json=payload)
resp.raise_for_status()
return resp.json()
def _error_event(message: str) -> str:
error = {
"id": "chatcmpl-gateway",
"object": "chat.completion.chunk",
"model": "error",
"choices": [
{
"index": 0,
"delta": {"content": f"[Error] {message}"},
"finish_reason": "stop",
}
],
}
return f"data: {json.dumps(error)}\n\ndata: [DONE]\n\n"