diff --git a/kb_writer.py b/kb_writer.py index 9e909c2..752b89e 100644 --- a/kb_writer.py +++ b/kb_writer.py @@ -9,6 +9,7 @@ import re import unicodedata from datetime import datetime, timezone, timedelta from pathlib import Path +from uuid import uuid4 from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -46,31 +47,25 @@ async def save(request: Request): date_str = now.strftime("%Y-%m-%d") iso_str = now.isoformat() - # 파일명 생성 + # 파일명 생성 (uuid4 short hash로 경합 방지) slug = _slugify(title) + uid = uuid4().hex[:6] if doc_type == "chat-memory": time_str = now.strftime("%H%M") - filename = f"{date_str}T{time_str}-{_slugify(topic)}.md" + filename = f"{date_str}T{time_str}-{uid}-{_slugify(topic)}.md" elif doc_type == "news": - filename = f"{date_str}-{_slugify(source)}-{slug}.md" + filename = f"{date_str}-{uid}-{_slugify(source)}-{slug}.md" else: - filename = f"{date_str}-{slug}.md" + filename = f"{date_str}-{uid}-{slug}.md" # 디렉토리 생성 type_dir = BASE_DIR / doc_type / month_dir type_dir.mkdir(parents=True, exist_ok=True) - - # 중복 파일명 처리 filepath = type_dir / filename - counter = 1 - while filepath.exists(): - stem = filename.rsplit(".", 1)[0] - filepath = type_dir / f"{stem}-{counter}.md" - counter += 1 # YAML frontmatter + 본문 tags_yaml = ", ".join(f'"{t}"' for t in tags) - qdrant_id = int(now.timestamp() * 1000) + qdrant_id = body.get("qdrant_id") or int(now.timestamp() * 1000) md_content = f"""--- title: "{title}" diff --git a/n8n/workflows/main-chat-pipeline.json b/n8n/workflows/main-chat-pipeline.json index 215bfab..389a934 100644 --- a/n8n/workflows/main-chat-pipeline.json +++ b/n8n/workflows/main-chat-pipeline.json @@ -428,6 +428,19 @@ 1200 ] }, + { + "parameters": { + "jsCode": "const cls = $input.first().json;\nreturn [{ json: { ...cls, pid: Date.now() } }];" + }, + "id": "b1000001-0000-0000-0000-000000000070", + "name": "Set pid", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [ + 1430, + 1200 + ] + }, { "parameters": { "operation": "executeQuery", @@ -1005,7 +1018,7 @@ }, { "parameters": { - "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst data = $input.first().json;\nconst prompt = `Q: ${data.userText}\\nA: ${data.aiText}`;\nconst emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model:'bge-m3', prompt });\nif (!emb.embedding||!Array.isArray(emb.embedding)) return [{json:{saved:false}}];\nconst pid = Date.now();\nconst qu = $env.QDRANT_URL||'http://host.docker.internal:6333';\nawait httpPut(`${qu}/collections/chat_memory/points`, { points:[{ id:pid, vector:emb.embedding, payload:{\n text:prompt, feature:'chat', intent:data.intent||'unknown',\n username:data.username||'unknown', topic:data.topic||'general', timestamp:pid\n}}]});\n// kb_writer 파일 저장 (graceful)\ntry {\n const kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\n await httpPost(`${kbUrl}/save`, {\n title: `${new Date().toISOString().split('T')[0]} 대화 메모`,\n content: prompt,\n type: 'chat-memory',\n tags: ['chat-memory', data.topic || 'general'],\n username: data.username || 'unknown',\n topic: data.topic || 'general'\n }, { timeout: 5000 });\n} catch(e) {}\n\nreturn [{json:{saved:true,pointId:pid,topic:data.topic}}];" + "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst data = $input.first().json;\nconst pid = $('Set pid').first().json.pid || Date.now();\nconst prompt = `Q: ${data.userText}\\nA: ${data.aiText}`;\nconst emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model:'bge-m3', prompt });\nif (!emb.embedding||!Array.isArray(emb.embedding)) return [{json:{saved:false}}];\nconst qu = $env.QDRANT_URL||'http://host.docker.internal:6333';\nawait httpPut(`${qu}/collections/chat_memory/points`, { points:[{ id:pid, vector:emb.embedding, payload:{\n text:prompt, feature:'chat', intent:data.intent||'unknown',\n username:data.username||'unknown', topic:data.topic||'general', timestamp:pid\n}}]});\n// kb_writer 파일 저장 (graceful)\ntry {\n const kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\n await httpPost(`${kbUrl}/save`, {\n title: `${new Date().toISOString().split('T')[0]} 대화 메모`,\n content: prompt,\n type: 'chat-memory',\n tags: ['chat-memory', data.topic || 'general'],\n username: data.username || 'unknown',\n topic: data.topic || 'general',\n qdrant_id: pid\n }, { timeout: 5000 });\n} catch(e) {}\n\nreturn [{json:{saved:true,pointId:pid,topic:data.topic}}];" }, "id": "b1000001-0000-0000-0000-000000000037", "name": "Embed & Save Memory", @@ -1186,7 +1199,7 @@ }, { "parameters": { - "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' \\u2192 ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' \\u2192 timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst cls = $('Qwen Classify v2').first().json;\nconst userText = cls.userText;\nconst username = cls.username;\n\n// \"기록해\", \"메모해\", \"저장해\", \"적어둬\" 등 제거\nconst content = userText\n .replace(/[.]\\s*(기록해|메모해|저장해|적어둬|기록|메모|저장)[.]?\\s*$/g, '')\n .replace(/^\\s*(기록해|메모해|저장해|적어둬)[.:]\\s*/g, '')\n .trim() || userText;\n\n// 제목: 앞 30자\nconst title = content.substring(0, 30).replace(/\\n/g, ' ') + (content.length > 30 ? '...' : '');\nconst now = new Date();\nconst dateStr = `${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}`;\nconst fullTitle = `${dateStr} ${title}`;\n\n// kb_writer 파일 저장\nconst kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\nlet saved = false;\ntry {\n const result = await httpPost(`${kbUrl}/save`, {\n title: fullTitle,\n content: content,\n type: 'note',\n tags: ['synology-chat', 'note'],\n username: username || 'unknown',\n topic: 'general'\n }, { timeout: 10000 });\n saved = result.success === true;\n} catch(e) {}\n\n// Qdrant 임베딩 (graceful)\ntry {\n const emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model: 'bge-m3', prompt: content });\n if (emb.embedding && Array.isArray(emb.embedding)) {\n const pid = Date.now();\n const qu = $env.QDRANT_URL || 'http://host.docker.internal:6333';\n await httpPut(`${qu}/collections/chat_memory/points`, { points: [{ id: pid, vector: emb.embedding, payload: {\n text: content, feature: 'note', intent: 'note',\n username: username || 'unknown', topic: 'general', timestamp: pid\n }}]});\n }\n} catch(e) {}\n\nconst responseText = saved\n ? `기록했습니다: ${title}`\n : `기록을 시도했지만 저장에 실패했습니다. 내용: ${title}`;\n\nreturn [{ json: { text: responseText, userText, username, response_tier: 'local', intent: 'note', model: 'id-9b:latest', inputTokens: 0, outputTokens: 0 } }];" + "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' \\u2192 ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' \\u2192 timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst cls = $('Qwen Classify v2').first().json;\nconst pid = $json.pid || Date.now();\nconst userText = cls.userText;\nconst username = cls.username;\n\n// \"기록해\", \"메모해\", \"저장해\", \"적어둬\" 등 제거\nconst content = userText\n .replace(/[.]\\s*(기록해|메모해|저장해|적어둬|기록|메모|저장)[.]?\\s*$/g, '')\n .replace(/^\\s*(기록해|메모해|저장해|적어둬)[.:]\\s*/g, '')\n .trim() || userText;\n\n// 제목: 앞 30자\nconst title = content.substring(0, 30).replace(/\\n/g, ' ') + (content.length > 30 ? '...' : '');\nconst now = new Date();\nconst dateStr = `${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}`;\nconst fullTitle = `${dateStr} ${title}`;\n\n// kb_writer 파일 저장\nconst kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\nlet saved = false;\ntry {\n const result = await httpPost(`${kbUrl}/save`, {\n title: fullTitle,\n content: content,\n type: 'note',\n tags: ['synology-chat', 'note'],\n username: username || 'unknown',\n topic: 'general',\n qdrant_id: pid\n }, { timeout: 10000 });\n saved = result.success === true;\n} catch(e) {}\n\n// Qdrant 임베딩 (graceful)\ntry {\n const emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model: 'bge-m3', prompt: content });\n if (emb.embedding && Array.isArray(emb.embedding)) {\n const qu = $env.QDRANT_URL || 'http://host.docker.internal:6333';\n await httpPut(`${qu}/collections/chat_memory/points`, { points: [{ id: pid, vector: emb.embedding, payload: {\n text: content, feature: 'note', intent: 'note',\n username: username || 'unknown', topic: 'general', timestamp: pid\n }}]});\n }\n} catch(e) {}\n\nconst responseText = saved\n ? `기록했습니다: ${title}`\n : `기록을 시도했지만 저장에 실패했습니다. 내용: ${title}`;\n\nreturn [{ json: { text: responseText, userText, username, response_tier: 'local', intent: 'note', model: 'id-9b:latest', inputTokens: 0, outputTokens: 0 } }];" }, "id": "b1000001-0000-0000-0000-000000000069", "name": "Handle Note", @@ -1480,7 +1493,7 @@ "main": [ [ { - "node": "Route by Intent", + "node": "Set pid", "type": "main", "index": 0 }, @@ -1492,6 +1505,17 @@ ] ] }, + "Set pid": { + "main": [ + [ + { + "node": "Route by Intent", + "type": "main", + "index": 0 + } + ] + ] + }, "Route by Intent": { "main": [ [ diff --git a/restore_kb.sh b/restore_kb.sh new file mode 100755 index 0000000..b790d68 --- /dev/null +++ b/restore_kb.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# DS1525+ → 맥미니 knowledge-base 복구 스크립트 +rsync -az hyungi@192.168.1.227:/volume4/Document_Server/Main/knowledge-base-backup/ \ + /Users/hyungi/Documents/code/syn-chat-bot/knowledge-base/