From a050f2e7d52ffd54ef69031e27309b7b7e8b0509 Mon Sep 17 00:00:00 2001 From: Hyungi Ahn Date: Thu, 19 Mar 2026 07:50:34 +0900 Subject: [PATCH] =?UTF-8?q?fix:=20kb=5Fwriter=20=ED=8C=8C=EC=9D=BC?= =?UTF-8?q?=EB=AA=85=20=EA=B2=BD=ED=95=A9=20=EC=88=98=EC=A0=95=20+=20qdran?= =?UTF-8?q?t=5Fid=20=EC=9D=BC=EA=B4=80=EC=84=B1=20=EB=B3=B4=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - kb_writer: uuid4 short hash로 파일명 경합 방지, counter 기반 중복 방어 제거 - kb_writer: qdrant_id 외부 수신 지원 (body.qdrant_id) - n8n: Set pid 노드 추가 — 분기 전 pid 한 번 생성, Handle Note/Embed & Save Memory에 전달 - Handle Note/Embed & Save Memory: 동일 pid를 kb_writer(qdrant_id)와 Qdrant point ID에 사용 - restore_kb.sh: DS1525+ → 맥미니 knowledge-base 복구 스크립트 추가 Co-Authored-By: Claude Opus 4.6 --- kb_writer.py | 19 +++++++---------- n8n/workflows/main-chat-pipeline.json | 30 ++++++++++++++++++++++++--- restore_kb.sh | 4 ++++ 3 files changed, 38 insertions(+), 15 deletions(-) create mode 100755 restore_kb.sh diff --git a/kb_writer.py b/kb_writer.py index 9e909c2..752b89e 100644 --- a/kb_writer.py +++ b/kb_writer.py @@ -9,6 +9,7 @@ import re import unicodedata from datetime import datetime, timezone, timedelta from pathlib import Path +from uuid import uuid4 from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -46,31 +47,25 @@ async def save(request: Request): date_str = now.strftime("%Y-%m-%d") iso_str = now.isoformat() - # 파일명 생성 + # 파일명 생성 (uuid4 short hash로 경합 방지) slug = _slugify(title) + uid = uuid4().hex[:6] if doc_type == "chat-memory": time_str = now.strftime("%H%M") - filename = f"{date_str}T{time_str}-{_slugify(topic)}.md" + filename = f"{date_str}T{time_str}-{uid}-{_slugify(topic)}.md" elif doc_type == "news": - filename = f"{date_str}-{_slugify(source)}-{slug}.md" + filename = f"{date_str}-{uid}-{_slugify(source)}-{slug}.md" else: - filename = f"{date_str}-{slug}.md" + filename = f"{date_str}-{uid}-{slug}.md" # 디렉토리 생성 type_dir = BASE_DIR / doc_type / month_dir type_dir.mkdir(parents=True, exist_ok=True) - - # 중복 파일명 처리 filepath = type_dir / filename - counter = 1 - while filepath.exists(): - stem = filename.rsplit(".", 1)[0] - filepath = type_dir / f"{stem}-{counter}.md" - counter += 1 # YAML frontmatter + 본문 tags_yaml = ", ".join(f'"{t}"' for t in tags) - qdrant_id = int(now.timestamp() * 1000) + qdrant_id = body.get("qdrant_id") or int(now.timestamp() * 1000) md_content = f"""--- title: "{title}" diff --git a/n8n/workflows/main-chat-pipeline.json b/n8n/workflows/main-chat-pipeline.json index 215bfab..389a934 100644 --- a/n8n/workflows/main-chat-pipeline.json +++ b/n8n/workflows/main-chat-pipeline.json @@ -428,6 +428,19 @@ 1200 ] }, + { + "parameters": { + "jsCode": "const cls = $input.first().json;\nreturn [{ json: { ...cls, pid: Date.now() } }];" + }, + "id": "b1000001-0000-0000-0000-000000000070", + "name": "Set pid", + "type": "n8n-nodes-base.code", + "typeVersion": 1, + "position": [ + 1430, + 1200 + ] + }, { "parameters": { "operation": "executeQuery", @@ -1005,7 +1018,7 @@ }, { "parameters": { - "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst data = $input.first().json;\nconst prompt = `Q: ${data.userText}\\nA: ${data.aiText}`;\nconst emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model:'bge-m3', prompt });\nif (!emb.embedding||!Array.isArray(emb.embedding)) return [{json:{saved:false}}];\nconst pid = Date.now();\nconst qu = $env.QDRANT_URL||'http://host.docker.internal:6333';\nawait httpPut(`${qu}/collections/chat_memory/points`, { points:[{ id:pid, vector:emb.embedding, payload:{\n text:prompt, feature:'chat', intent:data.intent||'unknown',\n username:data.username||'unknown', topic:data.topic||'general', timestamp:pid\n}}]});\n// kb_writer 파일 저장 (graceful)\ntry {\n const kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\n await httpPost(`${kbUrl}/save`, {\n title: `${new Date().toISOString().split('T')[0]} 대화 메모`,\n content: prompt,\n type: 'chat-memory',\n tags: ['chat-memory', data.topic || 'general'],\n username: data.username || 'unknown',\n topic: data.topic || 'general'\n }, { timeout: 5000 });\n} catch(e) {}\n\nreturn [{json:{saved:true,pointId:pid,topic:data.topic}}];" + "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst data = $input.first().json;\nconst pid = $('Set pid').first().json.pid || Date.now();\nconst prompt = `Q: ${data.userText}\\nA: ${data.aiText}`;\nconst emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model:'bge-m3', prompt });\nif (!emb.embedding||!Array.isArray(emb.embedding)) return [{json:{saved:false}}];\nconst qu = $env.QDRANT_URL||'http://host.docker.internal:6333';\nawait httpPut(`${qu}/collections/chat_memory/points`, { points:[{ id:pid, vector:emb.embedding, payload:{\n text:prompt, feature:'chat', intent:data.intent||'unknown',\n username:data.username||'unknown', topic:data.topic||'general', timestamp:pid\n}}]});\n// kb_writer 파일 저장 (graceful)\ntry {\n const kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\n await httpPost(`${kbUrl}/save`, {\n title: `${new Date().toISOString().split('T')[0]} 대화 메모`,\n content: prompt,\n type: 'chat-memory',\n tags: ['chat-memory', data.topic || 'general'],\n username: data.username || 'unknown',\n topic: data.topic || 'general',\n qdrant_id: pid\n }, { timeout: 5000 });\n} catch(e) {}\n\nreturn [{json:{saved:true,pointId:pid,topic:data.topic}}];" }, "id": "b1000001-0000-0000-0000-000000000037", "name": "Embed & Save Memory", @@ -1186,7 +1199,7 @@ }, { "parameters": { - "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' \\u2192 ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' \\u2192 timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst cls = $('Qwen Classify v2').first().json;\nconst userText = cls.userText;\nconst username = cls.username;\n\n// \"기록해\", \"메모해\", \"저장해\", \"적어둬\" 등 제거\nconst content = userText\n .replace(/[.]\\s*(기록해|메모해|저장해|적어둬|기록|메모|저장)[.]?\\s*$/g, '')\n .replace(/^\\s*(기록해|메모해|저장해|적어둬)[.:]\\s*/g, '')\n .trim() || userText;\n\n// 제목: 앞 30자\nconst title = content.substring(0, 30).replace(/\\n/g, ' ') + (content.length > 30 ? '...' : '');\nconst now = new Date();\nconst dateStr = `${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}`;\nconst fullTitle = `${dateStr} ${title}`;\n\n// kb_writer 파일 저장\nconst kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\nlet saved = false;\ntry {\n const result = await httpPost(`${kbUrl}/save`, {\n title: fullTitle,\n content: content,\n type: 'note',\n tags: ['synology-chat', 'note'],\n username: username || 'unknown',\n topic: 'general'\n }, { timeout: 10000 });\n saved = result.success === true;\n} catch(e) {}\n\n// Qdrant 임베딩 (graceful)\ntry {\n const emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model: 'bge-m3', prompt: content });\n if (emb.embedding && Array.isArray(emb.embedding)) {\n const pid = Date.now();\n const qu = $env.QDRANT_URL || 'http://host.docker.internal:6333';\n await httpPut(`${qu}/collections/chat_memory/points`, { points: [{ id: pid, vector: emb.embedding, payload: {\n text: content, feature: 'note', intent: 'note',\n username: username || 'unknown', topic: 'general', timestamp: pid\n }}]});\n }\n} catch(e) {}\n\nconst responseText = saved\n ? `기록했습니다: ${title}`\n : `기록을 시도했지만 저장에 실패했습니다. 내용: ${title}`;\n\nreturn [{ json: { text: responseText, userText, username, response_tier: 'local', intent: 'note', model: 'id-9b:latest', inputTokens: 0, outputTokens: 0 } }];" + "jsCode": "function httpPost(url, body, { timeout = 15000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'POST',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' \\u2192 ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' \\u2192 timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\nfunction httpPut(url, body, { timeout = 10000, headers = {} } = {}) {\n return new Promise((resolve, reject) => {\n const data = JSON.stringify(body);\n const u = require('url').parse(url);\n const mod = require(u.protocol === 'https:' ? 'https' : 'http');\n const req = mod.request({\n hostname: u.hostname, port: u.port, path: u.path,\n method: 'PUT',\n headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data), ...headers }\n }, (res) => {\n let body = '';\n res.on('data', c => body += c);\n res.on('end', () => {\n if (res.statusCode >= 400) return reject(new Error(url + ' → ' + res.statusCode + ': ' + body.slice(0, 200)));\n try { resolve(JSON.parse(body)); } catch(e) { reject(new Error('JSON parse error: ' + body.slice(0, 200))); }\n });\n });\n req.on('error', reject);\n req.setTimeout(timeout, () => { req.destroy(); reject(new Error(url + ' → timeout after ' + timeout + 'ms')); });\n req.write(data);\n req.end();\n });\n}\n\nconst cls = $('Qwen Classify v2').first().json;\nconst pid = $json.pid || Date.now();\nconst userText = cls.userText;\nconst username = cls.username;\n\n// \"기록해\", \"메모해\", \"저장해\", \"적어둬\" 등 제거\nconst content = userText\n .replace(/[.]\\s*(기록해|메모해|저장해|적어둬|기록|메모|저장)[.]?\\s*$/g, '')\n .replace(/^\\s*(기록해|메모해|저장해|적어둬)[.:]\\s*/g, '')\n .trim() || userText;\n\n// 제목: 앞 30자\nconst title = content.substring(0, 30).replace(/\\n/g, ' ') + (content.length > 30 ? '...' : '');\nconst now = new Date();\nconst dateStr = `${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}`;\nconst fullTitle = `${dateStr} ${title}`;\n\n// kb_writer 파일 저장\nconst kbUrl = $env.KB_WRITER_URL || 'http://host.docker.internal:8095';\nlet saved = false;\ntry {\n const result = await httpPost(`${kbUrl}/save`, {\n title: fullTitle,\n content: content,\n type: 'note',\n tags: ['synology-chat', 'note'],\n username: username || 'unknown',\n topic: 'general',\n qdrant_id: pid\n }, { timeout: 10000 });\n saved = result.success === true;\n} catch(e) {}\n\n// Qdrant 임베딩 (graceful)\ntry {\n const emb = await httpPost(`${$env.LOCAL_OLLAMA_URL}/api/embeddings`, { model: 'bge-m3', prompt: content });\n if (emb.embedding && Array.isArray(emb.embedding)) {\n const qu = $env.QDRANT_URL || 'http://host.docker.internal:6333';\n await httpPut(`${qu}/collections/chat_memory/points`, { points: [{ id: pid, vector: emb.embedding, payload: {\n text: content, feature: 'note', intent: 'note',\n username: username || 'unknown', topic: 'general', timestamp: pid\n }}]});\n }\n} catch(e) {}\n\nconst responseText = saved\n ? `기록했습니다: ${title}`\n : `기록을 시도했지만 저장에 실패했습니다. 내용: ${title}`;\n\nreturn [{ json: { text: responseText, userText, username, response_tier: 'local', intent: 'note', model: 'id-9b:latest', inputTokens: 0, outputTokens: 0 } }];" }, "id": "b1000001-0000-0000-0000-000000000069", "name": "Handle Note", @@ -1480,7 +1493,7 @@ "main": [ [ { - "node": "Route by Intent", + "node": "Set pid", "type": "main", "index": 0 }, @@ -1492,6 +1505,17 @@ ] ] }, + "Set pid": { + "main": [ + [ + { + "node": "Route by Intent", + "type": "main", + "index": 0 + } + ] + ] + }, "Route by Intent": { "main": [ [ diff --git a/restore_kb.sh b/restore_kb.sh new file mode 100755 index 0000000..b790d68 --- /dev/null +++ b/restore_kb.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# DS1525+ → 맥미니 knowledge-base 복구 스크립트 +rsync -az hyungi@192.168.1.227:/volume4/Document_Server/Main/knowledge-base-backup/ \ + /Users/hyungi/Documents/code/syn-chat-bot/knowledge-base/