7cdeac20cf
Instead of requiring DEVONthink export, reads files directly from .dtBase2 bundle's Files.noindex/ directory structure. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
236 lines
8.3 KiB
Python
236 lines
8.3 KiB
Python
"""DEVONthink → NAS PKM 마이그레이션 스크립트
|
|
|
|
.dtBase2 번들의 Files.noindex/ 디렉토리에서 직접 파일을 추출하여
|
|
NAS PKM 폴더 구조로 복사하고 DB에 등록합니다.
|
|
|
|
사용법:
|
|
# Dry-run (실제 복사/DB 등록 없이 시뮬레이션)
|
|
python scripts/migrate_from_devonthink.py --source-dir ~/Documents/Databases --dry-run
|
|
|
|
# 실제 실행
|
|
python scripts/migrate_from_devonthink.py \
|
|
--source-dir ~/Documents/Databases \
|
|
--target-dir /mnt/nas/Document_Server \
|
|
--database-url postgresql+asyncpg://pkm:PASSWORD@localhost:15432/pkm
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import os
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "app"))
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from core.utils import file_hash, setup_logger
|
|
|
|
logger = setup_logger("migrate")
|
|
|
|
# ─── DEVONthink DB → NAS PKM 폴더 매핑 ───
|
|
FOLDER_MAPPING = {
|
|
"00_Inbox DB": "PKM/Inbox",
|
|
"Inbox": "PKM/Inbox",
|
|
"00_Note_BOX": "PKM/Knowledge",
|
|
"01_Philosophie": "PKM/Knowledge/Philosophy",
|
|
"02_Language": "PKM/Knowledge/Language",
|
|
"03_Engineering": "PKM/Knowledge/Engineering",
|
|
"04_Industrial safety": "PKM/Knowledge/Industrial_Safety",
|
|
"05_Programming": "PKM/Knowledge/Programming",
|
|
"07_General Book": "PKM/Knowledge/General",
|
|
"97_Production drawing": "PKM/References",
|
|
"99_Reference Data": "PKM/References",
|
|
"99_Home File": "PKM/References",
|
|
"Archive": "PKM/Archive",
|
|
"Projects": "PKM/Knowledge",
|
|
"99_Technicalkorea": "Technicalkorea",
|
|
# 스킵 대상
|
|
"98_명일방주 엔드필드": None,
|
|
}
|
|
|
|
# 무시할 파일
|
|
SKIP_NAMES = {".DS_Store", "Thumbs.db", "desktop.ini", "Icon\r", "Icon"}
|
|
SKIP_EXTENSIONS = {".dtMeta", ".dtBase2", ".sparseimage", ".dtStore", ".dtCloud"}
|
|
|
|
|
|
def should_skip(path: Path) -> bool:
|
|
if path.name in SKIP_NAMES or path.name.startswith("._"):
|
|
return True
|
|
if path.suffix.lower() in SKIP_EXTENSIONS:
|
|
return True
|
|
if path.stat().st_size == 0:
|
|
return True
|
|
return False
|
|
|
|
|
|
async def migrate(
|
|
source_dir: str,
|
|
target_dir: str,
|
|
database_url: str,
|
|
dry_run: bool = False,
|
|
batch_size: int = 100,
|
|
):
|
|
"""마이그레이션 실행"""
|
|
source = Path(source_dir)
|
|
target = Path(target_dir)
|
|
|
|
if not source.exists():
|
|
logger.error(f"소스 디렉토리 없음: {source}")
|
|
return
|
|
|
|
engine = create_async_engine(database_url)
|
|
async_session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
stats = {"total": 0, "copied": 0, "skipped": 0, "duplicates": 0, "errors": 0}
|
|
batch = []
|
|
|
|
# .dtBase2 번들 탐색
|
|
for dtbase in sorted(source.glob("*.dtBase2")):
|
|
db_name = dtbase.stem # "04_Industrial safety"
|
|
target_prefix = FOLDER_MAPPING.get(db_name)
|
|
|
|
if target_prefix is None:
|
|
logger.info(f"[스킵] {db_name} (매핑: None)")
|
|
continue
|
|
|
|
files_dir = dtbase / "Files.noindex"
|
|
if not files_dir.exists():
|
|
logger.warning(f"[스킵] {db_name}: Files.noindex 없음")
|
|
continue
|
|
|
|
logger.info(f"[DB] {db_name} → {target_prefix}")
|
|
|
|
# Files.noindex 하위의 모든 파일 (format/hash/filename.ext 구조)
|
|
files = [f for f in files_dir.rglob("*") if f.is_file() and not should_skip(f)]
|
|
|
|
for source_file in files:
|
|
stats["total"] += 1
|
|
|
|
# 대상 경로: PKM/{domain}/{파일명}
|
|
dest_rel = f"{target_prefix}/{source_file.name}"
|
|
dest_path = target / dest_rel
|
|
|
|
if dry_run:
|
|
logger.info(f"[DRY-RUN] {source_file.name} → {dest_rel}")
|
|
stats["copied"] += 1
|
|
continue
|
|
|
|
try:
|
|
# 파일 복사
|
|
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 중복 파일명 처리
|
|
counter = 1
|
|
stem, suffix = dest_path.stem, dest_path.suffix
|
|
while dest_path.exists():
|
|
dest_path = dest_path.parent / f"{stem}_{counter}{suffix}"
|
|
dest_rel = str(dest_path.relative_to(target))
|
|
counter += 1
|
|
|
|
shutil.copy2(source_file, dest_path)
|
|
|
|
ext = source_file.suffix.lstrip(".").lower() or "unknown"
|
|
fhash = file_hash(dest_path)
|
|
fsize = dest_path.stat().st_size
|
|
|
|
batch.append({
|
|
"file_path": dest_rel,
|
|
"file_hash": fhash,
|
|
"file_format": ext,
|
|
"file_size": fsize,
|
|
"file_type": "immutable",
|
|
"import_source": f"devonthink:{db_name}",
|
|
"title": source_file.stem,
|
|
"source_channel": "manual",
|
|
})
|
|
stats["copied"] += 1
|
|
|
|
except Exception as e:
|
|
logger.error(f"[오류] {source_file}: {e}")
|
|
stats["errors"] += 1
|
|
|
|
# 배치 커밋
|
|
if len(batch) >= batch_size:
|
|
dups = await _insert_batch(async_session_factory, batch)
|
|
stats["duplicates"] += dups
|
|
batch.clear()
|
|
logger.info(f" 진행: {stats['copied']}건 처리됨")
|
|
|
|
# 남은 배치 처리
|
|
if batch and not dry_run:
|
|
dups = await _insert_batch(async_session_factory, batch)
|
|
stats["duplicates"] += dups
|
|
|
|
await engine.dispose()
|
|
|
|
logger.info("=" * 50)
|
|
logger.info(f"마이그레이션 {'시뮬레이션' if dry_run else '완료'}")
|
|
logger.info(f" 전체 파일: {stats['total']}")
|
|
logger.info(f" 복사/등록: {stats['copied']}")
|
|
logger.info(f" 스킵: {stats['skipped']}")
|
|
logger.info(f" 중복: {stats['duplicates']}")
|
|
logger.info(f" 오류: {stats['errors']}")
|
|
|
|
|
|
async def _insert_batch(async_session_factory, batch: list[dict]) -> int:
|
|
"""배치 단위로 documents + processing_queue 삽입"""
|
|
duplicates = 0
|
|
async with async_session_factory() as session:
|
|
for item in batch:
|
|
try:
|
|
result = await session.execute(
|
|
text("""
|
|
INSERT INTO documents (file_path, file_hash, file_format, file_size,
|
|
file_type, import_source, title, source_channel)
|
|
VALUES (:file_path, :file_hash, :file_format, :file_size,
|
|
:file_type, :import_source, :title, :source_channel)
|
|
ON CONFLICT (file_path) DO NOTHING
|
|
RETURNING id
|
|
"""),
|
|
item,
|
|
)
|
|
row = result.fetchone()
|
|
if row is None:
|
|
duplicates += 1
|
|
continue
|
|
|
|
doc_id = row[0]
|
|
await session.execute(
|
|
text("""
|
|
INSERT INTO processing_queue (document_id, stage, status)
|
|
VALUES (:doc_id, 'extract', 'pending')
|
|
ON CONFLICT DO NOTHING
|
|
"""),
|
|
{"doc_id": doc_id},
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"등록 실패: {item['file_path']}: {e}")
|
|
|
|
await session.commit()
|
|
return duplicates
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="DEVONthink → NAS PKM 마이그레이션")
|
|
parser.add_argument("--source-dir", required=True, help="DEVONthink Databases 디렉토리")
|
|
parser.add_argument("--target-dir", default="/mnt/nas/Document_Server", help="NAS 루트 경로")
|
|
parser.add_argument(
|
|
"--database-url",
|
|
default="postgresql+asyncpg://pkm:pkm@localhost:15432/pkm",
|
|
help="PostgreSQL 연결 URL",
|
|
)
|
|
parser.add_argument("--dry-run", action="store_true", help="시뮬레이션만 실행")
|
|
parser.add_argument("--batch-size", type=int, default=100, help="배치 커밋 크기")
|
|
|
|
args = parser.parse_args()
|
|
asyncio.run(migrate(
|
|
source_dir=args.source_dir,
|
|
target_dir=args.target_dir,
|
|
database_url=args.database_url,
|
|
dry_run=args.dry_run,
|
|
batch_size=args.batch_size,
|
|
))
|