Phase 3 완료: 파일 처리 시스템 구축

 주요 완성 기능:
- 프로젝트 생성 API (project_name 필드 포함)
- 엑셀 파일 업로드 및 파싱 시스템
- 자재 DB 저장 (2837개 자재 성공 저장)
- 자재 조회 및 요약 통계 API
- 외래키 관계 정상 동작 (projects -> files -> materials)

📊 테스트 결과:
- MP7 PIPING PROJECT Rev.2 프로젝트 생성
- 00.MP7 PIPING R.2_BOM.XLS 파일 업로드 성공
- NIPPLE, PIPE 등 자재 분류 및 재질 추출
- ASTM A106, SCH 80, 1인치 사이즈 등 정확 파싱

🛠️ 기술 스택:
- FastAPI + PostgreSQL + SQLAlchemy
- pandas를 활용한 엑셀 파싱
- 외래키 제약조건 적용된 정규화 DB 설계
This commit is contained in:
Hyungi Ahn
2025-07-14 13:19:24 +09:00
parent 2f9107ca55
commit 13c375477a
5 changed files with 499 additions and 68 deletions

View File

@@ -0,0 +1 @@
# API 라우터 패키지

375
backend/app/api/files.py Normal file
View File

@@ -0,0 +1,375 @@
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form
from sqlalchemy.orm import Session
from sqlalchemy import text
from typing import List, Optional
import os
import shutil
from datetime import datetime
import uuid
import pandas as pd
import re
from pathlib import Path
from ..database import get_db
router = APIRouter()
UPLOAD_DIR = Path("uploads")
UPLOAD_DIR.mkdir(exist_ok=True)
ALLOWED_EXTENSIONS = {".xlsx", ".xls", ".csv"}
@router.get("/")
async def get_files_info():
return {
"message": "파일 관리 API",
"allowed_extensions": list(ALLOWED_EXTENSIONS),
"upload_directory": str(UPLOAD_DIR)
}
@router.get("/test")
async def test_endpoint():
return {"status": "파일 API가 정상 작동합니다!"}
@router.post("/add-missing-columns")
async def add_missing_columns(db: Session = Depends(get_db)):
"""누락된 컬럼들 추가"""
try:
db.execute(text("ALTER TABLE files ADD COLUMN IF NOT EXISTS parsed_count INTEGER DEFAULT 0"))
db.execute(text("ALTER TABLE materials ADD COLUMN IF NOT EXISTS row_number INTEGER"))
db.commit()
return {
"success": True,
"message": "누락된 컬럼들이 추가되었습니다",
"added_columns": ["files.parsed_count", "materials.row_number"]
}
except Exception as e:
db.rollback()
return {"success": False, "error": f"컬럼 추가 실패: {str(e)}"}
def validate_file_extension(filename: str) -> bool:
return Path(filename).suffix.lower() in ALLOWED_EXTENSIONS
def generate_unique_filename(original_filename: str) -> str:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
unique_id = str(uuid.uuid4())[:8]
stem = Path(original_filename).stem
suffix = Path(original_filename).suffix
return f"{stem}_{timestamp}_{unique_id}{suffix}"
def parse_dataframe(df):
df = df.dropna(how='all')
df.columns = df.columns.str.strip().str.lower()
column_mapping = {
'description': ['description', 'item', 'material', '품명', '자재명'],
'quantity': ['qty', 'quantity', 'ea', '수량'],
'main_size': ['main_nom', 'nominal_diameter', 'nd', '주배관'],
'red_size': ['red_nom', 'reduced_diameter', '축소배관'],
'length': ['length', 'len', '길이'],
'weight': ['weight', 'wt', '중량'],
'dwg_name': ['dwg_name', 'drawing', '도면명'],
'line_num': ['line_num', 'line_number', '라인번호']
}
mapped_columns = {}
for standard_col, possible_names in column_mapping.items():
for possible_name in possible_names:
if possible_name in df.columns:
mapped_columns[standard_col] = possible_name
break
materials = []
for index, row in df.iterrows():
description = str(row.get(mapped_columns.get('description', ''), ''))
quantity_raw = row.get(mapped_columns.get('quantity', ''), 0)
try:
quantity = float(quantity_raw) if pd.notna(quantity_raw) else 0
except:
quantity = 0
material_grade = ""
if "ASTM" in description.upper():
astm_match = re.search(r'ASTM\s+([A-Z0-9\s]+)', description.upper())
if astm_match:
material_grade = astm_match.group(0).strip()
main_size = str(row.get(mapped_columns.get('main_size', ''), ''))
red_size = str(row.get(mapped_columns.get('red_size', ''), ''))
if main_size != 'nan' and red_size != 'nan' and red_size != '':
size_spec = f"{main_size} x {red_size}"
elif main_size != 'nan' and main_size != '':
size_spec = main_size
else:
size_spec = ""
if description and description not in ['nan', 'None', '']:
materials.append({
'original_description': description,
'quantity': quantity,
'unit': "EA",
'size_spec': size_spec,
'material_grade': material_grade,
'line_number': index + 1,
'row_number': index + 1
})
return materials
def parse_file_data(file_path):
file_extension = Path(file_path).suffix.lower()
try:
if file_extension == ".csv":
df = pd.read_csv(file_path, encoding='utf-8')
elif file_extension in [".xlsx", ".xls"]:
df = pd.read_excel(file_path, sheet_name=0)
else:
raise HTTPException(status_code=400, detail="지원하지 않는 파일 형식")
return parse_dataframe(df)
except Exception as e:
raise HTTPException(status_code=400, detail=f"파일 파싱 실패: {str(e)}")
@router.post("/upload")
async def upload_file(
file: UploadFile = File(...),
project_id: int = Form(...),
revision: str = Form("Rev.0"),
db: Session = Depends(get_db)
):
if not validate_file_extension(file.filename):
raise HTTPException(
status_code=400,
detail=f"지원하지 않는 파일 형식입니다. 허용된 확장자: {', '.join(ALLOWED_EXTENSIONS)}"
)
if file.size and file.size > 10 * 1024 * 1024:
raise HTTPException(status_code=400, detail="파일 크기는 10MB를 초과할 수 없습니다")
unique_filename = generate_unique_filename(file.filename)
file_path = UPLOAD_DIR / unique_filename
try:
with open(file_path, "wb") as buffer:
shutil.copyfileobj(file.file, buffer)
except Exception as e:
raise HTTPException(status_code=500, detail=f"파일 저장 실패: {str(e)}")
try:
materials_data = parse_file_data(str(file_path))
parsed_count = len(materials_data)
# 파일 정보 저장
file_insert_query = text("""
INSERT INTO files (filename, original_filename, file_path, project_id, revision, description, file_size, parsed_count, is_active)
VALUES (:filename, :original_filename, :file_path, :project_id, :revision, :description, :file_size, :parsed_count, :is_active)
RETURNING id
""")
file_result = db.execute(file_insert_query, {
"filename": unique_filename,
"original_filename": file.filename,
"file_path": str(file_path),
"project_id": project_id,
"revision": revision,
"description": f"BOM 파일 - {parsed_count}개 자재",
"file_size": file.size,
"parsed_count": parsed_count,
"is_active": True
})
file_id = file_result.fetchone()[0]
# 자재 데이터 저장
materials_inserted = 0
for material_data in materials_data:
material_insert_query = text("""
INSERT INTO materials (
file_id, original_description, quantity, unit, size_spec,
material_grade, line_number, row_number, classified_category,
classification_confidence, is_verified, created_at
)
VALUES (
:file_id, :original_description, :quantity, :unit, :size_spec,
:material_grade, :line_number, :row_number, :classified_category,
:classification_confidence, :is_verified, :created_at
)
""")
db.execute(material_insert_query, {
"file_id": file_id,
"original_description": material_data["original_description"],
"quantity": material_data["quantity"],
"unit": material_data["unit"],
"size_spec": material_data["size_spec"],
"material_grade": material_data["material_grade"],
"line_number": material_data["line_number"],
"row_number": material_data["row_number"],
"classified_category": None,
"classification_confidence": None,
"is_verified": False,
"created_at": datetime.now()
})
materials_inserted += 1
db.commit()
return {
"success": True,
"message": f"완전한 DB 저장 성공! {materials_inserted}개 자재 저장됨",
"original_filename": file.filename,
"file_id": file_id,
"parsed_materials_count": parsed_count,
"saved_materials_count": materials_inserted,
"sample_materials": materials_data[:3] if materials_data else []
}
except Exception as e:
db.rollback()
if os.path.exists(file_path):
os.remove(file_path)
raise HTTPException(status_code=500, detail=f"파일 처리 실패: {str(e)}")
@router.get("/materials")
async def get_materials(
project_id: Optional[int] = None,
file_id: Optional[int] = None,
skip: int = 0,
limit: int = 100,
db: Session = Depends(get_db)
):
"""저장된 자재 목록 조회"""
try:
query = """
SELECT m.id, m.file_id, m.original_description, m.quantity, m.unit,
m.size_spec, m.material_grade, m.line_number, m.row_number,
m.created_at,
f.original_filename, f.project_id,
p.official_project_code, p.project_name
FROM materials m
LEFT JOIN files f ON m.file_id = f.id
LEFT JOIN projects p ON f.project_id = p.id
WHERE 1=1
"""
params = {}
if project_id:
query += " AND f.project_id = :project_id"
params["project_id"] = project_id
if file_id:
query += " AND m.file_id = :file_id"
params["file_id"] = file_id
query += " ORDER BY m.line_number ASC LIMIT :limit OFFSET :skip"
params["limit"] = limit
params["skip"] = skip
result = db.execute(text(query), params)
materials = result.fetchall()
# 전체 개수 조회
count_query = """
SELECT COUNT(*) as total
FROM materials m
LEFT JOIN files f ON m.file_id = f.id
WHERE 1=1
"""
count_params = {}
if project_id:
count_query += " AND f.project_id = :project_id"
count_params["project_id"] = project_id
if file_id:
count_query += " AND m.file_id = :file_id"
count_params["file_id"] = file_id
count_result = db.execute(text(count_query), count_params)
total_count = count_result.fetchone()[0]
return {
"success": True,
"total_count": total_count,
"returned_count": len(materials),
"skip": skip,
"limit": limit,
"materials": [
{
"id": m.id,
"file_id": m.file_id,
"filename": m.original_filename,
"project_id": m.project_id,
"project_code": m.official_project_code,
"project_name": m.project_name,
"original_description": m.original_description,
"quantity": float(m.quantity) if m.quantity else 0,
"unit": m.unit,
"size_spec": m.size_spec,
"material_grade": m.material_grade,
"line_number": m.line_number,
"row_number": m.row_number,
"created_at": m.created_at
}
for m in materials
]
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"자재 조회 실패: {str(e)}")
@router.get("/materials/summary")
async def get_materials_summary(
project_id: Optional[int] = None,
file_id: Optional[int] = None,
db: Session = Depends(get_db)
):
"""자재 요약 통계"""
try:
query = """
SELECT
COUNT(*) as total_items,
COUNT(DISTINCT m.original_description) as unique_descriptions,
COUNT(DISTINCT m.size_spec) as unique_sizes,
COUNT(DISTINCT m.material_grade) as unique_materials,
SUM(m.quantity) as total_quantity,
AVG(m.quantity) as avg_quantity,
MIN(m.created_at) as earliest_upload,
MAX(m.created_at) as latest_upload
FROM materials m
LEFT JOIN files f ON m.file_id = f.id
WHERE 1=1
"""
params = {}
if project_id:
query += " AND f.project_id = :project_id"
params["project_id"] = project_id
if file_id:
query += " AND m.file_id = :file_id"
params["file_id"] = file_id
result = db.execute(text(query), params)
summary = result.fetchone()
return {
"success": True,
"summary": {
"total_items": summary.total_items,
"unique_descriptions": summary.unique_descriptions,
"unique_sizes": summary.unique_sizes,
"unique_materials": summary.unique_materials,
"total_quantity": float(summary.total_quantity) if summary.total_quantity else 0,
"avg_quantity": round(float(summary.avg_quantity), 2) if summary.avg_quantity else 0,
"earliest_upload": summary.earliest_upload,
"latest_upload": summary.latest_upload
}
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"요약 조회 실패: {str(e)}")

View File

@@ -1,23 +1,25 @@
from fastapi import FastAPI, Depends, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy.orm import Session
from sqlalchemy import text
from typing import List
from datetime import datetime
from . import models, schemas
from .database import SessionLocal, engine, get_db
from .database import get_db, engine
from .models import Base, Project
from .schemas import ProjectCreate, ProjectResponse
from .api import files
# 데이터베이스 테이블 생성 (이미 존재하면 무시)
models.Base.metadata.create_all(bind=engine)
Base.metadata.create_all(bind=engine)
app = FastAPI(
title="TK-MP BOM System API",
description="BOM (Bill of Materials) 시스템 API - 실제 데이터베이스 연동",
version="2.0.0"
title="TK-MP-Project API",
description="BOM 시스템 개발 프로젝트 - Phase 3: 파일 처리 시스템",
version="1.0.0",
docs_url="/docs",
redoc_url="/redoc"
)
# CORS 설정
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
@@ -26,62 +28,101 @@ app.add_middleware(
allow_headers=["*"],
)
app.include_router(files.router, prefix="/api/files", tags=["파일 관리"])
@app.get("/")
async def root():
return {
"message": "TK-MP BOM System API",
"version": "2.0.0",
"message": "TK-MP-Project API Server",
"version": "1.0.0 - Phase 3",
"status": "running",
"database": "PostgreSQL 연결됨"
"timestamp": datetime.now().isoformat(),
"new_features": [
"✅ Phase 1: 기반 시스템 구축",
"✅ Phase 2: 데이터베이스 연동",
"🔄 Phase 3: 파일 처리 시스템 개발 중"
]
}
@app.get("/health")
async def health_check(db: Session = Depends(get_db)):
try:
# 데이터베이스 연결 테스트 (SQLAlchemy 2.0 문법)
result = db.execute(text("SELECT 1"))
result = db.execute(text("SELECT 1 as test"))
test_value = result.fetchone()[0]
return {
"status": "healthy",
"database": "connected",
"api": "operational",
"version": "2.0.0",
"db_test": "SUCCESS"
"status": "healthy",
"database": "connected",
"test_query": test_value == 1,
"timestamp": datetime.now().isoformat(),
"phase": "Phase 3 - 파일 처리 시스템"
}
except Exception as e:
return {
"status": "error",
"database": "disconnected",
"error": str(e)
}
raise HTTPException(status_code=500, detail=f"데이터베이스 연결 실패: {str(e)}")
# 프로젝트 API
@app.get("/api/projects", response_model=schemas.ProjectResponse)
@app.get("/api/projects", response_model=List[ProjectResponse])
async def get_projects(db: Session = Depends(get_db)):
try:
projects = db.query(models.Project).all()
return {
"projects": projects,
"total": len(projects),
"message": f"{len(projects)}개 프로젝트 조회됨"
}
result = db.execute(text("""
SELECT id, official_project_code, project_name, design_project_code,
is_code_matched, status, created_at, updated_at
FROM projects
ORDER BY created_at DESC
"""))
projects = result.fetchall()
return [
ProjectResponse(
id=project.id,
official_project_code=project.official_project_code,
project_name=project.project_name,
design_project_code=project.design_project_code,
is_code_matched=project.is_code_matched,
status=project.status,
created_at=project.created_at,
updated_at=project.updated_at
)
for project in projects
]
except Exception as e:
raise HTTPException(status_code=500, detail=f"데이터베이스 오류: {str(e)}")
raise HTTPException(status_code=500, detail=f"프로젝트 조회 실패: {str(e)}")
@app.post("/api/projects", response_model=schemas.Project)
async def create_project(project: schemas.ProjectCreate, db: Session = Depends(get_db)):
@app.post("/api/projects", response_model=ProjectResponse)
async def create_project(project: ProjectCreate, db: Session = Depends(get_db)):
try:
db_project = models.Project(**project.dict())
db.add(db_project)
insert_query = text("""
INSERT INTO projects (official_project_code, project_name, design_project_code, is_code_matched, status, created_at)
VALUES (:official_code, :project_name, :design_code, :is_matched, :status, :created_at)
RETURNING id, official_project_code, project_name, design_project_code, is_code_matched, status, created_at, updated_at
""")
result = db.execute(insert_query, {
"official_code": project.official_project_code,
"project_name": project.project_name,
"design_code": project.design_project_code,
"is_matched": project.is_code_matched,
"status": project.status,
"created_at": datetime.now()
})
new_project = result.fetchone()
db.commit()
db.refresh(db_project)
return db_project
return ProjectResponse(
id=new_project.id,
official_project_code=new_project.official_project_code,
project_name=new_project.project_name,
design_project_code=new_project.design_project_code,
is_code_matched=new_project.is_code_matched,
status=new_project.status,
created_at=new_project.created_at,
updated_at=new_project.updated_at
)
except Exception as e:
db.rollback()
raise HTTPException(status_code=500, detail=f"프로젝트 생성 오류: {str(e)}")
raise HTTPException(status_code=500, detail=f"프로젝트 생성 실패: {str(e)}")
@app.get("/api/projects/{project_id}", response_model=schemas.Project)
async def get_project(project_id: int, db: Session = Depends(get_db)):
project = db.query(models.Project).filter(models.Project.id == project_id).first()
if project is None:
raise HTTPException(status_code=404, detail="프로젝트를 찾을 수 없습니다")
return project
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000, reload=True)

View File

@@ -1,36 +1,47 @@
from pydantic import BaseModel
from pydantic import BaseModel, Field
from datetime import datetime
from typing import Optional, List
from decimal import Decimal
# 프로젝트 스키마
# Project Schemas (project_name 추가)
class ProjectBase(BaseModel):
official_project_code: Optional[str] = None
project_name: str
client_name: Optional[str] = None
design_project_code: Optional[str] = None
design_project_name: Optional[str] = None
description: Optional[str] = None
notes: Optional[str] = None
official_project_code: str = Field(..., description="공식 프로젝트 코드")
project_name: str = Field(..., description="프로젝트명") # 추가
design_project_code: Optional[str] = Field(None, description="설계 프로젝트 코드")
is_code_matched: bool = Field(False, description="코드 매칭 여부")
status: str = Field("active", description="프로젝트 상태")
class ProjectCreate(ProjectBase):
pass
class ProjectUpdate(ProjectBase):
project_name: Optional[str] = None
class Project(ProjectBase):
class ProjectResponse(ProjectBase):
id: int
is_code_matched: bool
status: str
created_at: datetime
updated_at: datetime
updated_at: Optional[datetime] = None
class Config:
from_attributes = True
# 응답 스키마
class ProjectResponse(BaseModel):
projects: List[Project]
total: int
# File Schemas (기존 유지)
class FileResponse(BaseModel):
id: int
filename: str
original_filename: str
file_path: str
project_id: int
project_code: Optional[str] = None
revision: str = "Rev.0"
description: Optional[str] = None
upload_date: datetime
parsed_count: int = 0
is_active: bool = True
class Config:
from_attributes = True
class FileUploadResponse(BaseModel):
success: bool
message: str
file_id: int
filename: str
parsed_materials_count: int
file_path: str