feat(observability): 큐 밖 백그라운드 작업(backfill)을 처리 머신 보드에 노출

processing_queue 는 파이프라인 stage 전용이라 hier_overnight_backfill 같은 off-queue
관리 스크립트 작업이 대시보드 보드에 안 잡혀, 다른 세션이 모르고 fastapi 를 재생성해
in-flight 재분해를 끊는 사고가 발생(2026-06-14). 사각지대 해소.

- migrations/357_background_jobs.sql: background_jobs 테이블(kind/label/state/processed/
  total/heartbeat). worker_jobs(user_id 필수, worker-pool 전용)와 별개.
- services/background_jobs.py: start/heartbeat/finish 헬퍼 — 자율 트랜잭션(즉시 commit →
  실시간 가시화) + best-effort(관측 실패가 본작업 안 깸).
- hier_overnight_backfill: 작업 시작/절 ~10개마다 heartbeat/종료 계측.
- queue_overview: /api/queue/overview 응답에 background_jobs 추가(running + 최근 6h 완료,
  stale=heartbeat 끊김 추정). SAVEPOINT 로 테이블 부재/오류 시 보드 본체 무영향.
- ProcessingFlowBoard: "백그라운드 작업" 패널(진행/경과/state, stale 끊김 경고).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/hier_overnight_backfill.py

@@ -32,6 +32,7 @@ from core.config import settings
 from services.hier_decomp.builder import build_hier_tree
 from services.hier_decomp.persist import persist_hier_tree
 from services.search.llm_gate import Priority, acquire_mlx_gate
+from services.background_jobs import finish_job, heartbeat, start_job
 
 # 단일 진실: 절 분석 상수/헬퍼 (PROMPT_VERSION 일치 = 멱등 보존)
 from section_summary_pilot import (
@@ -140,8 +141,10 @@ def _make_engine():
     return create_async_engine(os.environ["DATABASE_URL"], pool_pre_ping=True)
 
 
-async def _analyze_doc_leaves(session, client, doc_id, doc_domain, model_name, stop_at):
-    """doc 의 미분석 hier leaf 분석 → upsert. stop_at(epoch) 넘으면 leaf 경계 중단."""
+async def _analyze_doc_leaves(session, client, doc_id, doc_domain, model_name, stop_at,
+                              engine=None, job_id=None, base_processed=0):
+    """doc 의 미분석 hier leaf 분석 → upsert. stop_at(epoch) 넘으면 leaf 경계 중단.
+    engine/job_id 주어지면 background_jobs 에 ~10절마다 진행 heartbeat(보드 가시화)."""
     rows = (await session.execute(LEAF_SQL, {"doc": doc_id, "pv": PROMPT_VERSION})).mappings().all()
     ok = fail = skip = 0
     timings, types = [], []
@@ -187,6 +190,8 @@ async def _analyze_doc_leaves(session, client, doc_id, doc_domain, model_name, s
             "content_hash": r["content_hash"], "error": err,
         })
         await session.commit()
+        if job_id and (ok + fail + skip) % 10 == 0:
+            await heartbeat(engine, job_id, processed=base_processed + ok + fail + skip)
     await session.commit()
     return {"ok": ok, "fail": fail, "skip": skip, "leaves": len(rows),
             "timings": timings, "types": types, "aborted": aborted}
@@ -256,6 +261,12 @@ async def cmd_run(args):
                      _candidate_params(allowlist, doc_ids))).mappings().all()
         _log(f"후보 doc {len(cands)} 선별. 시작.")
 
+        # 관측: 큐 밖 작업이라 대시보드 보드가 못 보므로 background_jobs 에 진행 노출(best-effort)
+        _job_kind = "hier_redecompose" if reprocess else "hier_backfill"
+        _job_label = (f"doc {args.doc} {'재분해' if reprocess else '분해'}" if doc_ids
+                      else f"{len(cands)}개 문서 {'재분해' if reprocess else '분해'}")
+        job_id = await start_job(engine, _job_kind, _job_label, total=None)
+
         for c in cands:
             if time.time() >= stop_at:
                 _log(f"⏰ deadline 버퍼 도달 — doc 경계에서 중단 (처리 {tot_docs} doc)")
@@ -272,7 +283,10 @@ async def cmd_run(args):
                              "timings": [], "types": [], "aborted": False}
                 else:
                     async with sm() as session:
-                        astat = await _analyze_doc_leaves(session, client, doc_id, doc_domain, model_name, stop_at)
+                        astat = await _analyze_doc_leaves(
+                            session, client, doc_id, doc_domain, model_name, stop_at,
+                            engine=engine, job_id=job_id,
+                            base_processed=(tot_ok + tot_fail + tot_skip))
             except Exception as exc:
                 _log(f"  ✗ doc={doc_id} 처리 실패(건너뜀): {type(exc).__name__}: {repr(exc)[:160]}")
                 continue
@@ -280,6 +294,8 @@ async def cmd_run(args):
             tot_docs += 1
             tot_ok += astat["ok"]; tot_fail += astat["fail"]; tot_skip += astat["skip"]
             all_timings += astat["timings"]; all_types += astat["types"]
+            await heartbeat(engine, job_id, processed=(tot_ok + tot_fail + tot_skip),
+                            total=tot_leaves_created)
             avg = statistics.mean(astat["timings"]) if astat["timings"] else 0
             _log(f"  ✓ doc={doc_id} ({len(body):,}자 {doc_domain.split('/')[0]}) "
                  f"leaf생성={leaves_created} 분석ok={astat['ok']} fail={astat['fail']} skip={astat['skip']} "
@@ -287,6 +303,7 @@ async def cmd_run(args):
             if astat["aborted"]:
                 _log("⏰ leaf 분석 중 deadline 도달 — 중단")
                 break
+        await finish_job(engine, job_id, state="done")
     finally:
         await client.close()
         await engine.dispose()