hyungi
d05e41128a
- 흐름 컨테이너 pt/px 헤드룸 — -top/-right 돌출 뱃지가 overflow-x-auto 에 잘리던 문제 - 단계별 현황 details = 흐름 보드가 대체(R2 통합안 의도) — 전용 파생값/헬퍼/chevron 동반 제거 - etaShort: 48시간 이상은 일 단위 (약 131시간 → 약 5.5일) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
420 lines
20 KiB
Svelte
420 lines
20 KiB
Svelte
<script lang="ts">
|
|
// 처리 머신 보드 v2 — 파이프라인 흐름 뷰 (plan ds-board-engines-1, R2 통합안).
|
|
// 메인 = 좌→우 흐름 노드(병목 amber·실패 뱃지), 노드 클릭 = 상세 패널(안1 변형),
|
|
// 실패 뱃지 클릭 = 실패 처리 드로어 (재시도/건너뛰기 — 영구 실패의 유일한 조치 경로).
|
|
// 데이터 = GET /api/queue/overview (60s 폴링 store) + GET /api/queue/failed (드로어 열 때).
|
|
import { api } from '$lib/api';
|
|
import { refreshQueueOverview } from '$lib/stores/queueOverview';
|
|
import { addToast } from '$lib/stores/toast';
|
|
import {
|
|
AUX_NODES,
|
|
FLOW_NODES,
|
|
MACHINE_META,
|
|
type FlowNodeDef,
|
|
etaShort,
|
|
flowStageLabel,
|
|
formatAgeSec,
|
|
formatRate,
|
|
} from '$lib/utils/queueDisplay';
|
|
import type {
|
|
FailedItem,
|
|
FailedListResponse,
|
|
MachineCurrentItem,
|
|
QueueOverview,
|
|
QueueStageRow,
|
|
RetryResponse,
|
|
SkipResponse,
|
|
} from '$lib/types/queue';
|
|
|
|
let { overview }: { overview: QueueOverview } = $props();
|
|
|
|
// ─── 노드 통계 합성 ───
|
|
interface NodeStats {
|
|
def: FlowNodeDef;
|
|
/** 다중 stage 노드(청크·임베딩)는 같은 문서가 양쪽 큐에 있어 max — 합산 = 이중계산 */
|
|
pending: number;
|
|
processing: number;
|
|
failed: number; // 실패는 행 단위 사실이라 합산
|
|
done1h: number;
|
|
created1h: number;
|
|
doneToday: number;
|
|
oldestAgeSec: number | null;
|
|
etaMinutes: number | null;
|
|
inflowDominant: boolean;
|
|
perStage: QueueStageRow[];
|
|
}
|
|
|
|
const stageBy = $derived(new Map(overview.stages.map((s) => [s.stage, s])));
|
|
|
|
function nodeStats(def: FlowNodeDef): NodeStats {
|
|
const rows = def.stages
|
|
.map((s) => stageBy.get(s))
|
|
.filter((r): r is QueueStageRow => r != null);
|
|
const pending = rows.reduce((m, r) => Math.max(m, r.pending), 0);
|
|
const done1h = rows.reduce((m, r) => Math.max(m, r.done_1h), 0);
|
|
const created1h = rows.reduce((m, r) => Math.max(m, r.created_1h), 0);
|
|
const oldest = rows.reduce<number | null>(
|
|
(m, r) => (r.oldest_pending_age_sec == null ? m : Math.max(m ?? 0, r.oldest_pending_age_sec)),
|
|
null,
|
|
);
|
|
return {
|
|
def,
|
|
pending,
|
|
processing: rows.reduce((s, r) => s + r.processing, 0),
|
|
failed: rows.reduce((s, r) => s + r.failed, 0),
|
|
done1h,
|
|
created1h,
|
|
doneToday: rows.reduce((m, r) => Math.max(m, r.done_today), 0),
|
|
oldestAgeSec: oldest,
|
|
etaMinutes: pending > 0 && done1h > 0 ? Math.round((pending / done1h) * 60) : null,
|
|
inflowDominant: pending > 0 && created1h > done1h,
|
|
perStage: rows,
|
|
};
|
|
}
|
|
|
|
const mainNodes = $derived(FLOW_NODES.map(nodeStats));
|
|
const auxAll = $derived(AUX_NODES.map(nodeStats));
|
|
const auxActive = $derived(
|
|
auxAll.filter((n) => n.pending + n.processing + n.failed + n.doneToday > 0),
|
|
);
|
|
const auxIdle = $derived(
|
|
auxAll.filter((n) => n.pending + n.processing + n.failed + n.doneToday === 0),
|
|
);
|
|
const totalFailed = $derived(overview.totals.failed);
|
|
|
|
// 머신 스트립 — overview.machines 의 state/처리율 + 정적 모델 메타
|
|
const machineStrip = $derived(
|
|
overview.machines.map((m) => ({
|
|
...m,
|
|
meta: MACHINE_META[m.key],
|
|
})),
|
|
);
|
|
|
|
// ─── 선택 상태 (노드 상세 / 실패 드로어 — 동시에 하나만) ───
|
|
let selected = $state<string | null>(null);
|
|
let failOpen = $state(false);
|
|
|
|
function toggleNode(key: string) {
|
|
selected = selected === key ? null : key;
|
|
if (selected) failOpen = false;
|
|
}
|
|
|
|
const selectedNode = $derived(
|
|
[...mainNodes, ...auxAll].find((n) => n.def.key === selected) ?? null,
|
|
);
|
|
|
|
function nodeCurrent(def: FlowNodeDef): MachineCurrentItem[] {
|
|
return overview.machines.flatMap((m) => m.current.filter((c) => def.stages.includes(c.stage)));
|
|
}
|
|
|
|
// ─── 실패 드로어 ───
|
|
let failItems = $state<FailedItem[]>([]);
|
|
let failLoading = $state(false);
|
|
let busy = $state(false);
|
|
let expanded = $state<Record<string, boolean>>({});
|
|
|
|
async function openFailures() {
|
|
failOpen = true;
|
|
selected = null;
|
|
await loadFailures();
|
|
}
|
|
|
|
async function loadFailures() {
|
|
failLoading = true;
|
|
try {
|
|
const r = await api<FailedListResponse>('/queue/failed');
|
|
failItems = r.items;
|
|
} catch {
|
|
addToast('error', '실패 목록을 불러오지 못했습니다');
|
|
} finally {
|
|
failLoading = false;
|
|
}
|
|
}
|
|
|
|
interface FailGroup {
|
|
key: string;
|
|
stage: string;
|
|
pattern: string;
|
|
items: FailedItem[];
|
|
}
|
|
|
|
// 그룹핑 = stage + 에러 메시지 prefix(36자) — 같은 원인(ReadTimeout 등) 묶음
|
|
const failGroups = $derived.by(() => {
|
|
const map = new Map<string, FailGroup>();
|
|
for (const it of failItems) {
|
|
const pattern = (it.error_message ?? '(메시지 없음)').slice(0, 36);
|
|
const key = `${it.stage}::${pattern}`;
|
|
const g = map.get(key);
|
|
if (g) g.items.push(it);
|
|
else map.set(key, { key, stage: it.stage, pattern, items: [it] });
|
|
}
|
|
return [...map.values()].sort(
|
|
(a, b) => a.stage.localeCompare(b.stage) || b.items.length - a.items.length,
|
|
);
|
|
});
|
|
|
|
async function retryIds(ids: number[]) {
|
|
if (busy || ids.length === 0) return;
|
|
busy = true;
|
|
try {
|
|
const r = await api<RetryResponse>('/queue/retry', {
|
|
method: 'POST',
|
|
body: JSON.stringify({ ids }),
|
|
});
|
|
addToast(
|
|
'success',
|
|
`재시도 ${r.retried}건 큐 재진입${r.not_retried > 0 ? ` (${r.not_retried}건 제외 — 이미 활성/처리됨)` : ''}`,
|
|
);
|
|
await afterAction();
|
|
} catch {
|
|
addToast('error', '재시도 요청 실패');
|
|
} finally {
|
|
busy = false;
|
|
}
|
|
}
|
|
|
|
async function skipIds(ids: number[]) {
|
|
if (busy || ids.length === 0) return;
|
|
busy = true;
|
|
try {
|
|
const r = await api<SkipResponse>('/queue/skip', {
|
|
method: 'POST',
|
|
body: JSON.stringify({ ids }),
|
|
});
|
|
addToast('success', `건너뛰기 ${r.skipped}건 처리 (해당 단계 제외)`);
|
|
await afterAction();
|
|
} catch {
|
|
addToast('error', '건너뛰기 요청 실패');
|
|
} finally {
|
|
busy = false;
|
|
}
|
|
}
|
|
|
|
async function afterAction() {
|
|
await Promise.all([loadFailures(), refreshQueueOverview()]);
|
|
}
|
|
|
|
// ─── trend_24h 스파크라인 (summarize 유입 vs 소화 — API 가 주는데 미렌더이던 슬롯) ───
|
|
const spark = $derived.by(() => {
|
|
const t = overview.trend_24h;
|
|
if (!t || t.length === 0) return null;
|
|
const max = Math.max(1, ...t.map((b) => Math.max(b.inflow, b.done)));
|
|
const w = 120;
|
|
const h = 24;
|
|
const step = w / Math.max(1, t.length - 1);
|
|
const pts = (sel: (b: (typeof t)[number]) => number) =>
|
|
t.map((b, i) => `${(i * step).toFixed(1)},${(h - (sel(b) / max) * (h - 3) + 1).toFixed(1)}`).join(' ');
|
|
return { inflow: pts((b) => b.inflow), done: pts((b) => b.done) };
|
|
});
|
|
</script>
|
|
|
|
<div class="mt-5">
|
|
<!-- 헤더: 타이틀 + 요약 24h 스파크라인 + 실패 합계 -->
|
|
<div class="flex items-center justify-between gap-3 mb-3">
|
|
<div class="text-[11px] font-bold text-dim uppercase tracking-wider">처리 머신</div>
|
|
<div class="flex items-center gap-3">
|
|
{#if totalFailed > 0}
|
|
<button
|
|
class="text-[11px] font-semibold text-error hover:underline cursor-pointer"
|
|
onclick={openFailures}
|
|
>실패 {totalFailed}건 처리</button>
|
|
{/if}
|
|
{#if spark}
|
|
<div class="flex items-center gap-2 text-[10px] text-faint tabular-nums" title="요약(summarize) 단계 24시간 — 유입(회색) vs 소화(녹색)">
|
|
<svg width="120" height="24" viewBox="0 0 120 24" class="block">
|
|
<polyline points={spark.inflow} fill="none" stroke="currentColor" stroke-width="1.5" class="text-faint" />
|
|
<polyline points={spark.done} fill="none" stroke="currentColor" stroke-width="1.5" class="text-success" />
|
|
</svg>
|
|
<span>요약 24h 유입/소화</span>
|
|
</div>
|
|
{/if}
|
|
</div>
|
|
</div>
|
|
|
|
<!-- 머신 스트립 -->
|
|
<div class="flex flex-wrap gap-2 mb-3">
|
|
{#each machineStrip as m (m.key)}
|
|
<div class="flex items-center gap-2 bg-surface border border-default rounded-full px-3.5 py-1.5 text-xs">
|
|
<span class="w-2 h-2 rounded-full shrink-0 {m.state === 'active' ? 'bg-success' : m.state === 'deferred' ? 'bg-warning' : 'bg-faint'}"></span>
|
|
<span class="font-bold text-text">{m.meta?.label ?? m.label}</span>
|
|
<span class="text-[10px] text-faint font-mono">{m.meta?.model}</span>
|
|
<span class="text-[11px] text-dim tabular-nums">{formatRate(m.done_1h)}/h</span>
|
|
{#if m.key === 'macbook' && m.deferred_pending > 0}
|
|
<span class="text-[10px] font-semibold text-warning tabular-nums">보류 {m.deferred_pending}</span>
|
|
{/if}
|
|
</div>
|
|
{/each}
|
|
</div>
|
|
|
|
<!-- 흐름 노드 — pt/px 헤드룸 = 실패 뱃지(-top/-right 돌출)가 스크롤 컨테이너에 잘리지 않게 -->
|
|
<div class="flex items-stretch overflow-x-auto pt-2.5 pb-1 px-2 -mx-2">
|
|
{#each mainNodes as n, i (n.def.key)}
|
|
{#if i > 0}
|
|
<div class="flex items-center text-faint text-sm px-1.5 shrink-0" aria-hidden="true">→</div>
|
|
{/if}
|
|
<div
|
|
class="relative bg-surface border-[1.5px] rounded-card px-3 py-2.5 min-w-[124px] shrink-0 text-left transition-colors cursor-pointer hover:bg-surface-hover
|
|
{n.inflowDominant ? 'border-warning' : n.etaMinutes != null && n.def.stages.includes('chunk') ? 'border-success' : 'border-default'}
|
|
{selected === n.def.key ? 'node-sel' : ''}"
|
|
role="button"
|
|
tabindex="0"
|
|
onclick={() => toggleNode(n.def.key)}
|
|
onkeydown={(e) => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); toggleNode(n.def.key); } }}
|
|
title="{n.def.label} — 클릭하면 상세"
|
|
>
|
|
{#if n.failed > 0}
|
|
<button
|
|
class="absolute -top-2 -right-1.5 text-[9px] font-extrabold bg-error text-white rounded-full px-1.5 py-px shadow cursor-pointer"
|
|
onclick={(e) => { e.stopPropagation(); openFailures(); }}
|
|
title="실패 {n.failed}건 — 클릭하면 실패 처리"
|
|
>{n.failed}</button>
|
|
{/if}
|
|
<span class="inline-block text-[9px] font-bold rounded px-1.5 py-px mb-1.5 mtag-{n.def.machine}">
|
|
{MACHINE_META[n.def.machine].label} · {n.def.engine}
|
|
</span>
|
|
<div class="text-xs font-bold text-text flex items-center gap-1.5">
|
|
{n.def.label}
|
|
{#if n.processing > 0}
|
|
<span class="inline-block w-1.5 h-1.5 rounded-full bg-accent animate-pulse" title="처리 중 {n.processing}"></span>
|
|
{/if}
|
|
{#if n.inflowDominant}
|
|
<span class="text-[9px] font-bold text-warning">유입 우세</span>
|
|
{/if}
|
|
</div>
|
|
<div class="text-base font-extrabold tabular-nums tracking-tight leading-tight mt-0.5 text-text">
|
|
{n.pending.toLocaleString()}
|
|
</div>
|
|
<div class="text-[10px] text-dim tabular-nums">
|
|
{formatRate(n.done1h)}/h · 오늘 {n.doneToday.toLocaleString()}
|
|
{#if n.etaMinutes != null && !n.inflowDominant && n.pending > 0}
|
|
· <span class="text-accent font-semibold">{etaShort(n.etaMinutes)}</span>
|
|
{/if}
|
|
</div>
|
|
</div>
|
|
{/each}
|
|
</div>
|
|
|
|
<!-- 보조 라인 -->
|
|
<p class="text-[10px] text-faint mt-1.5 tabular-nums">
|
|
{#each auxActive as n, i (n.def.key)}
|
|
{i > 0 ? ' · ' : '보조: '}{n.def.label}({n.def.engine}) 대기 {n.pending.toLocaleString()} · {formatRate(n.done1h)}/h{n.failed > 0 ? ` · 실패 ${n.failed}` : ''}
|
|
{/each}
|
|
{#if auxIdle.length > 0}
|
|
{auxActive.length > 0 ? ' — ' : ''}한가: {auxIdle.map((n) => n.def.label).join(' · ')}
|
|
{/if}
|
|
— 뉴스 등 일부 소스는 분류/추출을 건너뜀 (흐름 그림은 대표 경로)
|
|
</p>
|
|
|
|
<!-- 상세 패널 (노드 클릭) -->
|
|
{#if selectedNode}
|
|
<div class="border rounded-card mt-3 overflow-hidden bg-surface detail-frame">
|
|
<div class="flex items-center gap-2.5 px-4 py-2.5 text-xs font-bold detail-head">
|
|
{selectedNode.def.label} — {selectedNode.def.engine}
|
|
<span class="text-[10px] font-mono font-medium text-dim bg-surface border border-default rounded px-1.5">{selectedNode.def.sub} · {MACHINE_META[selectedNode.def.machine].label}</span>
|
|
<button class="ml-auto text-[11px] text-dim font-normal cursor-pointer hover:text-text" onclick={() => (selected = null)}>닫기</button>
|
|
</div>
|
|
<div class="px-4 pb-3.5">
|
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-2.5 my-2.5">
|
|
<div class="bg-bg border border-default rounded-card px-3 py-2">
|
|
<div class="text-[9px] text-faint uppercase tracking-wide">대기</div>
|
|
<div class="text-lg font-extrabold tabular-nums text-text">{selectedNode.pending.toLocaleString()}</div>
|
|
</div>
|
|
<div class="bg-bg border border-default rounded-card px-3 py-2">
|
|
<div class="text-[9px] text-faint uppercase tracking-wide">처리율 (1h)</div>
|
|
<div class="text-lg font-extrabold tabular-nums text-text">{formatRate(selectedNode.done1h)}<span class="text-[11px] text-dim font-semibold">/h</span></div>
|
|
</div>
|
|
<div class="bg-bg border border-default rounded-card px-3 py-2">
|
|
<div class="text-[9px] text-faint uppercase tracking-wide">오늘 완료</div>
|
|
<div class="text-lg font-extrabold tabular-nums text-text">{selectedNode.doneToday.toLocaleString()}</div>
|
|
</div>
|
|
<div class="bg-bg border border-default rounded-card px-3 py-2">
|
|
<div class="text-[9px] text-faint uppercase tracking-wide">소진 예상</div>
|
|
<div class="text-lg font-extrabold tabular-nums {selectedNode.inflowDominant ? 'text-warning' : 'text-accent'}">
|
|
{#if selectedNode.inflowDominant}유입 우세{:else if selectedNode.etaMinutes != null}{etaShort(selectedNode.etaMinutes)}{:else if selectedNode.pending === 0}한가{:else}—{/if}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{#if selectedNode.perStage.length > 1}
|
|
{#each selectedNode.perStage as row (row.stage)}
|
|
<div class="flex items-center gap-2.5 py-1.5 border-t border-default text-xs">
|
|
<span class="font-semibold text-text min-w-[72px]">{flowStageLabel(row.stage)}</span>
|
|
<span class="ml-auto text-dim tabular-nums">
|
|
대기 <strong class="text-text">{row.pending.toLocaleString()}</strong>
|
|
· {formatRate(row.done_1h)}/h · 오늘 {row.done_today.toLocaleString()}
|
|
{#if row.failed > 0}· <span class="text-error font-semibold">실패 {row.failed}</span>{/if}
|
|
</span>
|
|
</div>
|
|
{/each}
|
|
{/if}
|
|
<div class="text-[11px] text-dim border-t border-dashed border-default mt-2 pt-2 tabular-nums">
|
|
{#if selectedNode.oldestAgeSec != null && selectedNode.oldestAgeSec > 600}
|
|
가장 오래 기다린 항목 {formatAgeSec(selectedNode.oldestAgeSec)}
|
|
{/if}
|
|
{#each nodeCurrent(selectedNode.def) as c, i (c.document_id + c.stage)}
|
|
{i === 0 && !(selectedNode.oldestAgeSec != null && selectedNode.oldestAgeSec > 600) ? '' : ' · '}지금: {c.title} ({flowStageLabel(c.stage)})
|
|
{/each}
|
|
{#if selectedNode.failed > 0}
|
|
· <button class="text-error font-semibold cursor-pointer hover:underline" onclick={openFailures}>실패 {selectedNode.failed}건 처리</button>
|
|
{/if}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{/if}
|
|
|
|
<!-- 실패 처리 드로어 -->
|
|
{#if failOpen}
|
|
<div class="border border-error/40 rounded-card mt-3 overflow-hidden bg-surface">
|
|
<div class="flex items-center gap-2.5 px-4 py-2.5 bg-error/5 text-xs font-bold text-text">
|
|
실패 처리
|
|
<span class="text-[10px] font-semibold text-error">영구 실패 {failItems.length}건 — 자동 재시도 3회 소진, 수동 조치 대기</span>
|
|
<button class="ml-auto text-[11px] text-dim font-normal cursor-pointer hover:text-text" onclick={() => (failOpen = false)}>닫기</button>
|
|
</div>
|
|
{#if failLoading}
|
|
<p class="text-xs text-dim text-center py-4">불러오는 중…</p>
|
|
{:else if failItems.length === 0}
|
|
<p class="text-xs text-dim text-center py-4">영구 실패 항목 없음</p>
|
|
{:else}
|
|
{#each failGroups as g (g.key)}
|
|
<div class="px-4 py-2.5 border-t border-default">
|
|
<div class="flex items-center gap-2 flex-wrap text-xs font-bold text-text mb-1">
|
|
{flowStageLabel(g.stage)} {g.items.length}건
|
|
<span class="text-[10px] font-mono font-medium text-error bg-error/10 rounded px-1.5 py-px">{g.pattern}{g.items[0]?.error_message && g.items[0].error_message.length > 36 ? '…' : ''}</span>
|
|
</div>
|
|
{#each expanded[g.key] ? g.items : g.items.slice(0, 4) as it (it.id)}
|
|
<div class="flex items-center gap-2.5 py-1 border-t border-dashed border-default/60 text-xs">
|
|
<span class="flex-1 min-w-0 truncate text-text" title={it.title}>{it.title}</span>
|
|
<span class="text-[10px] font-mono text-faint shrink-0 tabular-nums">시도 {it.attempts}/{it.max_attempts}</span>
|
|
<span class="text-[10px] font-mono text-error shrink-0 max-w-[260px] truncate" title={it.error_message ?? ''}>{it.error_message ?? ''}</span>
|
|
<button class="text-[10px] font-bold border border-accent text-accent rounded px-2 py-0.5 shrink-0 cursor-pointer hover:bg-accent/10 disabled:opacity-40" disabled={busy} onclick={() => retryIds([it.id])}>재시도</button>
|
|
<button class="text-[10px] font-bold border border-default text-faint rounded px-2 py-0.5 shrink-0 cursor-pointer hover:bg-surface-hover disabled:opacity-40" disabled={busy} onclick={() => skipIds([it.id])}>건너뛰기</button>
|
|
</div>
|
|
{/each}
|
|
{#if g.items.length > 4 && !expanded[g.key]}
|
|
<button class="text-[10px] text-dim cursor-pointer hover:text-text mt-1" onclick={() => (expanded = { ...expanded, [g.key]: true })}>… 외 {g.items.length - 4}건 펼치기</button>
|
|
{/if}
|
|
{#if g.items.length > 1}
|
|
<div class="flex gap-2 mt-1.5">
|
|
<button class="text-[10px] font-bold border border-accent text-accent rounded px-2.5 py-0.5 cursor-pointer hover:bg-accent/10 disabled:opacity-40" disabled={busy} onclick={() => retryIds(g.items.map((x) => x.id))}>그룹 전체 재시도 ({g.items.length})</button>
|
|
<button class="text-[10px] font-bold border border-default text-faint rounded px-2.5 py-0.5 cursor-pointer hover:bg-surface-hover disabled:opacity-40" disabled={busy} onclick={() => skipIds(g.items.map((x) => x.id))}>그룹 전체 건너뛰기</button>
|
|
</div>
|
|
{/if}
|
|
</div>
|
|
{/each}
|
|
<p class="text-[10px] text-faint px-4 py-2 border-t border-default">
|
|
재시도 = 시도 횟수 리셋 후 큐 재진입 (자동 재시도 3회 새로 부여) · 건너뛰기 = 이 단계 완료 처리(후속 단계 연쇄 없음, 감사 마킹) · 같은 오류가 반복되는 항목(빈 텍스트 등)은 건너뛰기 권장
|
|
</p>
|
|
{/if}
|
|
</div>
|
|
{/if}
|
|
</div>
|
|
|
|
<style>
|
|
/* 머신 색 — 디자인 토큰 외 3색 (gpu 청/macmini 보라/macbook 황) — 이 컴포넌트 한정 */
|
|
.mtag-gpu { background: #e7eef6; color: #3b6ea5; }
|
|
.mtag-macmini { background: #efe9f7; color: #8a5fbf; }
|
|
.mtag-macbook { background: #f7eedd; color: #b07a10; }
|
|
.node-sel { outline: 2px solid #3b6ea5; outline-offset: 1px; }
|
|
.detail-frame { border-color: #3b6ea5; }
|
|
.detail-head { background: #e7eef6; }
|
|
</style>
|