diff --git a/reports/freshness_decay_observation_baseline.json b/reports/freshness_decay_observation_baseline.json new file mode 100644 index 0000000..3f96591 --- /dev/null +++ b/reports/freshness_decay_observation_baseline.json @@ -0,0 +1,368 @@ +[ + { + "query": "중대재해 사고", + "n_results": 5, + "results": [ + { + "id": 3854, + "title": "산업안전보건법 (20251001) 제4장_유해ㆍ위험_방지_조치", + "score": 1.0, + "policy": "law_365d", + "age_days": 29, + "decay_factor": 0.9451, + "base_score": 0.7133, + "adjusted_score": 0.7016 + }, + { + "id": 10571, + "title": "대표_중대재해_유형과_재발방지", + "score": 0.7625, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.6859, + "adjusted_score": 0.6859 + }, + { + "id": 10573, + "title": "산업안전보건법_개요", + "score": 0.525, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.6834, + "adjusted_score": 0.6834 + }, + { + "id": 3922, + "title": "중대재해 처벌 등에 관한 법률 시행령 (20251001) 제3장_중대시민재해", + "score": 0.2875, + "policy": "law_365d", + "age_days": 29, + "decay_factor": 0.9451, + "base_score": 0.6872, + "adjusted_score": 0.6759 + }, + { + "id": 3877, + "title": "산업안전보건법 시행규칙 (20250530) 제4장_유해ㆍ위험_방지_조치", + "score": 0.05, + "policy": "law_365d", + "age_days": 29, + "decay_factor": 0.9451, + "base_score": 0.673, + "adjusted_score": 0.662 + } + ], + "timing_ms": { + "freshness_ms": 0.486, + "total_ms": 250.096, + "rerank_ms": 4.69 + } + }, + { + "query": "최근 중대재해", + "n_results": 5, + "results": [ + { + "id": 10571, + "title": "대표_중대재해_유형과_재발방지", + "score": 1.0, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.6274, + "adjusted_score": 0.6274 + }, + { + "id": 11566, + "title": "06_분진폭발", + "score": 0.7625, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.5959, + "adjusted_score": 0.5959 + }, + { + "id": 3922, + "title": "중대재해 처벌 등에 관한 법률 시행령 (20251001) 제3장_중대시민재해", + "score": 0.525, + "policy": "law_365d", + "age_days": 29, + "decay_factor": 0.9451, + "base_score": 0.6012, + "adjusted_score": 0.5913 + }, + { + "id": 6695, + "title": "정부 중대재해 근절 기조 통했나…올 1분기 산재 사망자 ‘역대 최저’", + "score": 0.2875, + "policy": "news_90d", + "age_days": 18, + "decay_factor": 0.8675, + "base_score": 0.4892, + "adjusted_score": 0.4698 + }, + { + "id": 11825, + "title": "‘아리셀’ 대폭 감형에 “중대재해법 양형 기준 설정” 목소리", + "score": 0.05, + "policy": "news_90d", + "age_days": 5, + "decay_factor": 0.958, + "base_score": 0.0159, + "adjusted_score": 0.0157 + } + ], + "timing_ms": { + "freshness_ms": 0.487, + "total_ms": 286.772, + "rerank_ms": 4.604 + } + }, + { + "query": "산안법 개정", + "n_results": 5, + "results": [ + { + "id": 10572, + "title": "밀폐공간_작업_안전기준", + "score": 1.0, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.6013, + "adjusted_score": 0.6013 + }, + { + "id": 4026, + "title": "고압가스 안전관리법 시행령 (20260317) 전문", + "score": 0.7625, + "policy": "law_365d", + "age_days": 29, + "decay_factor": 0.9451, + "base_score": 0.606, + "adjusted_score": 0.596 + }, + { + "id": 10573, + "title": "산업안전보건법_개요", + "score": 0.525, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.572, + "adjusted_score": 0.572 + }, + { + "id": 5229, + "title": "사업체들의 산업안전 활동 최근 동향과 과제", + "score": 0.2875, + "policy": null, + "age_days": 24, + "decay_factor": null, + "base_score": 0.0161, + "adjusted_score": 0.0161 + }, + { + "id": 10569, + "title": "MSDS_읽는법", + "score": 0.05, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.0159, + "adjusted_score": 0.0159 + } + ], + "timing_ms": { + "freshness_ms": 3.055, + "total_ms": 199.741, + "rerank_ms": 4.592 + } + }, + { + "query": "KGS Code 개정", + "n_results": 5, + "results": [ + { + "id": 11647, + "title": "04_KGS_Code", + "score": 1.0, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.717, + "adjusted_score": 0.717 + }, + { + "id": 13914, + "title": "KGS FP112 § 1.5~1.6 — 경과조치·용품 사용제한", + "score": 0.7625, + "policy": null, + "age_days": 0, + "decay_factor": null, + "base_score": 0.6865, + "adjusted_score": 0.6865 + }, + { + "id": 11692, + "title": "05_KGS_GC_도시가스", + "score": 0.525, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.6585, + "adjusted_score": 0.6585 + }, + { + "id": 11693, + "title": "06_KGS_체계종합", + "score": 0.2875, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.5594, + "adjusted_score": 0.5594 + }, + { + "id": 11691, + "title": "04_KGS_AC_용기", + "score": 0.05, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.4961, + "adjusted_score": 0.4961 + } + ], + "timing_ms": { + "freshness_ms": 0.434, + "total_ms": 271.985, + "rerank_ms": 4.739 + } + }, + { + "query": "위험성평가 최근 동향", + "n_results": 5, + "results": [ + { + "id": 5243, + "title": "위험성평가 사업장 구축 및 실행방안", + "score": 1.0, + "policy": null, + "age_days": 24, + "decay_factor": null, + "base_score": 0.6768, + "adjusted_score": 0.6768 + }, + { + "id": 5229, + "title": "사업체들의 산업안전 활동 최근 동향과 과제", + "score": 0.7625, + "policy": null, + "age_days": 24, + "decay_factor": null, + "base_score": 0.6587, + "adjusted_score": 0.6587 + }, + { + "id": 10574, + "title": "위험성평가_KRAS_절차", + "score": 0.525, + "policy": null, + "age_days": 8, + "decay_factor": null, + "base_score": 0.6365, + "adjusted_score": 0.6365 + }, + { + "id": 11685, + "title": "03_위험성평가기법", + "score": 0.2875, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.4886, + "adjusted_score": 0.4886 + }, + { + "id": 11568, + "title": "08_위험성평가지표", + "score": 0.05, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.4755, + "adjusted_score": 0.4755 + } + ], + "timing_ms": { + "freshness_ms": 0.43, + "total_ms": 284.218, + "rerank_ms": 4.936 + } + }, + { + "query": "가스 사고 최근 사례", + "n_results": 5, + "results": [ + { + "id": 11684, + "title": "02_사고사례", + "score": 1.0, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.8395, + "adjusted_score": 0.8395 + }, + { + "id": 11564, + "title": "04_BLEVE", + "score": 0.7625, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.7673, + "adjusted_score": 0.7673 + }, + { + "id": 11565, + "title": "05_UVCE", + "score": 0.525, + "policy": null, + "age_days": 5, + "decay_factor": null, + "base_score": 0.7629, + "adjusted_score": 0.7629 + }, + { + "id": 7107, + "title": "청주 가스 폭발 사고 전날 ‘경보기 오작동’", + "score": 0.2875, + "policy": "news_90d", + "age_days": 17, + "decay_factor": 0.8742, + "base_score": 0.5377, + "adjusted_score": 0.5174 + }, + { + "id": 6707, + "title": "“가스 냄새 신고했는데 이상 없다더니”... 청주 폭발 사고 ‘인재’ 논란", + "score": 0.05, + "policy": "news_90d", + "age_days": 18, + "decay_factor": 0.8675, + "base_score": 0.5257, + "adjusted_score": 0.5048 + } + ], + "timing_ms": { + "freshness_ms": 0.444, + "total_ms": 349.87, + "rerank_ms": 4.863 + } + } +] diff --git a/reports/freshness_decay_observation_baseline.md b/reports/freshness_decay_observation_baseline.md new file mode 100644 index 0000000..11c8c10 --- /dev/null +++ b/reports/freshness_decay_observation_baseline.md @@ -0,0 +1,103 @@ +# PR-RAG-Time-1 — 1주 운영 관찰 Baseline (Snapshot) + +**측정일**: 2026-05-03 (배포 직후, main `5185501`) +**대상 endpoint**: `GET /api/search/?q=...&debug=true&limit=5` +**비교 시점**: 2026-05-10 경 1주 후 동일 6 쿼리 재측정 → ranking / freshness_ms / total_ms 회귀 점검 +**원본 JSON**: `reports/freshness_decay_observation_baseline.json` (1주 후 자동 비교 가능) + +## 4 관찰 포인트 + +1. **news/law_monitor 가 과도하게 boost 되지 않는지** — multiplier floor 0.7 안에서 동작하므로 base score 가 약하면 freshness 만으로 상위 못 올라가야 함. +2. **오래된 핵심 문서가 부당하게 밀리지 않는지** — 산안법/KGS Code 원문(law_monitor) 이 floor 0.7 보장에 의해 사라지지 않아야 함. +3. **drive_sync / manual / Study / ai_drafted 비적용 가드 유지** — policy=None 으로 base 그대로. +4. **freshness_ms 와 total latency 회귀 없음** — 현 baseline freshness_ms 0.4~3.1ms / total 200~350ms. + +회귀 신호 (1주 후 실측): +- top 3 ranking 의 doc_id 변동이 6 쿼리 중 3 이상 발생 +- freshness_ms p95 > 10ms (현재 max 3.06ms 대비 3× 초과) +- total_ms p95 > 500ms +- 사용자 명시 피드백 ("검색 결과가 이상하다") +- `policy != None` 인 row 가 비정상적으로 적게/많게 나오는 분포 변화 + +## 6 쿼리 baseline 요약 + +### 쿼리별 정책 적용 분포 +| 쿼리 | n | law_365d | news_90d | None | freshness_ms | total_ms | +|---|---:|---:|---:|---:|---:|---:| +| 중대재해 사고 | 5 | 3 | 0 | 2 | 0.486 | 250 | +| 최근 중대재해 | 5 | 1 | 2 | 2 | 0.487 | 287 | +| 산안법 개정 | 5 | 1 | 0 | 4 | 3.055 | 200 | +| KGS Code 개정 | 5 | 0 | 0 | 5 | 0.434 | 272 | +| 위험성평가 최근 동향 | 5 | 0 | 0 | 5 | 0.430 | 284 | +| 가스 사고 최근 사례 | 5 | 0 | 2 | 3 | 0.444 | 350 | +| **합계 30** | | **5** | **4** | **21** | avg 0.89 | avg 274 | + +### 쿼리별 top-5 doc_id (1주 후 비교 기준) +``` +중대재해 사고: 3854 → 10571 → 10573 → 3922 → 3877 +최근 중대재해: 10571 → 11566 → 3922 → 6695 → 11825 +산안법 개정: 10572 → 4026 → 10573 → 5229 → 10569 +KGS Code 개정: 11647 → 13914 → 11692 → 11693 → 11691 +위험성평가 최근 동향: 5243 → 5229 → 10574 → 11685 → 11568 +가스 사고 최근 사례: 11684 → 11564 → 11565 → 7107 → 6707 +``` + +## 발현된 정책 sample (검산) + +| query | doc_id | source | age (일) | base | adj | ratio | policy | +|---|---:|---|---:|---:|---:|---:|---| +| 중대재해 사고 | 3854 | law_monitor | 29 | 0.7133 | 0.7016 | 0.9835 | law_365d | +| 중대재해 사고 | 10571 | drive_sync | 8 | 0.6859 | 0.6859 | 1.0000 | None | +| 중대재해 사고 | 3922 | law_monitor | 29 | 0.6872 | 0.6759 | 0.9835 | law_365d | +| 최근 중대재해 | 6695 | news | 18 | (계산) | (계산) | (계산) | news_90d | +| 가스 사고 최근 사례 | 7107 | news | 17 | (계산) | (계산) | (계산) | news_90d | + +검산: age=29, half_life=365 → decay = exp(-ln(2) × 29/365) = **0.9451**. +multiplier = 0.7 + 0.3 × 0.9451 = **0.9835** (실측 ratio 와 일치). ✓ + +## 관찰 신호 (현재 시점 메모) + +**중립**: +- news/law_monitor 가 30 rows 중 9건 (30%) 발현 — 6 쿼리가 시사 도메인 키워드 위주이므로 적정. +- floor 0.7 가드 안에 있어 base score 가 약한 row 는 절대 상위 침투 못 함 (관찰 포인트 1 통과 조건). + +**잠재 회귀 후보**: +- "최근 중대재해" 쿼리에서 top 2 가 manual/drive_sync 8일 → 5일 짜리 학습 자료. news/law (18일) 는 4-5위. 사용자가 "최근" 키워드로 뉴스를 기대했는데 manual 이 우선이라면 1주 후 사용 패턴에 따라 base score(reranker) 재조정 필요. 단 이건 freshness 문제 아닌 reranker semantic match 의 문제. +- "가스 사고 최근 사례" 도 동일 패턴 — manual(BLEVE/UVCE 학습) 이 news(폭발 사고 기사) 보다 위. + +## 1주 후 비교 절차 + +```bash +# GPU 서버에서 1주 후 재실행 +TOKEN=$(...) +python3 /tmp/observe_freshness.py "$TOKEN" > /tmp/obs_week1.json + +# 로컬에서 baseline vs week1 diff +python3 -c " +import json +b = json.load(open('reports/freshness_decay_observation_baseline.json')) +w = json.load(open('/tmp/obs_week1.json')) +for qb, qw in zip(b, w): + bids = [r['id'] for r in qb['results'][:3]] + wids = [r['id'] for r in qw['results'][:3]] + if bids != wids: + print(f'{qb[\"query\"]}: top3 변동 {bids} → {wids}') + fb, fw = qb['timing_ms']['freshness_ms'], qw['timing_ms']['freshness_ms'] + if fw > 3 * fb: + print(f'{qb[\"query\"]}: freshness_ms 회귀 {fb}ms → {fw}ms') +" +``` + +회귀 발견 시 → rollback 검토 (search_pipeline.py:303~307 hook 비활성 또는 freshness_decay.py 의 `apply_freshness_decay` early-return). + +## 관련 파일 +- 구현: `app/services/search/freshness_decay.py` +- hook: `app/services/search/search_pipeline.py:303-307` +- schema: `app/api/search.py` `SearchResult.freshness_debug` +- tests: `tests/test_freshness_decay.py` (31 passed) +- plan: `~/.claude/plans/pr-rag-time-1-freshness-decay.md` +- 배포 commit: `5185501` + +## 1주 후 결과 (2026-05-12) + +PASS. top3 변동 0/6, freshness_ms max 0.54ms, total_ms max 413ms, policy 분포 동일 (9/30). 상세: `reports/freshness_decay_observation_week1.md`. diff --git a/reports/freshness_decay_observation_week1.json b/reports/freshness_decay_observation_week1.json new file mode 100644 index 0000000..44f4d32 --- /dev/null +++ b/reports/freshness_decay_observation_week1.json @@ -0,0 +1,368 @@ +[ + { + "query": "중대재해 사고", + "n_results": 5, + "results": [ + { + "id": 3854, + "title": "산업안전보건법 (20251001) 제4장_유해ㆍ위험_방지_조치", + "score": 1.0, + "policy": "law_365d", + "age_days": 39, + "decay_factor": 0.9285731073999445, + "base_score": 0.7132124121225515, + "adjusted_score": 0.6979296482140402 + }, + { + "id": 10571, + "title": "대표_중대재해_유형과_재발방지", + "score": 0.7625, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.685895461549119, + "adjusted_score": 0.685895461549119 + }, + { + "id": 10573, + "title": "산업안전보건법_개요", + "score": 0.525, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.6832846108871227, + "adjusted_score": 0.6832846108871227 + }, + { + "id": 3922, + "title": "중대재해 처벌 등에 관한 법률 시행령 (20251001) 제3장_중대시민재해", + "score": 0.2875, + "policy": "law_365d", + "age_days": 39, + "decay_factor": 0.9285763583755632, + "base_score": 0.6871882281328004, + "adjusted_score": 0.6724637824123938 + }, + { + "id": 3877, + "title": "산업안전보건법 시행규칙 (20250530) 제4장_유해ㆍ위험_방지_조치", + "score": 0.05, + "policy": "law_365d", + "age_days": 39, + "decay_factor": 0.9285744087487491, + "base_score": 0.6729559732730971, + "adjusted_score": 0.6585360897899696 + } + ], + "timing_ms": { + "freshness_ms": 0.5000130040571094, + "rerank_ms": 4.807750985492021, + "total_ms": 372.6260180119425 + } + }, + { + "query": "최근 중대재해", + "n_results": 5, + "results": [ + { + "id": 10571, + "title": "대표_중대재해_유형과_재발방지", + "score": 1.0, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.6274382672459488, + "adjusted_score": 0.6274382672459488 + }, + { + "id": 11566, + "title": "06_분진폭발", + "score": 0.7625, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.5959534486577056, + "adjusted_score": 0.5959534486577056 + }, + { + "id": 3922, + "title": "중대재해 처벌 등에 관한 법률 시행령 (20251001) 제3장_중대시민재해", + "score": 0.525, + "policy": "law_365d", + "age_days": 39, + "decay_factor": 0.9285763520501911, + "base_score": 0.6011851065468198, + "adjusted_score": 0.5883034565260193 + }, + { + "id": 6695, + "title": "정부 중대재해 근절 기조 통했나…올 1분기 산재 사망자 ‘역대 최저’", + "score": 0.2875, + "policy": "news_90d", + "age_days": 27, + "decay_factor": 0.8076323203924516, + "base_score": 0.4892148272700555, + "adjusted_score": 0.4609820909245911 + }, + { + "id": 11825, + "title": "‘아리셀’ 대폭 감형에 “중대재해법 양형 기준 설정” 목소리", + "score": 0.05, + "policy": "news_90d", + "age_days": 14, + "decay_factor": 0.8918135950791707, + "base_score": 0.015873015873015872, + "adjusted_score": 0.015357842516250018 + } + ], + "timing_ms": { + "freshness_ms": 0.5435149651020765, + "rerank_ms": 5.434333987068385, + "total_ms": 309.91031700978056 + } + }, + { + "query": "산안법 개정", + "n_results": 5, + "results": [ + { + "id": 10572, + "title": "밀폐공간_작업_안전기준", + "score": 1.0, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.6013045048418865, + "adjusted_score": 0.6013045048418865 + }, + { + "id": 4026, + "title": "고압가스 안전관리법 시행령 (20260317) 전문", + "score": 0.7625, + "policy": "law_365d", + "age_days": 39, + "decay_factor": 0.9285866891843126, + "base_score": 0.6059691664879325, + "adjusted_score": 0.5929868871585948 + }, + { + "id": 10573, + "title": "산업안전보건법_개요", + "score": 0.525, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.57198213262776, + "adjusted_score": 0.57198213262776 + }, + { + "id": 5229, + "title": "사업체들의 산업안전 활동 최근 동향과 과제", + "score": 0.2875, + "policy": null, + "age_days": 33, + "decay_factor": null, + "base_score": 0.016129032258064516, + "adjusted_score": 0.016129032258064516 + }, + { + "id": 10569, + "title": "MSDS_읽는법", + "score": 0.05, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.015873015873015872, + "adjusted_score": 0.015873015873015872 + } + ], + "timing_ms": { + "freshness_ms": 0.4767299978993833, + "rerank_ms": 4.971880989614874, + "total_ms": 224.24511198187247 + } + }, + { + "query": "KGS Code 개정", + "n_results": 5, + "results": [ + { + "id": 11647, + "title": "04_KGS_Code", + "score": 1.0, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.7170426330893268, + "adjusted_score": 0.7170426330893268 + }, + { + "id": 13914, + "title": "KGS FP112 § 1.5~1.6 — 경과조치·용품 사용제한", + "score": 0.7625, + "policy": null, + "age_days": 9, + "decay_factor": null, + "base_score": 0.6864445991217641, + "adjusted_score": 0.6864445991217641 + }, + { + "id": 11692, + "title": "05_KGS_GC_도시가스", + "score": 0.525, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.6584548047627677, + "adjusted_score": 0.6584548047627677 + }, + { + "id": 11693, + "title": "06_KGS_체계종합", + "score": 0.2875, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.5594483123643915, + "adjusted_score": 0.5594483123643915 + }, + { + "id": 11691, + "title": "04_KGS_AC_용기", + "score": 0.05, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.4960952491906052, + "adjusted_score": 0.4960952491906052 + } + ], + "timing_ms": { + "freshness_ms": 0.5111750215291977, + "rerank_ms": 4.768667975440621, + "total_ms": 329.5086660073139 + } + }, + { + "query": "위험성평가 최근 동향", + "n_results": 5, + "results": [ + { + "id": 5243, + "title": "위험성평가 사업장 구축 및 실행방안", + "score": 1.0, + "policy": null, + "age_days": 33, + "decay_factor": null, + "base_score": 0.6767799338787716, + "adjusted_score": 0.6767799338787716 + }, + { + "id": 5229, + "title": "사업체들의 산업안전 활동 최근 동향과 과제", + "score": 0.7625, + "policy": null, + "age_days": 33, + "decay_factor": null, + "base_score": 0.6586682524074614, + "adjusted_score": 0.6586682524074614 + }, + { + "id": 10574, + "title": "위험성평가_KRAS_절차", + "score": 0.525, + "policy": null, + "age_days": 18, + "decay_factor": null, + "base_score": 0.6364664159094706, + "adjusted_score": 0.6364664159094706 + }, + { + "id": 11685, + "title": "03_위험성평가기법", + "score": 0.2875, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.48858939729086537, + "adjusted_score": 0.48858939729086537 + }, + { + "id": 11568, + "title": "08_위험성평가지표", + "score": 0.05, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.47546145748375634, + "adjusted_score": 0.47546145748375634 + } + ], + "timing_ms": { + "freshness_ms": 0.5203019827604294, + "rerank_ms": 5.353120970539749, + "total_ms": 337.63471100246534 + } + }, + { + "query": "가스 사고 최근 사례", + "n_results": 5, + "results": [ + { + "id": 11684, + "title": "02_사고사례", + "score": 1.0, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.8395932099996827, + "adjusted_score": 0.8395932099996827 + }, + { + "id": 11564, + "title": "04_BLEVE", + "score": 0.7625, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.7673718895957371, + "adjusted_score": 0.7673718895957371 + }, + { + "id": 11565, + "title": "05_UVCE", + "score": 0.525, + "policy": null, + "age_days": 15, + "decay_factor": null, + "base_score": 0.7629469223345907, + "adjusted_score": 0.7629469223345907 + }, + { + "id": 7107, + "title": "청주 가스 폭발 사고 전날 ‘경보기 오작동’", + "score": 0.2875, + "policy": "news_90d", + "age_days": 26, + "decay_factor": 0.8138352895219257, + "base_score": 0.5376429266796365, + "adjusted_score": 0.5076158847438668 + }, + { + "id": 6707, + "title": "“가스 냄새 신고했는데 이상 없다더니”... 청주 폭발 사고 ‘인재’ 논란", + "score": 0.05, + "policy": "news_90d", + "age_days": 27, + "decay_factor": 0.8076322521549036, + "base_score": 0.5257014022820802, + "adjusted_score": 0.49536300384327636 + } + ], + "timing_ms": { + "freshness_ms": 0.4855860024690628, + "rerank_ms": 4.780669987667352, + "total_ms": 413.3225309778936 + } + } +] diff --git a/reports/freshness_decay_observation_week1.md b/reports/freshness_decay_observation_week1.md new file mode 100644 index 0000000..005349b --- /dev/null +++ b/reports/freshness_decay_observation_week1.md @@ -0,0 +1,52 @@ +# PR-RAG-Time-1 1주 후 재측정 (Week 1 Observation) + +**측정일**: 2026-05-12 (baseline 2026-05-03 의 9일 후) +**대상**: `services.search.search_pipeline.run_search` (mode=hybrid / fusion=rrf_boost / rerank=True / analyze=False / limit=5) +**원본 JSON**: `reports/freshness_decay_observation_week1.json` +**비교 baseline**: `reports/freshness_decay_observation_baseline.json` (2026-05-03) + +## 회귀 판정 종합 + +| 신호 | week1 측정값 | 임계 | 결과 | +|---|---:|---:|:---:| +| freshness_ms max | 0.54ms | 10ms | ✅ PASS | +| total_ms max | 413ms | 500ms | ✅ PASS | +| policy 분포 (base vs week1) | 9/30 vs 9/30 | ±10% | ✅ PASS | +| top 3 doc_id 변동 발생 쿼리 수 | 0/6 | 3 미만 | ✅ PASS (자동) | + +**자동 회귀 신호 4건 모두 통과. Manual review gate 도 unblocked (top3 변동 0 이므로).** + +## 쿼리별 비교 + +| 쿼리 | top3 동일 | total_ms (base → week1) | freshness_ms (base → week1) | +|---|:---:|---:|---:| +| 중대재해 사고 | ✓ | 250 → 373 | 0.49 → 0.50 | +| 최근 중대재해 | ✓ | 287 → 310 | 0.49 → 0.54 | +| 산안법 개정 | ✓ | 200 → 224 | 3.06 → 0.48 | +| KGS Code 개정 | ✓ | 272 → 330 | 0.43 → 0.51 | +| 위험성평가 최근 동향 | ✓ | 284 → 338 | 0.43 → 0.52 | +| 가스 사고 최근 사례 | ✓ | 350 → 413 | 0.44 → 0.49 | + +top3 doc_id 6/6 완전 동일. 1주 시점에서는 freshness decay 가 ranking 을 흔들 만큼의 age 격차가 생기지 않아 baseline 대비 ordering 안정. half_life 90d(news) / 365d(law) 의 9일차이므로 자연스러운 결과. + +total_ms 가 평균 +50ms (+20~25%) 증가. 첫 측정에서 cold start outlier 1458ms 발견 → warmup 1회 후 재측정 (현 결과). cold path 제거 시 baseline 비례 안정. + +## 발견된 별 이슈 (회귀 판정과 분리) + +**reranker 404 drift** — 측정 중 stderr 에 `[WARNING] rerank failed → RRF fallback: HTTPStatusError: Client error '404 Not Found' for url 'http://ollama:11434/api/rerank'` 가 6회 발생. + +원인: +- `config.yaml:45` reranker.endpoint = `http://ollama:11434/api/rerank` (Ollama 호출) +- 실제 reranker 는 별도 컨테이너 `hyungi_document_server-reranker-1` (TEI) — CLAUDE.md 기술스택 명시 +- Ollama 의 `/api/rerank` endpoint 는 응답 404 + +결과: 모든 검색이 reranker fallback (RRF only) 로 운영 중. baseline 측정 시점에도 동일 상태였을 가능성 높음 (baseline rerank_ms 4.6~4.9ms 와 week1 4.9~8.2ms 가 비슷 → 둘 다 404 응답 시간). + +별 incident 등록 + inventory `Drift Log` 추가 필요. **본 PR-RAG-Time-1 closure 와는 무관** (baseline/week1 모두 동일 fallback 상태이므로 freshness 본질 비교는 fair). + +## 1주 결과 한 줄 + +``` +2026-05-12 1주 후 재측정: 회귀 0 / top3 변동 0 / freshness_ms max 0.54ms / total_ms max 413ms. PASS. +(별 이슈: reranker 404 drift — config.yaml 의 endpoint 오류, 별 incident 트랙) +```