From 6a9142a2e5348560c00aed18e40fc5810399184d Mon Sep 17 00:00:00 2001 From: hyungi Date: Mon, 25 May 2026 05:46:14 +0000 Subject: [PATCH] docs(search): hier vs legacy go/no-go = NO-GO (replace-diagnose c6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-DocSrv-Hier-Replace-Diagnose-1 c6 측정+결정. prehier exact vs hier_sim exact, dedup 0/51. 결정權(분해-subset n=41): prehier 0.748 -> hier_sim_clean 0.675 (-0.074 회귀). raw 0.673 (robust). 카테고리: standards(법령, hier 최적가설) flat -0.002 / exam -0.183 / korean -0.109 / english -0.088. 법령 제N조조차 개선 없음 + 대체로 회귀 → 짧은 절 leaf 가 맥락 손실. dedup clean = 실제값. => NO-GO: 검색 코퍼스 hier 교체 안 함. Apply PR 미진입. hier leaf 는 in_corpus=false 잔존 (section-outline UI 재료, doc-level 검색 무관). 측정은 doc-level NDCG 한정. 산출물: decision md + 4 eval csv(sanity/prehier/clean/raw exact) + subset analysis script. in_corpus 634 전 구간 불변. default 검색 path 회귀 0. Co-Authored-By: Claude Opus 4.7 (1M context) --- ..._2_hier_replace_clean_exact_2026-05-25.csv | 52 ++++++++++++ ..._hier_replace_prehier_exact_2026-05-25.csv | 52 ++++++++++++ ...v0_2_hier_replace_raw_exact_2026-05-25.csv | 52 ++++++++++++ .../v0_2_hier_replace_sanity_2026-05-25.csv | 52 ++++++++++++ .../v0_2_hier_replace_decision_2026-05-25.md | 79 +++++++++++++++++++ .../scripts/hier_replace_subset_analysis.py | 65 +++++++++++++++ 6 files changed, 352 insertions(+) create mode 100644 reports/v0_2_hier_replace_clean_exact_2026-05-25.csv create mode 100644 reports/v0_2_hier_replace_prehier_exact_2026-05-25.csv create mode 100644 reports/v0_2_hier_replace_raw_exact_2026-05-25.csv create mode 100644 reports/v0_2_hier_replace_sanity_2026-05-25.csv create mode 100644 tests/search_eval/baselines/v0_2_hier_replace_decision_2026-05-25.md create mode 100644 tests/search_eval/scripts/hier_replace_subset_analysis.py diff --git a/reports/v0_2_hier_replace_clean_exact_2026-05-25.csv b/reports/v0_2_hier_replace_clean_exact_2026-05-25.csv new file mode 100644 index 0000000..0f83f3c --- /dev/null +++ b/reports/v0_2_hier_replace_clean_exact_2026-05-25.csv @@ -0,0 +1,52 @@ +label,id,category,legacy_category,intent,domain_hint,language,ocr_derived,failure_expected,query,relevant_ids,graded_relevance,returned_ids_top10,latency_ms,recall_at_10,mrr_at_10,ndcg_at_10,top3_hit,graded_ndcg_at_10,graded_recall_at_10_t2,graded_recall_at_10_t3,dedup_count,error +single,kw_001,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건법 제6장,3856;3868;3879,3856:3;3868:2;3879:2,3868;3890;3863;3879;3856;3908;3851;3900;4041;5227,1003.0,1.000,1.000,0.853,0,0.674,1.000,1.000,0, +single,kw_002,standards,exact_keyword,fact_lookup,document,ko,0,0,중대재해 처벌 등에 관한 법률 제2장 중대산업재해,3917;3921,3917:3;3921:2,3921;3917;5249;3919;3923;3920;10573;10571;3874;3918,1007.7,1.000,1.000,1.000,1,0.834,1.000,1.000,0, +single,kw_003,standards,exact_keyword,fact_lookup,document,ko,0,0,화학물질관리법 유해화학물질 영업자,3981,3981:3,3981;3985;3980;3984;3993;3857;3978;3986;3983;3957,856.9,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_004,standards,exact_keyword,fact_lookup,document,ko,0,0,근로기준법 안전과 보건,4041,4041:3,4041;3852;3851;3877;3905;3858;3903;3855;3881;4040,1024.9,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_005,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 보호구,3888,3888:3,10570;3888;3912;3913;3911;3905;3895;3909;3889;3910,1057.3,1.000,0.500,0.631,1,0.631,1.000,1.000,0, +single,nl_001,korean_only,natural_language_ko,semantic_search,document,ko,0,0,기계로 인한 산업재해 관련 법령,3856;3868;3879;3854,3854:1;3856:3;3868:2;3879:2,5244;3897;5249;3863;3879;3854;3856;3868;3851;3895,1064.5,1.000,0.200,0.543,0,0.443,1.000,1.000,0, +single,nl_002,korean_only,natural_language_ko,semantic_search,document,ko,0,0,사업주가 도급을 줄 때 산업재해를 예방하기 위해 해야 할 일,3855;3867;3878,3855:3;3867:2;3878:2,3855;3917;5236;3854;5227;5244;3867;5249;3851;3895,1006.5,0.667,1.000,0.626,1,0.770,0.667,1.000,0, +single,nl_003,korean_only,natural_language_ko,semantic_search,document,ko,0,0,유해화학물질을 다루는 회사가 지켜야 할 안전 의무,3980;3981;3982,3980:2;3981:2;3982:2,3980;3903;3896;3909;3981;5240;3985;3904;3910;3917,970.2,0.667,1.000,0.651,1,0.651,0.667,0.000,0, +single,nl_004,korean_only,natural_language_ko,semantic_search,document,ko,0,0,중대재해가 발생했을 때 경영책임자가 처벌받는 기준,3916;3917;3920;3921,3916:2;3917:3;3920:2;3921:2,10572;10573;3917;3918;3923;3919;10571;5244;5229;3854,953.3,0.250,0.333,0.195,1,0.300,0.250,1.000,0, +single,nl_005,korean_only,natural_language_ko,semantic_search,document,ko,0,0,안전보건교육은 누가 받아야 하고 어떤 내용을 다루는가,3853;3865,3853:3;3865:2,5249;3853;4025;3876;5234;6675;11677;10573;3859;5244,1139.5,0.500,0.500,0.387,1,0.497,0.500,1.000,0, +single,news_001,korean_only,news_ko,semantic_search,news,ko,0,0,이란과 미국의 군사 충돌,4303;4304;4307;4316;4322;4323;4327;4335,4303:2;4304:2;4307:2;4316:2;4322:2;4323:2;4327:2;4335:2,16081;18077;22048;12213;4317;15793;4321;21273;21276;4307,1039.6,0.125,0.100,0.073,1,0.073,0.125,0.000,0, +single,news_002,korean_only,news_ko,semantic_search,news,ko,0,0,호르무즈 해협 봉쇄,4316;4320;4322;4327,4316:3;4320:2;4322:2;4327:2,22055;22049;16392;9022;17123;11945;6657;9105;6774;6780,981.5,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,misc_001,korean_only,other_domain,fact_lookup,document,ko,0,0,강체의 평면 운동학,4063;4065,4063:3;4065:2,4065;4066;4063;4064;4071;4058;4067;4069;5099;5096,1091.3,1.000,1.000,0.920,1,0.731,1.000,1.000,0, +single,misc_002,korean_only,other_domain,semantic_search,document,ko,0,0,질점의 운동역학,4060;4061;4062,4060:2;4061:2;4062:2,4060;4070;4062;4061;4059;4065;4064;4058;4063;4067,1136.0,1.000,1.000,0.906,1,0.906,1.000,0.000,0, +single,news_003,english_only,news_en,semantic_search,news,en,0,0,Trump Iran ultimatum,4258;4260;4262,4258:2;4260:2;4262:2,21155;21441;4775;4202;4776;4679;4199;4519;4668;25592,988.6,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_001,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,기계 안전 가드 설계 원리,3770;3856,3770:3;3856:2,5239;3770;5244;3769;3773;3791;3774;4540;4541;3758,1187.8,0.500,0.500,0.387,1,0.497,0.500,1.000,0, +single,cl_002,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,산업 안전 입문서,3755;3775;3776;3777,3755:2;3775:2;3776:2;3777:2,5244;5230;5249;3774;3755;10573;22248;3851;3775;3779,1031.5,0.500,0.200,0.269,1,0.269,0.500,0.000,0, +single,cl_003,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,전기 안전 위험,3772;3790,3772:2;3790:2,3772;5260;3790;3897;11669;10568;3773;4018;13936;13938,1471.7,1.000,1.000,0.920,1,0.920,1.000,0.000,0, +single,news_004,mixed,news_fr,semantic_search,news,mixed,0,0,guerre en Iran,4199;4202;4210;4361;4363;4507;4519;4521,4199:2;4202:2;4210:2;4361:2;4363:2;4507:2;4519:2;4521:2,7740;5925;5840;16010;16457;6945;4507;5398;16842;4199,960.2,0.250,0.143,0.157,1,0.157,0.250,0.000,0, +single,news_005,mixed,news_crosslingual,semantic_search,news,mixed,0,0,이란 미국 전쟁 글로벌 반응,4202;4258;4262;4536;4303;4304;4316,4202:2;4258:2;4262:2;4303:2;4304:2;4316:2;4536:2,21275;16927;4457;19111;16761;6789;5520;17242;9469;4765,1191.3,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_001,failure_expected,failure_expected,semantic_search,document,mixed,0,1,Rust async runtime tokio scheduler 내부 구조,,,11612;23732;24155;13755;5262;5072;5063;20758;3819;3810,931.0,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_002,failure_expected,failure_expected,semantic_search,document,ko,0,1,양자컴퓨터 큐비트 디코히어런스,,,5135;16289;16384;4068;20507;5104;5210;5108;25198;5092,905.5,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_003,failure_expected,failure_expected,semantic_search,news,ko,0,1,재즈 보컬리스트 빌리 홀리데이,,,20022;20470;4634;15361;16059;9102;23082;23336;18286;16218,855.9,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,kw_006,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 작업장 통로,3886;3887,3886:3;3887:2,3898;5236;3886;3887;3881;3908;3912;3892;3900;3909,990.7,1.000,0.333,0.571,1,0.539,1.000,1.000,0, +single,kw_007,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 폭발 화재 위험물 누출 방지,3896;3766,3766:1;3896:3,3896;3898;3895;3903;13930;3897;13931;13929;3894;3886,1006.4,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,kw_008,standards,standards,fact_lookup,document,ko,0,0,고압가스 안전관리법 전문,4025;4026,4025:3;4026:2,11579;4025;4026;11644;11693;4034;13299;13749;13766;13658,1009.8,1.000,0.500,0.693,1,0.665,1.000,1.000,0, +single,kw_009,standards,standards,fact_lookup,document,ko,0,0,KGS FP111 가스설비 배관설비 기준,13305,13305:3,13305;13311;13306;13312;13302;13304;13309;13313;13918;13310,1004.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_010,standards,standards,fact_lookup,document,ko,0,0,KGS FU551 가스설비 압력조정기 가스계량기,13652,13652:3,13652;11689;13657;13655;13656;13649;13651;13752;13659;13650,896.9,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_011,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 전기로 인한 위험 방지,3897;3772,3772:1;3897:3,3897;3895;3886;3887;13935;3902;3877;3896;3898;3854,1040.7,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,en_001,english_only,english_only,semantic_search,document,en,0,0,pressure vessel flange design,5144;5136,5136:2;5144:3,5144;5140;5193;5206;5133;5187;5190;5137;5212;5149,2192.2,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,en_002,english_only,english_only,semantic_search,document,en,0,0,ASME Section VIII introduction,5204;5206,5204:3;5206:2,5204;5143;5208;5210;5180;5206;5212;11634;5207;5140,1983.6,1.000,1.000,0.832,1,0.907,1.000,1.000,0, +single,en_003,english_only,english_only,semantic_search,document,en,0,0,impact test requirements ASME,5205;5148,5148:1;5205:3,5205;5224;5178;5140;5145;5186;5180;5212;5209;5192,2201.7,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,en_004,english_only,english_only,semantic_search,document,en,0,0,design of vessel supports,5149,5149:3,5149;5186;5137;5139;5140;5145;5178;5206;5210;5141,2118.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_005,english_only,english_only,semantic_search,document,en,0,0,hydrogen piping ASME code,5178,5178:3,5178;5180;5148;5145;5143;5179;5210;5212;5204;5182,2074.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_006,english_only,english_only,semantic_search,document,en,0,0,ASME welding qualification requirements,5209;3771,3771:1;5209:3,5180;5205;5225;5209;5224;5178;5192;4835;4826;5222,2004.6,0.500,0.250,0.264,0,0.395,1.000,1.000,0, +single,en_007,english_only,english_only,semantic_search,document,en,0,0,pressure vessel fabrication and inspection,5208;5145,5145:2;5208:3,5208;5189;5204;5225;5188;5185;5212;5186;5182;5187,2209.5,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,en_008,english_only,english_only,semantic_search,document,en,0,0,Industrial Safety and Health Management ergonomics,3763;3755,3755:1;3763:3,3759;3763;3774;3818;3756;3812;3769;3757;3762;3761,1645.4,0.500,0.500,0.387,1,0.579,1.000,1.000,0, +single,cl_004,mixed,mixed,semantic_search,document,mixed,0,0,ASME 압력용기 설계 실무,5207;5210;5139;5135,5135:2;5139:2;5207:3;5210:3,5207;5139;5206;5208;5204;5135;5210;5182;5225;5140,1304.5,1.000,1.000,0.906,1,0.865,1.000,1.000,0, +single,cl_005,mixed,mixed,semantic_search,document,mixed,0,0,ASME 용접 코드 해설,5224;5209,5209:2;5224:3,5224;5222;5209;5225;5139;5178;5208;5204;5210;5205,1273.3,1.000,1.000,0.920,1,0.956,1.000,1.000,0, +single,cl_006,mixed,mixed,semantic_search,document,mixed,0,0,pressure vessel Korean industrial safety regulation,4025;3856;5136,3856:2;4025:2;5136:1,5244;5186;5225;5204;5182;3895;5177;5180;13749;13760,1496.2,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_007,mixed,mixed,semantic_search,document,mixed,0,0,KGS 코드 LPG safety standard,11647;11689;11645;4025,4025:1;11645:2;11647:3;11689:2,13760;13674;13669;13774;13773;13675;13755;13924;11647;11688,947.3,0.250,0.111,0.118,0,0.195,0.333,1.000,0, +single,cl_008,mixed,mixed,semantic_search,document,mixed,0,0,수소 가스 안전 기준 hydrogen safety,5178;5169,5169:2;5178:3,10575;11671;5173;5177;11649;11648;13915;5241;11653;13303,1218.1,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,exam_001,exam,exam,fact_lookup,document,ko,0,0,레이놀즈수 정의 공식,11504;11505,11504:3;11505:2,11533;11504;5082;5090;4544;11509;11505;11476;11486;5140,1086.9,1.000,0.500,0.591,1,0.609,1.000,1.000,0, +single,exam_002,exam,exam,fact_lookup,document,ko,0,0,탱크 바닥 구멍 유체 유속 토리첼리,11500;11495;11496,11495:2;11496:2;11500:3,11500;11495;11501;5085;5140;11503;5139;11515;11517;5210,893.3,0.667,1.000,0.765,1,0.856,0.667,1.000,0, +single,exam_003,exam,exam,fact_lookup,document,ko,0,0,이상기체 음속 마하수,11514;11515,11514:3;11515:2,11514;5088;11475;11531;11476;5090;11473;5093;11479;11526,1137.7,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,exam_004,exam,exam,fact_lookup,document,ko,0,0,고압가스 용기 내압시험 영구증가량,11591;11644;11691,11591:3;11644:2;11691:2,11591;11664;13660;5177;11665;13943;13942;13917;11660;13653,913.2,0.333,1.000,0.469,1,0.674,0.333,1.000,0, +single,exam_005,exam,exam,fact_lookup,document,ko,0,0,도시가스 배관 매설 이격거리,11627;11625;11646,11625:2;11627:3;11646:1,11658;11600;11625;13918;11692;11627;13751;11655;13653;13753,938.9,0.667,0.333,0.402,0,0.425,1.000,1.000,0, +single,exam_006,exam,exam,fact_lookup,document,ko,0,0,LPG 저장탱크 안전거리 분말소화기,11617;11669;11620,11617:3;11620:1;11669:2,11595;11616;13669;11649;11605;11655;11690;13916;11658;13915,890.1,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,exam_007,exam,exam,fact_lookup,document,ko,0,0,오리피스 차압식 유량계,11712;11711;11503,11503:2;11711:2;11712:3,11711;11503;11500;11712;11714;11713;13930;11717;11701;11715,936.0,1.000,1.000,0.967,0,0.761,1.000,1.000,0, +single,fail_004,failure_expected,failure_expected,fact_lookup,document,ko,0,1,KGS AC999 임의 가스 코드,,,11691;5177;11693;11692;13665;13661;13664;13666;13670;13773,968.3,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_005,failure_expected,failure_expected,fact_lookup,document,ko,0,1,초전도 안전 관리법 시행규칙,,,4026;3966;3977;3971;4018;3972;3973;3974;4019;4021,1005.1,0.000,0.000,0.000,1,0.000,0.000,0.000,0, diff --git a/reports/v0_2_hier_replace_prehier_exact_2026-05-25.csv b/reports/v0_2_hier_replace_prehier_exact_2026-05-25.csv new file mode 100644 index 0000000..ea892e8 --- /dev/null +++ b/reports/v0_2_hier_replace_prehier_exact_2026-05-25.csv @@ -0,0 +1,52 @@ +label,id,category,legacy_category,intent,domain_hint,language,ocr_derived,failure_expected,query,relevant_ids,graded_relevance,returned_ids_top10,latency_ms,recall_at_10,mrr_at_10,ndcg_at_10,top3_hit,graded_ndcg_at_10,graded_recall_at_10_t2,graded_recall_at_10_t3,dedup_count,error +single,kw_001,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건법 제6장,3856;3868;3879,3856:3;3868:2;3879:2,3879;3868;3890;3863;3856;3908;3851;4041;3900;3895,354.4,1.000,1.000,0.947,0,0.731,1.000,1.000,0, +single,kw_002,standards,exact_keyword,fact_lookup,document,ko,0,0,중대재해 처벌 등에 관한 법률 제2장 중대산업재해,3917;3921,3917:3;3921:2,3921;3917;3919;3923;10573;10571;3916;3874;3918;3854,391.4,1.000,1.000,1.000,1,0.834,1.000,1.000,0, +single,kw_003,standards,exact_keyword,fact_lookup,document,ko,0,0,화학물질관리법 유해화학물질 영업자,3981,3981:3,3981;3985;3980;3984;3993;3857;3978;3986;3983;3957,255.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_004,standards,exact_keyword,fact_lookup,document,ko,0,0,근로기준법 안전과 보건,4041,4041:3,4041;3852;3851;3877;3905;3903;3858;3781;3881;4040,418.9,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_005,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 보호구,3888,3888:3,10570;3888;3912;3913;3911;3905;3909;3889;3910;3893,434.0,1.000,0.500,0.631,1,0.631,1.000,1.000,0, +single,nl_001,korean_only,natural_language_ko,semantic_search,document,ko,0,0,기계로 인한 산업재해 관련 법령,3856;3868;3879;3854,3854:1;3856:3;3868:2;3879:2,5244;3878;5249;3897;3863;3868;5253;3856;3879;3895,447.3,0.750,0.167,0.380,0,0.386,1.000,1.000,0, +single,nl_002,korean_only,natural_language_ko,semantic_search,document,ko,0,0,사업주가 도급을 줄 때 산업재해를 예방하기 위해 해야 할 일,3855;3867;3878,3855:3;3867:2;3878:2,3855;5227;3917;3854;5244;3867;3878;3851;3908;3903,392.0,1.000,1.000,0.793,1,0.873,1.000,1.000,0, +single,nl_003,korean_only,natural_language_ko,semantic_search,document,ko,0,0,유해화학물질을 다루는 회사가 지켜야 할 안전 의무,3980;3981;3982,3980:2;3981:2;3982:2,3980;3903;3904;3981;3760;5253;3985;3896;3917;3857,345.3,0.667,1.000,0.671,1,0.671,0.667,0.000,0, +single,nl_004,korean_only,natural_language_ko,semantic_search,document,ko,0,0,중대재해가 발생했을 때 경영책임자가 처벌받는 기준,3916;3917;3920;3921,3916:2;3917:3;3920:2;3921:2,10572;10573;3917;3918;3916;3923;5244;3919;10571;5227,317.4,0.500,0.333,0.346,1,0.399,0.500,1.000,0, +single,nl_005,korean_only,natural_language_ko,semantic_search,document,ko,0,0,안전보건교육은 누가 받아야 하고 어떤 내용을 다루는가,3853;3865,3853:3;3865:2,3853;3876;5249;5234;4025;6675;11677;10573;3757;3859,528.5,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,news_001,korean_only,news_ko,semantic_search,news,ko,0,0,이란과 미국의 군사 충돌,4303;4304;4307;4316;4322;4323;4327;4335,4303:2;4304:2;4307:2;4316:2;4322:2;4323:2;4327:2;4335:2,16081;18077;22048;12213;4317;15793;4321;21273;21276;4307,428.2,0.125,0.100,0.073,1,0.073,0.125,0.000,0, +single,news_002,korean_only,news_ko,semantic_search,news,ko,0,0,호르무즈 해협 봉쇄,4316;4320;4322;4327,4316:3;4320:2;4322:2;4327:2,22055;22049;16392;9022;17123;11945;6657;9105;6774;6780,349.2,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,misc_001,korean_only,other_domain,fact_lookup,document,ko,0,0,강체의 평면 운동학,4063;4065,4063:3;4065:2,4063;4065;4064;4071;4066;4068;4058;4067;5063;5105,537.2,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,misc_002,korean_only,other_domain,semantic_search,document,ko,0,0,질점의 운동역학,4060;4061;4062,4060:2;4061:2;4062:2,4060;4062;4070;4059;4061;4064;4065;4063;4058;5095,512.4,1.000,1.000,0.947,1,0.947,1.000,0.000,0, +single,news_003,english_only,news_en,semantic_search,news,en,0,0,Trump Iran ultimatum,4258;4260;4262,4258:2;4260:2;4262:2,21155;21441;4775;4202;4776;4679;4199;4519;4668;25592,309.6,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_001,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,기계 안전 가드 설계 원리,3770;3856,3770:3;3856:2,5239;5244;3770;3817;3774;4540;3762;3789;5249;5253,498.0,0.500,0.333,0.307,1,0.394,0.500,1.000,0, +single,cl_002,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,산업 안전 입문서,3755;3775;3776;3777,3755:2;3775:2;3776:2;3777:2,5249;3760;5229;3755;3774;5230;10573;3775;3787;3818,433.8,0.500,0.250,0.291,1,0.291,0.500,0.000,0, +single,cl_003,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,전기 안전 위험,3772;3790,3772:2;3790:2,3772;3790;5260;3897;3755;5235;3773;10568;13936;13938,889.3,1.000,1.000,1.000,1,1.000,1.000,0.000,0, +single,news_004,mixed,news_fr,semantic_search,news,mixed,0,0,guerre en Iran,4199;4202;4210;4361;4363;4507;4519;4521,4199:2;4202:2;4210:2;4361:2;4363:2;4507:2;4519:2;4521:2,7740;5925;5840;16010;16457;6945;4507;5398;16842;4199,360.3,0.250,0.143,0.157,1,0.157,0.250,0.000,0, +single,news_005,mixed,news_crosslingual,semantic_search,news,mixed,0,0,이란 미국 전쟁 글로벌 반응,4202;4258;4262;4536;4303;4304;4316,4202:2;4258:2;4262:2;4303:2;4304:2;4316:2;4536:2,21275;16927;4457;19111;16761;6789;5520;17242;9469;4765,579.1,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_001,failure_expected,failure_expected,semantic_search,document,mixed,0,1,Rust async runtime tokio scheduler 내부 구조,,,5161;11612;5262;23732;24155;20758;4547;25771;3774;20036,337.1,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_002,failure_expected,failure_expected,semantic_search,document,ko,0,1,양자컴퓨터 큐비트 디코히어런스,,,5070;16289;16384;5066;20507;5092;5118;25198;5173;23605,274.2,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_003,failure_expected,failure_expected,semantic_search,news,ko,0,1,재즈 보컬리스트 빌리 홀리데이,,,18567;20022;20470;4634;15361;16059;9102;23082;23336;18286,228.1,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,kw_006,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 작업장 통로,3886;3887,3886:3;3887:2,3898;3886;3887;3881;3908;3912;3892;3900;3909;3890,381.5,1.000,0.500,0.693,1,0.665,1.000,1.000,0, +single,kw_007,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 폭발 화재 위험물 누출 방지,3896;3766,3766:1;3896:3,3896;3895;3903;13930;13931;13929;3894;3886;13944;3892,330.1,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,kw_008,standards,standards,fact_lookup,document,ko,0,0,고압가스 안전관리법 전문,4025;4026,4025:3;4026:2,11644;11579;4026;4025;11693;4034;11645;13299;13749;13766,400.1,1.000,0.333,0.571,0,0.508,1.000,1.000,0, +single,kw_009,standards,standards,fact_lookup,document,ko,0,0,KGS FP111 가스설비 배관설비 기준,13305,13305:3,13305;13311;13306;13312;13302;13304;13309;13299;13313;13918,396.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_010,standards,standards,fact_lookup,document,ko,0,0,KGS FU551 가스설비 압력조정기 가스계량기,13652,13652:3,13652;11689;13657;13655;13656;13649;13651;13752;13659;13650,291.6,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_011,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 전기로 인한 위험 방지,3897;3772,3772:1;3897:3,3897;3895;3887;13935;3902;3896;3877;3900;3899;3854,415.8,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,en_001,english_only,english_only,semantic_search,document,en,0,0,pressure vessel flange design,5144;5136,5136:2;5144:3,5144;5136;5180;5140;5137;5149;5178;5207;5148;5186,1598.7,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_002,english_only,english_only,semantic_search,document,en,0,0,ASME Section VIII introduction,5204;5206,5204:3;5206:2,5204;5212;5180;5208;5210;5143;5141;5206;5137;11634,1421.7,1.000,1.000,0.807,1,0.894,1.000,1.000,0, +single,en_003,english_only,english_only,semantic_search,document,en,0,0,impact test requirements ASME,5205;5148,5148:1;5205:3,5205;5204;5178;5214;5224;5148;5209;5145;5186;5190,1558.8,1.000,1.000,0.832,1,0.964,1.000,1.000,0, +single,en_004,english_only,english_only,semantic_search,document,en,0,0,design of vessel supports,5149,5149:3,5149;5141;5137;5139;5136;5140;5178;5186;5145;5143,1549.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_005,english_only,english_only,semantic_search,document,en,0,0,hydrogen piping ASME code,5178,5178:3,5178;5180;5210;5179;5212;5182;5133;5208;5207;5148,1508.8,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_006,english_only,english_only,semantic_search,document,en,0,0,ASME welding qualification requirements,5209;3771,3771:1;5209:3,5224;5208;5209;5205;5178;5204;5180;5225;5187;5186,1382.9,0.500,0.333,0.307,1,0.459,1.000,1.000,0, +single,en_007,english_only,english_only,semantic_search,document,en,0,0,pressure vessel fabrication and inspection,5208;5145,5145:2;5208:3,5208;5189;5192;5180;5187;5186;5188;5182;5137;5193,1645.3,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,en_008,english_only,english_only,semantic_search,document,en,0,0,Industrial Safety and Health Management ergonomics,3763;3755,3755:1;3763:3,3763;3759;3774;3755;3818;3812;3778;3756;3761;3769,1022.4,1.000,1.000,0.877,1,0.974,1.000,1.000,0, +single,cl_004,mixed,mixed,semantic_search,document,mixed,0,0,ASME 압력용기 설계 실무,5207;5210;5139;5135,5135:2;5139:2;5207:3;5210:3,5139;5207;5204;5225;5206;5208;5210;5137;5182;5145,733.4,0.750,1.000,0.767,1,0.686,0.750,1.000,0, +single,cl_005,mixed,mixed,semantic_search,document,mixed,0,0,ASME 용접 코드 해설,5224;5209,5209:2;5224:3,5224;5225;5222;5209;5204;5210;5205;5178;5143;5207,701.7,1.000,1.000,0.877,1,0.932,1.000,1.000,0, +single,cl_006,mixed,mixed,semantic_search,document,mixed,0,0,pressure vessel Korean industrial safety regulation,4025;3856;5136,3856:2;4025:2;5136:1,4026;5145;5143;3895;5210;13749;5204;5186;13760;5244,881.8,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_007,mixed,mixed,semantic_search,document,mixed,0,0,KGS 코드 LPG safety standard,11647;11689;11645;4025,4025:1;11645:2;11647:3;11689:2,11647;13760;13674;13669;13774;13773;13675;13755;13924;11688,340.2,0.250,1.000,0.390,1,0.647,0.333,1.000,0, +single,cl_008,mixed,mixed,semantic_search,document,mixed,0,0,수소 가스 안전 기준 hydrogen safety,5178;5169,5169:2;5178:3,10575;5177;11671;11649;11648;13915;5241;5173;11653;5170,586.7,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,exam_001,exam,exam,fact_lookup,document,ko,0,0,레이놀즈수 정의 공식,11504;11505,11504:3;11505:2,11504;11533;5090;4544;5081;11509;11505;11476;11486;5132,461.1,1.000,1.000,0.818,1,0.900,1.000,1.000,0, +single,exam_002,exam,exam,fact_lookup,document,ko,0,0,탱크 바닥 구멍 유체 유속 토리첼리,11500;11495;11496,11495:2;11496:2;11500:3,11500;11495;11501;5139;5090;5178;11515;11517;5210;11493,293.4,0.667,1.000,0.765,1,0.856,0.667,1.000,0, +single,exam_003,exam,exam,fact_lookup,document,ko,0,0,이상기체 음속 마하수,11514;11515,11514:3;11515:2,11514;5083;11475;5090;5084;11531;11476;11473;11479;5124,528.8,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,exam_004,exam,exam,fact_lookup,document,ko,0,0,고압가스 용기 내압시험 영구증가량,11591;11644;11691,11591:3;11644:2;11691:2,11591;11664;13660;5177;11691;11665;13652;13942;13917;11660,305.8,0.667,1.000,0.651,1,0.785,0.667,1.000,0, +single,exam_005,exam,exam,fact_lookup,document,ko,0,0,도시가스 배관 매설 이격거리,11627;11625;11646,11625:2;11627:3;11646:1,11627;11658;11600;11625;13918;11692;13751;5177;11655;13653,328.5,0.667,1.000,0.671,1,0.883,1.000,1.000,0, +single,exam_006,exam,exam,fact_lookup,document,ko,0,0,LPG 저장탱크 안전거리 분말소화기,11617;11669;11620,11617:3;11620:1;11669:2,11595;11616;13669;11617;11649;11605;11655;11690;11658;13915,279.5,0.333,0.250,0.202,0,0.321,0.500,1.000,0, +single,exam_007,exam,exam,fact_lookup,document,ko,0,0,오리피스 차압식 유량계,11712;11711;11503,11503:2;11711:2;11712:3,11711;11712;11503;11500;11714;11713;13930;11717;11701;11715,322.3,1.000,1.000,1.000,1,0.858,1.000,1.000,0, +single,fail_004,failure_expected,failure_expected,fact_lookup,document,ko,0,1,KGS AC999 임의 가스 코드,,,11691;11647;11693;11692;13665;13661;13664;13666;13670;13773,326.0,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_005,failure_expected,failure_expected,fact_lookup,document,ko,0,1,초전도 안전 관리법 시행규칙,,,4026;5236;3966;3977;5260;3971;4018;3972;3973;3974,391.2,0.000,0.000,0.000,1,0.000,0.000,0.000,0, diff --git a/reports/v0_2_hier_replace_raw_exact_2026-05-25.csv b/reports/v0_2_hier_replace_raw_exact_2026-05-25.csv new file mode 100644 index 0000000..bebd69a --- /dev/null +++ b/reports/v0_2_hier_replace_raw_exact_2026-05-25.csv @@ -0,0 +1,52 @@ +label,id,category,legacy_category,intent,domain_hint,language,ocr_derived,failure_expected,query,relevant_ids,graded_relevance,returned_ids_top10,latency_ms,recall_at_10,mrr_at_10,ndcg_at_10,top3_hit,graded_ndcg_at_10,graded_recall_at_10_t2,graded_recall_at_10_t3,dedup_count,error +single,kw_001,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건법 제6장,3856;3868;3879,3856:3;3868:2;3879:2,3921;3879;3856;3917;3886;3878;3851;3855;3853;3863,854.8,0.667,0.500,0.531,1,0.519,0.667,1.000,0, +single,kw_002,standards,exact_keyword,fact_lookup,document,ko,0,0,중대재해 처벌 등에 관한 법률 제2장 중대산업재해,3917;3921,3917:3;3921:2,3874;3922;3923;3918;3917;10573;10571;5249;3877;3920,537.4,0.500,0.200,0.237,0,0.305,0.500,1.000,0, +single,kw_003,standards,exact_keyword,fact_lookup,document,ko,0,0,화학물질관리법 유해화학물질 영업자,3981,3981:3,3981;3985;3980;3857;3904;3984;3880;3896;3993;3895,404.6,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_004,standards,exact_keyword,fact_lookup,document,ko,0,0,근로기준법 안전과 보건,4041,4041:3,4041;3855;10573;3854;3858;3852;3851;3867;3881;3905,520.8,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_005,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 보호구,3888,3888:3,3910;3888;3905;3890;3885;3913;3915;3908;3894;3911,567.4,1.000,0.500,0.631,1,0.631,1.000,1.000,0, +single,nl_001,korean_only,natural_language_ko,semantic_search,document,ko,0,0,기계로 인한 산업재해 관련 법령,3856;3868;3879;3854,3854:1;3856:3;3868:2;3879:2,3879;3856;5249;3897;3851;3895;3878;3917;3874;5244,598.5,0.500,1.000,0.637,1,0.685,0.667,1.000,0, +single,nl_002,korean_only,natural_language_ko,semantic_search,document,ko,0,0,사업주가 도급을 줄 때 산업재해를 예방하기 위해 해야 할 일,3855;3867;3878,3855:3;3867:2;3878:2,3878;3855;5227;3917;3854;10571;5244;3896;5236;3895,544.8,0.667,1.000,0.765,1,0.714,0.667,1.000,0, +single,nl_003,korean_only,natural_language_ko,semantic_search,document,ko,0,0,유해화학물질을 다루는 회사가 지켜야 할 안전 의무,3980;3981;3982,3980:2;3981:2;3982:2,3980;3917;3985;3760;3981;3904;3896;5238;5227;3921,485.8,0.667,1.000,0.651,1,0.651,0.667,0.000,0, +single,nl_004,korean_only,natural_language_ko,semantic_search,document,ko,0,0,중대재해가 발생했을 때 경영책임자가 처벌받는 기준,3916;3917;3920;3921,3916:2;3917:3;3920:2;3921:2,3917;3854;10573;3918;10571;10572;3877;5249;5244;3919,459.3,0.250,1.000,0.390,1,0.599,0.250,1.000,0, +single,nl_005,korean_only,natural_language_ko,semantic_search,document,ko,0,0,안전보건교육은 누가 받아야 하고 어떤 내용을 다루는가,3853;3865,3853:3;3865:2,3917;5249;3921;3853;4025;3876;11677;10573;6675;3811,660.3,0.500,0.250,0.264,0,0.339,0.500,1.000,0, +single,news_001,korean_only,news_ko,semantic_search,news,ko,0,0,이란과 미국의 군사 충돌,4303;4304;4307;4316;4322;4323;4327;4335,4303:2;4304:2;4307:2;4316:2;4322:2;4323:2;4327:2;4335:2,20893;22048;21276;22054;15793;16081;18088;15922;18077;21273,563.9,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,news_002,korean_only,news_ko,semantic_search,news,ko,0,0,호르무즈 해협 봉쇄,4316;4320;4322;4327,4316:3;4320:2;4322:2;4327:2,20240;16532;21706;15922;17123;21890;22049;3895;4346;9022,458.6,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,misc_001,korean_only,other_domain,fact_lookup,document,ko,0,0,강체의 평면 운동학,4063;4065,4063:3;4065:2,4071;4064;4065;4063;4058;4066;4067;4068;5105;5099,602.9,1.000,0.333,0.571,0,0.508,1.000,1.000,0, +single,misc_002,korean_only,other_domain,semantic_search,document,ko,0,0,질점의 운동역학,4060;4061;4062,4060:2;4061:2;4062:2,4060;4058;4062;4059;4070;4061;4068;4067;4066;4064,553.7,1.000,1.000,0.871,1,0.871,1.000,0.000,0, +single,news_003,english_only,news_en,semantic_search,news,en,0,0,Trump Iran ultimatum,4258;4260;4262,4258:2;4260:2;4262:2,21155;21441;4775;4202;4776;4679;4199;4519;4668;25592,515.6,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_001,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,기계 안전 가드 설계 원리,3770;3856,3770:3;3856:2,5239;3770;5244;3769;3773;3791;3774;4540;4541;3758,690.2,0.500,0.500,0.387,1,0.497,0.500,1.000,0, +single,cl_002,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,산업 안전 입문서,3755;3775;3776;3777,3755:2;3775:2;3776:2;3777:2,5230;5249;3755;10573;22248;11644;3775;3853;3779;3855,601.0,0.500,0.333,0.325,1,0.325,0.500,0.000,0, +single,cl_003,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,전기 안전 위험,3772;3790,3772:2;3790:2,3772;5260;3790;3897;11669;10568;3773;13936;13938;13937,1068.8,1.000,1.000,0.920,1,0.920,1.000,0.000,0, +single,news_004,mixed,news_fr,semantic_search,news,mixed,0,0,guerre en Iran,4199;4202;4210;4361;4363;4507;4519;4521,4199:2;4202:2;4210:2;4361:2;4363:2;4507:2;4519:2;4521:2,7740;5925;5840;16010;16457;6945;4507;5398;16842;4199,560.0,0.250,0.143,0.157,1,0.157,0.250,0.000,0, +single,news_005,mixed,news_crosslingual,semantic_search,news,mixed,0,0,이란 미국 전쟁 글로벌 반응,4202;4258;4262;4536;4303;4304;4316,4202:2;4258:2;4262:2;4303:2;4304:2;4316:2;4536:2,21275;16927;4457;19111;16761;6789;5520;17242;9469;4765,770.2,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_001,failure_expected,failure_expected,semantic_search,document,mixed,0,1,Rust async runtime tokio scheduler 내부 구조,,,5092;23732;5139;13755;5262;5227;20758;5135;5105;20036,492.0,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_002,failure_expected,failure_expected,semantic_search,document,ko,0,1,양자컴퓨터 큐비트 디코히어런스,,,5135;20507;5104;5210;5108;25198;5092;5168;5209;5078,478.7,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_003,failure_expected,failure_expected,semantic_search,news,ko,0,1,재즈 보컬리스트 빌리 홀리데이,,,4634;16059;5139;9102;23082;23336;5103;5141;4058;5210,420.3,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,kw_006,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 작업장 통로,3886;3887,3886:3;3887:2,3886;3898;5236;3887;3895;3881;3908;3912;3892;3900,581.4,1.000,1.000,0.877,1,0.932,1.000,1.000,0, +single,kw_007,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 폭발 화재 위험물 누출 방지,3896;3766,3766:1;3896:3,3896;3895;3897;3903;13930;13931;13929;3894;3886;13944,548.2,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,kw_008,standards,standards,fact_lookup,document,ko,0,0,고압가스 안전관리법 전문,4025;4026,4025:3;4026:2,11579;11644;4025;4026;11693;4034;13299;13749;13766;13658,599.9,1.000,0.333,0.571,1,0.539,1.000,1.000,0, +single,kw_009,standards,standards,fact_lookup,document,ko,0,0,KGS FP111 가스설비 배관설비 기준,13305,13305:3,13305;13311;13306;13312;13302;13304;13309;13313;13918;13310,581.2,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_010,standards,standards,fact_lookup,document,ko,0,0,KGS FU551 가스설비 압력조정기 가스계량기,13652,13652:3,13652;11689;13657;13655;13656;13649;13651;13752;13659;13650,496.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_011,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 전기로 인한 위험 방지,3897;3772,3772:1;3897:3,3897;3886;3887;13935;3902;3895;3877;3896;3854;3900,639.1,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,en_001,english_only,english_only,semantic_search,document,en,0,0,pressure vessel flange design,5144;5136,5136:2;5144:3,5144;5140;5193;5206;5133;5187;5190;5137;5212;5149,1767.9,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,en_002,english_only,english_only,semantic_search,document,en,0,0,ASME Section VIII introduction,5204;5206,5204:3;5206:2,5204;5208;5210;5180;5206;5212;11634;5207;5140;5205,1580.8,1.000,1.000,0.850,1,0.918,1.000,1.000,0, +single,en_003,english_only,english_only,semantic_search,document,en,0,0,impact test requirements ASME,5205;5148,5148:1;5205:3,5205;5224;5178;5140;5145;5186;5180;5212;5209;5192,1807.7,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,en_004,english_only,english_only,semantic_search,document,en,0,0,design of vessel supports,5149,5149:3,5149;5186;5137;5139;5140;5145;5178;5206;5210;5141,1708.8,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_005,english_only,english_only,semantic_search,document,en,0,0,hydrogen piping ASME code,5178,5178:3,5178;5180;5148;5145;5143;5179;5210;5212;5204;5182,1668.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_006,english_only,english_only,semantic_search,document,en,0,0,ASME welding qualification requirements,5209;3771,3771:1;5209:3,5180;5205;5225;5209;5224;5178;5192;4835;4826;5222,1586.1,0.500,0.250,0.264,0,0.395,1.000,1.000,0, +single,en_007,english_only,english_only,semantic_search,document,en,0,0,pressure vessel fabrication and inspection,5208;5145,5145:2;5208:3,5208;5189;5204;5225;5188;5185;5212;5186;5182;5187,1849.4,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,en_008,english_only,english_only,semantic_search,document,en,0,0,Industrial Safety and Health Management ergonomics,3763;3755,3755:1;3763:3,3763;3759;3774;3818;3756;3812;3769;3757;3762;3770,1285.1,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,cl_004,mixed,mixed,semantic_search,document,mixed,0,0,ASME 압력용기 설계 실무,5207;5210;5139;5135,5135:2;5139:2;5207:3;5210:3,5207;5206;5208;5204;5135;5210;5182;5225;5140;5212,876.6,0.750,1.000,0.680,1,0.750,0.750,1.000,0, +single,cl_005,mixed,mixed,semantic_search,document,mixed,0,0,ASME 용접 코드 해설,5224;5209,5209:2;5224:3,5224;5222;5209;5225;5178;5208;5204;5210;5205;5180,849.9,1.000,1.000,0.920,1,0.956,1.000,1.000,0, +single,cl_006,mixed,mixed,semantic_search,document,mixed,0,0,pressure vessel Korean industrial safety regulation,4025;3856;5136,3856:2;4025:2;5136:1,5244;5186;5225;5204;5182;3895;5177;5180;13749;13760,1058.9,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_007,mixed,mixed,semantic_search,document,mixed,0,0,KGS 코드 LPG safety standard,11647;11689;11645;4025,4025:1;11645:2;11647:3;11689:2,13760;13674;13669;13774;13773;13675;13755;13924;11647;11688,538.9,0.250,0.111,0.118,0,0.195,0.333,1.000,0, +single,cl_008,mixed,mixed,semantic_search,document,mixed,0,0,수소 가스 안전 기준 hydrogen safety,5178;5169,5169:2;5178:3,10575;11671;5173;5177;11649;11648;13915;5241;11653;13303,770.2,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,exam_001,exam,exam,fact_lookup,document,ko,0,0,레이놀즈수 정의 공식,11504;11505,11504:3;11505:2,11533;11504;5082;5090;4544;11509;11505;11476;11486;5140,669.9,1.000,0.500,0.591,1,0.609,1.000,1.000,0, +single,exam_002,exam,exam,fact_lookup,document,ko,0,0,탱크 바닥 구멍 유체 유속 토리첼리,11500;11495;11496,11495:2;11496:2;11500:3,11500;11495;11501;5085;11503;11515;11517;5210;11719;5133,474.9,0.667,1.000,0.765,1,0.856,0.667,1.000,0, +single,exam_003,exam,exam,fact_lookup,document,ko,0,0,이상기체 음속 마하수,11514;11515,11514:3;11515:2,11514;5088;11475;11531;11476;5090;11473;5093;11479;11526,727.2,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,exam_004,exam,exam,fact_lookup,document,ko,0,0,고압가스 용기 내압시험 영구증가량,11591;11644;11691,11591:3;11644:2;11691:2,11591;11664;13660;11665;13943;13942;13917;11660;13653;13752,490.1,0.333,1.000,0.469,1,0.674,0.333,1.000,0, +single,exam_005,exam,exam,fact_lookup,document,ko,0,0,도시가스 배관 매설 이격거리,11627;11625;11646,11625:2;11627:3;11646:1,11658;11600;11625;13918;11692;11627;13751;11655;13653;13753,524.9,0.667,0.333,0.402,0,0.425,1.000,1.000,0, +single,exam_006,exam,exam,fact_lookup,document,ko,0,0,LPG 저장탱크 안전거리 분말소화기,11617;11669;11620,11617:3;11620:1;11669:2,11595;11616;13669;11649;11605;11655;11690;11658;13916;13915,475.4,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,exam_007,exam,exam,fact_lookup,document,ko,0,0,오리피스 차압식 유량계,11712;11711;11503,11503:2;11711:2;11712:3,11711;11503;11500;11712;11714;11713;13930;11717;11701;11715,519.3,1.000,1.000,0.967,0,0.761,1.000,1.000,0, +single,fail_004,failure_expected,failure_expected,fact_lookup,document,ko,0,1,KGS AC999 임의 가스 코드,,,11691;11693;11692;13665;13661;13664;13666;13670;13773;13934,523.5,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_005,failure_expected,failure_expected,fact_lookup,document,ko,0,1,초전도 안전 관리법 시행규칙,,,4026;3966;3977;3971;4025;3886;3972;3973;3974;3895,646.9,0.000,0.000,0.000,1,0.000,0.000,0.000,0, diff --git a/reports/v0_2_hier_replace_sanity_2026-05-25.csv b/reports/v0_2_hier_replace_sanity_2026-05-25.csv new file mode 100644 index 0000000..8db0bbd --- /dev/null +++ b/reports/v0_2_hier_replace_sanity_2026-05-25.csv @@ -0,0 +1,52 @@ +label,id,category,legacy_category,intent,domain_hint,language,ocr_derived,failure_expected,query,relevant_ids,graded_relevance,returned_ids_top10,latency_ms,recall_at_10,mrr_at_10,ndcg_at_10,top3_hit,graded_ndcg_at_10,graded_recall_at_10_t2,graded_recall_at_10_t3,dedup_count,error +single,kw_001,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건법 제6장,3856;3868;3879,3856:3;3868:2;3879:2,3879;3868;3890;3863;3856;3908;3851;4041;3900;3895,387.0,1.000,1.000,0.947,0,0.731,1.000,1.000,0, +single,kw_002,standards,exact_keyword,fact_lookup,document,ko,0,0,중대재해 처벌 등에 관한 법률 제2장 중대산업재해,3917;3921,3917:3;3921:2,3921;3917;3919;3923;10573;10571;3916;3874;3918;3854,500.8,1.000,1.000,1.000,1,0.834,1.000,1.000,0, +single,kw_003,standards,exact_keyword,fact_lookup,document,ko,0,0,화학물질관리법 유해화학물질 영업자,3981,3981:3,3981;3985;3980;3984;3993;3857;3978;3986;3983;3957,332.3,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_004,standards,exact_keyword,fact_lookup,document,ko,0,0,근로기준법 안전과 보건,4041,4041:3,4041;3852;3851;3877;3905;3903;3858;3781;3881;4040,480.3,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_005,standards,exact_keyword,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 보호구,3888,3888:3,10570;3888;3912;3913;3911;3905;3909;3889;3910;3893,496.7,1.000,0.500,0.631,1,0.631,1.000,1.000,0, +single,nl_001,korean_only,natural_language_ko,semantic_search,document,ko,0,0,기계로 인한 산업재해 관련 법령,3856;3868;3879;3854,3854:1;3856:3;3868:2;3879:2,5244;3878;5249;3897;3863;3868;5253;3879;3856;3895,526.3,0.750,0.167,0.380,0,0.381,1.000,1.000,0, +single,nl_002,korean_only,natural_language_ko,semantic_search,document,ko,0,0,사업주가 도급을 줄 때 산업재해를 예방하기 위해 해야 할 일,3855;3867;3878,3855:3;3867:2;3878:2,3855;5227;3917;3854;5244;3867;3878;3851;3908;3903,479.6,1.000,1.000,0.793,1,0.873,1.000,1.000,0, +single,nl_003,korean_only,natural_language_ko,semantic_search,document,ko,0,0,유해화학물질을 다루는 회사가 지켜야 할 안전 의무,3980;3981;3982,3980:2;3981:2;3982:2,3980;3903;3904;3981;3760;5253;3985;3896;3917;3857,419.5,0.667,1.000,0.671,1,0.671,0.667,0.000,0, +single,nl_004,korean_only,natural_language_ko,semantic_search,document,ko,0,0,중대재해가 발생했을 때 경영책임자가 처벌받는 기준,3916;3917;3920;3921,3916:2;3917:3;3920:2;3921:2,10572;10573;3917;3918;3916;3923;5244;3919;10571;5227,408.9,0.500,0.333,0.346,1,0.399,0.500,1.000,0, +single,nl_005,korean_only,natural_language_ko,semantic_search,document,ko,0,0,안전보건교육은 누가 받아야 하고 어떤 내용을 다루는가,3853;3865,3853:3;3865:2,3853;3876;5249;5234;4025;6675;11677;10573;3757;3859,606.2,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,news_001,korean_only,news_ko,semantic_search,news,ko,0,0,이란과 미국의 군사 충돌,4303;4304;4307;4316;4322;4323;4327;4335,4303:2;4304:2;4307:2;4316:2;4322:2;4323:2;4327:2;4335:2,16081;18077;22048;12213;4317;15793;4321;21273;21276;4307,494.8,0.125,0.100,0.073,1,0.073,0.125,0.000,0, +single,news_002,korean_only,news_ko,semantic_search,news,ko,0,0,호르무즈 해협 봉쇄,4316;4320;4322;4327,4316:3;4320:2;4322:2;4327:2,22055;22049;16392;9022;17123;11945;6657;9105;6774;6780,414.0,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,misc_001,korean_only,other_domain,fact_lookup,document,ko,0,0,강체의 평면 운동학,4063;4065,4063:3;4065:2,4063;4065;4064;4071;4066;4068;4058;4067;5063;5105,596.6,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,misc_002,korean_only,other_domain,semantic_search,document,ko,0,0,질점의 운동역학,4060;4061;4062,4060:2;4061:2;4062:2,4060;4062;4070;4059;4061;4064;4065;4063;4058;5095,573.0,1.000,1.000,0.947,1,0.947,1.000,0.000,0, +single,news_003,english_only,news_en,semantic_search,news,en,0,0,Trump Iran ultimatum,4258;4260;4262,4258:2;4260:2;4262:2,21155;21441;4775;4202;4776;4679;4199;4519;4668;25592,379.4,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_001,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,기계 안전 가드 설계 원리,3770;3856,3770:3;3856:2,5239;5244;3770;3817;3774;4540;3762;3789;5249;5253,578.0,0.500,0.333,0.307,1,0.394,0.500,1.000,0, +single,cl_002,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,산업 안전 입문서,3755;3775;3776;3777,3755:2;3775:2;3776:2;3777:2,5249;3760;5229;3755;3774;5230;10573;3775;3787;3818,509.2,0.500,0.250,0.291,1,0.291,0.500,0.000,0, +single,cl_003,mixed,crosslingual_ko_en,semantic_search,document,mixed,0,0,전기 안전 위험,3772;3790,3772:2;3790:2,3772;3790;5260;3897;3755;5235;3773;10568;13936;13938,951.6,1.000,1.000,1.000,1,1.000,1.000,0.000,0, +single,news_004,mixed,news_fr,semantic_search,news,mixed,0,0,guerre en Iran,4199;4202;4210;4361;4363;4507;4519;4521,4199:2;4202:2;4210:2;4361:2;4363:2;4507:2;4519:2;4521:2,7740;5925;5840;16010;16457;6945;4507;5398;16842;4199,427.7,0.250,0.143,0.157,1,0.157,0.250,0.000,0, +single,news_005,mixed,news_crosslingual,semantic_search,news,mixed,0,0,이란 미국 전쟁 글로벌 반응,4202;4258;4262;4536;4303;4304;4316,4202:2;4258:2;4262:2;4303:2;4304:2;4316:2;4536:2,21275;16927;4457;19111;16761;6789;5520;17242;9469;4765,634.7,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_001,failure_expected,failure_expected,semantic_search,document,mixed,0,1,Rust async runtime tokio scheduler 내부 구조,,,5161;11612;5262;23732;24155;20758;4547;25771;3774;20036,399.1,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_002,failure_expected,failure_expected,semantic_search,document,ko,0,1,양자컴퓨터 큐비트 디코히어런스,,,5070;16289;16384;5066;20507;5092;5118;25198;5173;23605,336.1,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_003,failure_expected,failure_expected,semantic_search,news,ko,0,1,재즈 보컬리스트 빌리 홀리데이,,,18567;20022;20470;4634;15361;16059;9102;23082;23336;18286,322.6,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,kw_006,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준에 관한 규칙 작업장 통로,3886;3887,3886:3;3887:2,3898;3886;3887;3881;3908;3912;3892;3900;3909;3890,450.9,1.000,0.500,0.693,1,0.665,1.000,1.000,0, +single,kw_007,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 폭발 화재 위험물 누출 방지,3896;3766,3766:1;3896:3,3896;3895;3903;13930;13931;13929;3894;3886;13944;3892,406.3,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,kw_008,standards,standards,fact_lookup,document,ko,0,0,고압가스 안전관리법 전문,4025;4026,4025:3;4026:2,11644;11579;4026;4025;11693;4034;11645;13299;13749;13766,463.2,1.000,0.333,0.571,0,0.508,1.000,1.000,0, +single,kw_009,standards,standards,fact_lookup,document,ko,0,0,KGS FP111 가스설비 배관설비 기준,13305,13305:3,13305;13311;13306;13312;13302;13304;13309;13299;13313;13918,451.6,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_010,standards,standards,fact_lookup,document,ko,0,0,KGS FU551 가스설비 압력조정기 가스계량기,13652,13652:3,13652;11689;13657;13655;13656;13649;13651;13752;13659;13650,373.4,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,kw_011,standards,standards,fact_lookup,document,ko,0,0,산업안전보건기준 전기로 인한 위험 방지,3897;3772,3772:1;3897:3,3897;3895;3887;13935;3902;3896;3877;3900;3899;3854,485.3,0.500,1.000,0.613,1,0.917,1.000,1.000,0, +single,en_001,english_only,english_only,semantic_search,document,en,0,0,pressure vessel flange design,5144;5136,5136:2;5144:3,5144;5140;5136;5180;5137;5149;5178;5207;5148;5212,1668.8,1.000,1.000,0.920,1,0.956,1.000,1.000,0, +single,en_002,english_only,english_only,semantic_search,document,en,0,0,ASME Section VIII introduction,5204;5206,5204:3;5206:2,5204;5212;5180;5208;5210;5143;5141;5206;5137;11634,1483.3,1.000,1.000,0.807,1,0.894,1.000,1.000,0, +single,en_003,english_only,english_only,semantic_search,document,en,0,0,impact test requirements ASME,5205;5148,5148:1;5205:3,5205;5204;5178;5214;5224;5148;5140;5209;5145;5186,1649.4,1.000,1.000,0.832,1,0.964,1.000,1.000,0, +single,en_004,english_only,english_only,semantic_search,document,en,0,0,design of vessel supports,5149,5149:3,5149;5186;5141;5137;5140;5139;5136;5178;5145;5143,1600.5,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_005,english_only,english_only,semantic_search,document,en,0,0,hydrogen piping ASME code,5178,5178:3,5178;5180;5210;5179;5212;5182;5133;5208;5207;5148,1571.5,1.000,1.000,1.000,1,1.000,1.000,1.000,0, +single,en_006,english_only,english_only,semantic_search,document,en,0,0,ASME welding qualification requirements,5209;3771,3771:1;5209:3,5224;5208;5209;5205;5178;5204;5180;5225;5187;4835,1461.3,0.500,0.333,0.307,1,0.459,1.000,1.000,0, +single,en_007,english_only,english_only,semantic_search,document,en,0,0,pressure vessel fabrication and inspection,5208;5145,5145:2;5208:3,5208;5189;5192;5180;5187;5225;5188;5186;5182;5137,1695.7,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,en_008,english_only,english_only,semantic_search,document,en,0,0,Industrial Safety and Health Management ergonomics,3763;3755,3755:1;3763:3,3763;3759;3774;3755;3818;3812;3778;3756;3761;3769,1094.0,1.000,1.000,0.877,1,0.974,1.000,1.000,0, +single,cl_004,mixed,mixed,semantic_search,document,mixed,0,0,ASME 압력용기 설계 실무,5207;5210;5139;5135,5135:2;5139:2;5207:3;5210:3,5139;5207;5204;5225;5206;5208;5210;5137;5182;5145,796.6,0.750,1.000,0.767,1,0.686,0.750,1.000,0, +single,cl_005,mixed,mixed,semantic_search,document,mixed,0,0,ASME 용접 코드 해설,5224;5209,5209:2;5224:3,5224;5222;5225;5209;5204;5210;5205;5178;5143;5207,753.3,1.000,1.000,0.877,1,0.932,1.000,1.000,0, +single,cl_006,mixed,mixed,semantic_search,document,mixed,0,0,pressure vessel Korean industrial safety regulation,4025;3856;5136,3856:2;4025:2;5136:1,4026;5145;5186;5143;3895;5210;13749;5204;13760;5244,932.6,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,cl_007,mixed,mixed,semantic_search,document,mixed,0,0,KGS 코드 LPG safety standard,11647;11689;11645;4025,4025:1;11645:2;11647:3;11689:2,11647;13760;13674;13669;13774;13773;13675;13755;13924;11688,407.7,0.250,1.000,0.390,1,0.647,0.333,1.000,0, +single,cl_008,mixed,mixed,semantic_search,document,mixed,0,0,수소 가스 안전 기준 hydrogen safety,5178;5169,5169:2;5178:3,10575;5177;11671;11649;11648;13915;5241;5173;11653;5170,642.5,0.000,0.000,0.000,0,0.000,0.000,0.000,0, +single,exam_001,exam,exam,fact_lookup,document,ko,0,0,레이놀즈수 정의 공식,11504;11505,11504:3;11505:2,11504;11533;5090;4544;11509;11505;11476;11486;5132;5140,526.2,1.000,1.000,0.832,1,0.907,1.000,1.000,0, +single,exam_002,exam,exam,fact_lookup,document,ko,0,0,탱크 바닥 구멍 유체 유속 토리첼리,11500;11495;11496,11495:2;11496:2;11500:3,11500;11495;11501;5139;5090;5140;5178;11515;11517;5210,353.1,0.667,1.000,0.765,1,0.856,0.667,1.000,0, +single,exam_003,exam,exam,fact_lookup,document,ko,0,0,이상기체 음속 마하수,11514;11515,11514:3;11515:2,11514;5083;11475;5090;5084;11531;11476;11473;11479;5124,603.1,0.500,1.000,0.613,1,0.787,0.500,1.000,0, +single,exam_004,exam,exam,fact_lookup,document,ko,0,0,고압가스 용기 내압시험 영구증가량,11591;11644;11691,11591:3;11644:2;11691:2,11591;11664;13660;5177;11691;11665;13652;13942;13917;11660,368.0,0.667,1.000,0.651,1,0.785,0.667,1.000,0, +single,exam_005,exam,exam,fact_lookup,document,ko,0,0,도시가스 배관 매설 이격거리,11627;11625;11646,11625:2;11627:3;11646:1,11627;11658;11600;11625;13918;11692;13751;5177;11655;13653,398.7,0.667,1.000,0.671,1,0.883,1.000,1.000,0, +single,exam_006,exam,exam,fact_lookup,document,ko,0,0,LPG 저장탱크 안전거리 분말소화기,11617;11669;11620,11617:3;11620:1;11669:2,11595;11616;13669;11617;11649;11605;11655;11690;11658;13915,344.9,0.333,0.250,0.202,0,0.321,0.500,1.000,0, +single,exam_007,exam,exam,fact_lookup,document,ko,0,0,오리피스 차압식 유량계,11712;11711;11503,11503:2;11711:2;11712:3,11711;11712;11503;11500;11714;11713;13930;11717;11701;11715,385.9,1.000,1.000,1.000,1,0.858,1.000,1.000,0, +single,fail_004,failure_expected,failure_expected,fact_lookup,document,ko,0,1,KGS AC999 임의 가스 코드,,,11691;11647;11693;11692;13665;13661;13664;13666;13670;13773,386.7,0.000,0.000,0.000,1,0.000,0.000,0.000,0, +single,fail_005,failure_expected,failure_expected,fact_lookup,document,ko,0,1,초전도 안전 관리법 시행규칙,,,4026;5236;3966;3977;5260;3971;4018;3972;3973;3974,470.2,0.000,0.000,0.000,1,0.000,0.000,0.000,0, diff --git a/tests/search_eval/baselines/v0_2_hier_replace_decision_2026-05-25.md b/tests/search_eval/baselines/v0_2_hier_replace_decision_2026-05-25.md new file mode 100644 index 0000000..d659176 --- /dev/null +++ b/tests/search_eval/baselines/v0_2_hier_replace_decision_2026-05-25.md @@ -0,0 +1,79 @@ +# Hier vs Legacy 검색 교체 go/no-go — Decision + +**PR**: PR-DocSrv-Hier-Replace-Diagnose-1 (plan `hier-hazy-waffle.md`) +**측정일**: 2026-05-25 | **branch**: `feat/hier-replace-diagnose` +**결정**: 🔴 **NO-GO** — 절(leaf) 단위 청크로 검색 코퍼스를 교체하지 않는다. + +--- + +## 질문 + +대형/구조화 문서를 절(clause/leaf) 단위로 분해한 hier 청크가 기존 고정윈도우(legacy) 청크보다 **doc-level 검색 품질이 좋은가?** 좋으면 검색 코퍼스를 교체(`in_corpus` 플립)한다. + +## 측정 설계 (비파괴) + +- **변종 3종 (chunk leg 뷰 교체, doc-level + fts/trgm = documents 무관)**: `prehier`(legacy baseline, legacy 30952 + null-source 375) / `hier_sim_raw`(post-replace 시뮬, builder 산출 그대로) / `hier_sim_clean`(+ childless-tiny<30자 leaf 제외). +- **exact KNN 양쪽 강제** (`enable_indexscan/bitmapscan=off`) → ivfflat 근사 변수 제거, 청킹 전략만 분리. **공정 비교 = prehier exact vs hier_sim exact.** +- **커버리지 보정 (Phase A)**: eval 타깃 미분해 non-news 47 doc(law 21+library 24+document 2) 분해+임베딩(additive, in_corpus=false). 법령 native 장→절→조 포함 (DOC_MIN_CHARS=4000 게이트 우회). builder spot-check = **held-out 법령**(eval 타깃 제외)으로 검수 → eval 과적합 차단(B1). +- snapshot freeze (doc_id_max=25912, chunk_id_max=71164). in_corpus 634 전 구간 불변. +- **dedup audit (B4)**: 전 변종 0/51 cases — inflation 0, NDCG ≤ 1.0. **회귀는 artifact 아닌 실제값.** + +## 결과 + +### Overall (full-51, graded NDCG@10) +| variant | NDCG | Δ vs prehier | p50 lat | +|---|---:|---:|---:| +| **prehier exact** (legacy 기준) | **0.672** | — | 416ms | +| hier_sim_clean exact | 0.606 | **−0.066** | 1025ms | +| hier_sim_raw exact | 0.603 | −0.069 | 581ms | +| (harness sanity: production path) | 0.671 | — | 485ms | + +- harness sanity 0.671 ≈ PR-Eval-V0_2 baseline 0.659 (+0.012 = 634 pilot leaf 편입 + corpus 성장). 측정 체인 정상. +- prehier exact ≈ sanity (0.672 vs 0.671) → 소형 corpus 라 ivfflat ≈ exact. + +### 결정權 = 분해-subset (B2) +full-51 은 변종 미영향 doc(news/단일청크 등)으로 희석됨. **hier 가 실제 영향 주는 query(graded 타깃이 분해된 doc) 41건**: + +| subset | n | prehier | clean | Δ | raw | Δ | +|---|---:|---:|---:|---:|---:|---:| +| **AFFECTED (결정權)** | 41 | **0.748** | **0.675** | **−0.074** | 0.673 | −0.075 | +| UNCHANGED (sanity) | 10 | 0.023 | 0.023 | +0.000 | 0.016 | −0.007 | + +→ 분해된 doc query 에서 hier 가 **−0.074 NDCG 회귀**. unchanged subset 은 변동 0 (변종이 안 건드릴 곳은 안 건드림 ✓). + +### 카테고리별 (affected subset, prehier → clean) +| category | n | prehier | clean | Δ | +|---|---:|---:|---:|---:| +| **standards (법령, hier 최적 가설)** | 11 | 0.837 | 0.834 | **−0.002 (flat)** | +| mixed | 8 | 0.494 | 0.463 | −0.031 | +| english_only | 8 | 0.885 | 0.796 | −0.088 | +| korean_only | 7 | 0.723 | 0.614 | −0.109 | +| **exam** | 7 | 0.770 | 0.587 | **−0.183** | + +**핵심**: hier 가 가장 유리할 것으로 가정한 **standards(법령 제N조)조차 flat** — 개선 아님. 나머지 전부 회귀, 특히 **exam −0.183**. + +### 왜 hier 가 지는가 (per-query losers) +- exam_005(도시가스 매설 이격거리)/exam_006(LPG 안전거리)/exam_001(레이놀즈수 공식) — exam 질문은 **사실이 맥락 속에 있어야** 매칭. 짧은 절 leaf 가 답을 조각냄. +- cl_007(KGS LPG standard)/en_008(ISH Management) — 넓은 개념 query 는 절 leaf 보다 **주변 맥락 보유한 고정윈도우 청크가 더 잘 매칭**. +- winners(소수): cl_004(ASME 설계 실무 +0.18)/kw_008(고압가스법 전문 +0.16) — 특정 절이 정확히 맞는 케이스. 절-단위가 도움되는 query 타입 존재하나 net 부정. + +## 판정 (B1~B4) +- **B2 결정權 = AFFECTED subset −0.074** → 명확한 회귀. +- **B3 exact 마진**: hier 가 exact 에서 이미 prehier 에 뒤짐 → production ivfflat(hier leaf 미색인, REINDEX 필요, 짧은 leaf 多 → 근사 품질 악화 경향)에서 뒤집힐 여지 없음. exact 우위 마진이 음수. +- **B4 dedup**: clean/raw 모두 0/51, inflation 0 → 회귀는 실제. +- **B1 과적합**: held-out 튜닝 + raw≈clean → 결과는 builder 튜닝 산물 아님. + +## 결정: 🔴 NO-GO + +**검색 코퍼스를 hier leaf 로 교체하지 않는다.** 절 단위 청크는 doc-level 검색에서 고정윈도우 대비 개선이 없고(법령 flat) 대체로 회귀(exam/korean/english). `replace_doc_corpus` 대량 적용 + ivfflat REINDEX(`PR-DocSrv-Hier-Replace-Apply-1`)는 **진입 안 함**. + +### hier leaf 의 잔존 가치 (폐기 아님) +- **section-outline UI** (`PR-DocSrv-Hier-Section-UI-1`, 배포됨) — 문서 상세의 절 목차/요약. retrieval 과 무관, 그대로 유지. +- 12,697 leaf + 절 분석은 **in_corpus=false 로 잔존** (검색 무영향). 향후 clause-level 답변(Ask/RAG 정밀 passage) 또는 deep-link 기능의 재료. +- **본 측정은 doc-level NDCG 한정.** 절 단위가 도움될 다른 granularity(짧은 passage 반환형 RAG)는 별도 평가 영역 — 본 NO-GO 는 "doc 검색 코퍼스 교체" 한정. + +## Out-of-scope / 잔여 +- Phase A 가 추가한 47 doc 의 hier leaf 는 in_corpus=false 로 둠 (UI 절 목차에는 노출 — additive). +- 측정 뷰 3종(`corpus_chunks_prehier/hier_sim_raw/hier_sim_clean`)은 reference 로 보존 또는 DROP(사용자 결정). 운영 검색 무영향. +- 47 신규 분해 doc 의 per-leaf 절 분석(Mac mini) = 미실행(retrieval 무관, defer). +- builder 의 법령 childless-tiny 제외(clean predicate)는 hier UI 품질엔 유효 — 향후 UI 트랙에서 builder.py 정식 반영 검토 가능(별 PR). diff --git a/tests/search_eval/scripts/hier_replace_subset_analysis.py b/tests/search_eval/scripts/hier_replace_subset_analysis.py new file mode 100644 index 0000000..34be97d --- /dev/null +++ b/tests/search_eval/scripts/hier_replace_subset_analysis.py @@ -0,0 +1,65 @@ +import csv, subprocess, statistics +# decomposed doc ids (348) +import os +os.chdir(os.path.expanduser("~/Documents/code/hyungi_Document_Server")) +out = subprocess.check_output(["docker","compose","exec","-T","postgres","psql","-U","pkm","-d","pkm","-t","-A","-c", + "SELECT doc_id FROM document_chunks WHERE source_type='hier_section' AND is_leaf GROUP BY doc_id"]).decode() +decomposed = {int(x) for x in out.split() if x.strip().isdigit()} + +def load(path): + rows={} + with open(path) as f: + for r in csv.DictReader(f): + rows[r["id"]] = r + return rows + +pre = load("/tmp/hier_prehier_exact.csv") +clean = load("/tmp/hier_clean_exact.csv") +raw = load("/tmp/hier_raw_exact.csv") + +def targets(r): + gr = r["graded_relevance"] + ids=[] + for part in gr.split(";"): + if ":" in part: + ids.append(int(part.split(":")[0])) + return ids + +affected=[]; unchanged=[] +for qid,r in pre.items(): + tg=targets(r) + if any(t in decomposed for t in tg): + affected.append(qid) + else: + unchanged.append(qid) + +def mean_ndcg(rows, ids): + vals=[float(rows[i]["graded_ndcg_at_10"]) for i in ids if i in rows] + return statistics.mean(vals) if vals else 0.0, len(vals) + +print(f"decomposed docs: {len(decomposed)}") +print(f"queries affected (target in decomposed): {len(affected)} | unchanged: {len(unchanged)}") +print() +for name,ids in [("AFFECTED-SUBSET (decision權 B2)",affected),("UNCHANGED (sanity: should be ~equal)",unchanged),("FULL-51",list(pre.keys()))]: + p,_=mean_ndcg(pre,ids); c,_=mean_ndcg(clean,ids); rw,n=mean_ndcg(raw,ids) + print(f"{name} (n={n}):") + print(f" prehier={p:.3f} clean={c:.3f} (Δ{c-p:+.3f}) raw={rw:.3f} (Δ{rw-p:+.3f})") +print() +# per-category within affected subset +cats={} +for qid in affected: + cats.setdefault(pre[qid]["category"],[]).append(qid) +print("AFFECTED subset by category (prehier -> clean):") +for cat,ids in sorted(cats.items()): + p,_=mean_ndcg(pre,ids); c,n=mean_ndcg(clean,ids) + print(f" {cat:14s} n={n:2d} prehier={p:.3f} clean={c:.3f} (Δ{c-p:+.3f})") +print() +# biggest per-query losers/winners (clean vs prehier) within affected +deltas=[(float(clean[q]["graded_ndcg_at_10"])-float(pre[q]["graded_ndcg_at_10"]), q, pre[q]["category"], pre[q]["query"][:34]) for q in affected] +deltas.sort() +print("Biggest LOSERS (clean - prehier):") +for d,q,c,qry in deltas[:6]: + print(f" {d:+.3f} {q} [{c}] {qry}") +print("Biggest WINNERS:") +for d,q,c,qry in deltas[-5:][::-1]: + print(f" {d:+.3f} {q} [{c}] {qry}")