{"id":"https://openalex.org/W7164917139","doi":"https://doi.org/10.48550/arxiv.2606.15029","title":"Metric Match: A Subset Selection Approach to Evaluating LLM Judge Reliability","display_name":"Metric Match: A Subset Selection Approach to Evaluating LLM Judge Reliability","publication_year":2026,"publication_date":"2026-06-12","ids":{"openalex":"https://openalex.org/W7164917139","doi":"https://doi.org/10.48550/arxiv.2606.15029"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.15029","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15029","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.15029","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138746303","display_name":"Alyssa Unell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Unell, Alyssa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018536380","display_name":"Natalie Dullerud","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dullerud, Natalie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138721970","display_name":"Naomi Boneh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boneh, Naomi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005798722","display_name":"Meena Jagadeesan","orcid":"https://orcid.org/0000-0002-3521-440X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jagadeesan, Meena","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038596598","display_name":"Tatsu Hashimoto","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hashimoto, Tatsu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136221977","display_name":"Nigam Shah","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shah, Nigam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138695244","display_name":"Sanmi Koyejo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koyejo, Sanmi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6428999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6428999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04740000143647194,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.04699999839067459,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7954999804496765},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7261000275611877},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.6875},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5479999780654907},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.40389999747276306},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.38190001249313354},{"id":"https://openalex.org/keywords/human-reliability","display_name":"Human reliability","score":0.3723999857902527}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7954999804496765},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7261000275611877},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.6875},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6164000034332275},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5479999780654907},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4106000065803528},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4090999960899353},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.40389999747276306},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.38190001249313354},{"id":"https://openalex.org/C191147762","wikidata":"https://www.wikidata.org/wiki/Q186289","display_name":"Human reliability","level":3,"score":0.3723999857902527},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.3716999888420105},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.35589998960494995},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.35530000925064087},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.3499999940395355},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31949999928474426},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.28870001435279846},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.2549000084400177},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.15029","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15029","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.15029","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.15029","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.7745224833488464,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM":[0,56],"judges":[1,22,57],"are":[2],"used":[3],"to":[4,82,140,154],"reduce":[5],"the":[6,18,73,76],"need":[7],"for":[8,50,68,143,186],"costly":[9,38],"human":[10,29,39,69],"labor":[11],"in":[12,113],"evaluating":[13],"open-ended":[14],"text":[15],"generation.":[16],"However,":[17],"reliability":[19,53,78,152,155],"of":[20,55,66,95,157,188],"these":[21],"depends":[23,36],"critically":[24],"on":[25,37],"their":[26],"alignment":[27],"with":[28,80,109,170],"raters":[30],"--":[31],"a":[32,46,64,93,125,130,159,164],"property":[33],"that":[34,72,89],"itself":[35],"annotations.":[40,60],"In":[41],"this":[42],"work,":[43],"we":[44,147,180],"develop":[45],"method":[47,136],"(Metric":[48],"Match)":[49],"estimating":[51],"correlation-based":[52],"metrics":[54,105],"from":[58,151],"limited":[59],"Metric":[61,90,171],"Match":[62,91],"selects":[63],"subset":[65,74,99],"samples":[67],"annotation":[70,119],"such":[71],"matches":[75],"population":[77],"metric":[79],"respect":[81],"acquired":[83],"synthetic":[84],"labels.":[85],"We":[86,123],"empirically":[87],"show":[88],"achieves":[92],"win-rate":[94],"0.838":[96],"against":[97],"random":[98,141,168],"selection":[100,142,169],"across":[101],"four":[102],"different":[103],"correlation":[104],"and":[106,117,128,179],"15":[107],"datasets,":[108],"an":[110,183],"18.7%":[111],"decrease":[112],"average":[114],"estimation":[115,153],"error":[116],"reduces":[118],"needs":[120],"by":[121],"32.5%.":[122],"provide":[124,182],"cost":[126],"model":[127],"highlight":[129],"medical":[131],"case":[132],"study":[133],"where":[134],"our":[135,149],"saves":[137],"$1,041.67":[138],"compared":[139],"expert":[144],"annotation.":[145],"Further,":[146],"shift":[148],"task":[150],"classification":[156],"whether":[158],"given":[160],"judge":[161],"is":[162,176],"above":[163],"deployment":[165],"threshold,":[166],"outperforming":[167],"Match.":[172],"All":[173],"project":[174],"code":[175],"publicly":[177],"available,":[178],"additionally":[181],"installable":[184],"package":[185],"ease":[187],"use.":[189]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-17T00:00:00"}