{"id":"https://openalex.org/W7204444844","doi":"https://doi.org/10.18653/2026.findings-emnlp.47","title":"Localize-Then-Decide Guarantees for LLM Judgments","display_name":"Localize-Then-Decide Guarantees for LLM Judgments","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7204444844","doi":"https://doi.org/10.18653/2026.findings-emnlp.47"},"language":"en","primary_location":{"id":"doi:10.18653/2026.findings-emnlp.47","doi":"https://doi.org/10.18653/2026.findings-emnlp.47","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.findings-emnlp.47","pdf_url":"https://aclanthology.org/2026.findings-emnlp.47.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2026","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://aclanthology.org/2026.findings-emnlp.47.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148463168","display_name":"Xinyu Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I23923803","display_name":"University of Exeter","ror":"https://ror.org/03yghzc09","country_code":"GB","type":"education","lineage":["https://openalex.org/I23923803"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Xinyu Li","raw_affiliation_strings":["University of Exeter"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Exeter","institution_ids":["https://openalex.org/I23923803"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148418868","display_name":"Yi Zhou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I79510175","display_name":"Cardiff University","ror":"https://ror.org/03kk7td41","country_code":"GB","type":"education","lineage":["https://openalex.org/I79510175"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yi Zhou","raw_affiliation_strings":["Cardiff University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cardiff University","institution_ids":["https://openalex.org/I79510175"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148422403","display_name":"Guanqun Cao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I178535277","display_name":"University of the West of England","ror":"https://ror.org/02nwg5t34","country_code":"GB","type":"education","lineage":["https://openalex.org/I178535277"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Guanqun Cao","raw_affiliation_strings":["University of the West of England"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of the West of England","institution_ids":["https://openalex.org/I178535277"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148441722","display_name":"Zeyu Fu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I23923803","display_name":"University of Exeter","ror":"https://ror.org/03yghzc09","country_code":"GB","type":"education","lineage":["https://openalex.org/I23923803"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zeyu FU","raw_affiliation_strings":["University of Exeter"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Exeter","institution_ids":["https://openalex.org/I23923803"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123896017","display_name":"Tianjin Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I23923803","display_name":"University of Exeter","ror":"https://ror.org/03yghzc09","country_code":"GB","type":"education","lineage":["https://openalex.org/I23923803"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Tianjin Huang","raw_affiliation_strings":["University of Exeter"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Exeter","institution_ids":["https://openalex.org/I23923803"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123993160","display_name":"Gaojie Jin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Gaojie Jin","raw_affiliation_strings":["University of Macau"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Macau","institution_ids":["https://openalex.org/I204512498"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3743,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.89891939,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"943","last_page":"965"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.21369999647140503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.21369999647140503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1428000032901764,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.09610000252723694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/confidence-thresholding","display_name":"confidence thresholding","score":0.5360000133514404},{"id":"https://openalex.org/keywords/conformal-prediction","display_name":"conformal prediction","score":0.8180000185966492},{"id":"https://openalex.org/keywords/preference-alignment","display_name":"preference alignment","score":0.6439999938011169},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.5080000162124634},{"id":"https://openalex.org/keywords/confidence-calibration","display_name":"confidence calibration","score":0.6629999876022339},{"id":"https://openalex.org/keywords/abstention","display_name":"abstention","score":0.6100000143051147}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.741100013256073},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5414999723434448},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5282999873161316},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.4659000039100647},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.4203999936580658},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.36640000343322754},{"id":"https://openalex.org/C2776292839","wikidata":"https://www.wikidata.org/wiki/Q5179217","display_name":"Coverage probability","level":3,"score":0.3610000014305115},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3564999997615814},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3352999985218048},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3336000144481659},{"id":"https://openalex.org/C183905921","wikidata":"https://www.wikidata.org/wiki/Q1038757","display_name":"Multiple comparisons problem","level":2,"score":0.3264000117778778},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.325300008058548},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3075000047683716},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30550000071525574},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2921999990940094},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2712000012397766},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.26339998841285706},{"id":"https://openalex.org/C2778067643","wikidata":"https://www.wikidata.org/wiki/Q166507","display_name":"Interval (graph theory)","level":2,"score":0.2621000111103058}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/2026.findings-emnlp.47","doi":"https://doi.org/10.18653/2026.findings-emnlp.47","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.findings-emnlp.47","pdf_url":"https://aclanthology.org/2026.findings-emnlp.47.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2026","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2608.25824","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.25824","pdf_url":"https://arxiv.org/pdf/2608.25824","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.25824","doi":"https://doi.org/10.48550/arxiv.2608.25824","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.25824","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/2026.findings-emnlp.47","doi":"https://doi.org/10.18653/2026.findings-emnlp.47","is_oa":true,"landing_page_url":"https://doi.org/10.18653/2026.findings-emnlp.47","pdf_url":"https://aclanthology.org/2026.findings-emnlp.47.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204444844.pdf","grobid_xml":"https://content.openalex.org/works/W7204444844.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W3210904070","https://openalex.org/W4221161695","https://openalex.org/W4223908421","https://openalex.org/W4285429195","https://openalex.org/W4290055968","https://openalex.org/W4378942327","https://openalex.org/W4378945636","https://openalex.org/W4381551003","https://openalex.org/W4385006113","https://openalex.org/W4387560040","https://openalex.org/W4391988394","https://openalex.org/W4394652919","https://openalex.org/W4396570275","https://openalex.org/W4396763774","https://openalex.org/W4396777837","https://openalex.org/W4402466848","https://openalex.org/W4403583114","https://openalex.org/W4404782348","https://openalex.org/W7055216282","https://openalex.org/W7078268262","https://openalex.org/W7129012486","https://openalex.org/W7197047392"],"related_works":["https://openalex.org/W4402466848","https://openalex.org/W7161588979","https://openalex.org/W7213244650","https://openalex.org/W4417163898","https://openalex.org/W4407310229","https://openalex.org/W7203791510","https://openalex.org/W7129012486","https://openalex.org/W7090848549","https://openalex.org/W4402670754","https://openalex.org/W4416036640"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"used":[6],"as":[7],"evaluators":[8],"to":[9],"assess":[10],"output":[11],"quality":[12],"and":[13,118,121,133,146],"preference":[14],"alignment,":[15],"yet":[16],"providing":[17],"reliable":[18],"guarantees":[19,34],"of":[20,60],"agreement":[21,124],"with":[22,50,94,126],"human":[23],"judgments":[24],"remains":[25],"challenging.Recent":[26],"work":[27],"introduces":[28],"confidence-thresholding":[29],"methods":[30],"that":[31,42,89,137],"provide":[32],"such":[33],"for":[35],"pairwise":[36],"comparisons,":[37],"relying":[38],"on":[39],"the":[40,58,91,113],"assumption":[41,53],"higher":[43,142,148],"estimated":[44],"confidence":[45,73,117],"implies":[46],"lower":[47],"disagreement":[48,119],"risk":[49,120],"humans.However,":[51],"this":[52,76,107],"can":[54,71],"break":[55],"down":[56],"when":[57],"number":[59],"candidate":[61,128],"responses":[62],"increases,":[63],"since":[64],"distributing":[65],"probability":[66],"mass":[67],"across":[68,130],"many":[69],"alternatives":[70],"distort":[72],"estimates.To":[74],"address":[75],"issue,":[77],"we":[78],"propose":[79],"a":[80,86,97,103],"Localize-Then-Decide":[81],"framework.First,":[82],"conformal":[83],"prediction":[84],"localizes":[85],"small":[87],"shortlist":[88,108],"contains":[90],"human-preferred":[92],"response":[93,105],"high":[95],"probability.Then,":[96],"calibrated":[98],"confidence-based":[99],"rule":[100],"selectively":[101],"chooses":[102],"single":[104],"from":[106],"or":[109],"abstains.This":[110],"design":[111],"restores":[112],"monotonic":[114],"relationship":[115],"between":[116],"enables":[122],"high-probability":[123],"guarantees.Experiments":[125],"multiple":[127],"sizes":[129],"several":[131],"datasets":[132],"judge":[134],"LLMs":[135],"demonstrate":[136],"our":[138],"framework":[139],"consistently":[140],"achieves":[141],"guarantee":[143],"success":[144],"rates":[145],"substantially":[147],"coverage":[149],"than":[150],"single-stage":[151],"baselines.":[152]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-28T00:00:00"}