{"id":"https://openalex.org/W4412888522","doi":"https://doi.org/10.18653/v1/2025.findings-acl.423","title":"UBench: Benchmarking Uncertainty in Large Language Models with Multiple Choice Questions","display_name":"UBench: Benchmarking Uncertainty in Large Language Models with Multiple Choice Questions","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888522","doi":"https://doi.org/10.18653/v1/2025.findings-acl.423"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.423","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.423","pdf_url":"https://aclanthology.org/2025.findings-acl.423.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.423.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104331801","display_name":"Xunzhi Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xunzhi Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070892867","display_name":"Zhuowei Zhang","orcid":"https://orcid.org/0009-0009-9874-1002"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhuowei Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111823043","display_name":"G.C. Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gaonan Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011611587","display_name":"Qiongyu Li","orcid":"https://orcid.org/0000-0003-3654-0378"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiongyu Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109735402","display_name":"Bitong Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bitong Luo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102691717","display_name":"Zhixin Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhixin Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006114209","display_name":"Haotian Wang","orcid":"https://orcid.org/0000-0001-9478-5862"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haotian Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113013369","display_name":"Zhiyu Li","orcid":"https://orcid.org/0000-0003-3771-0199"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhiyu Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100781021","display_name":"Hang Gao","orcid":"https://orcid.org/0000-0003-3613-4011"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hang Gao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5048141765","display_name":"Mengting Hu","orcid":"https://orcid.org/0000-0003-1536-5400"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mengting Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"8076","last_page":"8107"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9621000289916992,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8579879999160767},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7116078734397888},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38311243057250977},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3689171075820923},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33925724029541016},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.07402276992797852}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8579879999160767},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7116078734397888},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38311243057250977},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3689171075820923},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33925724029541016},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.07402276992797852},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.423","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.423","pdf_url":"https://aclanthology.org/2025.findings-acl.423.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.423","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.423","pdf_url":"https://aclanthology.org/2025.findings-acl.423.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5099999904632568,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888522.pdf","grobid_xml":"https://content.openalex.org/works/W4412888522.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Despite":[0],"recent":[1],"progress":[2],"in":[3],"systematic":[4],"evaluation":[5],"frameworks,":[6],"benchmarking":[7,22],"the":[8,23,31,59,96,99,108,163],"uncertainty":[9,24,60,93,100,136],"of":[10,25,61,98,101,107,110,165],"large":[11],"language":[12],"models":[13,143],"(LLMs)":[14],"remains":[15],"a":[16,54],"highly":[17,133],"challenging":[18],"task.Existing":[19],"methods":[20,131],"for":[21,33,46,57,135],"LLMs":[26],"face":[27],"three":[28],"key":[29],"challenges:":[30],"need":[32],"internal":[34],"model":[35,120,160],"access,":[36],"additional":[37],"training,":[38],"or":[39],"high":[40],"computational":[41],"costs.This":[42],"is":[43,66,173],"particularly":[44],"unfavorable":[45],"closed-source":[47,148],"models.To":[48],"this":[49],"end,":[50],"we":[51,84],"introduce":[52],"UBENCH,":[53],"new":[55],"benchmark":[56],"evaluating":[58],"LLMs.Unlike":[62],"other":[63,91],"benchmarks,":[64],"UBENCH":[65],"based":[67],"on":[68,82,119],"confidence":[69,129],"intervals.It":[70],"encompasses":[71],"11,978":[72],"multiple-choice":[73],"questions":[74],"spanning":[75],"knowledge,":[76],"language,":[77],"understanding,":[78],"and":[79,104,117,152],"reasoning":[80],"capabilities.Based":[81],"this,":[83],"conduct":[85],"extensive":[86],"experiments.This":[87],"includes":[88],"comparisons":[89],"with":[90],"advanced":[92],"estimation":[94],"methods,":[95],"assessment":[97],"20":[102],"LLMs,":[103],"an":[105],"exploration":[106],"effects":[109],"Chain-of-Thought":[111],"(CoT)":[112],"prompts,":[113,116],"role-playing":[114],"(RP)":[115],"temperature":[118,166],"uncertainty.Our":[121],"analysis":[122],"reveals":[123],"several":[124],"crucial":[125],"insights:":[126],"1)":[127],"Our":[128],"interval-based":[130],"are":[132],"effective":[134],"quantification;":[137],"2)":[138],"Regarding":[139],"uncertainty,":[140],"outstanding":[141],"open-source":[142],"show":[144],"competitive":[145],"performance":[146],"versus":[147],"models;":[149],"3)":[150],"CoT":[151],"RP":[153],"prompts":[154],"present":[155],"potential":[156],"ways":[157],"to":[158],"improve":[159],"reliability,":[161],"while":[162],"influence":[164],"changes":[167],"follows":[168],"no":[169],"universal":[170],"rule.Our":[171],"implementation":[172],"available":[174],"at":[175],"https://github.com/Cyno2232/UBENCH.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}