{"id":"https://openalex.org/W7150847780","doi":"https://doi.org/10.48550/arxiv.2604.03216","title":"BAS: A Decision-Theoretic Approach to Evaluating Large Language Model Confidence","display_name":"BAS: A Decision-Theoretic Approach to Evaluating Large Language Model Confidence","publication_year":2026,"publication_date":"2026-04-03","ids":{"openalex":"https://openalex.org/W7150847780","doi":"https://doi.org/10.48550/arxiv.2604.03216"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.03216","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03216","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.03216","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133023389","display_name":"Sean Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Sean","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058002446","display_name":"Fredrik Gustafsson","orcid":"https://orcid.org/0000-0003-3270-171X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gustafsson, Fredrik K.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133011496","display_name":"Edward Phillips","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Phillips, Edward","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069203302","display_name":"Boyan Gao","orcid":"https://orcid.org/0000-0003-4866-9988"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Boyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035845115","display_name":"Anshul Thakur","orcid":"https://orcid.org/0000-0002-7006-1947"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thakur, Anshul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133001381","display_name":"David A. Clifton","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Clifton, David A.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2079000025987625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2079000025987625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.17949999868869781,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09849999845027924,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overconfidence-effect","display_name":"Overconfidence effect","score":0.8273000121116638},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6118999719619751},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5842000246047974},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5716000199317932},{"id":"https://openalex.org/keywords/confidence-interval","display_name":"Confidence interval","score":0.4546999931335449},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.4510999917984009},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.41260001063346863},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.390500009059906}],"concepts":[{"id":"https://openalex.org/C51110983","wikidata":"https://www.wikidata.org/wiki/Q16503490","display_name":"Overconfidence effect","level":2,"score":0.8273000121116638},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6118999719619751},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6057999730110168},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5842000246047974},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5716000199317932},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.4546999931335449},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4510999917984009},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4172999858856201},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.41260001063346863},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.40779998898506165},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.390500009059906},{"id":"https://openalex.org/C2909755999","wikidata":"https://www.wikidata.org/wiki/Q4751126","display_name":"Low Confidence","level":2,"score":0.3596999943256378},{"id":"https://openalex.org/C205706631","wikidata":"https://www.wikidata.org/wiki/Q2319304","display_name":"Expected utility hypothesis","level":2,"score":0.35010001063346863},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34940001368522644},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.32989999651908875},{"id":"https://openalex.org/C41426520","wikidata":"https://www.wikidata.org/wiki/Q1192065","display_name":"Point estimation","level":2,"score":0.30889999866485596},{"id":"https://openalex.org/C207609745","wikidata":"https://www.wikidata.org/wiki/Q4944086","display_name":"Bootstrapping (finance)","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2922999858856201},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.28769999742507935},{"id":"https://openalex.org/C13662513","wikidata":"https://www.wikidata.org/wiki/Q5160087","display_name":"Confidence distribution","level":3,"score":0.28299999237060547},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.27549999952316284},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.27149999141693115},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2712000012397766},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.03216","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03216","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.03216","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03216","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7827993631362915,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,186,194,201],"(LLMs)":[3],"often":[4],"produce":[5],"confident":[6],"but":[7,125],"incorrect":[8],"answers":[9],"in":[10,177],"settings":[11],"where":[12],"abstention":[13],"would":[14],"be":[15],"safer.":[16],"Standard":[17],"evaluation":[18],"protocols,":[19],"however,":[20],"require":[21],"a":[22,49,76,82,161,246,250],"response":[23],"and":[24,71,93,132,156,170,180,183,233,249],"do":[25],"not":[26],"account":[27],"for":[28,52,253],"how":[29,54],"confidence":[30,57,102,165,231,239,256],"should":[31],"guide":[32],"decisions":[33],"under":[34],"different":[35,210],"risk":[36,79],"preferences.":[37],"To":[38],"address":[39],"this":[40],"gap,":[41],"we":[42,158],"introduce":[43],"the":[44,91],"Behavioral":[45],"Alignment":[46],"Score":[47],"(BAS),":[48],"decision-theoretic":[50],"metric":[51,248],"evaluating":[53,254],"well":[55],"LLM":[56,255],"supports":[58],"abstention-aware":[59],"decision":[60],"making.":[61],"BAS":[62,107,114,136,148,211],"is":[63,115],"derived":[64],"from":[65],"an":[66,138],"explicit":[67],"answer-or-abstain":[68],"utility":[69,74],"model":[70],"aggregates":[72],"realized":[73],"across":[75,167],"continuum":[77],"of":[78,84,95,163,219],"thresholds,":[80],"yielding":[81],"measure":[83],"decision-level":[85],"reliability":[86,166],"that":[87,100,141,225],"depends":[88],"on":[89],"both":[90,245],"magnitude":[92],"ordering":[94],"confidence.":[96],"We":[97,222],"show":[98,224],"theoretically":[99],"truthful":[101],"estimates":[103],"uniquely":[104],"maximize":[105],"expected":[106],"utility,":[108],"linking":[109],"calibration":[110],"to":[111,117,188,197,213],"decision-optimal":[112],"behavior.":[113],"related":[116],"proper":[118],"scoring":[119],"rules":[120],"such":[121,153,228],"as":[122,154,229],"log":[123,128],"loss,":[124],"differs":[126],"structurally:":[127],"loss":[129],"penalizes":[130],"underconfidence":[131],"overconfidence":[133],"symmetrically,":[134],"whereas":[135],"imposes":[137],"asymmetric":[139],"penalty":[140],"strongly":[142],"prioritizes":[143],"avoiding":[144],"overconfident":[145,215],"errors.":[146],"Using":[147],"alongside":[149],"widely":[150],"used":[151],"metrics":[152],"ECE":[155,204],"AURC,":[157],"then":[159],"construct":[160],"benchmark":[162,252],"self-reported":[164],"multiple":[168],"LLMs":[169],"tasks.":[171],"Our":[172],"results":[173],"reveal":[174],"substantial":[175],"variation":[176],"decision-useful":[178],"confidence,":[179],"while":[181],"larger":[182],"more":[184],"accurate":[185],"tend":[187],"achieve":[189],"higher":[190],"BAS,":[191],"even":[192],"frontier":[193],"remain":[195],"prone":[196],"severe":[198],"overconfidence.":[199],"Importantly,":[200],"with":[202],"similar":[203],"or":[205],"AURC":[206],"can":[207,236],"exhibit":[208],"very":[209],"due":[212],"highly":[214],"errors,":[216],"highlighting":[217],"limitations":[218],"standard":[220],"metrics.":[221],"further":[223],"simple":[226],"interventions,":[227],"top-$k$":[230],"elicitation":[232],"post-hoc":[234],"calibration,":[235],"meaningfully":[237],"improve":[238],"reliability.":[240,257],"Overall,":[241],"our":[242],"work":[243],"provides":[244],"principled":[247],"comprehensive":[251]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-07T00:00:00"}