{"id":"https://openalex.org/W7163874244","doi":"https://doi.org/10.48550/arxiv.2606.06622","title":"UnpredictaBench: A Benchmark for Evaluating Distributional Randomness in LLMs","display_name":"UnpredictaBench: A Benchmark for Evaluating Distributional Randomness in LLMs","publication_year":2026,"publication_date":"2026-06-04","ids":{"openalex":"https://openalex.org/W7163874244","doi":"https://doi.org/10.48550/arxiv.2606.06622"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.06622","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06622","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.06622","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138144597","display_name":"Amirhossein Abaskohi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abaskohi, Amirhossein","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095094630","display_name":"Amirhossein Dabiriaghdam","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dabiriaghdam, Amirhossein","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138126597","display_name":"Liang Luo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Liang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113135629","display_name":"Ellie Dingqiao Wen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wen, Ellie Dingqiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138113210","display_name":"Lele Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Lele","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138126771","display_name":"Giuseppe Carenini","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Carenini, Giuseppe","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138125574","display_name":"Peter West","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"West, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09640000015497208,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.09640000015497208,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08020000159740448,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.05530000105500221,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.8069999814033508},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5252000093460083},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5231999754905701},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.4440999925136566},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.43149998784065247},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.41620001196861267},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.3727000057697296},{"id":"https://openalex.org/keywords/probability-distribution","display_name":"Probability distribution","score":0.367900013923645},{"id":"https://openalex.org/keywords/sampling-distribution","display_name":"Sampling distribution","score":0.34850001335144043}],"concepts":[{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.8069999814033508},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.607699990272522},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5252000093460083},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5231999754905701},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.503600001335144},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4440999925136566},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.43149998784065247},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.41620001196861267},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.39309999346733093},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.3727000057697296},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.367900013923645},{"id":"https://openalex.org/C167723999","wikidata":"https://www.wikidata.org/wiki/Q3773214","display_name":"Sampling distribution","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.3481999933719635},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.32510000467300415},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C127491075","wikidata":"https://www.wikidata.org/wiki/Q7617825","display_name":"Stochastic modelling","level":2,"score":0.30329999327659607},{"id":"https://openalex.org/C25915931","wikidata":"https://www.wikidata.org/wiki/Q4180825","display_name":"Stochastic simulation","level":2,"score":0.29750001430511475},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.29580000042915344},{"id":"https://openalex.org/C122123141","wikidata":"https://www.wikidata.org/wiki/Q176623","display_name":"Random variable","level":2,"score":0.2939000129699707},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C119693030","wikidata":"https://www.wikidata.org/wiki/Q2180497","display_name":"Economic model","level":2,"score":0.2782000005245209},{"id":"https://openalex.org/C45555294","wikidata":"https://www.wikidata.org/wiki/Q28113351","display_name":"Inequality","level":2,"score":0.27720001339912415},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2768000066280365},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2743000090122223},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.2680000066757202},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.25440001487731934},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2540000081062317},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25130000710487366},{"id":"https://openalex.org/C20353970","wikidata":"https://www.wikidata.org/wiki/Q1056998","display_name":"Simple random sample","level":3,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.06622","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06622","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.06622","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.06622","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,114],"introduce":[1,115],"UnpredictaBench,":[2],"an":[3],"evaluation":[4,124],"that":[5,71,110,126,237],"tests":[6],"the":[7,35,52,138,144],"ability":[8],"of":[9,37,54,89,154,186],"large":[10,175],"language":[11],"models":[12,39,183],"(LLMs)":[13],"to":[14,40,50,74,150,198,205],"capture":[15,51],"true":[16],"underlying":[17],"distributions.":[18],"As":[19],"LLMs":[20,252],"are":[21,72,262],"increasingly":[22],"used":[23],"as":[24,217,253],"substitutes":[25],"for":[26,30,65,232,255],"other":[27],"entities":[28],"(e.g.,":[29],"humans":[31],"in":[32,177,214],"economic":[33],"simulations),":[34],"tendency":[36],"many":[38],"collapse":[41],"towards":[42],"a":[43,48,75,84,122,130,174,218,246],"single":[44],"plausible":[45],"answer":[46],"means":[47],"failure":[49],"unpredictability":[53],"real":[55],"systems.":[56,257],"Recent":[57],"work":[58],"on":[59],"improving":[60],"output":[61],"diversity":[62],"is":[63,143,203],"insufficient":[64],"this":[66,90,233],"setting:":[67],"simulation":[68,241],"requires":[69],"samples":[70,153,185],"calibrated":[73],"target":[76,96,135],"distribution,":[77],"not":[78],"merely":[79],"varied":[80],"outputs.":[81],"UnpredictaBench":[82,235],"isolates":[83],"simplified":[85],"but":[86],"fundamental":[87],"version":[88],"problem:":[91],"sampling":[92,216],"outcomes":[93],"from":[94,195],"individual":[95],"distributions,":[97,101],"including":[98],"canonical":[99],"statistical":[100,140],"distributions":[102,136],"induced":[103],"by":[104],"stochastic":[105],"programs,":[106],"and":[107,169,260],"natural-language":[108],"scenarios":[109],"describe":[111],"random":[112],"processes.":[113],"448":[116],"such":[117],"problems":[118],"together":[119],"with":[120,160],"KS@N,":[121],"general-purpose":[123],"metric":[125],"quantifies":[127],"how":[128],"well":[129],"model":[131,152,202],"outputs":[132],"approximate":[133],"black-box":[134],"via":[137],"Kolmogorov-Smirnov":[139],"test.":[141],"This":[142],"rate":[145],"at":[146,209,264],"which":[147],"we":[148,172,227],"fail":[149],"reject":[151],"size":[155,187],"N":[156,162],"against":[157],"ground-truth":[158],"samples,":[159],"larger":[161],"indicating":[163],"greater":[164],"difficulty.":[165],"Tested":[166],"across":[167],"open":[168],"proprietary":[170],"models,":[171],"find":[173,228],"spread":[176],"distributional":[178,215,240],"capabilities.":[179],"For":[180],"instance,":[181],"when":[182],"generate":[184],"100":[188],"(KS@100,":[189],"our":[190],"standard":[191],"metric),":[192],"scores":[193],"range":[194],"near":[196],"0":[197],"over":[199,207],"20%.":[200],"No":[201],"able":[204],"achieve":[206],"40%":[208],"KS@100,":[210],"showing":[211],"significant":[212],"headroom":[213],"capability.":[219],"Although":[220],"adding":[221],"reasoning":[222],"can":[223],"somewhat":[224],"increase":[225],"scores,":[226],"no":[229],"immediate":[230],"solution":[231],"issue.":[234],"shows":[236],"even":[238],"simple":[239],"remains":[242],"challenging,":[243],"making":[244],"it":[245],"necessary":[247],"first":[248],"step":[249],"toward":[250],"using":[251],"stand-ins":[254],"complex":[256],"Project":[258],"website":[259],"resources":[261],"available":[263],"https://unpredictabenchmark.github.io/.":[265]},"counts_by_year":[],"updated_date":"2026-07-08T06:17:01.165560","created_date":"2026-06-09T00:00:00"}