{"id":"https://openalex.org/W7203705162","doi":"https://doi.org/10.48550/arxiv.2608.15798","title":"The Hardness of Estimating Cross-Entropy Risk for Language Models","display_name":"The Hardness of Estimating Cross-Entropy Risk for Language Models","publication_year":2026,"publication_date":"2026-08-16","ids":{"openalex":"https://openalex.org/W7203705162","doi":"https://doi.org/10.48550/arxiv.2608.15798"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.15798","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.15798","pdf_url":"https://arxiv.org/pdf/2608.15798","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.15798","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110742589","display_name":"Hanti Lin","orcid":"https://orcid.org/0000-0001-6123-3033","observed_orcids":["https://orcid.org/0000-0001-6123-3033"]},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Lin, Hanti","raw_affiliation_strings":["University of California , Davis"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California , Davis","institution_ids":["https://openalex.org/I84218800"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5110742589"],"corresponding_institution_ids":["https://openalex.org/I84218800"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6660000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6660000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.16580000519752502,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.04320000112056732,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/language-models","display_name":"language models","score":0.8700000047683716},{"id":"https://openalex.org/keywords/pointwise-consistency","display_name":"pointwise consistency","score":0.7649999856948853},{"id":"https://openalex.org/keywords/impossibility-theorem","display_name":"impossibility theorem","score":0.878000020980835},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.6110000014305115},{"id":"https://openalex.org/keywords/risk-estimation","display_name":"risk estimation","score":0.5109999775886536},{"id":"https://openalex.org/keywords/statistical-learning-theory","display_name":"statistical learning theory","score":0.367000013589859}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6150000095367432},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.5881999731063843},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5692999958992004},{"id":"https://openalex.org/C63584917","wikidata":"https://www.wikidata.org/wiki/Q333286","display_name":"Bounding overwatch","level":2,"score":0.53329998254776},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5116999745368958},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.45750001072883606},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.44110000133514404},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.41679999232292175},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.41530001163482666},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3993000090122223},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.39070001244544983},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.38769999146461487},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.37130001187324524},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.33180001378059387},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33149999380111694},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.32170000672340393},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.31540000438690186},{"id":"https://openalex.org/C205706631","wikidata":"https://www.wikidata.org/wiki/Q2319304","display_name":"Expected utility hypothesis","level":2,"score":0.31130000948905945},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C162392398","wikidata":"https://www.wikidata.org/wiki/Q272404","display_name":"Finite set","level":2,"score":0.2964000105857849},{"id":"https://openalex.org/C2777317252","wikidata":"https://www.wikidata.org/wiki/Q18393516","display_name":"Rare events","level":2,"score":0.2784999907016754},{"id":"https://openalex.org/C86426650","wikidata":"https://www.wikidata.org/wiki/Q7452504","display_name":"Sequential estimation","level":2,"score":0.27720001339912415},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.265500009059906},{"id":"https://openalex.org/C148845407","wikidata":"https://www.wikidata.org/wiki/Q5141358","display_name":"Coherent risk measure","level":4,"score":0.25529998540878296}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.15798","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.15798","pdf_url":"https://arxiv.org/pdf/2608.15798","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.15798","doi":"https://doi.org/10.48550/arxiv.2608.15798","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.15798","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.15798","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.15798","pdf_url":"https://arxiv.org/pdf/2608.15798","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203705162.pdf","grobid_xml":"https://content.openalex.org/works/W7203705162.grobid-xml"},"referenced_works_count":4,"referenced_works":["https://openalex.org/W3001279689","https://openalex.org/W3160336054","https://openalex.org/W4403814362","https://openalex.org/W7125223929"],"related_works":["https://openalex.org/W2098737404","https://openalex.org/W24513791","https://openalex.org/W7197000284","https://openalex.org/W1540024815","https://openalex.org/W2093478938","https://openalex.org/W6903972594","https://openalex.org/W7110180832","https://openalex.org/W6943766387","https://openalex.org/W7196927778","https://openalex.org/W4404782002"],"abstract_inverted_index":{"Language":[0],"models,":[1],"once":[2],"pretrained,":[3],"are":[4,148],"routinely":[5],"assessed":[6],"by":[7,54],"the":[8,32,44,50,55,80,105,139,146,175,195],"holdout":[9,140],"method,":[10],"which":[11,182],"estimates":[12,174],"their":[13],"per-token":[14],"cross-entropy":[15],"risk---a":[16],"quantity":[17],"that":[18,30],"may":[19],"be":[20],"finite":[21,160,199],"or":[22,71,206],"infinite.":[23],"We":[24],"prove":[25],"possibility":[26],"and":[27,49,113,198],"impossibility":[28],"theorems":[29],"characterize":[31],"hardness":[33],"of":[34,62,69,74,134],"this":[35],"estimation":[36],"problem.":[37],"The":[38],"estimand":[39],"depends":[40],"on":[41,130],"two":[42],"objects:":[43],"unknown":[45],"data":[46,115],"distribution":[47,51],"$p$":[48,63],"$q$":[52],"induced":[53],"language":[56],"model":[57],"under":[58],"study.":[59],"Our":[60],"ignorance":[61,68],"is":[64,77,100,108,121,168],"empirical,":[65],"whereas":[66],"our":[67,93],"$q$,":[70],"at":[72,178],"least":[73],"its":[75],"tail,":[76],"mathematical;":[78],"so":[79],"parameter":[81,106],"space":[82,107],"{\\em":[83,95,136,190,211],"should}":[84],"range":[85],"over":[86],"pairs":[87],"$(p,q)$---a":[88],"product":[89],"space---to":[90],"fully":[91],"capture":[92],"ignorance.":[94],"Negative":[96],"results}:":[97,138],"no":[98,165],"estimator":[99,128,141],"pointwise":[101,143,218],"consistent,":[102],"even":[103],"when":[104,145],"restricted":[109,150],"to":[110,114,151,155,216],"full-support":[111,156],"models":[112,147,157,187],"distributions":[116],"whose":[117],"expected":[118],"text":[119],"length":[120],"below":[122],"a":[123,131,159,210,228],"fixed":[124,180],"bound;":[125],"worse,":[126],"every":[127],"fails":[129],"dense":[132],"set":[133],"pairs.":[135],"Positive":[137],"becomes":[142],"consistent":[144],"further":[149],"Transformers":[152],"or,":[153],"architecture-neutrally,":[154],"with":[158],"context":[161,200],"window;":[162],"and,":[163],"where":[164],"such":[166,193],"design":[167],"in":[169,188,224],"place,":[170],"it":[171],"still":[172],"consistently":[173],"risk":[176],"capped":[177],"any":[179],"threshold,":[181],"suffices":[183],"for":[184,204],"screening":[185],"candidate":[186],"practice.":[189],"Lesson}:":[191],"designs":[192],"as":[194],"Transformer":[196],"architecture":[197],"windows,":[201],"although":[202],"adopted":[203],"computational":[205],"engineering":[207],"reasons,":[208],"pay":[209],"statistical":[212],"dividend}:":[213],"they":[214,226],"suffice":[215],"make":[217],"consistency":[219],"possible---a":[220],"dividend":[221],"visible":[222],"only":[223],"how":[225],"turn":[227],"negative":[229],"result":[230],"into":[231],"positive":[232],"ones.":[233]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-19T00:00:00"}