{"id":"https://openalex.org/W4412571773","doi":"https://doi.org/10.1162/tacl_a_00757","title":"REAL Sampling: Boosting Factuality and Diversity of Open-ended Generation by Extrapolating the Entropy of an Infinitely Large LM","display_name":"REAL Sampling: Boosting Factuality and Diversity of Open-ended Generation by Extrapolating the Entropy of an Infinitely Large LM","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412571773","doi":"https://doi.org/10.1162/tacl_a_00757"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00757","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00757","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00757/2538234/tacl_a_00757.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00757/2538234/tacl_a_00757.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080115221","display_name":"Haw-Shiuan Chang","orcid":"https://orcid.org/0000-0003-4607-936X"},"institutions":[{"id":"https://openalex.org/I1303243448","display_name":"UMass Memorial Health Care","ror":"https://ror.org/00pq96s96","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1303243448"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haw-Shiuan Chang","raw_affiliation_strings":["UMass Amherst CICS, USA. hschang@cs.umass.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UMass Amherst CICS, USA. hschang@cs.umass.edu","institution_ids":["https://openalex.org/I1303243448"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030248499","display_name":"Nanyun Peng","orcid":"https://orcid.org/0000-0002-8509-6595"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nanyun Peng","raw_affiliation_strings":["Amazon AGI Foundations, USA. pengnany@amazon.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon AGI Foundations, USA. pengnany@amazon.com","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001987532","display_name":"Mohit Bansal","orcid":"https://orcid.org/0000-0001-5522-1351"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohit Bansal","raw_affiliation_strings":["Amazon AGI Foundations, USA. mobansal@amazon.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon AGI Foundations, USA. mobansal@amazon.com","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072018684","display_name":"Anil Ramakrishna","orcid":"https://orcid.org/0000-0002-7999-0531"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]},{"id":"https://openalex.org/I4210089985","display_name":"Amazon (Germany)","ror":"https://ror.org/00b9ktm87","country_code":"DE","type":"company","lineage":["https://openalex.org/I1311688040","https://openalex.org/I4210089985"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Anil Ramakrishna","raw_affiliation_strings":["Amazon AGI Foundations, USA. aniramak@amazon.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon AGI Foundations, USA. aniramak@amazon.com","institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I4210089985"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047508601","display_name":"Tagyoung Chung","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]},{"id":"https://openalex.org/I4210089985","display_name":"Amazon (Germany)","ror":"https://ror.org/00b9ktm87","country_code":"DE","type":"company","lineage":["https://openalex.org/I1311688040","https://openalex.org/I4210089985"]}],"countries":["DE","US"],"is_corresponding":false,"raw_author_name":"Tagyoung Chung","raw_affiliation_strings":["Amazon AGI Foundations, USA. tagyoung@amazon.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon AGI Foundations, USA. tagyoung@amazon.com","institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I4210089985"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.396,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.8481551,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"13","issue":null,"first_page":"760","last_page":"783"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9854999780654907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9746999740600586,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.8418110609054565},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7145786285400391},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.5907045006752014},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49455973505973816},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.48469236493110657},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42535415291786194},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.4140772223472595},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.1405288279056549},{"id":"https://openalex.org/keywords/thermodynamics","display_name":"Thermodynamics","score":0.06468632817268372},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.06267872452735901}],"concepts":[{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.8418110609054565},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7145786285400391},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5907045006752014},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49455973505973816},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.48469236493110657},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42535415291786194},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.4140772223472595},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.1405288279056549},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.06468632817268372},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.06267872452735901},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1162/tacl_a_00757","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00757","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00757/2538234/tacl_a_00757.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00757","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00757","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00757/2538234/tacl_a_00757.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320307791","display_name":"Cisco Systems","ror":"https://ror.org/03yt1ez60"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412571773.pdf","grobid_xml":"https://content.openalex.org/works/W4412571773.grobid-xml"},"referenced_works_count":94,"referenced_works":["https://openalex.org/W2290195878","https://openalex.org/W2502759836","https://openalex.org/W2781528640","https://openalex.org/W2892163801","https://openalex.org/W2962788902","https://openalex.org/W2963096510","https://openalex.org/W2963961878","https://openalex.org/W3001279689","https://openalex.org/W3047430695","https://openalex.org/W3153046263","https://openalex.org/W3155332104","https://openalex.org/W3173343821","https://openalex.org/W4229005866","https://openalex.org/W4281679115","https://openalex.org/W4298149550","https://openalex.org/W4307418160","https://openalex.org/W4311887664","https://openalex.org/W4320836790","https://openalex.org/W4321012036","https://openalex.org/W4366736258","https://openalex.org/W4366974303","https://openalex.org/W4377130677","https://openalex.org/W4378464611","https://openalex.org/W4378771713","https://openalex.org/W4379473824","https://openalex.org/W4379928343","https://openalex.org/W4383987670","https://openalex.org/W4385571645","https://openalex.org/W4385571740","https://openalex.org/W4385572775","https://openalex.org/W4385573518","https://openalex.org/W4385573613","https://openalex.org/W4386555477","https://openalex.org/W4386566737","https://openalex.org/W4387595998","https://openalex.org/W4389518340","https://openalex.org/W4389518874","https://openalex.org/W4389519449","https://openalex.org/W4389519579","https://openalex.org/W4389520749","https://openalex.org/W4389524379","https://openalex.org/W4389718641","https://openalex.org/W4390602555","https://openalex.org/W4390833061","https://openalex.org/W4393119187","https://openalex.org/W4394867973","https://openalex.org/W4395687471","https://openalex.org/W4401042371","https://openalex.org/W4401043276","https://openalex.org/W4402670439","https://openalex.org/W4403928636","https://openalex.org/W4404534210","https://openalex.org/W4404783149","https://openalex.org/W4411630296","https://openalex.org/W6640842352","https://openalex.org/W6704091791","https://openalex.org/W6728984900","https://openalex.org/W6761551260","https://openalex.org/W6772383348","https://openalex.org/W6789307833","https://openalex.org/W6810162553","https://openalex.org/W6811340617","https://openalex.org/W6838297376","https://openalex.org/W6846249729","https://openalex.org/W6846777431","https://openalex.org/W6849910325","https://openalex.org/W6850077679","https://openalex.org/W6850854928","https://openalex.org/W6851579256","https://openalex.org/W6851934322","https://openalex.org/W6851946153","https://openalex.org/W6852505888","https://openalex.org/W6853043925","https://openalex.org/W6853076579","https://openalex.org/W6853186635","https://openalex.org/W6853260906","https://openalex.org/W6853282104","https://openalex.org/W6853465110","https://openalex.org/W6854165772","https://openalex.org/W6854612734","https://openalex.org/W6855707979","https://openalex.org/W6856948088","https://openalex.org/W6857108601","https://openalex.org/W6857199164","https://openalex.org/W6858522248","https://openalex.org/W6859967303","https://openalex.org/W6860028411","https://openalex.org/W6860318346","https://openalex.org/W6861694376","https://openalex.org/W6863756980","https://openalex.org/W6864664432","https://openalex.org/W6866188099","https://openalex.org/W6874349812","https://openalex.org/W6882246716"],"related_works":["https://openalex.org/W2125652721","https://openalex.org/W1540371141","https://openalex.org/W1549363203","https://openalex.org/W2147697413","https://openalex.org/W2154063878","https://openalex.org/W4231274751","https://openalex.org/W2556012038","https://openalex.org/W1489772951","https://openalex.org/W3082059448","https://openalex.org/W4313640622"],"abstract_inverted_index":{"Abstract":[0],"Decoding":[1],"methods":[2],"for":[3],"large":[4,102],"language":[5,103],"models":[6],"(LLMs)":[7],"usually":[8],"struggle":[9],"with":[10,110,183,210],"the":[11,33,49,59,64,68,84,91,96,120,124,133,152,173,200,207],"tradeoff":[12],"between":[13],"ensuring":[14],"factuality":[15,174],"and":[16,175,192,203],"maintaining":[17],"diversity.":[18,65],"In":[19,151],"this":[20],"paper,":[21],"we":[22,74,159],"propose":[23],"REAL":[24,47,56,149,162,186],"(Residual":[25],"Entropy":[26],"from":[27,105],"Asymptotic":[28],"Line)":[29],"sampling,1":[30],"which":[31,82,141],"predicts":[32,83,136],"step-wise":[34,69],"hallucination":[35,70,139],"likelihood":[36,71],"of":[37,90,99,108,177],"an":[38,41,100,114],"LLM.":[39],"When":[40],"LLM":[42,125],"is":[43,117,126],"likely":[44],"to":[45,62,143],"hallucinate,":[46],"lowers":[48],"p":[50,60,146,211],"threshold":[51,61,147],"in":[52,148],"nucleus":[53,208],"sampling.":[54,150],"Otherwise,":[55],"sampling":[57,163,187,190,202,209],"increases":[58],"boost":[63],"To":[66],"predict":[67],"without":[72],"supervision,":[73],"construct":[75],"a":[76,106,137,144,166],"THF":[77,134,168],"(Token-level":[78],"Hallucination":[79],"Forecasting)":[80],"model,":[81],"asymptotic":[85,121],"entropy":[86,116,122],"(i.e.,":[87,123],"inherent":[88],"uncertainty)":[89],"next":[92],"token":[93],"by":[94],"extrapolating":[95],"next-token":[97],"entropies":[98],"infinitely":[101],"model":[104,135,169],"series":[107],"LLMs":[109,179],"different":[111],"sizes.":[112],"If":[113],"LLM\u2019s":[115],"higher":[118],"than":[119,129,199,206],"more":[127,197,204],"uncertain":[128],"it":[130],"should":[131],"be),":[132],"high":[138],"hazard,":[140],"leads":[142],"lower":[145],"FactualityPrompts":[153],"benchmark":[154],"(Lee":[155],"et":[156],"al.,":[157],"2022),":[158],"demonstrate":[160],"that":[161,195],"based":[164],"on":[165],"70M":[167],"can":[170],"substantially":[171],"improve":[172],"diversity":[176],"7B":[178],"simultaneously.":[180],"After":[181],"combined":[182],"contrastive":[184],"decoding,":[185],"outperforms":[188],"13":[189],"methods,":[191],"generates":[193],"texts":[194],"are":[196],"factual":[198],"greedy":[201],"diverse":[205],"=":[212],"0.5.":[213]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}