{"id":"https://openalex.org/W7163015123","doi":"https://doi.org/10.48550/arxiv.2605.30844","title":"Fine-Tuning Improves Information Conveyance in Language Models","display_name":"Fine-Tuning Improves Information Conveyance in Language Models","publication_year":2026,"publication_date":"2026-05-29","ids":{"openalex":"https://openalex.org/W7163015123","doi":"https://doi.org/10.48550/arxiv.2605.30844"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.30844","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30844","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.30844","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137557149","display_name":"Yuwei Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Yuwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137577209","display_name":"Weiyi Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Weiyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137548420","display_name":"Haifeng Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Haifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15489999949932098,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15489999949932098,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.15299999713897705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.13109999895095825,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.6403999924659729},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.4893999993801117},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4754999876022339},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4077000021934509},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.3889999985694885},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.37880000472068787},{"id":"https://openalex.org/keywords/information-theory","display_name":"Information theory","score":0.329800009727478},{"id":"https://openalex.org/keywords/mutual-information","display_name":"Mutual information","score":0.31450000405311584}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6561999917030334},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.6403999924659729},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4893999993801117},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4754999876022339},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43529999256134033},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4077000021934509},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.3889999985694885},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.37880000472068787},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35109999775886536},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.329800009727478},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.31450000405311584},{"id":"https://openalex.org/C67935508","wikidata":"https://www.wikidata.org/wiki/Q17078448","display_name":"Information diagram","level":5,"score":0.31119999289512634},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2928999960422516},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2922999858856201},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.2892000079154968},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.28380000591278076},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C101721835","wikidata":"https://www.wikidata.org/wiki/Q813908","display_name":"Conditional entropy","level":3,"score":0.26499998569488525},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26109999418258667},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.25450000166893005},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.25440001487731934}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.30844","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30844","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.30844","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.30844","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Fine-tuning":[0],"is":[1,29,122,227],"often":[2],"believed":[3],"to":[4,25,96],"reduce":[5,212],"uncertainty":[6,28,77,197],"and":[7,22,84,145,173,188,221],"diversity":[8,200],"in":[9,78],"large":[10],"language":[11,48],"models,":[12],"but":[13,214],"existing":[14],"analyses":[15],"overlook":[16],"output":[17,81],"length,":[18],"a":[19,44,51,114],"key":[20],"confounder,":[21],"therefore":[23],"fail":[24],"capture":[26],"how":[27],"distributed":[30],"across":[31,143],"an":[32],"entire":[33],"generation":[34,49,72],"rollout.":[35],"To":[36],"address":[37],"this,":[38],"we":[39,91,148,176],"propose":[40],"Canopy":[41],"Entropy":[42],"($\\mathrm{CE}^\\star$),":[43],"measure":[45],"that":[46,93,150,178,192,207],"views":[47],"from":[50],"tree":[52],"perspective,":[53],"where":[54,103,120],"``canopy''":[55],"represents":[56],"the":[57,67,71,80,85,106,123,182],"space":[58],"of":[59,70],"all":[60],"possible":[61],"rollouts,":[62],"making":[63],"$\\mathrm{CE}^\\star$":[64,74],"naturally":[65],"quantify":[66],"effective":[68],"size":[69],"space.":[73],"jointly":[75],"captures":[76],"both":[79],"length":[82],"$N$":[83],"generated":[86],"sequence":[87],"$Y_{1:N}$":[88],"--":[89],"indeed,":[90],"show":[92],"it":[94,217],"equals":[95],"total":[97,162],"Shannon":[98],"entropy":[99,124,163,186],"$H(N,":[100],"Y_{1:N}\\mid":[101],"X)$,":[102],"$X$":[104],"denotes":[105],"prompt.":[107],"This":[108],"formulation":[109],"yields":[110],"interpretable":[111],"metrics,":[112],"including":[113],"length-entropy":[115],"correlation":[116,157,183],"term":[117],"$\u03c1(N,":[118,158],"r_N)$,":[119,159],"$r_N$":[121],"rate,":[125],"quantifying":[126],"information":[127],"conveyance":[128],"efficiency":[129],"by":[130],"indicating":[131],"whether":[132],"longer":[133],"outputs":[134],"are":[135],"more":[136,201,219],"or":[137],"less":[138],"informative":[139,220],"per":[140],"token.":[141],"Empirically,":[142],"tasks":[144],"model":[146,169],"families,":[147],"find":[149,177],"fine-tuned":[151],"models":[152,194],"consistently":[153],"exhibit":[154],"stronger":[155],"positive":[156],"even":[160],"when":[161],"decreases.":[164],"Furthermore,":[165],"after":[166],"controlling":[167],"for":[168],"family,":[170],"task,":[171],"prompt,":[172],"output-length":[174],"effects,":[175],"fine-tuning":[179,208],"nearly":[180],"triples":[181],"strength":[184],"between":[185],"rate":[187],"semantic":[189,199],"diversity,":[190],"suggesting":[191],"aligned":[193],"convert":[195],"token":[196],"into":[198,218],"efficiently.":[202],"Overall,":[203],"these":[204],"results":[205],"demonstrate":[206],"does":[209],"not":[210],"simply":[211],"uncertainty,":[213],"fundamentally":[215],"reorganizes":[216],"semantically":[222],"meaningful":[223],"generations.":[224],"Our":[225],"code":[226],"available":[228],"at":[229],"https://github.com/WeiyiTian/canopy-entropy.":[230]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-02T00:00:00"}