{"id":"https://openalex.org/W7216292525","doi":"https://doi.org/10.48550/arxiv.2610.01493","title":"No Model Required: Text Entropy Rate Filtering Mitigates Iterative Fine-Tuning Collapse","display_name":"No Model Required: Text Entropy Rate Filtering Mitigates Iterative Fine-Tuning Collapse","publication_year":2026,"publication_date":"2026-10-01","ids":{"openalex":"https://openalex.org/W7216292525","doi":"https://doi.org/10.48550/arxiv.2610.01493"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.01493","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.01493","pdf_url":"https://arxiv.org/pdf/2610.01493","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.01493","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055805371","display_name":"Lewis Mitchell","orcid":"https://orcid.org/0000-0001-8191-1997","observed_orcids":["https://orcid.org/0000-0001-8191-1997"]},"institutions":[{"id":"https://openalex.org/I4405259444","display_name":"Adelaide University","ror":"https://ror.org/028g18b61","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444"]},{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Mitchell, Lewis","raw_affiliation_strings":["Adelaide Data Science Centre School of Mathematical Sciences Adelaide University Adelaide SA 5005 , Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Adelaide Data Science Centre School of Mathematical Sciences Adelaide University Adelaide SA 5005 , Australia","institution_ids":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5055805371"],"corresponding_institution_ids":["https://openalex.org/I4405259444","https://openalex.org/I5681781"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9653000235557556,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9653000235557556,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.008999999612569809,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.00430000014603138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-collapse","display_name":"model collapse","score":0.9169999957084656},{"id":"https://openalex.org/keywords/iterative-fine-tuning","display_name":"iterative fine-tuning","score":0.9340000152587891},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"synthetic data","score":0.6179999709129333},{"id":"https://openalex.org/keywords/text-diversity","display_name":"text diversity","score":0.6029999852180481},{"id":"https://openalex.org/keywords/repetition","display_name":"repetition","score":0.3440000116825104},{"id":"https://openalex.org/keywords/qlora","display_name":"QLoRA","score":0.703000009059906},{"id":"https://openalex.org/keywords/llama-3-1","display_name":"Llama 3.1","score":0.8519999980926514},{"id":"https://openalex.org/keywords/collapse-detection","display_name":"collapse detection","score":0.6330000162124634},{"id":"https://openalex.org/keywords/information-theory","display_name":"information theory","score":0.5989999771118164}],"concepts":[{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.6097999811172485},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5940999984741211},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5641000270843506},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.539900004863739},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.4323999881744385},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.41690000891685486},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.3971000015735626},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.382099986076355},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.37709999084472656},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.375900000333786},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.3407999873161316},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.3237000107765198},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3222000002861023},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.3151000142097473},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.2921000123023987},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.28529998660087585},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2840999960899353},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2732999920845032},{"id":"https://openalex.org/C2987896495","wikidata":"https://www.wikidata.org/wiki/Q5416716","display_name":"Event data","level":3,"score":0.27079999446868896},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.2702000141143799},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26989999413490295},{"id":"https://openalex.org/C2779696439","wikidata":"https://www.wikidata.org/wiki/Q7512811","display_name":"Signature (topology)","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C182049051","wikidata":"https://www.wikidata.org/wiki/Q17147155","display_name":"Transfer entropy","level":3,"score":0.2644999921321869},{"id":"https://openalex.org/C101721835","wikidata":"https://www.wikidata.org/wiki/Q813908","display_name":"Conditional entropy","level":3,"score":0.26330000162124634},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C125252325","wikidata":"https://www.wikidata.org/wiki/Q1345213","display_name":"Entropy rate","level":4,"score":0.26269999146461487},{"id":"https://openalex.org/C206468330","wikidata":"https://www.wikidata.org/wiki/Q834373","display_name":"Code rate","level":3,"score":0.2587999999523163},{"id":"https://openalex.org/C113336015","wikidata":"https://www.wikidata.org/wiki/Q574010","display_name":"Complete information","level":2,"score":0.2531999945640564}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.01493","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.01493","pdf_url":"https://arxiv.org/pdf/2610.01493","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.01493","doi":"https://doi.org/10.48550/arxiv.2610.01493","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.01493","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.01493","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.01493","pdf_url":"https://arxiv.org/pdf/2610.01493","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320315885","display_name":"Australian Government","ror":"https://ror.org/0314h5y94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7216292525.pdf","grobid_xml":"https://content.openalex.org/works/W7216292525.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W4389156994","https://openalex.org/W4392679398","https://openalex.org/W4393924649","https://openalex.org/W4401306886","https://openalex.org/W4403578020","https://openalex.org/W4405655184","https://openalex.org/W4415955458","https://openalex.org/W7155040213"],"related_works":["https://openalex.org/W7169845352","https://openalex.org/W4403578020","https://openalex.org/W7152510654","https://openalex.org/W7213427798","https://openalex.org/W7167508801","https://openalex.org/W7171681265","https://openalex.org/W6979243400","https://openalex.org/W6922441616","https://openalex.org/W4416766928","https://openalex.org/W4415251841"],"abstract_inverted_index":{"Iterative":[0],"fine-tuning":[1,90],"on":[2,83,98,112],"synthetic":[3],"data":[4],"causes":[5],"\\emph{model":[6],"collapse}:":[7],"output":[8],"diversity":[9],"narrows":[10],"as":[11,21,136],"rare":[12],"patterns":[13],"are":[14,174],"progressively":[15],"lost,":[16],"a":[17,43,79,87,93,137],"signature":[18],"most":[19,103],"visible":[20],"phrase-level":[22],"repetition.":[23],"Existing":[24],"mitigations":[25],"either":[26],"require":[27],"model":[28,68,158],"log-probabilities,":[29],"an":[30],"external":[31],"oracle,":[32],"or":[33],"continued":[34],"access":[35],"to":[36,171],"real":[37],"human":[38],"data.":[39],"Here":[40],"we":[41],"develop":[42],"new":[44,178],"approach":[45],"grounded":[46],"in":[47,77,86],"mathematical":[48],"information":[49,168],"theory:":[50],"the":[51],"non-parametric":[52],"Kontoyiannis":[53],"entropy":[54,139],"rate":[55],"estimator":[56],"$h_k$,":[57],"computed":[58],"entirely":[59],"from":[60],"raw":[61],"text":[62],"via":[63],"match-length":[64],"statistics,":[65],"with":[66],"no":[67,108],"of":[69],"any":[70,113],"kind.":[71],"We":[72,133],"show":[73],"that":[74,167],"this":[75],"is":[76],"fact":[78],"\\emph{superior}":[80],"training-data":[81],"filter":[82],"text-diversity":[84,110],"metrics":[85],"fully-synthetic,":[88],"single-lineage":[89],"setting.":[91],"In":[92],"six-generation":[94],"QLoRA":[95],"collapse":[96,147,172],"experiment":[97],"Llama-3.1-8B,":[99],"logprob-based":[100],"filtering":[101],"(the":[102],"established":[104],"model-access-requiring":[105],"baseline)":[106],"provides":[107],"significant":[109],"benefit":[111],"metric":[114],"($p":[115],">":[116],"0.23$),":[117],"whereas":[118],"$h_k$-filtering":[119],"yields":[120],"$+42\\%$":[121],"unique":[122],"trigrams,":[123],"$+30\\%$":[124],"vocabulary,":[125],"and":[126,146,160,176],"$-19\\%$":[127],"repetition":[128],"(all":[129],"$p":[130,151],"<":[131,152],"0.001$).":[132],"validate":[134],"$h_k$":[135],"cross-domain":[138],"proxy":[140],"($\u03b2=":[141],"0.924$,":[142],"$R^2":[143],"=":[144],"0.746$)":[145],"detector":[148],"($\u03c1=":[149],"+0.454$,":[150],"0.0001$)":[153],"across":[154],"4~domains,":[155],"2~temperatures,":[156],"2~generator--scorer":[157],"pairs,":[159],"1{,}520":[161],"generated":[162],"documents.":[163],"Our":[164],"results":[165],"demonstrate":[166],"theoretic":[169],"approaches":[170,179],"mitigation":[173],"efficient,":[175],"suggest":[177],"for":[180],"maintaining":[181],"multi-agent":[182],"diversity.":[183]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-03T00:00:00"}