{"id":"https://openalex.org/W7204525498","doi":"https://doi.org/10.48550/arxiv.2608.26676","title":"FOCUS & RePAIR: Mitigating Text Degeneration via Token-Level Guidance for Pruned Large Language Models","display_name":"FOCUS & RePAIR: Mitigating Text Degeneration via Token-Level Guidance for Pruned Large Language Models","publication_year":2026,"publication_date":"2026-08-27","ids":{"openalex":"https://openalex.org/W7204525498","doi":"https://doi.org/10.48550/arxiv.2608.26676"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.26676","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.26676","pdf_url":"https://arxiv.org/pdf/2608.26676","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.26676","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132821733","display_name":"Junyoung Lee","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I123900574","display_name":"Pohang University of Science and Technology","ror":"https://ror.org/04xysgw12","country_code":"KR","type":"education","lineage":["https://openalex.org/I123900574"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Lee, Junyoung","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , POSTECH , Pohang , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , POSTECH , Pohang , Republic of Korea","institution_ids":["https://openalex.org/I123900574"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012447154","display_name":"Sehyeon Park","orcid":"https://orcid.org/0000-0002-6222-665X","observed_orcids":["https://orcid.org/0000-0002-6222-665X"]},"institutions":[{"id":"https://openalex.org/I193352282","display_name":"Daegu Gyeongbuk Institute of Science and Technology","ror":"https://ror.org/03frjya69","country_code":"KR","type":"education","lineage":["https://openalex.org/I193352282"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Park, Sehyeon","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea","institution_ids":["https://openalex.org/I193352282"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101269027","display_name":"Shinhyoung Jang","orcid":"https://orcid.org/0009-0008-0378-3801","observed_orcids":["https://orcid.org/0009-0008-0378-3801"]},"institutions":[{"id":"https://openalex.org/I193352282","display_name":"Daegu Gyeongbuk Institute of Science and Technology","ror":"https://ror.org/03frjya69","country_code":"KR","type":"education","lineage":["https://openalex.org/I193352282"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jang, Shinhyoung","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea","institution_ids":["https://openalex.org/I193352282"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148491999","display_name":"Seonha Ryu","orcid":"https://orcid.org/0009-0001-5128-3368","observed_orcids":["https://orcid.org/0009-0001-5128-3368"]},"institutions":[{"id":"https://openalex.org/I193352282","display_name":"Daegu Gyeongbuk Institute of Science and Technology","ror":"https://ror.org/03frjya69","country_code":"KR","type":"education","lineage":["https://openalex.org/I193352282"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Ryu, Seonha","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea","institution_ids":["https://openalex.org/I193352282"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039214401","display_name":"\uae40\ud638\uc815","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I193352282","display_name":"Daegu Gyeongbuk Institute of Science and Technology","ror":"https://ror.org/03frjya69","country_code":"KR","type":"education","lineage":["https://openalex.org/I193352282"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kim, Hojeong","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea","institution_ids":["https://openalex.org/I193352282"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004378291","display_name":"Hyunsei Lee","orcid":"https://orcid.org/0009-0009-7780-8167","observed_orcids":["https://orcid.org/0009-0009-7780-8167"]},"institutions":[{"id":"https://openalex.org/I193352282","display_name":"Daegu Gyeongbuk Institute of Science and Technology","ror":"https://ror.org/03frjya69","country_code":"KR","type":"education","lineage":["https://openalex.org/I193352282"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Lee, Hyunsei","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , Daegu Gyeongbuk Institute of Science and Technology (DGIST) , Daegu , Republic of Korea","institution_ids":["https://openalex.org/I193352282"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030410269","display_name":"Il Hong Suh","orcid":"https://orcid.org/0000-0002-0981-329X","observed_orcids":["https://orcid.org/0000-0002-0981-329X"]},"institutions":[{"id":"https://openalex.org/I4575257","display_name":"Hanyang University","ror":"https://ror.org/046865y68","country_code":"KR","type":"education","lineage":["https://openalex.org/I4575257"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Suh, Il Hong","raw_affiliation_strings":["Hanyang University , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hanyang University , Seoul , Republic of Korea","institution_ids":["https://openalex.org/I4575257"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067380102","display_name":"Yeseong Kim","orcid":"https://orcid.org/0000-0001-5947-9632","observed_orcids":["https://orcid.org/0000-0001-5947-9632"]},"institutions":[{"id":"https://openalex.org/I123900574","display_name":"Pohang University of Science and Technology","ror":"https://ror.org/04xysgw12","country_code":"KR","type":"education","lineage":["https://openalex.org/I123900574"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kim, Yeseong","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science , POSTECH , Pohang , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science , POSTECH , Pohang , Republic of Korea","institution_ids":["https://openalex.org/I123900574"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6079000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6079000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.20389999449253082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.11490000039339066,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/distillation","display_name":"distillation","score":0.5889999866485596},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.3790000081062317}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.8123000264167786},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7228000164031982},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.607699990272522},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5885999798774719},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.5648999810218811},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.5307000279426575},{"id":"https://openalex.org/C88626702","wikidata":"https://www.wikidata.org/wiki/Q1128903","display_name":"Continuation","level":2,"score":0.5221999883651733},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5131000280380249},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5012000203132629},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4569999873638153},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.45320001244544983},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4507000148296356},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.38679999113082886},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3605000078678131},{"id":"https://openalex.org/C2776141515","wikidata":"https://www.wikidata.org/wiki/Q1274479","display_name":"Repetition (rhetorical device)","level":2,"score":0.3531000018119812},{"id":"https://openalex.org/C2776331573","wikidata":"https://www.wikidata.org/wiki/Q178074","display_name":"Veto","level":3,"score":0.328000009059906},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3255999982357025},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3246999979019165},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3244999945163727},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31139999628067017},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.2987000048160553},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.289000004529953},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.25380000472068787},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.250900000333786}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.26676","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.26676","pdf_url":"https://arxiv.org/pdf/2608.26676","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.26676","doi":"https://doi.org/10.48550/arxiv.2608.26676","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.26676","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.26676","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.26676","pdf_url":"https://arxiv.org/pdf/2608.26676","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2694013364","display_name":null,"funder_award_id":"RS\u20102024\u201000445759","funder_id":"https://openalex.org/F4320321681","funder_display_name":"Ministry of Trade, Industry and Energy"}],"funders":[{"id":"https://openalex.org/F4320321681","display_name":"Ministry of Trade, Industry and Energy","ror":"https://ror.org/008nkqk13"},{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204525498.pdf","grobid_xml":"https://content.openalex.org/works/W7204525498.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W2936695845","https://openalex.org/W2938704169","https://openalex.org/W3126409997","https://openalex.org/W3139460307","https://openalex.org/W4225518699","https://openalex.org/W4308760184","https://openalex.org/W4313484599","https://openalex.org/W4321012036","https://openalex.org/W4385572634","https://openalex.org/W4389523755","https://openalex.org/W4391631876","https://openalex.org/W4392019445","https://openalex.org/W4393968834","https://openalex.org/W4409362950","https://openalex.org/W4412888815","https://openalex.org/W7125135101","https://openalex.org/W7133208539","https://openalex.org/W7133245307"],"related_works":["https://openalex.org/W7124585027","https://openalex.org/W7127350043","https://openalex.org/W7204100827","https://openalex.org/W7124626234","https://openalex.org/W4404782843","https://openalex.org/W4415068093","https://openalex.org/W7213371199","https://openalex.org/W7166847149","https://openalex.org/W4401042820","https://openalex.org/W6888068170"],"abstract_inverted_index":{"Pruning":[0],"is":[1,74],"a":[2,34,45,53,120],"practical":[3],"approach":[4],"to":[5,81,109,123,131],"compress":[6],"large":[7],"language":[8],"models":[9],"(LLMs),":[10],"but":[11],"it":[12],"can":[13],"amplify":[14],"text":[15],"degeneration,":[16],"especially":[17],"repetition":[18,132,147],"loops,":[19],"even":[20],"when":[21],"perplexity":[22],"and":[23,50,67,70,127,138,148],"task":[24],"accuracy":[25],"remain":[26],"largely":[27],"unchanged.":[28],"In":[29],"this":[30,38],"work,":[31],"we":[32,93],"present":[33],"token-level":[35,96],"analysis":[36,60],"of":[37,56],"failure":[39],"mode":[40],"by":[41,76,90],"viewing":[42],"decoding":[43],"as":[44],"dynamical":[46],"process":[47],"that":[48,72,142],"enters":[49],"persists":[51],"in":[52],"small":[54],"set":[55],"recurrent":[57],"contexts.":[58],"Our":[59],"decomposes":[61],"degeneration":[62],"into":[63],"loop":[64,68],"entry":[65],"risk":[66],"persistence,":[69],"shows":[71],"persistence":[73],"controlled":[75],"the":[77,85],"escape":[78],"mass":[79],"assigned":[80],"plausible":[82,125],"alternatives":[83,126],"within":[84],"token":[86],"sampling":[87],"set.":[88],"Motivated":[89],"these":[91],"findings,":[92],"propose":[94],"two":[95],"guidance":[97],"objectives":[98],"for":[99],"post-pruning":[100],"fine-tuning.":[101],"FOCUS":[102],"reweights":[103],"distillation":[104],"toward":[105],"high-confidence":[106],"teacher":[107],"regions":[108],"suppress":[110],"leakage,":[111],"while":[112],"RePAIR":[113],"uses":[114],"onset-centered":[115],"positive/negative":[116],"continuation":[117,137],"pairs":[118],"with":[119],"margin":[121],"loss":[122],"promote":[124],"prevent":[128],"early":[129],"commitment":[130],"loops.":[133],"Experiments":[134],"on":[135],"open-ended":[136],"instruction-based":[139],"generation":[140,150],"show":[141],"both":[143],"methods":[144],"consistently":[145],"reduce":[146],"improve":[149],"quality.":[151]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-29T00:00:00"}