{"id":"https://openalex.org/W4412944973","doi":"https://doi.org/10.18653/v1/2025.acl-long.972","title":"AlignDistil: Token-Level Language Model Alignment as Adaptive Policy Distillation","display_name":"AlignDistil: Token-Level Language Model Alignment as Adaptive Policy Distillation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412944973","doi":"https://doi.org/10.18653/v1/2025.acl-long.972"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.972","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.972","pdf_url":"https://aclanthology.org/2025.acl-long.972.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.972.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062529453","display_name":"Songming Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Songming Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100445851","display_name":"Xue Zhang","orcid":"https://orcid.org/0000-0002-6579-7845"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100378800","display_name":"Tong Zhang","orcid":"https://orcid.org/0000-0002-7025-6365"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tong Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113852554","display_name":"Bojie Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bojie Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100394297","display_name":"Yufeng Chen","orcid":"https://orcid.org/0000-0003-0437-6788"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yufeng Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101698034","display_name":"Jinan Xu","orcid":"https://orcid.org/0000-0003-0170-626X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jinan Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"19791","last_page":"19807"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9632999897003174,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9632999897003174,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9429000020027161,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.7709782123565674},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7393897175788879},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.6371304392814636},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4990062713623047},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3293009400367737},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.1795426309108734},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.05734449625015259}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.7709782123565674},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7393897175788879},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6371304392814636},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4990062713623047},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3293009400367737},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.1795426309108734},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.05734449625015259},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.972","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.972","pdf_url":"https://aclanthology.org/2025.acl-long.972.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.972","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.972","pdf_url":"https://aclanthology.org/2025.acl-long.972.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G310944059","display_name":null,"funder_award_id":"62476023","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5447623188","display_name":null,"funder_award_id":"62376019","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7838506768","display_name":null,"funder_award_id":"61976016","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412944973.pdf","grobid_xml":"https://content.openalex.org/works/W4412944973.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4388335561","https://openalex.org/W2970530566","https://openalex.org/W4288261899","https://openalex.org/W4307309205","https://openalex.org/W2967478618","https://openalex.org/W4385009901","https://openalex.org/W4385572700"],"abstract_inverted_index":{"In":[0],"modern":[1],"large":[2],"language":[3],"models":[4],"(LLMs),":[5],"LLM":[6,36],"alignment":[7],"is":[8,13],"of":[9,54,190],"crucial":[10],"importance":[11],"and":[12,26,70,99,107,124,143,157,196],"typically":[14],"achieved":[15],"through":[16],"methods":[17,34,195],"such":[18],"as":[19],"reinforcement":[20],"learning":[21],"from":[22,120,139],"human":[23],"feedback":[24],"(RLHF)":[25],"direct":[27],"preference":[28,51],"optimization":[29],"(DPO).However,":[30],"in":[31,40,67],"most":[32],"existing":[33,194],"for":[35,84,183],"alignment,":[37],"all":[38],"tokens":[39,61],"the":[41,90,96,102,113,118,121,133,137,140,144,188],"response":[42],"are":[43],"optimized":[44],"using":[45],"a":[46,108,125,150,155,158,171],"sparse,":[47],"response-level":[48],"reward":[49,86,91,138,146,153,205],"or":[50,62],"annotation.The":[52],"ignorance":[53],"token-level":[55,85,109],"rewards":[56],"may":[57],"erroneously":[58],"punish":[59],"high-quality":[60],"encourage":[63],"lowquality":[64],"tokens,":[65,168],"resulting":[66],"suboptimal":[68],"performance":[69],"slow":[71],"convergence":[72,199],"speed.To":[73],"address":[74],"this":[75,105,128],"issue,":[76],"we":[77,88,130,169],"propose":[78],"AlignDistil,":[79],"an":[80,179],"RLHFequivalent":[81],"distillation":[82,110],"method":[83],"optimization.Specifically,":[87],"introduce":[89],"learned":[92],"by":[93,148],"DPO":[94,122,141,152,160],"into":[95],"RLHF":[97],"objective":[98,106],"theoretically":[100],"prove":[101],"equivalence":[103],"between":[104,136],"process,":[111],"where":[112],"teacher":[114,181],"distribution":[115,182],"linearly":[116],"combines":[117],"logits":[119],"model":[123,142],"reference":[126],"model.On":[127],"basis,":[129],"further":[131],"bridge":[132],"accuracy":[134],"gap":[135],"pure":[145],"model,":[147],"building":[149],"contrastive":[151],"with":[154],"normal":[156],"reverse":[159],"model.Moreover,":[161],"to":[162,177,201],"avoid":[163],"under-and":[164],"over-optimization":[165],"on":[166],"different":[167],"design":[170],"token":[172],"adaptive":[173],"logit":[174],"extrapolation":[175],"mechanism":[176],"construct":[178],"appropriate":[180],"each":[184],"token.Experimental":[185],"results":[186],"demonstrate":[187],"superiority":[189],"our":[191],"AlignDistil":[192],"over":[193],"showcase":[197],"fast":[198],"due":[200],"its":[202],"tokenlevel":[203],"distributional":[204],"optimization.":[206]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}