{"id":"https://openalex.org/W4412889794","doi":"https://doi.org/10.18653/v1/2025.acl-long.1353","title":"T-REG: Preference Optimization with Token-Level Reward Regularization","display_name":"T-REG: Preference Optimization with Token-Level Reward Regularization","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412889794","doi":"https://doi.org/10.18653/v1/2025.acl-long.1353"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.acl-long.1353","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1353","pdf_url":"https://aclanthology.org/2025.acl-long.1353.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.acl-long.1353.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007359104","display_name":"Wenxuan Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenxuan Zhou","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056212977","display_name":"Shujian Zhang","orcid":"https://orcid.org/0000-0003-3216-722X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shujian Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101697859","display_name":"Lingxiao Zhao","orcid":"https://orcid.org/0000-0003-2553-3148"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lingxiao Zhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101997947","display_name":"Tao Meng","orcid":"https://orcid.org/0000-0002-5932-8364"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tao Meng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"27876","last_page":"27889"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.5776000022888184,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.5776000022888184,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.5767999887466431,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11596","display_name":"Constraint Satisfaction and Optimization","score":0.5551999807357788,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.5533332228660583},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4979984760284424},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.49706366658210754},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.24198901653289795},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.20100224018096924},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.1716797649860382}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5533332228660583},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4979984760284424},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49706366658210754},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.24198901653289795},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.20100224018096924},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.1716797649860382}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.acl-long.1353","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1353","pdf_url":"https://aclanthology.org/2025.acl-long.1353.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.acl-long.1353","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.acl-long.1353","pdf_url":"https://aclanthology.org/2025.acl-long.1353.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412889794.pdf","grobid_xml":"https://content.openalex.org/works/W4412889794.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4388335561","https://openalex.org/W2970530566","https://openalex.org/W4288261899","https://openalex.org/W4307309205","https://openalex.org/W2967478618","https://openalex.org/W4385009901","https://openalex.org/W4385572700"],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"from":[2],"human":[3,16],"feedback":[4],"(RLHF)":[5],"has":[6],"been":[7],"crucial":[8],"in":[9],"aligning":[10],"large":[11],"language":[12],"models":[13],"(LLMs)":[14],"with":[15],"values.Traditionally,":[17],"RLHF":[18],"involves":[19],"generating":[20],"responses":[21],"to":[22,30,34,43,58,68,75,138,141,155,193],"a":[23,27,32,47,89,115],"query":[24],"and":[25,102,122,168,181,195],"using":[26],"reward":[28,33,112,150],"model":[29,57,93,154],"assign":[31],"the":[35,56,63,69,100,105,128,153,173],"entire":[36],"response.However,":[37],"this":[38,77,107],"approach":[39,117],"faces":[40],"challenges":[41],"due":[42],"its":[44],"reliance":[45],"on":[46,87,172],"single,":[48],"sparse":[49],"reward,":[50],"which":[51,60],"makes":[52],"it":[53],"challenging":[54],"for":[55,125],"identify":[59],"parts":[61],"of":[62,104,131],"sequence":[64],"contribute":[65],"most":[66],"significantly":[67],"final":[70],"reward.Recent":[71],"methods":[72,84,190],"have":[73],"attempted":[74],"address":[76],"limitation":[78],"by":[79,191],"introducing":[80],"token-level":[81,111,123,143,165],"rewards.However,":[82],"these":[83],"often":[85],"rely":[86],"either":[88],"trained":[90],"credit":[91,166],"assignment":[92,167],"or":[94],"AI":[95],"annotators,":[96],"raising":[97],"concerns":[98],"about":[99],"quality":[101],"reliability":[103],"rewards.In":[106],"paper,":[108],"we":[109],"propose":[110],"regularization":[113],"(T-REG),":[114],"novel":[116],"that":[118,184],"leverages":[119],"both":[120],"sequence-level":[121,159],"rewards":[124,146,160],"preference":[126],"optimization.Harnessing":[127],"self-refinement":[129],"capabilities":[130],"LLMs,":[132],"our":[133,185],"method":[134,186],"uses":[135],"contrastive":[136],"prompting":[137],"enable":[139],"LLMs":[140],"self-generate":[142],"rewards.These":[144],"self-generated":[145],"then":[147],"act":[148],"as":[149],"regularization,":[151],"guiding":[152],"more":[156],"effectively":[157],"distribute":[158],"across":[161],"tokens.This":[162],"facilitates":[163],"better":[164],"enhances":[169],"alignment":[170],"performance.Experiments":[171],"instruction":[174],"following":[175],"benchmarks,":[176],"including":[177],"Alpaca":[178],"Eval":[179],"2":[180],"Arena-Hard,":[182],"show":[183],"consistently":[187],"outperforms":[188],"baseline":[189],"up":[192],"3.8%":[194],"4.4%,":[196],"respectively.":[197]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}