{"id":"https://openalex.org/W7161084346","doi":"https://doi.org/10.48550/arxiv.2605.12288","title":"TokenRatio: Principled Token-Level Preference Optimization via Ratio Matching","display_name":"TokenRatio: Principled Token-Level Preference Optimization via Ratio Matching","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161084346","doi":"https://doi.org/10.48550/arxiv.2605.12288"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.12288","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12288","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.12288","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136003260","display_name":"Truong Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Truong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005670374","display_name":"Tien-Phat Nguyen","orcid":"https://orcid.org/0000-0002-4204-842X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Tien-Phat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125251729","display_name":"Linh Ngo Van","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Van, Linh Ngo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136045237","display_name":"Duy Minh Ho Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Duy Minh Ho","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136005709","display_name":"Khoa Doan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Doan, Khoa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136062446","display_name":"Trung Le","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Le, Trung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2953000068664551,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2953000068664551,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.28130000829696655,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11596","display_name":"Constraint Satisfaction and Optimization","score":0.04699999839067459,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.8755000233650208},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7192000150680542},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.6427000164985657},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.6107000112533569},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.6037999987602234},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.5626999735832214},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.41200000047683716},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3686999976634979},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.31380000710487366}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.8755000233650208},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7192000150680542},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.6427000164985657},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.6107000112533569},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.6037999987602234},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.5626999735832214},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5332000255584717},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5105000138282776},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.41200000047683716},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3903999924659729},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3874000012874603},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3686999976634979},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3190999925136566},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C68781425","wikidata":"https://www.wikidata.org/wiki/Q2052203","display_name":"Multi-objective optimization","level":2,"score":0.31360000371932983},{"id":"https://openalex.org/C164752517","wikidata":"https://www.wikidata.org/wiki/Q5570875","display_name":"Global optimization","level":2,"score":0.30790001153945923},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3075000047683716},{"id":"https://openalex.org/C81386100","wikidata":"https://www.wikidata.org/wiki/Q7100792","display_name":"Ordinal optimization","level":3,"score":0.29420000314712524},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2912999987602234},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.27649998664855957},{"id":"https://openalex.org/C147764199","wikidata":"https://www.wikidata.org/wiki/Q6865248","display_name":"Minification","level":2,"score":0.27149999141693115},{"id":"https://openalex.org/C146393371","wikidata":"https://www.wikidata.org/wiki/Q620702","display_name":"Stable marriage problem","level":3,"score":0.2669999897480011},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.25780001282691956},{"id":"https://openalex.org/C2987595161","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Optimization algorithm","level":2,"score":0.2551000118255615},{"id":"https://openalex.org/C123853557","wikidata":"https://www.wikidata.org/wiki/Q7098946","display_name":"Optimal matching","level":3,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.12288","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12288","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.12288","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.12288","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Direct":[0],"Preference":[1,66],"Optimization":[2,67],"(DPO)":[3],"is":[4,27],"a":[5,37,71,85,117],"widely":[6],"used":[7],"RL-free":[8],"method":[9],"for":[10],"aligning":[11],"language":[12],"models":[13,19],"from":[14],"pairwise":[15,60],"preferences,":[16],"but":[17],"it":[18],"preferences":[20],"over":[21,76],"full":[22],"sequences":[23],"even":[24],"though":[25],"generation":[26],"driven":[28],"by":[29,101],"per-token":[30],"decisions.":[31],"Existing":[32],"token-level":[33,53,72,103,153],"extensions":[34],"typically":[35],"decompose":[36],"sequence-level":[38,59,151],"Bradley-Terry":[39,73],"objective":[40,89],"across":[41],"timesteps,":[42],"leaving":[43],"per-prefix":[44],"(state-wise)":[45],"optimality":[46,55],"implicit.":[47],"We":[48,62,109],"study":[49],"how":[50],"to":[51,149],"recover":[52],"preference":[54,74],"using":[56],"only":[57],"standard":[58],"comparisons.":[61],"introduce":[63,110],"Token-level":[64],"Bregman":[65],"(TBPO),":[68],"which":[69,114,123],"posits":[70],"model":[75,104],"next-token":[77],"actions":[78],"conditioned":[79],"on":[80],"the":[81,92,97,102,125],"prefix,":[82],"and":[83,105,121,134,141,144,152],"derive":[84],"Bregman-divergence":[86],"density-ratio":[87],"matching":[88],"that":[90],"generalizes":[91],"logistic/DPO":[93],"loss":[94],"while":[95],"preserving":[96],"optimal":[98],"policy":[99],"induced":[100],"maintaining":[106],"DPO-like":[107],"simplicity.":[108],"two":[111],"instantiations:":[112],"TBPO-Q,":[113],"explicitly":[115],"learns":[116],"lightweight":[118],"state":[119],"baseline,":[120],"TBPO-A,":[122],"removes":[124],"baseline":[126],"through":[127],"advantage":[128],"normalization.":[129],"Across":[130],"instruction":[131],"following,":[132],"helpfulness/harmlessness,":[133],"summarization":[135],"benchmarks,":[136],"TBPO":[137],"improves":[138],"alignment":[139],"quality":[140],"training":[142],"stability":[143],"increases":[145],"output":[146],"diversity":[147],"relative":[148],"strong":[150],"baselines.":[154]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}