{"id":"https://openalex.org/W4414978697","doi":"https://doi.org/10.48550/arxiv.2510.05703","title":"Provably Convergent Primal-Dual DPO for Constrained LLM Alignment","display_name":"Provably Convergent Primal-Dual DPO for Constrained LLM Alignment","publication_year":2025,"publication_date":"2025-10-07","ids":{"openalex":"https://openalex.org/W4414978697","doi":"https://doi.org/10.48550/arxiv.2510.05703"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.05703","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.05703","pdf_url":"https://arxiv.org/pdf/2510.05703","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.05703","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054293823","display_name":"Yihan Du","orcid":"https://orcid.org/0009-0009-1668-6493","observed_orcids":["https://orcid.org/0009-0009-1668-6493"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Yihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043550754","display_name":"Seo Taek Kong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kong, Seo Taek","raw_affiliation_strings":["UIUC ECE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UIUC ECE","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078518595","display_name":"R. Srikant","orcid":"https://orcid.org/0000-0003-1483-5204","observed_orcids":["https://orcid.org/0000-0003-1483-5204"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Srikant, R.","raw_affiliation_strings":["UIUC ECE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UIUC ECE","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.225600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.225600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.14180000126361847,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.12389999628067017,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-alignment","display_name":"large language model alignment","score":0.8600000143051147},{"id":"https://openalex.org/keywords/primal-dual-optimization","display_name":"primal-dual optimization","score":0.8009999990463257},{"id":"https://openalex.org/keywords/direct-preference-optimization","display_name":"direct preference optimization","score":0.9509999752044678},{"id":"https://openalex.org/keywords/lagrangian-relaxation","display_name":"Lagrangian relaxation","score":0.7039999961853027},{"id":"https://openalex.org/keywords/reward-modeling","display_name":"reward modeling","score":0.699999988079071},{"id":"https://openalex.org/keywords/online-exploration","display_name":"online exploration","score":0.7639999985694885},{"id":"https://openalex.org/keywords/memory-efficient-fine-tuning","display_name":"memory-efficient fine-tuning","score":0.6830000281333923},{"id":"https://openalex.org/keywords/llm-safety","display_name":"LLM safety","score":0.6690000295639038}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.656000018119812},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.48089998960494995},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.4799000024795532},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.37700000405311584},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.3723999857902527},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3628000020980835},{"id":"https://openalex.org/C55660270","wikidata":"https://www.wikidata.org/wiki/Q5164377","display_name":"Constrained optimization","level":2,"score":0.3140000104904175},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.31220000982284546},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.3059999942779541},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.29580000042915344},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2702000141143799},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.267300009727478},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2644999921321869},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.26109999418258667}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.05703","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.05703","pdf_url":"https://arxiv.org/pdf/2510.05703","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.05703","doi":"https://doi.org/10.48550/arxiv.2510.05703","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.05703","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.05703","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.05703","pdf_url":"https://arxiv.org/pdf/2510.05703","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1104240869","display_name":null,"funder_award_id":"FA9550-24-1-0002","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G1523888516","display_name":null,"funder_award_id":"FA9550-","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G6026246122","display_name":"NeTS: Medium: SLATE: Service Layer Traffic Engineering","funder_award_id":"2312714","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6296865152","display_name":"Collaborative Research: CIF: Small: Nonasymptotic Analysis for Stochastic Networks and Systems: Foundations and Applications","funder_award_id":"2207547","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414978697.pdf","grobid_xml":"https://content.openalex.org/works/W4414978697.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W2978455699","https://openalex.org/W3088599783","https://openalex.org/W3201174429","https://openalex.org/W4322718191","https://openalex.org/W4327810158","https://openalex.org/W4384918448","https://openalex.org/W4392822465","https://openalex.org/W4401306979","https://openalex.org/W4414588213","https://openalex.org/W4416034308","https://openalex.org/W6947991961"],"related_works":["https://openalex.org/W4392539477","https://openalex.org/W4414586533","https://openalex.org/W4414588213","https://openalex.org/W4415796752","https://openalex.org/W7220466991","https://openalex.org/W7140288970","https://openalex.org/W7131667651","https://openalex.org/W4393336101","https://openalex.org/W4414969236","https://openalex.org/W4415795814"],"abstract_inverted_index":{"The":[0],"widespread":[1],"application":[2],"of":[3,73,187],"large":[4],"language":[5],"models":[6,44,133],"(LLMs)":[7],"raises":[8],"increasing":[9],"demands":[10],"on":[11,54,102,177],"ensuring":[12],"safety":[13,35],"or":[14,50],"imposing":[15],"constraints,":[16],"such":[17],"as":[18],"reducing":[19],"harmful":[20],"content":[21],"and":[22,45,110,142,154,167,180],"adhering":[23],"to":[24,41,79,106,123,130,163],"predefined":[25],"rules.":[26],"While":[27],"there":[28],"have":[29],"been":[30],"several":[31],"works":[32,38],"studying":[33],"LLM":[34],"alignment,":[36],"these":[37],"either":[39],"need":[40],"train":[42,131],"three":[43],"incur":[46],"high":[47],"memory":[48,140],"costs,":[49,141],"require":[51,145],"prior":[52,147],"knowledge":[53],"the":[55,63,71,77,119,169,174,178,184],"optimal":[56],"solution.":[57],"Witnessing":[58],"this":[59],"fact,":[60],"we":[61,150],"investigate":[62],"constrained":[64],"alignment":[65],"problem":[66],"for":[67],"LLMs,":[68],"i.e.,":[69],"maximizing":[70],"reward":[72,103,108,121],"outputs":[74],"while":[75],"restricting":[76],"cost":[78],"stay":[80],"below":[81],"a":[82,86,97,113],"threshold.":[83],"We":[84,158],"propose":[85],"novel":[87],"primal-dual":[88],"direct":[89],"preference":[90,104],"optimization":[91],"(DPO)":[92],"approach,":[93],"which":[94,137],"first":[95],"trains":[96],"model":[98],"using":[99],"standard":[100],"DPO":[101,116],"data":[105,170],"provide":[107],"information,":[109],"then":[111],"adopts":[112],"rearranged":[114],"Lagrangian":[115],"objective":[117],"utilizing":[118],"provided":[120],"information":[122],"fine-tune":[124],"LLMs.":[125],"Our":[126],"approach":[127,162],"only":[128],"needs":[129],"two":[132],"rather":[134],"than":[135],"three,":[136],"significantly":[138],"saves":[139],"does":[143],"not":[144],"extra":[146],"knowledge.":[148],"Moreover,":[149],"establish":[151],"rigorous":[152],"suboptimality":[153],"constraint":[155],"violation":[156],"guarantees.":[157],"also":[159],"extend":[160],"our":[161,188],"enable":[164],"online":[165],"exploration":[166],"drop":[168],"coverage":[171],"dependence":[172],"in":[173],"results.":[175],"Experiments":[176],"PKU-SafeRLHF":[179],"TruthfulQA":[181],"datasets":[182],"demonstrate":[183],"state-of-the-art":[185],"performance":[186],"approach.":[189]},"counts_by_year":[{"year":2026,"cited_by_count":4}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-10-10T00:00:00"}