{"id":"https://openalex.org/W4394591101","doi":"https://doi.org/10.48550/arxiv.2404.03715","title":"Direct Nash Optimization: Teaching Language Models to Self-Improve with General Preferences","display_name":"Direct Nash Optimization: Teaching Language Models to Self-Improve with General Preferences","publication_year":2024,"publication_date":"2024-04-04","ids":{"openalex":"https://openalex.org/W4394591101","doi":"https://doi.org/10.48550/arxiv.2404.03715"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2404.03715","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.03715","pdf_url":"https://arxiv.org/pdf/2404.03715","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2404.03715","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026219235","display_name":"Corby Rosset","orcid":"https://orcid.org/0000-0001-9167-6214"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rosset, Corby","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101825910","display_name":"Ching-An Cheng","orcid":"https://orcid.org/0000-0002-0610-2070"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Ching-An","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101651425","display_name":"Arindam Mitra","orcid":"https://orcid.org/0000-0003-0089-510X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mitra, Arindam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073314121","display_name":"Michael Santacroce","orcid":"https://orcid.org/0000-0002-9963-3574"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Santacroce, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021000040","display_name":"Ahmed Hassan Awadallah","orcid":"https://orcid.org/0000-0001-6426-3537"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Awadallah, Ahmed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5061481788","display_name":"Tengyang Xie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Tengyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.628600001335144,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.628600001335144,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.6126000285148621,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5918999910354614,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6195681691169739},{"id":"https://openalex.org/keywords/nash-equilibrium","display_name":"Nash equilibrium","score":0.556167483329773},{"id":"https://openalex.org/keywords/mathematical-economics","display_name":"Mathematical economics","score":0.3446611762046814},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.20250540971755981}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6195681691169739},{"id":"https://openalex.org/C46814582","wikidata":"https://www.wikidata.org/wiki/Q23389","display_name":"Nash equilibrium","level":2,"score":0.556167483329773},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.3446611762046814},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.20250540971755981}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2404.03715","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.03715","pdf_url":"https://arxiv.org/pdf/2404.03715","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2404.03715","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2404.03715","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2404.03715","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.03715","pdf_url":"https://arxiv.org/pdf/2404.03715","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394591101.pdf","grobid_xml":"https://content.openalex.org/works/W4394591101.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2803931294","https://openalex.org/W2282604764","https://openalex.org/W2500870930","https://openalex.org/W2218535193","https://openalex.org/W4389370903","https://openalex.org/W2468739564","https://openalex.org/W1554758635","https://openalex.org/W2171981925","https://openalex.org/W2955320052","https://openalex.org/W1721270588"],"abstract_inverted_index":{"This":[0],"paper":[1],"studies":[2],"post-training":[3,27],"large":[4],"language":[5],"models":[6,233],"(LLMs)":[7],"using":[8,161],"preference":[9,71],"feedback":[10],"from":[11,32,150],"a":[12,17,47,87,104,133,157,162,184,193],"powerful":[13],"oracle":[14],"to":[15,65,98,225],"help":[16,179],"model":[18,198],"iteratively":[19],"improve":[20,181],"over":[21,119,183,227],"itself.":[22],"The":[23],"typical":[24],"approach":[25,50],"for":[26,91,216],"LLMs":[28],"involves":[29],"Reinforcement":[30],"Learning":[31],"Human":[33],"Feedback":[34],"(RLHF),":[35],"which":[36,63],"traditionally":[37],"separates":[38],"reward":[39,48,78,100,111],"learning":[40,79,146],"and":[41,80,135,142,169,245],"subsequent":[42],"policy":[43,81],"optimization.":[44],"However,":[45],"such":[46],"maximization":[49,101,112],"is":[51,156,167],"limited":[52],"by":[53,200],"the":[54,99,110,140,203,228],"nature":[55],"of":[56,107,116,144,208,222,248],"\"point-wise\"":[57],"rewards":[58],"(such":[59,187],"as":[60,188],"Bradley-Terry":[61],"model),":[62],"fails":[64],"express":[66],"complex":[67],"intransitive":[68],"or":[69,121],"cyclic":[70],"relations.":[72],"While":[73],"advances":[74],"on":[75,210],"RLHF":[76],"show":[77],"optimization":[82],"can":[83],"be":[84],"merged":[85],"into":[86],"single":[88],"contrastive":[89,145],"objective":[90],"stability,":[92],"they":[93],"yet":[94],"still":[95],"remain":[96],"tethered":[97],"framework.":[102],"Recently,":[103],"new":[105],"wave":[106],"research":[108],"sidesteps":[109],"presumptions":[113],"in":[114],"favor":[115],"directly":[117],"optimizing":[118,151],"\"pair-wise\"":[120],"general":[122,152],"preferences.":[123,153],"In":[124,190],"this":[125],"paper,":[126],"we":[127],"introduce":[128],"Direct":[129],"Nash":[130],"Optimization":[131],"(DNO),":[132],"provable":[134],"scalable":[136],"algorithm":[137,160],"that":[138,178],"marries":[139],"simplicity":[141],"stability":[143],"with":[147,234],"theoretical":[148],"generality":[149],"Because":[154],"DNO":[155,172,201],"batched":[158],"on-policy":[159],"regression-based":[163],"objective,":[164],"its":[165],"implementation":[166],"straightforward":[168],"efficient.":[170],"Moreover,":[171],"enjoys":[173],"monotonic":[174],"improvement":[175],"across":[176],"iterations":[177],"it":[180],"even":[182],"strong":[185],"teacher":[186],"GPT-4).":[189],"our":[191],"experiments,":[192],"resulting":[194],"7B":[195],"parameter":[196],"Orca-2.5":[197],"aligned":[199],"achieves":[202],"state-of-the-art":[204],"win-rate":[205],"against":[206],"GPT-4-Turbo":[207],"33%":[209],"AlpacaEval":[211],"2.0":[212],"(even":[213],"after":[214],"controlling":[215],"response":[217],"length),":[218],"an":[219],"absolute":[220],"gain":[221],"26%":[223],"(7%":[224],"33%)":[226],"initializing":[229],"model.":[230],"It":[231],"outperforms":[232],"far":[235],"more":[236],"parameters,":[237],"including":[238],"Mistral":[239],"Large,":[240],"Self-Rewarding":[241],"LM":[242],"(70B":[243],"parameters),":[244],"older":[246],"versions":[247],"GPT-4.":[249]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2024-04-09T00:00:00"}