{"id":"https://openalex.org/W4394868032","doi":"https://doi.org/10.48550/arxiv.2404.09656","title":"Learn Your Reference Model for Real Good Alignment","display_name":"Learn Your Reference Model for Real Good Alignment","publication_year":2024,"publication_date":"2024-04-15","ids":{"openalex":"https://openalex.org/W4394868032","doi":"https://doi.org/10.48550/arxiv.2404.09656"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2404.09656","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.09656","pdf_url":"https://arxiv.org/pdf/2404.09656","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2404.09656","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015546377","display_name":"Alexey Gorbatovski","orcid":"https://orcid.org/0000-0003-3705-0047","observed_orcids":["https://orcid.org/0000-0003-3705-0047"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gorbatovski, Alexey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036870079","display_name":"Boris Shaposhnikov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shaposhnikov, Boris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Malakhov, Alexey","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Malakhov, Alexey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095740272","display_name":"Nikita Surnachev","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Surnachev, Nikita","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113673257","display_name":"Yaroslav Aksenov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aksenov, Yaroslav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095740273","display_name":"Ian Maksimov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maksimov, Ian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041590485","display_name":"Nikita Balagansky","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Balagansky, Nikita","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102488100","display_name":"Daniil Gavrilov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gavrilov, Daniil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2921999990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2921999990940094,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.21960000693798065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.12489999830722809,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-alignment","display_name":"large language model alignment","score":0.9330000281333923},{"id":"https://openalex.org/keywords/trust-region-method","display_name":"trust region method","score":0.7950000166893005},{"id":"https://openalex.org/keywords/reference-policy","display_name":"reference policy","score":0.6380000114440918},{"id":"https://openalex.org/keywords/summarization","display_name":"summarization","score":0.9509999752044678},{"id":"https://openalex.org/keywords/llama-3","display_name":"Llama 3","score":0.7620000243186951},{"id":"https://openalex.org/keywords/alpacaeval","display_name":"AlpacaEval","score":0.8539999723434448}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.556288480758667}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2404.09656","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.09656","pdf_url":"https://arxiv.org/pdf/2404.09656","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2404.09656","doi":"https://doi.org/10.48550/arxiv.2404.09656","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2404.09656","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2404.09656","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.09656","pdf_url":"https://arxiv.org/pdf/2404.09656","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394868032.pdf","grobid_xml":"https://content.openalex.org/works/W4394868032.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7212077798","https://openalex.org/W4400025402","https://openalex.org/W4406071657","https://openalex.org/W4417307829","https://openalex.org/W4414384997","https://openalex.org/W4399455118","https://openalex.org/W4404783001","https://openalex.org/W4415158783","https://openalex.org/W4393336101","https://openalex.org/W4415796384"],"abstract_inverted_index":{"Despite":[0],"the":[1,28,34,64,68,92,98,146,150,157],"fact":[2],"that":[3,74,108],"offline":[4,50],"methods":[5,77,162],"for":[6],"Large":[7],"Language":[8],"Models":[9],"(LLMs)":[10],"alignment":[11,51,76],"do":[12],"not":[13,103],"require":[14],"a":[15,39,46],"direct":[16],"reward":[17],"model,":[18],"they":[19,132],"remain":[20],"susceptible":[21],"to":[22,38,83],"overoptimization.":[23],"This":[24],"issue":[25],"arises":[26],"when":[27,88],"trained":[29],"model":[30,148],"deviates":[31],"excessively":[32],"from":[33,91],"reference":[35,65,94],"policy,":[36],"leading":[37],"decrease":[40],"in":[41,118,141],"sample":[42],"quality.":[43],"We":[44,96],"propose":[45],"new":[47],"paradigm":[48],"of":[49,100,159],"methods,":[52],"called":[53],"Trust":[54,160],"Region":[55,161],"(including":[56],"variants":[57],"TR-DPO,":[58],"TR-IPO,":[59],"TR-KTO),":[60],"which":[61],"dynamically":[62],"updates":[63],"policy":[66],"throughout":[67],"training":[69],"process.":[70],"Our":[71],"results":[72],"show":[73],"TR":[75],"effectively":[78],"mitigate":[79],"overoptimization,":[80,111],"enabling":[81],"models":[82],"maintain":[84],"strong":[85],"performance":[86],"even":[87],"substantially":[89],"deviating":[90],"initial":[93],"policy.":[95],"demonstrate":[97],"efficacy":[99],"these":[101],"approaches":[102],"only":[104],"through":[105,114],"toy":[106],"examples":[107],"exhibit":[109],"reduced":[110],"but":[112],"also":[113],"direct,":[115],"side-by-side":[116],"comparisons":[117],"specific":[119],"tasks":[120],"such":[121],"as":[122,127,129],"helpful":[123],"and":[124,153],"harmless":[125],"dialogue,":[126],"well":[128],"summarization,":[130],"where":[131],"surpass":[133],"conventional":[134],"methods.":[135],"Additionally,":[136],"we":[137],"report":[138],"significant":[139],"improvements":[140],"general-purpose":[142],"assistant":[143],"setups":[144],"with":[145],"Llama3":[147],"on":[149],"AlpacaEval":[151],"2":[152],"Arena-Hard":[154],"benchmarks,":[155],"highlighting":[156],"advantages":[158],"over":[163],"classical":[164],"approaches.":[165]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2024-04-17T00:00:00"}