{"id":"https://openalex.org/W6891824290","doi":"https://doi.org/10.48550/arxiv.2503.13538","title":"From Demonstrations to Rewards: Alignment Without Explicit Human Preferences","display_name":"From Demonstrations to Rewards: Alignment Without Explicit Human Preferences","publication_year":2025,"publication_date":"2025-03-15","ids":{"openalex":"https://openalex.org/W6891824290","doi":"https://doi.org/10.48550/arxiv.2503.13538"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2503.13538","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.13538","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2503.13538","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Zeng, Siliang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zeng, Siliang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Rangwala, Huzefa","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rangwala, Huzefa","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Karypis, George","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karypis, George","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Hong, Mingyi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hong, Mingyi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Fakoor, Rasool","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fakoor, Rasool","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.37229999899864197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.37229999899864197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.21559999883174896,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.051500000059604645,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7361000180244446},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.6265000104904175},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.5536999702453613},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5404999852180481},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.3955000042915344}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7361000180244446},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6650000214576721},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.6265000104904175},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5536999702453613},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5404999852180481},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5317999720573425},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47929999232292175},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.3955000042915344},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.32420000433921814},{"id":"https://openalex.org/C2779436431","wikidata":"https://www.wikidata.org/wiki/Q30672407","display_name":"Policy learning","level":2,"score":0.3190999925136566},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.2678999900817871}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2503.13538","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.13538","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2503.13538","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.13538","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"One":[0],"of":[1,4,21,36,107],"the":[2,14,18,65,96,115,125],"challenges":[3],"aligning":[5],"large":[6],"models":[7],"with":[8,74],"human":[9,46,75],"preferences":[10,47],"lies":[11],"in":[12,78],"both":[13],"data":[15,40,134,150],"requirements":[16],"and":[17,41,144,171],"technical":[19],"complexities":[20],"current":[22,140],"approaches.":[23],"Predominant":[24],"methods,":[25],"such":[26],"as":[27,57],"RLHF,":[28,45],"involve":[29],"multiple":[30],"steps,":[31],"each":[32],"demanding":[33],"distinct":[34],"types":[35],"data,":[37,111],"including":[38],"demonstration":[39,119,133,149,186],"preference":[42,110],"data.":[43,120,187],"In":[44],"are":[48],"typically":[49],"modeled":[50],"through":[51],"a":[52,58,71,83,137],"reward":[53,103,116,165],"model,":[54],"which":[55],"serves":[56],"proxy":[59],"to":[60,127,179],"guide":[61],"policy":[62,72,98],"learning":[63,67,87,93],"during":[64],"reinforcement":[66,92],"stage,":[68],"ultimately":[69],"producing":[70],"aligned":[73],"preferences.":[76],"However,":[77,105],"this":[79],"paper,":[80],"we":[81,112],"propose":[82],"fresh":[84],"perspective":[85],"on":[86,90,109,163,185],"alignment":[88],"based":[89,162],"inverse":[91],"principles,":[94],"where":[95],"optimal":[97],"is":[99,135],"still":[100],"derived":[101],"from":[102,118],"maximization.":[104],"instead":[106],"relying":[108],"directly":[113],"learn":[114],"model":[117],"This":[121],"new":[122],"formulation":[123],"offers":[124,151],"flexibility":[126],"be":[128],"applied":[129],"even":[130],"when":[131],"only":[132],"available,":[136],"capability":[138],"that":[139,148,174,182],"RLHF":[141],"methods":[142,181],"lack,":[143],"it":[145],"also":[146],"shows":[147],"more":[152],"utility":[153],"than":[154],"what":[155],"conventional":[156],"wisdom":[157],"suggests.":[158],"Our":[159],"extensive":[160],"evaluation,":[161],"public":[164],"benchmark,":[166],"HuggingFace":[167],"Open":[168],"LLM":[169],"Leaderboard":[170],"MT-Bench,":[172],"demonstrates":[173],"our":[175],"approach":[176],"compares":[177],"favorably":[178],"state-of-the-art":[180],"rely":[183],"solely":[184]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}