{"id":"https://openalex.org/W4281963856","doi":"https://doi.org/10.48550/arxiv.2206.02231","title":"Models of human preference for learning reward functions","display_name":"Models of human preference for learning reward functions","publication_year":2022,"publication_date":"2022-06-05","ids":{"openalex":"https://openalex.org/W4281963856","doi":"https://doi.org/10.48550/arxiv.2206.02231"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2206.02231","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.02231","pdf_url":"https://arxiv.org/pdf/2206.02231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2206.02231","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056649746","display_name":"W. Bradley Knox","orcid":"https://orcid.org/0000-0002-6006-9523"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Knox, W. Bradley","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068136137","display_name":"Stephane Hatgis-Kessell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hatgis-Kessell, Stephane","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084731369","display_name":"Serena Booth","orcid":"https://orcid.org/0000-0001-7738-4418"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Booth, Serena","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043572737","display_name":"Scott Niekum","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niekum, Scott","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001594330","display_name":"Peter Stone","orcid":"https://orcid.org/0000-0002-6795-420X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stone, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5059194708","display_name":"Alessandro Allievi","orcid":"https://orcid.org/0000-0001-5793-7679"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Allievi, Alessandro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13283","display_name":"Mental Health Research Topics","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13283","display_name":"Mental Health Research Topics","score":0.9779999852180481,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/regret","display_name":"Regret","score":0.9435312151908875},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7349358797073364},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.6941587328910828},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5671162009239197},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5445376634597778},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.543797492980957},{"id":"https://openalex.org/keywords/preference-elicitation","display_name":"Preference elicitation","score":0.5326589941978455},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4028150737285614},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32557982206344604},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.2110716700553894},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.1794014871120453}],"concepts":[{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.9435312151908875},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7349358797073364},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.6941587328910828},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5671162009239197},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5445376634597778},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.543797492980957},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.5326589941978455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4028150737285614},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32557982206344604},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.2110716700553894},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.1794014871120453},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2206.02231","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.02231","pdf_url":"https://arxiv.org/pdf/2206.02231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2206.02231","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2206.02231","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2206.02231","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2206.02231","pdf_url":"https://arxiv.org/pdf/2206.02231","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.800000011920929}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4292701710","https://openalex.org/W2159111852","https://openalex.org/W271352469","https://openalex.org/W2937325523","https://openalex.org/W64851098","https://openalex.org/W1562775108","https://openalex.org/W2050663403","https://openalex.org/W2952338588","https://openalex.org/W9932698","https://openalex.org/W2073850970"],"abstract_inverted_index":{"The":[0],"utility":[1],"of":[2,10,16,35,40,62,88,198,216],"reinforcement":[3,41],"learning":[4,42],"is":[5,24,201],"limited":[6],"by":[7,57,82],"the":[8,14,27,60,114,125,147,158,196,226],"alignment":[9,23],"reward":[11,28,110,115,178],"functions":[12,179],"with":[13,152],"interests":[15],"human":[17,44,48,77,173,227],"stakeholders.":[18],"One":[19],"promising":[20],"method":[21],"for":[22,239],"to":[25,53,71,101,113,185],"learn":[26],"function":[29,111,116],"from":[30,43,92,180],"human-generated":[31],"preferences":[32,49,78,98,174,182,228],"between":[33],"pairs":[34],"trajectory":[36],"segments,":[37],"a":[38,86,89,109,213,241,243],"type":[39],"feedback":[45],"(RLHF).":[46],"These":[47],"are":[50,188],"typically":[51],"assumed":[52],"be":[54,72],"informed":[55,81],"solely":[56],"partial":[58,127,148],"return,":[59],"sum":[61],"rewards":[63],"along":[64],"each":[65,83],"segment.":[66],"We":[67,137,219],"find":[68,163],"this":[69,131,192],"assumption":[70,215],"flawed":[73],"and":[74,121,175,203,232,235],"propose":[75],"modeling":[76],"instead":[79],"as":[80],"segment's":[84,90],"regret,":[85,102],"measure":[87],"deviation":[91],"optimal":[93],"decision-making.":[94],"Given":[95],"infinitely":[96],"many":[97],"generated":[99,118],"according":[100],"we":[103,106,122,162,230],"prove":[104,123],"that":[105,117,124,140,164,183,187,195],"can":[107],"identify":[108],"equivalent":[112],"those":[119],"preferences,":[120],"previous":[126],"return":[128,149],"model":[129,145,151,169,200,208],"lacks":[130],"identifiability":[132],"property":[133],"in":[134,156],"multiple":[135],"contexts.":[136],"empirically":[138],"show":[139],"our":[141,165,204,223,233],"proposed":[142,166,205],"regret":[143,167,206],"preference":[144,150,168,199,207,236],"outperforms":[146],"finite":[153],"training":[154,234],"data":[155],"otherwise":[157],"same":[159],"setting.":[160],"Additionally,":[161],"better":[170,189],"predicts":[171],"real":[172],"also":[176],"learns":[177],"these":[181],"lead":[184],"policies":[186],"human-aligned.":[190],"Overall,":[191],"work":[193],"establishes":[194],"choice":[197],"impactful,":[202],"provides":[209],"an":[210],"improvement":[211],"upon":[212],"core":[214],"recent":[217],"research.":[218],"have":[220],"open":[221],"sourced":[222],"experimental":[224],"code,":[225],"dataset":[229],"gathered,":[231],"elicitation":[237],"interfaces":[238],"gathering":[240],"such":[242],"dataset.":[244]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}