{"id":"https://openalex.org/W4416749603","doi":"https://doi.org/10.1109/iros60139.2025.11247027","title":"PrefMMT: Modeling Human Preferences in Preference-based Reinforcement Learning with Multimodal Transformers","display_name":"PrefMMT: Modeling Human Preferences in Preference-based Reinforcement Learning with Multimodal Transformers","publication_year":2025,"publication_date":"2025-10-19","ids":{"openalex":"https://openalex.org/W4416749603","doi":"https://doi.org/10.1109/iros60139.2025.11247027"},"language":null,"primary_location":{"id":"doi:10.1109/iros60139.2025.11247027","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros60139.2025.11247027","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101254875","display_name":"Dezhong Zhao","orcid":"https://orcid.org/0000-0003-4478-9595"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dezhong Zhao","raw_affiliation_strings":["Beijing University of Chemical Technology,College of Mechanical and Electrical Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology,College of Mechanical and Electrical Engineering,Beijing,China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100387947","display_name":"Ruiqi Wang","orcid":"https://orcid.org/0009-0001-0431-829X"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruiqi Wang","raw_affiliation_strings":["Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112464555","display_name":"Dayoon Suh","orcid":"https://orcid.org/0000-0002-4839-9773"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dayoon Suh","raw_affiliation_strings":["Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100774203","display_name":"Taehyeon Kim","orcid":"https://orcid.org/0000-0003-1169-3968"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Taehyeon Kim","raw_affiliation_strings":["Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109746539","display_name":"Ziqin Yuan","orcid":null},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ziqin Yuan","raw_affiliation_strings":["Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076467173","display_name":"Byung\u2010Cheol Min","orcid":"https://orcid.org/0000-0001-6458-4365"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Byung-Cheol Min","raw_affiliation_strings":["Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University,SMART Laboratory,Department of Computer and Information Technology,West Lafayette,IN,USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107143571","display_name":"Guohua Chen","orcid":"https://orcid.org/0000-0003-4167-467X"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guohua Chen","raw_affiliation_strings":["Beijing University of Chemical Technology,College of Mechanical and Electrical Engineering,Beijing,China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology,College of Mechanical and Electrical Engineering,Beijing,China","institution_ids":["https://openalex.org/I75390827"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.34287843,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"18692","last_page":"18699"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5097000002861023,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.5097000002861023,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.2395000010728836,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11099","display_name":"Autonomous Vehicle Technology and Safety","score":0.05429999902844429,"subfield":{"id":"https://openalex.org/subfields/2203","display_name":"Automotive Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.718500018119812},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6327999830245972},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.52920001745224},{"id":"https://openalex.org/keywords/hidden-markov-model","display_name":"Hidden Markov model","score":0.4715000092983246},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.42149999737739563},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.4027000069618225},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.382999986410141},{"id":"https://openalex.org/keywords/learning-classifier-system","display_name":"Learning classifier system","score":0.37119999527931213},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.36309999227523804}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.718500018119812},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6632000207901001},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6597999930381775},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6327999830245972},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.52920001745224},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48989999294281006},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.4715000092983246},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.42149999737739563},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4027000069618225},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.382999986410141},{"id":"https://openalex.org/C199190896","wikidata":"https://www.wikidata.org/wiki/Q3509276","display_name":"Learning classifier system","level":3,"score":0.37119999527931213},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.36309999227523804},{"id":"https://openalex.org/C145460709","wikidata":"https://www.wikidata.org/wiki/Q859951","display_name":"Human\u2013robot interaction","level":3,"score":0.3621000051498413},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.3612000048160553},{"id":"https://openalex.org/C40506919","wikidata":"https://www.wikidata.org/wiki/Q7452469","display_name":"Sequence learning","level":2,"score":0.3603000044822693},{"id":"https://openalex.org/C117035363","wikidata":"https://www.wikidata.org/wiki/Q3769299","display_name":"Human behavior","level":2,"score":0.3158000111579895},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.30469998717308044},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.29249998927116394},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2856999933719635},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C188888258","wikidata":"https://www.wikidata.org/wiki/Q7353390","display_name":"Robot learning","level":4,"score":0.2773999869823456},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2743000090122223},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.2653000056743622},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.2542000114917755}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iros60139.2025.11247027","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iros60139.2025.11247027","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W1977655452","https://openalex.org/W2087309216","https://openalex.org/W2736902602","https://openalex.org/W2763110165","https://openalex.org/W2912748933","https://openalex.org/W2966174181","https://openalex.org/W3119920397","https://openalex.org/W3162737204","https://openalex.org/W4312480554","https://openalex.org/W4312550960","https://openalex.org/W4313562996","https://openalex.org/W4385245566","https://openalex.org/W4389665973","https://openalex.org/W4389666460","https://openalex.org/W4391128508","https://openalex.org/W4391855070","https://openalex.org/W4399310716","https://openalex.org/W4406415458","https://openalex.org/W4413926103"],"related_works":[],"abstract_inverted_index":{"Preference-based":[0],"reinforcement":[1],"learning":[2,61,158],"(PbRL)":[3],"shows":[4],"promise":[5],"in":[6],"aligning":[7],"robot":[8,43,71],"behaviors":[9],"with":[10],"human":[11,23,48,101],"preferences,":[12],"but":[13],"its":[14],"success":[15],"depends":[16],"heavily":[17],"on":[18,160],"the":[19,39,67,92,164,171],"accurate":[20],"modeling":[21,35,56,111],"of":[22,70,75],"preferences":[24],"through":[25],"reward":[26],"models.":[27],"Most":[28],"methods":[29],"adopt":[30],"Markovian":[31],"assumptions":[32],"for":[33,113],"preference":[34,142],"(PM),":[36],"which":[37,73,129],"overlook":[38],"temporal":[40,133],"dependencies":[41,134],"within":[42],"behavior":[44],"trajectories":[45],"that":[46,98,148],"impact":[47],"evaluations.":[49],"While":[50],"recent":[51],"works":[52],"have":[53],"utilized":[54],"sequence":[55,110],"to":[57,90,139],"mitigate":[58],"this":[59,104],"by":[60,115],"sequential":[62],"non-Markovian":[63],"rewards,":[64],"they":[65,87],"ignore":[66],"multimodal":[68,109,124],"nature":[69],"trajectories,":[72],"consist":[74],"elements":[76],"from":[77,163,170],"two":[78],"distinctive":[79],"modalities:":[80],"state":[81,117],"and":[82,118,135,154,167,176],"action.":[83],"As":[84],"a":[85,108,123],"result,":[86],"often":[88],"struggle":[89],"capture":[91,140],"complex":[93,141],"interplay":[94],"between":[95],"these":[96],"modalities":[97],"significantly":[99],"shapes":[100],"preferences.":[102],"In":[103],"paper,":[105],"we":[106],"propose":[107],"approach":[112],"PM":[114,153],"disentangling":[116],"action":[119],"modalities.":[120],"We":[121],"introduce":[122],"transformer":[125],"network,":[126],"named":[127],"PrefMMT,":[128],"hierarchically":[130],"leverages":[131],"intra-modal":[132],"inter-modal":[136],"state-action":[137],"interactions":[138],"patterns.":[143],"Our":[144],"experimental":[145],"results":[146],"demonstrate":[147],"PrefMMT":[149],"consistently":[150],"outperforms":[151],"state-of-the-art":[152],"direct":[155],"preference-based":[156],"policy":[157],"baselines":[159],"locomotion":[161],"tasks":[162,169],"D4RL":[165],"benchmark":[166],"manipulation":[168],"MetaWorld":[172],"benchmark.":[173],"Source":[174],"code":[175],"supplementary":[177],"information":[178],"are":[179],"available":[180],"at":[181],"https://sites.google.com/view/prefmmt.":[182]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-11-28T00:00:00"}