{"id":"https://openalex.org/W4414360512","doi":"https://doi.org/10.24963/ijcai.2025/61","title":"Indirect Online Preference Optimization via Reinforcement Learning","display_name":"Indirect Online Preference Optimization via Reinforcement Learning","publication_year":2025,"publication_date":"2025-09-01","ids":{"openalex":"https://openalex.org/W4414360512","doi":"https://doi.org/10.24963/ijcai.2025/61"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2025/61","is_oa":false,"landing_page_url":"https://doi.org/10.24963/ijcai.2025/61","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Fourth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033953325","display_name":"En Wang","orcid":"https://orcid.org/0000-0001-6112-2923"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]},{"id":"https://openalex.org/I4210136497","display_name":"Jilin Medical University","ror":"https://ror.org/03mzw7781","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210136497"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"En Wang","raw_affiliation_strings":["Jilin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jilin University","institution_ids":["https://openalex.org/I194450716","https://openalex.org/I4210136497"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089190296","display_name":"Xingyu Lin","orcid":"https://orcid.org/0000-0002-0950-0736"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]},{"id":"https://openalex.org/I4210136497","display_name":"Jilin Medical University","ror":"https://ror.org/03mzw7781","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210136497"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xingyu Lin","raw_affiliation_strings":["Jilin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jilin University","institution_ids":["https://openalex.org/I194450716","https://openalex.org/I4210136497"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079104581","display_name":"Du Su","orcid":"https://orcid.org/0000-0002-6767-6019"},"institutions":[{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Du Su","raw_affiliation_strings":["Institute of Computing Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Computing Technology","institution_ids":["https://openalex.org/I4210090176"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027883532","display_name":"Chenfu Bao","orcid":null},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenfu Bao","raw_affiliation_strings":["Baidu Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu Inc","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007860458","display_name":"Zhonghou Lv","orcid":null},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhonghou Lv","raw_affiliation_strings":["Baidu Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu Inc","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022144626","display_name":"Funing Yang","orcid":"https://orcid.org/0000-0002-5286-9491"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]},{"id":"https://openalex.org/I4210136497","display_name":"Jilin Medical University","ror":"https://ror.org/03mzw7781","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210136497"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Funing Yang","raw_affiliation_strings":["Jilin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jilin University","institution_ids":["https://openalex.org/I194450716","https://openalex.org/I4210136497"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056077366","display_name":"Yuanbo Xu","orcid":"https://orcid.org/0000-0001-8370-5011"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]},{"id":"https://openalex.org/I4210136497","display_name":"Jilin Medical University","ror":"https://ror.org/03mzw7781","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210136497"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanbo Xu","raw_affiliation_strings":["Jilin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jilin University","institution_ids":["https://openalex.org/I194450716","https://openalex.org/I4210136497"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100373255","display_name":"Wenbin Liu","orcid":"https://orcid.org/0000-0002-4384-1446"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]},{"id":"https://openalex.org/I4210136497","display_name":"Jilin Medical University","ror":"https://ror.org/03mzw7781","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210136497"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenbin Liu","raw_affiliation_strings":["Jilin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jilin University","institution_ids":["https://openalex.org/I194450716","https://openalex.org/I4210136497"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.30482415,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"538","last_page":"546"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.7567999958992004,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.7567999958992004,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.7422999739646912,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.7265999913215637,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.6798999905586243},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.585099995136261},{"id":"https://openalex.org/keywords/preference-elicitation","display_name":"Preference elicitation","score":0.5716999769210815},{"id":"https://openalex.org/keywords/nash-equilibrium","display_name":"Nash equilibrium","score":0.45879998803138733},{"id":"https://openalex.org/keywords/framing","display_name":"Framing (construction)","score":0.4496999979019165},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.43309998512268066},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.40049999952316284},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.3912999927997589}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6952000260353088},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.6798999905586243},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.585099995136261},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.5716999769210815},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5044000148773193},{"id":"https://openalex.org/C46814582","wikidata":"https://www.wikidata.org/wiki/Q23389","display_name":"Nash equilibrium","level":2,"score":0.45879998803138733},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45179998874664307},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.4496999979019165},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.43309998512268066},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.40049999952316284},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.3912999927997589},{"id":"https://openalex.org/C2777745021","wikidata":"https://www.wikidata.org/wiki/Q646102","display_name":"Consequentialism","level":2,"score":0.3824999928474426},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.33469998836517334},{"id":"https://openalex.org/C140479938","wikidata":"https://www.wikidata.org/wiki/Q5254619","display_name":"Iterated function","level":2,"score":0.32440000772476196},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C153517567","wikidata":"https://www.wikidata.org/wiki/Q26090","display_name":"Mechanism design","level":2,"score":0.27950000762939453},{"id":"https://openalex.org/C117619785","wikidata":"https://www.wikidata.org/wiki/Q6094414","display_name":"Iterative learning control","level":3,"score":0.2752000093460083},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C177142836","wikidata":"https://www.wikidata.org/wiki/Q44455","display_name":"Game theory","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C2780490138","wikidata":"https://www.wikidata.org/wiki/Q7079636","display_name":"Offline learning","level":3,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2025/61","is_oa":false,"landing_page_url":"https://doi.org/10.24963/ijcai.2025/61","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Fourth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Human":[0],"preference":[1,61],"alignment":[2,122,173,184],"(HPA)":[3],"aims":[4],"to":[5,13,59,68,153,200],"ensure":[6],"Large":[7],"Language":[8],"Models":[9],"(LLMs)":[10],"responding":[11],"appropriately":[12],"meet":[14],"human":[15,31,47,75,187],"moral":[16],"and":[17,25,76,86,112,136,178,186,206],"ethical":[18],"requirements.":[19],"Existing":[20],"methods,":[21],"such":[22],"as":[23,79,81],"RLHF":[24],"DPO,":[26],"rely":[27],"heavily":[28],"on":[29],"high-quality":[30],"annotation,":[32],"which":[33],"restrict":[34],"the":[35,44,54,91,98,117,121,129,150,192],"efficiency":[36],"of":[37,46,56,72,93,100,123],"iterative":[38],"online":[39,51,179],"model":[40,77,196],"refinement.":[41],"To":[42,89],"address":[43],"inefficiencies":[45],"annotation":[48,209],"acquisition,":[49],"iterated":[50],"strategy":[52],"advocates":[53],"use":[55],"fine-tuned":[57],"LLMs":[58],"self-generate":[60],"data.":[62],"However,":[63],"this":[64],"approach":[65],"is":[66],"prone":[67],"distribution":[69,94,204],"bias,":[70,95],"because":[71],"differences":[73],"between":[74,84],"annotations,":[78],"well":[80],"modeling":[82],"errors":[83],"simulators":[85],"real-world":[87,166],"contexts.":[88],"mitigate":[90],"impact":[92],"we":[96,126,140],"adopt":[97],"principles":[99],"adversarial":[101,114,118],"training,":[102],"framing":[103],"a":[104,109],"zero-sum":[105],"two-player":[106],"game":[107],"with":[108],"protagonist":[110,124,151],"agent":[111,119,152],"an":[113],"agent.":[115],"With":[116],"challenging":[120],"agent,":[125],"continuously":[127],"refine":[128],"protagonist\u2019s":[130],"performance.":[131],"By":[132],"utilizing":[133],"min-max":[134],"equilibrium":[135,138],"Nash":[137],"strategies,":[139],"propose":[141],"Indirect":[142],"Online":[143],"Preference":[144],"Optimization":[145],"(IOPO)":[146],"mechanism":[147],"that":[148,169],"enables":[149],"converge":[154],"without":[155],"bias":[156],"while":[157],"maintaining":[158],"linear":[159],"computational":[160],"complexity.":[161],"Extensive":[162],"experiments":[163],"across":[164],"three":[165],"datasets":[167],"demonstrate":[168],"IOPO":[170],"outperforms":[171],"state-of-the-art":[172],"methods":[174],"in":[175],"both":[176],"offline":[177],"scenarios,":[180],"evidenced":[181],"by":[182],"standard":[183],"metrics":[185],"evaluations.":[188],"This":[189],"innovation":[190],"reduces":[191],"time":[193],"required":[194],"for":[195],"iterations":[197],"from":[198],"months":[199],"one":[201],"week,":[202],"alleviates":[203],"shifts,":[205],"significantly":[207],"cuts":[208],"costs.":[210]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}