{"id":"https://openalex.org/W4386728798","doi":"https://doi.org/10.1145/3604915.3608854","title":"Optimizing Long-term Value for Auction-Based Recommender Systems via On-Policy Reinforcement Learning","display_name":"Optimizing Long-term Value for Auction-Based Recommender Systems via On-Policy Reinforcement Learning","publication_year":2023,"publication_date":"2023-09-14","ids":{"openalex":"https://openalex.org/W4386728798","doi":"https://doi.org/10.1145/3604915.3608854"},"language":"en","primary_location":{"id":"doi:10.1145/3604915.3608854","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3604915.3608854","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023357082","display_name":"Ruiyang Xu","orcid":"https://orcid.org/0000-0002-4973-8458"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ruiyang Xu","raw_affiliation_strings":["Applied Reinforcement Learning, Meta AI, USA"],"affiliations":[{"raw_affiliation_string":"Applied Reinforcement Learning, Meta AI, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038161679","display_name":"Jalaj Bhandari","orcid":"https://orcid.org/0000-0002-7115-8986"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jalaj Bhandari","raw_affiliation_strings":["Applied Reinforcement Learning, Meta AI, USA"],"affiliations":[{"raw_affiliation_string":"Applied Reinforcement Learning, Meta AI, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058918505","display_name":"Dmytro Korenkevych","orcid":"https://orcid.org/0009-0007-5748-9571"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dmytro Korenkevych","raw_affiliation_strings":["Applied Reinforcement Learning, Meta AI, USA"],"affiliations":[{"raw_affiliation_string":"Applied Reinforcement Learning, Meta AI, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090348201","display_name":"Fan Liu","orcid":"https://orcid.org/0000-0001-5167-285X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan Liu","raw_affiliation_strings":["Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100642695","display_name":"Yangfan He","orcid":"https://orcid.org/0000-0002-3153-5177"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuchen He","raw_affiliation_strings":["Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092728750","display_name":"Alex Nikulkov","orcid":"https://orcid.org/0009-0008-2335-289X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alex Nikulkov","raw_affiliation_strings":["Applied Reinforcement Learning, Meta AI, USA"],"affiliations":[{"raw_affiliation_string":"Applied Reinforcement Learning, Meta AI, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050765125","display_name":"Zheqing Zhu","orcid":"https://orcid.org/0000-0002-1162-106X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheqing Zhu","raw_affiliation_strings":["Applied Reinforcement Learning, Meta AI, USA"],"affiliations":[{"raw_affiliation_string":"Applied Reinforcement Learning, Meta AI, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5023357082"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7653,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.74982452,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"955","last_page":"962"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8790645599365234},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8683894872665405},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8344851732254028},{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.4959031045436859},{"id":"https://openalex.org/keywords/double-auction","display_name":"Double auction","score":0.4296046495437622},{"id":"https://openalex.org/keywords/user-engagement","display_name":"User engagement","score":0.415947288274765},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3624972403049469},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3464288115501404},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.20730340480804443},{"id":"https://openalex.org/keywords/common-value-auction","display_name":"Common value auction","score":0.10757419466972351}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8790645599365234},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8683894872665405},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8344851732254028},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.4959031045436859},{"id":"https://openalex.org/C2539976","wikidata":"https://www.wikidata.org/wiki/Q12163285","display_name":"Double auction","level":3,"score":0.4296046495437622},{"id":"https://openalex.org/C2984870255","wikidata":"https://www.wikidata.org/wiki/Q5196451","display_name":"User engagement","level":2,"score":0.415947288274765},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3624972403049469},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3464288115501404},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.20730340480804443},{"id":"https://openalex.org/C163239763","wikidata":"https://www.wikidata.org/wiki/Q5153637","display_name":"Common value auction","level":2,"score":0.10757419466972351},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3604915.3608854","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3604915.3608854","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W1588965116","https://openalex.org/W2008886893","https://openalex.org/W2030808931","https://openalex.org/W2054141820","https://openalex.org/W2080122405","https://openalex.org/W2098432798","https://openalex.org/W2112420033","https://openalex.org/W2210543184","https://openalex.org/W2358698356","https://openalex.org/W2400213106","https://openalex.org/W2475334473","https://openalex.org/W2512971201","https://openalex.org/W2575705757","https://openalex.org/W2604822632","https://openalex.org/W2742272831","https://openalex.org/W2787933113","https://openalex.org/W2886209086","https://openalex.org/W2902572901","https://openalex.org/W2963428623","https://openalex.org/W2963842088","https://openalex.org/W3003416843","https://openalex.org/W3034853385","https://openalex.org/W4296604485"],"related_works":["https://openalex.org/W4390273403","https://openalex.org/W4386781444","https://openalex.org/W2150182025","https://openalex.org/W3092950680","https://openalex.org/W3197542405","https://openalex.org/W2056712470","https://openalex.org/W3125580266","https://openalex.org/W4317039510","https://openalex.org/W4238861846","https://openalex.org/W3172701938"],"abstract_inverted_index":{"Auction-based":[0],"recommender":[1,49,122],"systems":[2],"are":[3,11],"prevalent":[4],"in":[5,46,96,144],"online":[6,115],"advertising":[7],"platforms,":[8],"but":[9],"they":[10],"typically":[12],"optimized":[13],"to":[14,40],"allocate":[15],"recommendation":[16],"slots":[17],"based":[18],"on":[19,30,119],"immediate":[20],"expected":[21],"return":[22,44],"metrics,":[23],"neglecting":[24],"the":[25,69,89,108,140],"downstream":[26],"effects":[27],"of":[28,127,146],"recommendations":[29,72],"user":[31,76,148],"behavior.":[32],"In":[33],"this":[34],"study,":[35],"we":[36,60,132],"employ":[37],"reinforcement":[38,57],"learning":[39,58],"optimize":[41],"for":[42],"long-term":[43,75,147],"metrics":[45],"an":[47,114,120],"auction-based":[48,110,121],"system.":[50],"Utilizing":[51],"temporal":[52],"difference":[53],"learning,":[54],"a":[55,62],"fundamental":[56],"algorithm,":[59],"implement":[61],"one-step":[63],"policy":[64],"improvement":[65],"approach":[66,93],"that":[67,102,135],"biases":[68],"system":[70,123,143],"towards":[71],"with":[73,88],"higher":[74],"engagement":[77,149],"metrics.":[78,150],"This":[79],"optimizes":[80],"value":[81],"over":[82],"long":[83],"horizons":[84],"while":[85],"maintaining":[86],"compatibility":[87],"auction":[90],"framework.":[91],"Our":[92],"is":[94],"grounded":[95],"dynamic":[97],"programming":[98],"ideas":[99],"which":[100,124],"show":[101],"our":[103,136],"method":[104,138],"provably":[105],"improves":[106],"upon":[107],"existing":[109],"base":[111],"policy.":[112],"Through":[113],"A/B":[116],"test":[117],"conducted":[118],"handles":[125],"billions":[126],"impressions":[128],"and":[129],"users":[130],"daily,":[131],"empirically":[133],"establish":[134],"proposed":[137],"outperforms":[139],"current":[141],"production":[142],"terms":[145]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}