{"id":"https://openalex.org/W3211700439","doi":"https://doi.org/10.1109/tnnls.2021.3070484","title":"Hierarchical Multiagent Reinforcement Learning for Allocating Guaranteed Display Ads","display_name":"Hierarchical Multiagent Reinforcement Learning for Allocating Guaranteed Display Ads","publication_year":2021,"publication_date":"2021-05-17","ids":{"openalex":"https://openalex.org/W3211700439","doi":"https://doi.org/10.1109/tnnls.2021.3070484","mag":"3211700439","pmid":"https://pubmed.ncbi.nlm.nih.gov/33999823"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2021.3070484","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3070484","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083964854","display_name":"Lu Wang","orcid":"https://orcid.org/0000-0002-7305-1496"},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]},{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]},{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Lu Wang","raw_affiliation_strings":["School of Computer Science and Technology, East China Normal University, Shanghai, China","Tencent AI Lab, Bellevue, WA, USA","Shanghai Key Laboratory of Trustworthy Computing, Shanghai, China","School of Software Engineering, East China Normal University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-7305-1496","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]},{"raw_affiliation_string":"Tencent AI Lab, Bellevue, WA, USA","institution_ids":["https://openalex.org/I4210108985"]},{"raw_affiliation_string":"Shanghai Key Laboratory of Trustworthy Computing, Shanghai, China","institution_ids":["https://openalex.org/I4210139618"]},{"raw_affiliation_string":"School of Software Engineering, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101540919","display_name":"Lei Han","orcid":"https://orcid.org/0000-0003-1404-2415"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Han","raw_affiliation_strings":["Tencent AI Lab, Tencent, Inc., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Tencent, Inc., Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101579735","display_name":"Xinru Chen","orcid":"https://orcid.org/0009-0004-6582-7559"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinru Chen","raw_affiliation_strings":["Tencent AI Lab, Tencent, Inc., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Tencent, Inc., Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025415809","display_name":"Chengchang Li","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengchang Li","raw_affiliation_strings":["Tencent AI Lab, Tencent, Inc., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Tencent, Inc., Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068865316","display_name":"Junzhou Huang","orcid":"https://orcid.org/0000-0002-9548-1227"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junzhou Huang","raw_affiliation_strings":["Tencent AI Lab, Tencent, Inc., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Tencent, Inc., Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090720315","display_name":"Weinan Zhang","orcid":"https://orcid.org/0000-0002-0127-2425"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weinan Zhang","raw_affiliation_strings":["Department of Computer Science and Engineering, Shanghai Jiao Tong University, Shanghai, China","School of Computer Science and Technology, East China Normal University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-0127-2425","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"School of Computer Science and Technology, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101541045","display_name":"Wei Zhang","orcid":"https://orcid.org/0000-0001-6763-8146"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]},{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Zhang","raw_affiliation_strings":["Department of Computer Science and Engineering, Shanghai Jiao Tong University, Shanghai, China","School of Computer Science and Technology, East China Normal University, Shanghai, China","Shanghai Key Laboratory of Trustworthy Computing, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-6763-8146","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"School of Computer Science and Technology, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]},{"raw_affiliation_string":"Shanghai Key Laboratory of Trustworthy Computing, Shanghai, China","institution_ids":["https://openalex.org/I4210139618"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101434435","display_name":"Xiaofeng He","orcid":"https://orcid.org/0000-0002-6911-348X"},"institutions":[{"id":"https://openalex.org/I4210139618","display_name":"Shanghai Key Laboratory of Trustworthy Computing","ror":"https://ror.org/030qbr085","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210139618"]},{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaofeng He","raw_affiliation_strings":["School of Computer Science and Technology, East China Normal University, Shanghai, China","Shanghai Key Laboratory of Trustworthy Computing, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-6911-348X","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]},{"raw_affiliation_string":"Shanghai Key Laboratory of Trustworthy Computing, Shanghai, China","institution_ids":["https://openalex.org/I4210139618"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101186016","display_name":"Dijun Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dijun Luo","raw_affiliation_strings":["Tencent AI Lab, Tencent, Inc., Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tencent AI Lab, Tencent, Inc., Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8878,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.78532598,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"33","issue":"10","first_page":"5361","last_page":"5373"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10328","display_name":"Supply Chain and Inventory Management","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7910336256027222},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7233046889305115},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.6249490976333618},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5998438596725464},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.5916132926940918},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.47221308946609497},{"id":"https://openalex.org/keywords/operations-research","display_name":"Operations research","score":0.34148406982421875},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3402353525161743},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13055798411369324},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.09623178839683533}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7910336256027222},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7233046889305115},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.6249490976333618},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5998438596725464},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.5916132926940918},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.47221308946609497},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.34148406982421875},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3402353525161743},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13055798411369324},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.09623178839683533},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2021.3070484","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3070484","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:33999823","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33999823","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.5699999928474426}],"awards":[{"id":"https://openalex.org/G333974875","display_name":null,"funder_award_id":"2016YFB1000904","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W1120219466","https://openalex.org/W1585861384","https://openalex.org/W1641379095","https://openalex.org/W1769120835","https://openalex.org/W1907356258","https://openalex.org/W1972169135","https://openalex.org/W1973976093","https://openalex.org/W1982678075","https://openalex.org/W2044905589","https://openalex.org/W2096145798","https://openalex.org/W2107112577","https://openalex.org/W2109910161","https://openalex.org/W2112741516","https://openalex.org/W2121517924","https://openalex.org/W2148352389","https://openalex.org/W2149822245","https://openalex.org/W2170675255","https://openalex.org/W2183373924","https://openalex.org/W2539201403","https://openalex.org/W2602275733","https://openalex.org/W2606378794","https://openalex.org/W2768629321","https://openalex.org/W2974685809","https://openalex.org/W3124383212","https://openalex.org/W3193544356","https://openalex.org/W4252172911","https://openalex.org/W6634945376","https://openalex.org/W6683443546","https://openalex.org/W6683879207","https://openalex.org/W6684205842","https://openalex.org/W6734215269","https://openalex.org/W6734678876","https://openalex.org/W6738796088","https://openalex.org/W6741471465","https://openalex.org/W6755156419","https://openalex.org/W6758714175"],"related_works":["https://openalex.org/W4241523039","https://openalex.org/W2360028903","https://openalex.org/W4280543773","https://openalex.org/W178231042","https://openalex.org/W2366083136","https://openalex.org/W2387622493","https://openalex.org/W1932132538","https://openalex.org/W2357832196","https://openalex.org/W1566252468","https://openalex.org/W4200375594"],"abstract_inverted_index":{"In":[0],"this":[1,34,82,86],"article,":[2],"we":[3,84,161],"study":[4],"the":[5,29,38,57,62,69,75,94,112,120,123,126,129,135,141,148,151,173,184,196,211,227],"problem":[6,35,87,92],"of":[7,73,96,125,137,154,181,186,207,233,235,240],"guaranteed":[8],"display":[9,17,152],"ads":[10,18,182],"(GDAs)":[11],"allocation,":[12],"which":[13],"requires":[14],"proactively":[15],"allocate":[16],"to":[19,22,54,56,106,176,194,198],"different":[20,144],"impressions":[21,39,63,142],"fulfill":[23],"their":[24,217],"impression":[25],"demands":[26,124],"indicated":[27],"in":[28,93],"contracts.":[30],"Existing":[31],"methods":[32],"for":[33,114,133,157],"either":[36],"assume":[37],"that":[40,169,209],"are":[41,64,119,150],"static":[42],"or":[43],"solely":[44],"consider":[45],"a":[46,89,163,178,191,205],"specific":[47],"ad's":[48],"benefits.":[49],"Thus,":[50],"it":[51],"is":[52,78],"hard":[53],"generalize":[55],"industrial":[58],"production":[59],"scenario":[60],"where":[61],"dynamical":[65],"and":[66,68,109,128,140,147,183,203],"large-scale,":[67],"overall":[70],"allocation":[71,104],"optimality":[72],"all":[74,111],"considered":[76],"GDAs":[77],"required.":[79],"To":[80],"bridge":[81],"gap,":[83],"formulate":[85],"as":[88],"sequential":[90],"decision-making":[91],"scope":[95],"multiagent":[97],"reinforcement":[98],"learning":[99],"(MARL),":[100],"by":[101],"assigning":[102],"an":[103,200],"agent":[105,174,197],"each":[107,138,155,158],"ad":[108,127,139,156],"coordinating":[110],"agents":[113,212],"allocating":[115],"GDAs.":[116],"The":[117],"inputs":[118],"states":[121],"(e.g.,":[122],"remaining":[130],"time":[131,145],"steps":[132],"displaying":[134],"ads)":[136],"at":[143],"steps,":[146],"outputs":[149],"ratios":[153],"impression.":[159],"Specifically,":[160],"propose":[162],"novel":[164],"hierarchical":[165],"MARL":[166],"(HMARL)":[167],"method":[168],"creates":[170],"hierarchies":[171],"over":[172,242],"policies":[175],"handle":[177],"large":[179],"number":[180],"dynamics":[185],"impressions.":[187],"HMARL":[188,241],"contains:":[189],"1)":[190],"manager":[192],"policy":[193],"navigate":[195],"choose":[199],"appropriate":[201],"subpolicy":[202],"2)":[204],"set":[206],"subpolicies":[208],"let":[210],"perform":[213],"diverse":[214],"conditioning":[215],"on":[216,221],"states.":[218],"Extensive":[219],"experiments":[220],"three":[222],"real-world":[223],"data":[224],"sets":[225],"from":[226],"Tencent":[228],"advertising":[229],"platform":[230],"with":[231],"tens":[232],"millions":[234],"records":[236],"demonstrate":[237],"significant":[238],"improvements":[239],"state-of-the-art":[243],"approaches.":[244]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-25T09:21:30.201066","created_date":"2025-10-10T00:00:00"}