{"id":"https://openalex.org/W2921600809","doi":"https://doi.org/10.5220/0007395502560265","title":"A Reinforcement Learning Method to Select Ad Networks in Waterfall Strategy","display_name":"A Reinforcement Learning Method to Select Ad Networks in Waterfall Strategy","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2921600809","doi":"https://doi.org/10.5220/0007395502560265","mag":"2921600809"},"language":"en","primary_location":{"id":"doi:10.5220/0007395502560265","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0007395502560265","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th International Conference on Agents and Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.5220/0007395502560265","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069444765","display_name":"R. Afshar","orcid":"https://orcid.org/0000-0003-3384-3971"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Reza Afshar","raw_affiliation_strings":["School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077147157","display_name":"Yingqian Zhang","orcid":"https://orcid.org/0000-0002-5073-0787"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Yingqian Zhang","raw_affiliation_strings":["School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088250715","display_name":"Murat F\u0131rat","orcid":"https://orcid.org/0000-0001-8483-3533"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Murat Firat","raw_affiliation_strings":["School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080242338","display_name":"Uzay Kaymak","orcid":"https://orcid.org/0000-0002-4500-9098"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Uzay Kaymak","raw_affiliation_strings":["School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Industrial Engineering, Eindhoven University of Technology and The Netherlands, --- Select a Country ---","institution_ids":["https://openalex.org/I83019370"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I83019370"],"apc_list":null,"apc_paid":null,"fwci":4.7539,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.94724783,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"256","last_page":"265"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9854999780654907,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9854999780654907,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11161","display_name":"Consumer Market Behavior and Pricing","score":0.982200026512146,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10609","display_name":"Digital Marketing and Social Media","score":0.9815999865531921,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8606411218643188},{"id":"https://openalex.org/keywords/bidding","display_name":"Bidding","score":0.7913802862167358},{"id":"https://openalex.org/keywords/waterfall","display_name":"Waterfall","score":0.7669262886047363},{"id":"https://openalex.org/keywords/revenue","display_name":"Revenue","score":0.7190616130828857},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6450362205505371},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.5776950120925903},{"id":"https://openalex.org/keywords/waterfall-model","display_name":"Waterfall model","score":0.5628495812416077},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5172634720802307},{"id":"https://openalex.org/keywords/online-learning","display_name":"Online learning","score":0.43378984928131104},{"id":"https://openalex.org/keywords/operations-research","display_name":"Operations research","score":0.34952104091644287},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33516553044319153},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3305634558200836},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.2592419385910034},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.20929065346717834},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.20235469937324524},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.1712842583656311},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11201274394989014}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8606411218643188},{"id":"https://openalex.org/C9233905","wikidata":"https://www.wikidata.org/wiki/Q3276328","display_name":"Bidding","level":2,"score":0.7913802862167358},{"id":"https://openalex.org/C121012721","wikidata":"https://www.wikidata.org/wiki/Q34038","display_name":"Waterfall","level":2,"score":0.7669262886047363},{"id":"https://openalex.org/C195487862","wikidata":"https://www.wikidata.org/wiki/Q850210","display_name":"Revenue","level":2,"score":0.7190616130828857},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6450362205505371},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.5776950120925903},{"id":"https://openalex.org/C20505762","wikidata":"https://www.wikidata.org/wiki/Q478175","display_name":"Waterfall model","level":3,"score":0.5628495812416077},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5172634720802307},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.43378984928131104},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.34952104091644287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33516553044319153},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3305634558200836},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2592419385910034},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.20929065346717834},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.20235469937324524},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.1712842583656311},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11201274394989014},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.5220/0007395502560265","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0007395502560265","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th International Conference on Agents and Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.tue.nl:publications/54b4096d-ce5e-400b-a60a-d68c2b5a9275","is_oa":true,"landing_page_url":"https://research.tue.nl/en/publications/54b4096d-ce5e-400b-a60a-d68c2b5a9275","pdf_url":"https://pure.tue.nl/ws/files/121932422/ICAART_2019_113.pdf","source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Refaei Afshar, R, Zhang, Y, Firat, M & Kaymak, U 2019, A reinforcement learning method to select ad networks in Waterfall Strategy. in J van den Herik, L Steels & A Rocha (eds), ICAART 2019 - Proceedings of the 11th International Conference on Agents and Artificial Intelligence. vol. 2, SciTePress Digital Library, pp. 256-265, 11th International Conference on Agents and Artificial Intelligence, ICAART 2019, Prague, Czech Republic, 19/02/19. https://doi.org/10.5220/0007395502560265","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:tue:oai:pure.tue.nl:publications/54b4096d-ce5e-400b-a60a-d68c2b5a9275","is_oa":true,"landing_page_url":"https://research.tue.nl/nl/publications/54b4096d-ce5e-400b-a60a-d68c2b5a9275","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ICAART 2019 - Proceedings of the 11th International Conference on Agents and Artificial Intelligence, 2, 256 - 265","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"doi:10.5220/0007395502560265","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0007395502560265","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th International Conference on Agents and Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320338388","display_name":"Eurostars","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W129457532","https://openalex.org/W1554763265","https://openalex.org/W1557734215","https://openalex.org/W1669302834","https://openalex.org/W1989133735","https://openalex.org/W1992428795","https://openalex.org/W2073685064","https://openalex.org/W2077055010","https://openalex.org/W2138312809","https://openalex.org/W2138620854","https://openalex.org/W2148143831","https://openalex.org/W2149822245","https://openalex.org/W2188273680","https://openalex.org/W2345319452","https://openalex.org/W2564077743","https://openalex.org/W2758709491","https://openalex.org/W2974685809","https://openalex.org/W3105140685","https://openalex.org/W3214102110","https://openalex.org/W4211221179","https://openalex.org/W4214717370","https://openalex.org/W4236137412","https://openalex.org/W4306254152"],"related_works":["https://openalex.org/W2367672333","https://openalex.org/W3206992463","https://openalex.org/W4250558843","https://openalex.org/W4384202826","https://openalex.org/W2808522417","https://openalex.org/W2377167434","https://openalex.org/W315570438","https://openalex.org/W3129167933","https://openalex.org/W4312475296","https://openalex.org/W3110264322"],"abstract_inverted_index":{"A":[0],"high":[1],"percentage":[2],"of":[3,38,43,58,126,142],"online":[4,32,52],"advertising":[5],"is":[6,50,61],"currently":[7],"performed":[8],"through":[9,51],"real":[10],"time":[11],"bidding.":[12],"Impressions":[13],"are":[14,27],"generated":[15],"once":[16],"a":[17,55,90,118],"user":[18],"visits":[19],"the":[20,39,123,130,139,143],"websites":[21,49],"containing":[22],"empty":[23],"ad":[24,33,70,76,98,127],"slots,":[25],"which":[26,97],"subsequently":[28],"sold":[29],"in":[30,79,129],"an":[31],"exchange":[34],"market.":[35],"Nowadays,":[36],"one":[37],"most":[40,68],"important":[41],"sources":[42],"income":[44],"for":[45,92,102],"publishers":[46,94],"who":[47],"own":[48],"advertising.":[53],"From":[54],"publisher\u2019s":[56],"point":[57],"view":[59],"it":[60],"critical":[62],"to":[63,67,73,81,95,100,121],"send":[64],"its":[65,75],"impressions":[66],"profitable":[69],"networks":[71,99,128],"and":[72],"fill":[74],"slots":[77],"quickly":[78],"order":[80],"increase":[82],"their":[83],"revenue.":[84],"In":[85],"this":[86,136],"paper":[87],"we":[88],"present":[89],"method":[91,108,137],"helping":[93],"decide":[96],"use":[101],"each":[103],"available":[104],"impression.":[105],"Our":[106],"proposed":[107],"uses":[109],"reinforcement":[110],"learning":[111],"with":[112],"initial":[113],"state-action":[114],"values":[115],"obtained":[116],"from":[117],"prediction":[119],"model":[120],"find":[122],"best":[124],"ordering":[125],"waterfall":[131],"fashion.":[132],"We":[133],"show":[134],"that":[135],"increases":[138],"expected":[140],"revenue":[141],"publisher.":[144]},"counts_by_year":[{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}