{"id":"https://openalex.org/W3085534303","doi":"https://doi.org/10.1145/3411753","title":"MergeDTS","display_name":"MergeDTS","publication_year":2020,"publication_date":"2020-09-10","ids":{"openalex":"https://openalex.org/W3085534303","doi":"https://doi.org/10.1145/3411753","mag":"3085534303"},"language":"en","primary_location":{"id":"doi:10.1145/3411753","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3411753","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100429528","display_name":"Chang Li","orcid":"https://orcid.org/0000-0002-6978-2034"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Chang Li","raw_affiliation_strings":["University of Amsterdam, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103028673","display_name":"Ilya Markov","orcid":"https://orcid.org/0000-0001-9221-3043"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ilya Markov","raw_affiliation_strings":["University of Amsterdam, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, The Netherlands","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I4210112722","display_name":"Ahold Delhaize (Netherlands)","ror":"https://ror.org/01v6p2g18","country_code":"NL","type":"company","lineage":["https://openalex.org/I4210112722"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten De Rijke","raw_affiliation_strings":["University of Amsterdam and Ahold Delhaize, Zaandam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam and Ahold Delhaize, Zaandam, The Netherlands","institution_ids":["https://openalex.org/I4210112722","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066732643","display_name":"Masrour Zoghi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Masrour Zoghi","raw_affiliation_strings":["Microsoft, Redmond, WA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100429528"],"corresponding_institution_ids":["https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":0.5558,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.7213176,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":"38","issue":"4","first_page":"1","last_page":"28"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8040779829025269},{"id":"https://openalex.org/keywords/regret","display_name":"Regret","score":0.7161084413528442},{"id":"https://openalex.org/keywords/thompson-sampling","display_name":"Thompson sampling","score":0.6166150569915771},{"id":"https://openalex.org/keywords/interleaving","display_name":"Interleaving","score":0.6096532940864563},{"id":"https://openalex.org/keywords/condorcet-method","display_name":"Condorcet method","score":0.5644888877868652},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.5293204188346863},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4680319130420685},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4183768630027771}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8040779829025269},{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.7161084413528442},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.6166150569915771},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.6096532940864563},{"id":"https://openalex.org/C140940377","wikidata":"https://www.wikidata.org/wiki/Q868720","display_name":"Condorcet method","level":4,"score":0.5644888877868652},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.5293204188346863},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4680319130420685},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4183768630027771},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3411753","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3411753","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/5cee56c9-8de8-4ae6-801c-819a86f00634","is_oa":false,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/mergedts-a-method-for-effective-largescale-online-ranker-evaluation(5cee56c9-8de8-4ae6-801c-819a86f00634).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Li, C, Markov, I, de Rijke, M & Zoghi, M 2020, 'MergeDTS: A Method for Effective Large-scale Online Ranker Evaluation', ACM Transactions on Information Systems, vol. 38, no. 4, 40. https://doi.org/10.1145/3411753","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:uvapub:oai:dare.uva.nl:publications/5cee56c9-8de8-4ae6-801c-819a86f00634","is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/mergedts-a-method-for-effective-largescale-online-ranker-evaluation(5cee56c9-8de8-4ae6-801c-819a86f00634).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Transactions on Information Systems, 38(4):40. Association for Computing Machinery (ACM)","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4099999964237213,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G2586376837","display_name":null,"funder_award_id":"612.001.551","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1032589285","https://openalex.org/W1530210183","https://openalex.org/W1963548721","https://openalex.org/W1974360117","https://openalex.org/W1982530130","https://openalex.org/W2007105673","https://openalex.org/W2009979684","https://openalex.org/W2015236210","https://openalex.org/W2028554387","https://openalex.org/W2039522160","https://openalex.org/W2044493620","https://openalex.org/W2073248814","https://openalex.org/W2077902449","https://openalex.org/W2090589280","https://openalex.org/W2106628583","https://openalex.org/W2112508839","https://openalex.org/W2115711489","https://openalex.org/W2131958277","https://openalex.org/W2147892741","https://openalex.org/W2156541733","https://openalex.org/W2168405694","https://openalex.org/W2402441596","https://openalex.org/W2464171116","https://openalex.org/W2468270644","https://openalex.org/W2470088391","https://openalex.org/W2507731896","https://openalex.org/W2591957553","https://openalex.org/W2604653582","https://openalex.org/W2624553223","https://openalex.org/W2791379569","https://openalex.org/W2800969132","https://openalex.org/W2808544295","https://openalex.org/W2883737864","https://openalex.org/W2898640137","https://openalex.org/W2909643356","https://openalex.org/W2964081017","https://openalex.org/W4233413206","https://openalex.org/W4299828299","https://openalex.org/W4302322961"],"related_works":["https://openalex.org/W2043607118","https://openalex.org/W2082491017","https://openalex.org/W2256553487","https://openalex.org/W1967886354","https://openalex.org/W2052026775","https://openalex.org/W1850547517","https://openalex.org/W2964125852","https://openalex.org/W3176022311","https://openalex.org/W3046298489","https://openalex.org/W3048056964"],"abstract_inverted_index":{"Online":[0],"ranker":[1,31,171,184,266],"evaluation":[2,172,254,267],"is":[3,45,92,101,111,151,186,261],"one":[4,50],"of":[5,15,25,114,127,139,148,155,217,242,252],"the":[6,13,23,30,35,40,49,60,69,80,85,95,104,115,121,125,146,168,175,207,212,226,250,271],"key":[7],"challenges":[8],"in":[9,124,145],"information":[10],"retrieval.":[11],"Although":[12],"preferences":[14],"rankers":[16,54,74,140,150,230],"can":[17,63,134],"be":[18,64],"inferred":[19,61],"by":[20,103,211],"interleaving":[21,81],"methods,":[22],"problem":[24,173],"how":[26],"to":[27,188,214,224],"effectively":[28,144],"choose":[29],"pair":[32],"that":[33,185,205,262],"generates":[34],"interleaved":[36],"list":[37],"without":[38],"degrading":[39],"user":[41,86],"experience":[42,87],"too":[43,77,88],"much":[44],"still":[46],"challenging.":[47],"On":[48,68],"hand,":[51,71],"if":[52,72],"two":[53,73],"have":[55],"not":[56],"been":[57],"compared":[58,76],"enough,":[59],"preference":[62],"noisy":[65],"and":[66,142,219,238],"inaccurate.":[67],"other":[70,190],"are":[75,244],"many":[78],"times,":[79],"process":[82],"inevitably":[83],"hurts":[84],"much.":[89],"This":[90],"dilemma":[91],"known":[93],"as":[94],"exploration":[96],"versus":[97],"exploitation":[98],"tradeoff.":[99],"It":[100],"captured":[102],"K":[105,116,156],"-armed":[106,117,157],"dueling":[107,158,273],"bandit":[108,118,159,274],"problem,":[109,119],"which":[110,199],"a":[112,136,152,202],"variant":[113],"where":[120,179],"feedback":[122],"comes":[123],"form":[126],"pairwise":[128],"preferences.":[129],"Today\u2019s":[130],"deployed":[131],"search":[132],"systems":[133],"evaluate":[135],"large":[137],"number":[138],"concurrently,":[141],"scaling":[143],"presence":[147],"numerous":[149],"critical":[153],"aspect":[154],"problems.":[160],"In":[161],"this":[162],"article,":[163],"we":[164],"focus":[165],"on":[166],"solving":[167],"large-scale":[169,264],"online":[170,253],"under":[174],"so-called":[176],"Condorcet":[177,265],"assumption,":[178],"there":[180],"exists":[181],"an":[182],"optimal":[183],"preferred":[187],"all":[189],"rankers.":[191],"We":[192],"propose":[193],"Merge":[194],"Double":[195],"Thompson":[196,222],"Sampling":[197,223],"(MergeDTS),":[198],"first":[200],"utilizes":[201],"divide-and-conquer":[203],"strategy":[204],"localizes":[206],"comparisons":[208,227],"carried":[209],"out":[210],"algorithm":[213],"small":[215,233],"batches":[216],"rankers,":[218],"then":[220],"employs":[221],"reduce":[225],"between":[228],"suboptimal":[229],"inside":[231],"these":[232],"batches.":[234],"The":[235],"effectiveness":[236],"(regret)":[237],"efficiency":[239],"(time":[240],"complexity)":[241],"MergeDTS":[243,269],"extensively":[245],"evaluated":[246],"using":[247],"examples":[248],"from":[249],"domain":[251],"for":[255,263],"web":[256],"search.":[257],"Our":[258],"main":[259],"finding":[260],"problems,":[268],"outperforms":[270],"state-of-the-art":[272],"algorithms.":[275]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2020-09-21T00:00:00"}