{"id":"https://openalex.org/W2468270644","doi":"https://doi.org/10.1145/2911451.2914706","title":"An Improved Multileaving Algorithm for Online Ranker Evaluation","display_name":"An Improved Multileaving Algorithm for Online Ranker Evaluation","publication_year":2016,"publication_date":"2016-07-07","ids":{"openalex":"https://openalex.org/W2468270644","doi":"https://doi.org/10.1145/2911451.2914706","mag":"2468270644"},"language":"en","primary_location":{"id":"doi:10.1145/2911451.2914706","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2911451.2914706","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5118899784","display_name":"Brian Brost","orcid":"https://orcid.org/0000-0001-9074-5193"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Brian Brost","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047286157","display_name":"Ingemar J. Cox","orcid":"https://orcid.org/0000-0002-6662-417X"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]},{"id":"https://openalex.org/I4210088332","display_name":"University College Copenhagen","ror":"https://ror.org/004r9h172","country_code":"DK","type":"education","lineage":["https://openalex.org/I4210088332"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["DK","GB"],"is_corresponding":false,"raw_author_name":"Ingemar J. Cox","raw_affiliation_strings":["University of Copenhagen & University College London, Copenhagen & London, Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen & University College London, Copenhagen & London, Denmark","institution_ids":["https://openalex.org/I124055696","https://openalex.org/I4210088332","https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021828946","display_name":"Yevgeny Seldin","orcid":"https://orcid.org/0000-0003-3152-4635"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Yevgeny Seldin","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045425016","display_name":"Christina Lioma","orcid":"https://orcid.org/0000-0003-2600-2701"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Christina Lioma","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"745","last_page":"748"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9787999987602234,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9553999900817871,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9491000175476074,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6891209483146667},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.44700199365615845}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6891209483146667},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.44700199365615845}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2911451.2914706","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2911451.2914706","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/f5a06f7d-2852-4321-a433-a0556aea23c4","is_oa":false,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/f5a06f7d-2852-4321-a433-a0556aea23c4","pdf_url":null,"source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Brost , B , Cox , I J , Seldin , Y & Lioma , C 2016 , An improved multileaving algorithm for online ranker evaluation . in Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval : SIGIR '16 . Association for Computing Machinery , pp. 745-748 , International ACM SIGIR conference on Research and Development in Information Retrieval 2016 , Pisa , Italy , 17/07/2016 . https://doi.org/10.1145/2911451.2914706","raw_type":"contributionToPeriodical"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W1963548721","https://openalex.org/W1982530130","https://openalex.org/W2003473029","https://openalex.org/W2073248814","https://openalex.org/W2884475480"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2051487156","https://openalex.org/W2073681303","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109"],"abstract_inverted_index":{"Online":[0],"ranker":[1],"evaluation":[2,16,98],"is":[3,19,105],"a":[4,128],"key":[5],"challenge":[6],"in":[7,13,47,144],"information":[8],"retrieval.":[9],"An":[10],"important":[11],"task":[12],"the":[14,84,108,117],"online":[15],"of":[17,86],"rankers":[18,123],"using":[20],"implicit":[21],"user":[22],"feedback":[23],"for":[24,116,132],"inferring":[25],"preferences":[26],"between":[27,122],"rankers.":[28],"Interleaving":[29],"methods":[30,56,77],"have":[31],"been":[32,52],"found":[33],"to":[34],"be":[35,62],"efficient":[36,64],"and":[37,136],"sensitive,":[38],"i.e.":[39],"they":[40,111],"can":[41,61,91,120],"quickly":[42],"detect":[43],"even":[44],"small":[45],"differences":[46],"quality.":[48],"It":[49],"has":[50],"recently":[51],"shown":[53],"that":[54,74,110,119,138],"multileaving":[55,76,130],"exhibit":[57],"similar":[58],"sensitivity":[59],"but":[60],"more":[63,89],"than":[65],"interleaving":[66],"methods.":[67],"This":[68],"paper":[69],"presents":[70],"empirical":[71],"results":[72,93],"demonstrating":[73],"existing":[75,142],"either":[78],"do":[79,112],"not":[80,113],"scale":[81],"well":[82],"with":[83],"number":[85],"rankers,":[87],"or,":[88],"problematically,":[90],"produce":[92],"which":[94],"substantially":[95,140],"differ":[96],"from":[97],"measures":[99],"like":[100],"NDCG.":[101],"The":[102],"latter":[103],"problem":[104,135],"caused":[106],"by":[107,149],"fact":[109],"correctly":[114],"account":[115],"similarities":[118],"occur":[121],"being":[124],"multileaved.":[125],"We":[126],"propose":[127],"new":[129],"method":[131],"handling":[133],"this":[134],"demonstrate":[137],"it":[139],"outperforms":[141],"methods,":[143],"some":[145],"cases":[146],"reducing":[147],"errors":[148],"as":[150,152],"much":[151],"50%.":[153]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}