{"id":"https://openalex.org/W2073248814","doi":"https://doi.org/10.1145/2536736.2536737","title":"Fidelity, Soundness, and Efficiency of Interleaved Comparison Methods","display_name":"Fidelity, Soundness, and Efficiency of Interleaved Comparison Methods","publication_year":2013,"publication_date":"2013-11-01","ids":{"openalex":"https://openalex.org/W2073248814","doi":"https://doi.org/10.1145/2536736.2536737","mag":"2073248814"},"language":"en","primary_location":{"id":"doi:10.1145/2536736.2536737","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2536736.2536737","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027441885","display_name":"Katja Hofmann","orcid":"https://orcid.org/0000-0003-3697-407X"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Katja Hofmann","raw_affiliation_strings":["University of Amsterdam","University of Amsterdam#N#"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"University of Amsterdam#N#","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056879203","display_name":"Shimon Whiteson","orcid":null},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Shimon Whiteson","raw_affiliation_strings":["University of Amsterdam","University of Amsterdam#N#"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"University of Amsterdam#N#","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten De Rijke","raw_affiliation_strings":["University of Amsterdam","University of Amsterdam#N#"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"University of Amsterdam#N#","institution_ids":["https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5027441885"],"corresponding_institution_ids":["https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":16.9879,"has_fulltext":false,"cited_by_count":47,"citation_normalized_percentile":{"value":0.98884973,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"31","issue":"4","first_page":"1","last_page":"43"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9868999719619751,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8452964425086975},{"id":"https://openalex.org/keywords/fidelity","display_name":"Fidelity","score":0.6888009309768677},{"id":"https://openalex.org/keywords/soundness","display_name":"Soundness","score":0.5943503379821777},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.59332674741745},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5361437797546387},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.4838475286960602},{"id":"https://openalex.org/keywords/interleaving","display_name":"Interleaving","score":0.44562259316444397},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.43829748034477234},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.4176172912120819},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4054797887802124},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.40452995896339417},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.40105515718460083},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3776933252811432}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8452964425086975},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.6888009309768677},{"id":"https://openalex.org/C39920170","wikidata":"https://www.wikidata.org/wiki/Q693083","display_name":"Soundness","level":2,"score":0.5943503379821777},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.59332674741745},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5361437797546387},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.4838475286960602},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.44562259316444397},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.43829748034477234},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.4176172912120819},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4054797887802124},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.40452995896339417},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.40105515718460083},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3776933252811432},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2536736.2536737","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2536736.2536737","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/3541f3d2-fd9a-4e6b-8200-9fe202ad884e","is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/fidelity-soundness-and-efficiency-of-interleaved-comparison-methods(3541f3d2-fd9a-4e6b-8200-9fe202ad884e).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Hofmann, K, Whiteson, S & de Rijke, M 2013, 'Fidelity, Soundness, and Efficiency of Interleaved Comparison Methods', ACM Transactions on Information Systems, vol. 31, no. 4, 17. https://doi.org/10.1145/2536736.2536737","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4727451579","display_name":null,"funder_award_id":"612.061.814, 612.061.815, 640.004.802, 727.011.005, 612.001.116, HOR-11-10","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G4975220420","display_name":null,"funder_award_id":"258191 (PROMISE Network of Excellence), 288024 (LiMoSINe project)","funder_id":"https://openalex.org/F4320334960","funder_display_name":"Seventh Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320320934","display_name":"Koninklijke Nederlandse Akademie van Wetenschappen","ror":"https://ror.org/043c0p156"},{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320334960","display_name":"Seventh Framework Programme","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320338080","display_name":"European Social Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":52,"referenced_works":["https://openalex.org/W135190683","https://openalex.org/W139596442","https://openalex.org/W1489119587","https://openalex.org/W1493008453","https://openalex.org/W1514587017","https://openalex.org/W1515851193","https://openalex.org/W1746680969","https://openalex.org/W1809653203","https://openalex.org/W1968927634","https://openalex.org/W1969114939","https://openalex.org/W1972802558","https://openalex.org/W1982530130","https://openalex.org/W1996633087","https://openalex.org/W2004075725","https://openalex.org/W2011285051","https://openalex.org/W2023599408","https://openalex.org/W2037140704","https://openalex.org/W2041576073","https://openalex.org/W2047221353","https://openalex.org/W2069870183","https://openalex.org/W2075302537","https://openalex.org/W2094790959","https://openalex.org/W2099213975","https://openalex.org/W2104217798","https://openalex.org/W2105981469","https://openalex.org/W2106628583","https://openalex.org/W2108415592","https://openalex.org/W2110038066","https://openalex.org/W2113065326","https://openalex.org/W2113640060","https://openalex.org/W2123937625","https://openalex.org/W2125771191","https://openalex.org/W2126856725","https://openalex.org/W2129235726","https://openalex.org/W2135753071","https://openalex.org/W2137943964","https://openalex.org/W2138909795","https://openalex.org/W2139527693","https://openalex.org/W2144134691","https://openalex.org/W2144952090","https://openalex.org/W2147892741","https://openalex.org/W2148451394","https://openalex.org/W2153783077","https://openalex.org/W2154739689","https://openalex.org/W2156160882","https://openalex.org/W2156541733","https://openalex.org/W2160471965","https://openalex.org/W2163996478","https://openalex.org/W2293743194","https://openalex.org/W2318475177","https://openalex.org/W2407295445","https://openalex.org/W4300699176"],"related_works":["https://openalex.org/W1511249877","https://openalex.org/W2466303427","https://openalex.org/W1578828174","https://openalex.org/W2000943029","https://openalex.org/W2950376466","https://openalex.org/W4302559785","https://openalex.org/W2468270644","https://openalex.org/W2952254164","https://openalex.org/W4285187205","https://openalex.org/W4298524092"],"abstract_inverted_index":{"Ranker":[0],"evaluation":[1,24],"is":[2,47,101,115,150,167,184],"central":[3],"to":[4,12,16,21,42,92,186],"the":[5,48,84,93,97],"research":[6],"into":[7],"search":[8],"engines,":[9],"be":[10],"it":[11,166],"compare":[13,55],"rankers":[14,56],"or":[15],"provide":[17],"feedback":[18],"for":[19,72,141],"learning":[20],"rank.":[22],"Traditional":[23],"approaches":[25],"do":[26],"not":[27],"scale":[28],"well":[29],"because":[30],"they":[31],"require":[32],"explicit":[33],"relevance":[34,95],"judgments":[35],"of":[36,50,87,96,106,194],"document-query":[37],"pairs,":[38],"which":[39,54,149],"are":[40,110,120,164],"expensive":[41],"obtain.":[43],"A":[44],"promising":[45],"alternative":[46],"use":[49],"interleaved":[51,74,78,129,172],"comparison":[52,75,79,130,173],"methods,":[53],"using":[57],"click":[58],"data":[59,189],"obtained":[60],"when":[61],"interleaving":[62],"their":[63],"rankings.":[64],"In":[65],"this":[66],"article,":[67],"we":[68,178],"propose":[69,144],"a":[70,145,180],"framework":[71],"analyzing":[73],"methods.":[76,174],"An":[77],"method":[80],"has":[81,153],"fidelity":[82],"if":[83,103,117],"expected":[85,108],"outcome":[86,109],"ranker":[88,196],"comparisons":[89,193],"properly":[90],"corresponds":[91],"true":[94],"ranked":[98],"documents.":[99],"It":[100,114],"sound":[102,151,181],"its":[104],"estimates":[105,119],"that":[107,163,183],"unbiased":[111],"and":[112,132,152],"consistent.":[113],"efficient":[116,169],"those":[118],"accurate":[121],"with":[122],"only":[123],"little":[124],"data.":[125],"We":[126,143,155],"analyze":[127],"existing":[128,171],"methods":[131],"find":[133],"that,":[134,158],"while":[135],"sound,":[136],"none":[137],"meet":[138],"our":[139],"criteria":[140],"fidelity.":[142,154],"probabilistic":[146],"interleave":[147],"method,":[148],"show":[156],"empirically":[157],"by":[159],"marginalizing":[160],"out":[161],"variables":[162],"known,":[165],"more":[168],"than":[170],"Using":[175],"importance":[176],"sampling":[177],"derive":[179],"extension":[182],"able":[185],"reuse":[187],"historical":[188],"collected":[190],"in":[191],"previous":[192],"other":[195],"pairs.":[197]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":7},{"year":2016,"cited_by_count":8},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}