{"id":"https://openalex.org/W2342632098","doi":"https://doi.org/10.1145/2970398.2970410","title":"Unbiased Comparative Evaluation of Ranking Functions","display_name":"Unbiased Comparative Evaluation of Ranking Functions","publication_year":2016,"publication_date":"2016-09-09","ids":{"openalex":"https://openalex.org/W2342632098","doi":"https://doi.org/10.1145/2970398.2970410","mag":"2342632098"},"language":"en","primary_location":{"id":"doi:10.1145/2970398.2970410","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2970398.2970410","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2970410&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2970410&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063892002","display_name":"Tobias Schnabel","orcid":"https://orcid.org/0000-0002-9301-7631"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tobias Schnabel","raw_affiliation_strings":["Cornell University, Ithaca, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cornell University, Ithaca, NY, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034711562","display_name":"Adith Swaminathan","orcid":null},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adith Swaminathan","raw_affiliation_strings":["Cornell University, Ithaca, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cornell University, Ithaca, NY, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039019367","display_name":"Peter I. Frazier","orcid":"https://orcid.org/0000-0002-3501-3341"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter I. Frazier","raw_affiliation_strings":["Cornell University, Ithaca, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cornell University, Ithaca, NY, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014687727","display_name":"Thorsten Joachims","orcid":"https://orcid.org/0000-0003-3654-3683"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thorsten Joachims","raw_affiliation_strings":["Cornell University, Ithaca, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cornell University, Ithaca, NY, USA","institution_ids":["https://openalex.org/I205783295"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I205783295"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"109","last_page":"118"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7722971439361572},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.702111542224884},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.616521954536438},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.5938616394996643},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.5871120095252991},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.5351105332374573},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5328315496444702},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5306949615478516},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5266817808151245},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.49897170066833496},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.48325806856155396},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4616474211215973},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"Importance sampling","score":0.43804651498794556},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.41777336597442627},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3795117735862732},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.37496232986450195},{"id":"https://openalex.org/keywords/monte-carlo-method","display_name":"Monte Carlo method","score":0.36867666244506836},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.26509398221969604},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1657170057296753}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7722971439361572},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.702111542224884},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.616521954536438},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.5938616394996643},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.5871120095252991},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.5351105332374573},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5328315496444702},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5306949615478516},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5266817808151245},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.49897170066833496},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.48325806856155396},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4616474211215973},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.43804651498794556},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.41777336597442627},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3795117735862732},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.37496232986450195},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.36867666244506836},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.26509398221969604},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1657170057296753},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2970398.2970410","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2970398.2970410","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2970410&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/2970398.2970410","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2970398.2970410","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2970410&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 ACM International Conference on the Theory of Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.75,"display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G1057953870","display_name":null,"funder_award_id":"IIS-1247637","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1267113881","display_name":null,"funder_award_id":"FA9550-16-1-0046","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G1436116494","display_name":"III: Small: Collaborative Research: Learning to Model Sequences","funder_award_id":"1217686","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1523888516","display_name":null,"funder_award_id":"FA9550-","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G186882396","display_name":null,"funder_award_id":"CMMI-1536895","funder_id":"https://openalex.org/F4320337391","funder_display_name":"Division of Civil, Mechanical and Manufacturing Innovation"},{"id":"https://openalex.org/G1912645146","display_name":null,"funder_award_id":"CMMI-1254298","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1977972760","display_name":null,"funder_award_id":"DMR-1120296","funder_id":"https://openalex.org/F4320337367","funder_display_name":"Division of Materials Research"},{"id":"https://openalex.org/G230078430","display_name":null,"funder_award_id":"FA9550-16","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G2479931443","display_name":null,"funder_award_id":"CMMI-1254298","funder_id":"https://openalex.org/F4320337391","funder_display_name":"Division of Civil, Mechanical and Manufacturing Innovation"},{"id":"https://openalex.org/G2668764038","display_name":"BIGDATA: Mid-Scale: ESCE: Collaborative Research: Discovery and Social Analytics for Large-Scale Scientific Literature","funder_award_id":"1247637","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3392305090","display_name":null,"funder_award_id":"-16-1-","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G3616359867","display_name":null,"funder_award_id":"FA9550-16-1","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G4335483204","display_name":null,"funder_award_id":"IIS-1247696","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4656289215","display_name":null,"funder_award_id":"1120296","funder_id":"https://openalex.org/F4320337367","funder_display_name":"Division of Materials Research"},{"id":"https://openalex.org/G4687492762","display_name":null,"funder_award_id":"IIS-1217686","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G523149444","display_name":"Cornell Center for Materials Research - CEMRI","funder_award_id":"1120296","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5407182010","display_name":"CAREER: Methodology for Optimization via Simulation: Bayesian Methods, Frequentist Guarantees, and Applications to Cardiovascular Medicine","funder_award_id":"1254298","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5488607868","display_name":null,"funder_award_id":"IIS-1513692","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5809100787","display_name":null,"funder_award_id":"FA9550","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G5838217760","display_name":"Collaborative Research: Designing Functional Materials with Optimal Learning","funder_award_id":"1536895","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6618635139","display_name":null,"funder_award_id":"DMR-1120296","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8053764009","display_name":"BIGDATA: Mid-Scale: ESCE: Collaborative Research: Discovery and Social Analytics for Large-Scale Scientific Literature.","funder_award_id":"1247696","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8200652828","display_name":null,"funder_award_id":"DMR-1120296","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G8691988637","display_name":null,"funder_award_id":"FA9550-15-1-0038","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G8815179840","display_name":"III: Medium: Machine Learning with Humans in the Loop","funder_award_id":"1513692","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337367","display_name":"Division of Materials Research","ror":"https://ror.org/01pc7k308"},{"id":"https://openalex.org/F4320337391","display_name":"Division of Civil, Mechanical and Manufacturing Innovation","ror":"https://ror.org/028yd4c30"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2342632098.pdf","grobid_xml":"https://content.openalex.org/works/W2342632098.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W117871444","https://openalex.org/W1507150160","https://openalex.org/W1602484930","https://openalex.org/W1842094663","https://openalex.org/W1857290810","https://openalex.org/W1938221098","https://openalex.org/W1964675737","https://openalex.org/W1967879792","https://openalex.org/W1968927634","https://openalex.org/W1976076261","https://openalex.org/W1986572161","https://openalex.org/W1999331679","https://openalex.org/W2004918107","https://openalex.org/W2009954908","https://openalex.org/W2022995284","https://openalex.org/W2053100920","https://openalex.org/W2063471322","https://openalex.org/W2075893676","https://openalex.org/W2109244020","https://openalex.org/W2111310810","https://openalex.org/W2115536324","https://openalex.org/W2124504084","https://openalex.org/W2138909795","https://openalex.org/W2141175968","https://openalex.org/W2168469656","https://openalex.org/W2186354314","https://openalex.org/W2279176662","https://openalex.org/W2411735541","https://openalex.org/W2500119194","https://openalex.org/W2952613481"],"related_works":["https://openalex.org/W3024870410","https://openalex.org/W2280422768","https://openalex.org/W3143197806","https://openalex.org/W2410652950","https://openalex.org/W4380150146","https://openalex.org/W4252555497","https://openalex.org/W4283773154","https://openalex.org/W2900543860","https://openalex.org/W3171633752","https://openalex.org/W2266833899"],"abstract_inverted_index":{"Eliciting":[0],"relevance":[1,161],"judgments":[2,162],"for":[3],"ranking":[4,101],"evaluation":[5,58,89],"is":[6],"labor-intensive":[7],"and":[8,50,100,111,129,151],"costly,":[9],"motivating":[10],"careful":[11],"selection":[12,24],"of":[13,31,72,120,131,159],"which":[14],"documents":[15],"to":[16,68,137],"judge.":[17],"Unlike":[18],"traditional":[19],"approaches":[20,54],"that":[21,33,66,81,153],"make":[22],"this":[23,45,82],"deterministically,":[25],"probabilistic":[26],"sampling":[27,53,114,149],"enables":[28],"the":[29,57,78,118,138,157],"design":[30],"estimators":[32],"are":[34],"provably":[35],"unbiased":[36],"even":[37],"when":[38],"reusing":[39],"data":[40],"with":[41],"missing":[42,127],"judgments.":[43],"In":[44,135],"paper,":[46],"we":[47,85,107,141],"first":[48],"unify":[49],"extend":[51],"these":[52],"by":[55],"viewing":[56],"problem":[59],"as":[60],"a":[61,69,98,112],"Monte":[62],"Carlo":[63],"estimation":[64],"task":[65],"applies":[67],"large":[70],"number":[71,158],"common":[73],"IR":[74],"metrics.":[75],"Drawing":[76],"on":[77],"theoretical":[79,139],"clarity":[80],"view":[83],"offers,":[84],"tackle":[86],"three":[87],"practical":[88],"scenarios:":[90],"comparing":[91,94],"two":[92],"systems,":[93],"k":[95,102],"systems":[96],"against":[97,146],"baseline,":[99],"systems.":[103],"For":[104],"each":[105],"scenario,":[106],"derive":[108],"an":[109],"estimator":[110],"variance-optimizing":[113],"distribution":[115],"while":[116],"retaining":[117],"strengths":[119],"sampling-based":[121],"evaluation,":[122],"including":[123],"unbiasedness,":[124],"reusability":[125],"despite":[126],"data,":[128],"ease":[130],"use":[132],"in":[133,165],"practice.":[134],"addition":[136],"contribution,":[140],"empirically":[142],"evaluate":[143],"our":[144],"methods":[145],"previously":[147],"used":[148],"heuristics":[150],"find":[152],"they":[154],"often":[155],"cut":[156],"required":[160],"at":[163],"least":[164],"half.":[166]},"counts_by_year":[{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":3}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}