{"id":"https://openalex.org/W2110038066","doi":"https://doi.org/10.1145/1963405.1963463","title":"Evaluating new search engine configurations with pre-existing judgments and clicks","display_name":"Evaluating new search engine configurations with pre-existing judgments and clicks","publication_year":2011,"publication_date":"2011-03-28","ids":{"openalex":"https://openalex.org/W2110038066","doi":"https://doi.org/10.1145/1963405.1963463","mag":"2110038066"},"language":"en","primary_location":{"id":"doi:10.1145/1963405.1963463","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1963405.1963463","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th international conference on World wide web","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080908821","display_name":"Umut \u00d6zertem","orcid":null},"institutions":[{"id":"https://openalex.org/I4210134091","display_name":"Yahoo (United States)","ror":"https://ror.org/040dkzz12","country_code":"US","type":"company","lineage":["https://openalex.org/I4210134091"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Umut Ozertem","raw_affiliation_strings":["Yahoo Labs, Sunnyvale, CA, USA","Yahoo! Labs., Sunnyvale, CA, USA"],"affiliations":[{"raw_affiliation_string":"Yahoo Labs, Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I4210134091"]},{"raw_affiliation_string":"Yahoo! Labs., Sunnyvale, CA, USA","institution_ids":["https://openalex.org/I4210134091"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000992993","display_name":"Rosie Jones","orcid":"https://orcid.org/0009-0000-3821-1207"},"institutions":[{"id":"https://openalex.org/I10734018","display_name":"Akamai (United States)","ror":"https://ror.org/03tarb191","country_code":"US","type":"company","lineage":["https://openalex.org/I10734018"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rosie Jones","raw_affiliation_strings":["Akamai Technologies, Boston, MA, USA"],"affiliations":[{"raw_affiliation_string":"Akamai Technologies, Boston, MA, USA","institution_ids":["https://openalex.org/I10734018"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051559971","display_name":"Beno\u00eet Dumoulin","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benoit Dumoulin","raw_affiliation_strings":["Microsoft, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5080908821"],"corresponding_institution_ids":["https://openalex.org/I4210134091"],"apc_list":null,"apc_paid":null,"fwci":2.2168,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.91009867,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"397","last_page":"406"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.836909294128418},{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.7918539047241211},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7885335683822632},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6720198392868042},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6656020283699036},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6253768801689148},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.4961000382900238},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.49178627133369446},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.48264080286026},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.438951700925827},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.43570008873939514},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.43478935956954956},{"id":"https://openalex.org/keywords/result-set","display_name":"Result set","score":0.420911967754364},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.4162902235984802},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3485713005065918},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07529616355895996}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.836909294128418},{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.7918539047241211},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7885335683822632},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6720198392868042},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6656020283699036},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6253768801689148},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.4961000382900238},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.49178627133369446},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.48264080286026},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.438951700925827},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.43570008873939514},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.43478935956954956},{"id":"https://openalex.org/C4969071","wikidata":"https://www.wikidata.org/wiki/Q7316353","display_name":"Result set","level":3,"score":0.420911967754364},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.4162902235984802},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3485713005065918},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07529616355895996},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/1963405.1963463","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1963405.1963463","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th international conference on World wide web","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W1968927634","https://openalex.org/W1984103665","https://openalex.org/W1992549066","https://openalex.org/W2008285050","https://openalex.org/W2026784708","https://openalex.org/W2044072332","https://openalex.org/W2069870183","https://openalex.org/W2099213975","https://openalex.org/W2109244020","https://openalex.org/W2113640060","https://openalex.org/W2122946987","https://openalex.org/W2124504084","https://openalex.org/W2142927605","https://openalex.org/W2152314154","https://openalex.org/W2156160882","https://openalex.org/W2990138404"],"related_works":["https://openalex.org/W4380150146","https://openalex.org/W3024870410","https://openalex.org/W2410652950","https://openalex.org/W4283773154","https://openalex.org/W4379381520","https://openalex.org/W1990838575","https://openalex.org/W2390753775","https://openalex.org/W2017070768","https://openalex.org/W2253480670","https://openalex.org/W1971970850"],"abstract_inverted_index":{"We":[0,133,145,190],"provide":[1,100],"a":[2,124,135,148,204,210],"novel":[3,45],"method":[4,173,224],"of":[5,57,110,126,150,168,207,212],"evaluating":[6],"search":[7,59],"results,":[8],"which":[9],"allows":[10,51],"us":[11,52],"to":[12,53,64,116,141,197,203,228],"combine":[13],"existing":[14,78],"editorial":[15,38,71,79,97],"judgments":[16,39],"with":[17,129,178,209],"the":[18,32,47,55,90,95,107,165,171],"relevance":[19,101,131],"estimates":[20,102],"generated":[21],"by":[22],"click-based":[23],"user":[24,91],"browsing":[25,92],"models.":[26],"There":[27],"are":[28,75,123],"evaluation":[29,220],"methods":[30],"in":[31,46],"literature":[33],"that":[34,49,147,192],"use":[35],"clicks":[36,66,82],"and":[37,67,81,94,137,159,163,186],"together,":[40],"but":[41],"our":[42,216,223],"approach":[43],"is":[44,115,174,225],"sense":[48],"it":[50],"predict":[54],"impact":[56],"unseen":[58],"models":[60],"without":[61,68],"online":[62],"tests":[63],"collect":[65],"requesting":[69],"new":[70],"data,":[72,80],"since":[73],"we":[74],"only":[76,213],"re-using":[77],"observed":[83],"for":[84,103,106],"previous":[85],"result":[86],"set":[87,109],"configurations.":[88],"Since":[89],"model":[93],"pre-existing":[96],"data":[98],"cannot":[99],"all":[104],"documents":[105,128],"selected":[108],"queries,":[111,199,208],"one":[112],"important":[113],"challenge":[114],"obtain":[117],"this":[118,143],"performance":[119],"estimation":[120],"where":[121],"there":[122],"lot":[125],"ranked":[127],"missing":[130,169],"values.":[132],"introduce":[134],"query":[136,158],"rank":[138],"based":[139,161],"smoothing":[140,152],"overcome":[142],"problem.":[144],"show":[146,191],"hybrid":[149],"these":[151],"techniques":[153],"performs":[154],"better":[155],"than":[156],"both":[157],"position":[160],"smoothing,":[162],"despite":[164],"high":[166],"percentage":[167],"judgments,":[170],"resulting":[172],"significantly":[175],"correlated":[176],"(0.74)":[177],"DCG":[179],"values":[180],"evaluated":[181],"using":[182,221],"fully":[183],"judged":[184],"datasets,":[185],"approaches":[187],"inter-annotator":[188],"agreement.":[189],"previously":[193],"published":[194],"techniques,":[195],"applicable":[196,227],"frequent":[198],"degrade":[200],"when":[201],"applied":[202],"random":[205],"sample":[206],"correlation":[211],"0.29.":[214],"While":[215],"experiments":[217],"focus":[218],"on":[219],"DCG,":[222],"also":[226],"other":[229],"commonly":[230],"used":[231],"metrics.":[232]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}