{"id":"https://openalex.org/W2464171116","doi":"https://doi.org/10.1561/1500000051","title":"Online Evaluation for Information Retrieval","display_name":"Online Evaluation for Information Retrieval","publication_year":2016,"publication_date":"2016-06-22","ids":{"openalex":"https://openalex.org/W2464171116","doi":"https://doi.org/10.1561/1500000051","mag":"2464171116"},"language":"en","primary_location":{"id":"doi:10.1561/1500000051","is_oa":false,"landing_page_url":"https://doi.org/10.1561/1500000051","pdf_url":null,"source":{"id":"https://openalex.org/S197106261","display_name":"Foundations and Trends\u00ae in Information Retrieval","issn_l":"1554-0669","issn":["1554-0669","1554-0677"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Information Retrieval","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027441885","display_name":"Katja Hofmann","orcid":"https://orcid.org/0000-0003-3697-407X"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Katja Hofmann","raw_affiliation_strings":["Microsoft","Microsoft Research, Cambridge, UK"],"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research, Cambridge, UK","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100711454","display_name":"Lihong Li","orcid":"https://orcid.org/0000-0002-5440-8242"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Lihong Li","raw_affiliation_strings":["Microsoft","Microsoft Research, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081692910","display_name":"Filip Radlinski","orcid":"https://orcid.org/0000-0002-1386-9288"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Filip Radlinski","raw_affiliation_strings":["Microsoft","Microsoft Research, Cambridge, UK"],"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research, Cambridge, UK","institution_ids":["https://openalex.org/I4210164937"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5027441885"],"corresponding_institution_ids":["https://openalex.org/I4210164937"],"apc_list":null,"apc_paid":null,"fwci":25.577,"has_fulltext":false,"cited_by_count":119,"citation_normalized_percentile":{"value":0.99358363,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"10","issue":"1","first_page":"1","last_page":"117"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8221449255943298},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6451061367988586},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5508336424827576},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5060362219810486},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.49630552530288696},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.4444350302219391},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.41018950939178467},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32108113169670105}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8221449255943298},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6451061367988586},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5508336424827576},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5060362219810486},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.49630552530288696},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.4444350302219391},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.41018950939178467},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32108113169670105},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1561/1500000051","is_oa":false,"landing_page_url":"https://doi.org/10.1561/1500000051","pdf_url":null,"source":{"id":"https://openalex.org/S197106261","display_name":"Foundations and Trends\u00ae in Information Retrieval","issn_l":"1554-0669","issn":["1554-0669","1554-0677"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Information Retrieval","raw_type":"journal-article"},{"id":"pmh:oai:aleph.bib-bvb.de:BVB01-029248866","is_oa":false,"landing_page_url":"https://bvbr.bib-bvb.de:443/F?func=service&doc_library=BVB01&local_base=BVB01&doc_number=029248866&sequence=000002&line_number=0001&func_code=DB_RECORDS&service_type=MEDIA","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:cds.cern.ch:2762160","is_oa":false,"landing_page_url":"http://cds.cern.ch/record/2762160","pdf_url":null,"source":{"id":"https://openalex.org/S4306402195","display_name":"CERN Document Server (European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.41999998688697815,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":214,"referenced_works":["https://openalex.org/W135190683","https://openalex.org/W191658262","https://openalex.org/W417266313","https://openalex.org/W850914856","https://openalex.org/W1094752974","https://openalex.org/W1131710610","https://openalex.org/W1491542233","https://openalex.org/W1493008453","https://openalex.org/W1500478607","https://openalex.org/W1514587017","https://openalex.org/W1523791880","https://openalex.org/W1536091370","https://openalex.org/W1541055999","https://openalex.org/W1554944419","https://openalex.org/W1730782591","https://openalex.org/W1787140601","https://openalex.org/W1809653203","https://openalex.org/W1859938857","https://openalex.org/W1896407793","https://openalex.org/W1931027396","https://openalex.org/W1948287798","https://openalex.org/W1951724000","https://openalex.org/W1951936954","https://openalex.org/W1963548721","https://openalex.org/W1965690069","https://openalex.org/W1967498673","https://openalex.org/W1968380849","https://openalex.org/W1969056668","https://openalex.org/W1969184538","https://openalex.org/W1971712327","https://openalex.org/W1972802558","https://openalex.org/W1973602439","https://openalex.org/W1974360117","https://openalex.org/W1975566260","https://openalex.org/W1976876561","https://openalex.org/W1978108654","https://openalex.org/W1978658266","https://openalex.org/W1979483312","https://openalex.org/W1981457167","https://openalex.org/W1982530130","https://openalex.org/W1984770088","https://openalex.org/W1992549066","https://openalex.org/W1995945562","https://openalex.org/W1996633087","https://openalex.org/W1998381498","https://openalex.org/W1998955392","https://openalex.org/W2003473029","https://openalex.org/W2004089651","https://openalex.org/W2005917549","https://openalex.org/W2006551346","https://openalex.org/W2007105673","https://openalex.org/W2007529991","https://openalex.org/W2007750197","https://openalex.org/W2007815473","https://openalex.org/W2008251177","https://openalex.org/W2009979684","https://openalex.org/W2011285051","https://openalex.org/W2015236210","https://openalex.org/W2020160576","https://openalex.org/W2020331739","https://openalex.org/W2021735564","https://openalex.org/W2022657818","https://openalex.org/W2022995284","https://openalex.org/W2023473991","https://openalex.org/W2023599408","https://openalex.org/W2023616424","https://openalex.org/W2026784708","https://openalex.org/W2035569891","https://openalex.org/W2038135052","https://openalex.org/W2038385982","https://openalex.org/W2040257652","https://openalex.org/W2043382629","https://openalex.org/W2044493620","https://openalex.org/W2045228838","https://openalex.org/W2047221353","https://openalex.org/W2049934117","https://openalex.org/W2051744369","https://openalex.org/W2056044004","https://openalex.org/W2056705371","https://openalex.org/W2057744254","https://openalex.org/W2061503185","https://openalex.org/W2063438171","https://openalex.org/W2067472300","https://openalex.org/W2069739265","https://openalex.org/W2069870183","https://openalex.org/W2069916456","https://openalex.org/W2069997576","https://openalex.org/W2070815407","https://openalex.org/W2073248814","https://openalex.org/W2074380434","https://openalex.org/W2075156494","https://openalex.org/W2075186161","https://openalex.org/W2075585362","https://openalex.org/W2075874021","https://openalex.org/W2075893676","https://openalex.org/W2077902449","https://openalex.org/W2078681399","https://openalex.org/W2078957694","https://openalex.org/W2080224672","https://openalex.org/W2080706991","https://openalex.org/W2084664053","https://openalex.org/W2086206379","https://openalex.org/W2090715008","https://openalex.org/W2092092836","https://openalex.org/W2094419105","https://openalex.org/W2094790959","https://openalex.org/W2094984792","https://openalex.org/W2095056536","https://openalex.org/W2096254678","https://openalex.org/W2096946253","https://openalex.org/W2097246133","https://openalex.org/W2099213975","https://openalex.org/W2099471337","https://openalex.org/W2099966806","https://openalex.org/W2102693936","https://openalex.org/W2105256943","https://openalex.org/W2106576032","https://openalex.org/W2106628583","https://openalex.org/W2106630408","https://openalex.org/W2107170340","https://openalex.org/W2108279663","https://openalex.org/W2108368547","https://openalex.org/W2108415592","https://openalex.org/W2108598243","https://openalex.org/W2110038066","https://openalex.org/W2110228583","https://openalex.org/W2112420033","https://openalex.org/W2112508839","https://openalex.org/W2113065326","https://openalex.org/W2113304784","https://openalex.org/W2114418126","https://openalex.org/W2115283655","https://openalex.org/W2115632358","https://openalex.org/W2116435975","https://openalex.org/W2117600734","https://openalex.org/W2118395326","https://openalex.org/W2119074598","https://openalex.org/W2122124659","https://openalex.org/W2123937625","https://openalex.org/W2124108136","https://openalex.org/W2125001590","https://openalex.org/W2126002144","https://openalex.org/W2127713077","https://openalex.org/W2128475742","https://openalex.org/W2135114864","https://openalex.org/W2136399505","https://openalex.org/W2138909795","https://openalex.org/W2140190241","https://openalex.org/W2142830965","https://openalex.org/W2143891888","https://openalex.org/W2144673831","https://openalex.org/W2147892741","https://openalex.org/W2149590690","https://openalex.org/W2150761663","https://openalex.org/W2151592910","https://openalex.org/W2152886806","https://openalex.org/W2153222072","https://openalex.org/W2153253904","https://openalex.org/W2153500072","https://openalex.org/W2153783077","https://openalex.org/W2153872522","https://openalex.org/W2154739689","https://openalex.org/W2155587858","https://openalex.org/W2155843307","https://openalex.org/W2156160882","https://openalex.org/W2156186232","https://openalex.org/W2156541733","https://openalex.org/W2158254843","https://openalex.org/W2158312787","https://openalex.org/W2159665776","https://openalex.org/W2159927689","https://openalex.org/W2160471965","https://openalex.org/W2161598049","https://openalex.org/W2162379844","https://openalex.org/W2162651021","https://openalex.org/W2163996478","https://openalex.org/W2169068368","https://openalex.org/W2169113736","https://openalex.org/W2169783907","https://openalex.org/W2170443290","https://openalex.org/W2176930994","https://openalex.org/W2188353343","https://openalex.org/W2189217339","https://openalex.org/W2218578097","https://openalex.org/W2257238407","https://openalex.org/W2258789910","https://openalex.org/W2293233856","https://openalex.org/W2293743194","https://openalex.org/W2341328702","https://openalex.org/W2341805565","https://openalex.org/W2341909854","https://openalex.org/W2402441596","https://openalex.org/W2500756573","https://openalex.org/W2502051664","https://openalex.org/W2571880945","https://openalex.org/W2610685016","https://openalex.org/W2760537888","https://openalex.org/W2787894218","https://openalex.org/W2799002609","https://openalex.org/W2884475480","https://openalex.org/W2905472553","https://openalex.org/W2914354916","https://openalex.org/W2950929549","https://openalex.org/W2962694783","https://openalex.org/W2962802563","https://openalex.org/W2962867505","https://openalex.org/W2964068481","https://openalex.org/W2964297722","https://openalex.org/W3120740533","https://openalex.org/W3122193054","https://openalex.org/W3123750779","https://openalex.org/W3123895079","https://openalex.org/W4250212051","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2387743295","https://openalex.org/W4387426029","https://openalex.org/W2115758952","https://openalex.org/W4254162896","https://openalex.org/W4388792380","https://openalex.org/W3082787378","https://openalex.org/W2136007095","https://openalex.org/W1477999932","https://openalex.org/W2361979572"],"abstract_inverted_index":{"Online":[0],"evaluation":[1,61,67,98,107,133],"is":[2,108,208],"one":[3],"of":[4,13,83,96,127,141,153,169,216,257,264],"the":[5,11,21,38,63,222,265,270],"most":[6,201,266],"common":[7,64],"approaches":[8,68],"to":[9,25,49,177,183,188,242],"measure":[10],"effectiveness":[12],"an":[14,51,94,150],"information":[15,22,47,101,251],"retrieval":[16,23,56],"system.":[17,39],"It":[18],"involves":[19],"fielding":[20],"system":[24],"real":[26,45],"users,":[27,217],"and":[28,85,145,159,198,247,272],"observing":[29],"these":[30],"users\u2019":[31],"interactions":[32],"in-situ":[33],"while":[34],"they":[35],"engage":[36],"with":[37,44,214],"This":[40],"allows":[41],"actual":[42,86],"users":[43],"world":[46],"needs":[48],"play":[50],"important":[52],"part":[53,168],"in":[54,180,195,212,253,269],"assessing":[55],"quality.":[57],"As":[58],"such,":[59],"online":[60,97,106,132,196,205],"complements":[62],"alternative":[65],"offline":[66],"which":[69],"may":[70],"provide":[71,93],"more":[72],"easily":[73],"interpretable":[74],"outcomes,":[75],"yet":[76],"are":[77],"often":[78],"less":[79],"realistic":[80],"when":[81],"measuring":[82],"quality":[84,123],"user":[87],"experience.":[88],"In":[89],"this":[90,170,232],"survey,":[91],"we":[92,148,218,234,260],"overview":[95],"techniques":[99,224],"for":[100,110],"retrieval.":[102],"We":[103],"show":[104],"how":[105,182,187],"used":[109],"controlled":[111],"experiments,":[112],"segmenting":[113],"them":[114],"into":[115,190],"experiment":[116,160],"designs":[117],"that":[118,221,238],"allow":[119],"absolute":[120],"or":[121],"relative":[122],"assessments.":[124],"Our":[125],"presentation":[126],"different":[128,136],"metrics":[129],"further":[130],"partitions":[131],"based":[134,162],"on":[135,156,163,173,204],"sized":[137],"experimental":[138,185],"units":[139],"commonly":[140],"interest:":[142],"documents,":[143],"lists":[144],"sessions.":[146],"Additionally,":[147],"include":[149],"extensive":[151],"discussion":[152],"recent":[154,236,267],"work":[155,171,203,237,268],"data":[157],"re-use,":[158],"estimation":[161],"historical":[164],"data.":[165],"A":[166],"substantial":[167],"focuses":[172],"practical":[174],"issues:":[175],"How":[176],"run":[178],"evaluations":[179],"practice,":[181],"select":[184],"parameters,":[186],"take":[189],"account":[191],"ethical":[192],"considerations":[193],"inherent":[194],"evaluations,":[197],"limitations.":[199],"While":[200],"published":[202],"experimentation":[206],"today":[207],"at":[209,228,244],"large":[210],"scale":[211],"systems":[213],"millions":[215],"also":[219],"emphasize":[220,235],"same":[223],"can":[225],"be":[226],"applied":[227],"small":[229],"scale.":[230],"To":[231],"end,":[233],"makes":[239],"it":[240],"easier":[241],"use":[243],"smaller":[245],"scales":[246],"encourage":[248],"studying":[249],"real-world":[250],"seeking":[252],"a":[254,262],"wide":[255],"range":[256],"scenarios.":[258],"Finally,":[259],"present":[261],"summary":[263],"area,":[271],"describe":[273],"open":[274],"problems,":[275],"as":[276,278],"well":[277],"postulating":[279],"future":[280],"directions.":[281]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":30},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":10},{"year":2020,"cited_by_count":18},{"year":2019,"cited_by_count":15},{"year":2018,"cited_by_count":18},{"year":2017,"cited_by_count":10},{"year":2016,"cited_by_count":2}],"updated_date":"2026-04-04T06:10:10.580331","created_date":"2025-10-10T00:00:00"}