{"id":"https://openalex.org/W4384659117","doi":"https://doi.org/10.1145/3539618.3592020","title":"Measuring Service-Level Learning Effects in Search Via Query-Randomized Experiments","display_name":"Measuring Service-Level Learning Effects in Search Via Query-Randomized Experiments","publication_year":2023,"publication_date":"2023-07-18","ids":{"openalex":"https://openalex.org/W4384659117","doi":"https://doi.org/10.1145/3539618.3592020"},"language":"en","primary_location":{"id":"doi:10.1145/3539618.3592020","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3592020","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3592020","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3592020","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086714311","display_name":"P. W. Musgrave","orcid":"https://orcid.org/0009-0001-7807-3280"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]},{"id":"https://openalex.org/I4210133358","display_name":"Search","ror":"https://ror.org/03f78hn46","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210133358"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Paul Musgrave","raw_affiliation_strings":["Amazon Search, Palo Alto, CA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon Search, Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I4210133358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061088566","display_name":"Cuize Han","orcid":"https://orcid.org/0000-0003-3957-0687"},"institutions":[{"id":"https://openalex.org/I4210133358","display_name":"Search","ror":"https://ror.org/03f78hn46","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210133358"]},{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cuize Han","raw_affiliation_strings":["Amazon Search, Palo Alto, CA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon Search, Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I4210133358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102919542","display_name":"Parth Gupta","orcid":"https://orcid.org/0000-0003-0232-3412"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]},{"id":"https://openalex.org/I4210133358","display_name":"Search","ror":"https://ror.org/03f78hn46","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210133358"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Parth Gupta","raw_affiliation_strings":["Amazon Search, Palo Alto, CA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon Search, Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I4210133358"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5086714311"],"corresponding_institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I4210133358"],"apc_list":null,"apc_paid":null,"fwci":0.2769,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.65738954,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"2169","last_page":"2173"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9860000014305115,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7488492727279663},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7068257927894592},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.6192923188209534},{"id":"https://openalex.org/keywords/query-expansion","display_name":"Query expansion","score":0.5442193150520325},{"id":"https://openalex.org/keywords/web-search-query","display_name":"Web search query","score":0.5141990184783936},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.5098115801811218},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.5080282688140869},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4649905562400818},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.4263456165790558},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3283594846725464},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.3180380165576935},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.09897089004516602}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7488492727279663},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7068257927894592},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.6192923188209534},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.5442193150520325},{"id":"https://openalex.org/C164120249","wikidata":"https://www.wikidata.org/wiki/Q995982","display_name":"Web search query","level":3,"score":0.5141990184783936},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.5098115801811218},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.5080282688140869},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4649905562400818},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.4263456165790558},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3283594846725464},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.3180380165576935},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09897089004516602},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3539618.3592020","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3592020","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3592020","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3539618.3592020","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3539618.3592020","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3539618.3592020","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4384659117.pdf","grobid_xml":"https://content.openalex.org/works/W4384659117.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W2007331509","https://openalex.org/W2101829497","https://openalex.org/W2124692861","https://openalex.org/W2125771191","https://openalex.org/W2126002144","https://openalex.org/W2132917208","https://openalex.org/W2152886806","https://openalex.org/W2176930994","https://openalex.org/W2275236596","https://openalex.org/W2584822570","https://openalex.org/W2610858089","https://openalex.org/W2740607000","https://openalex.org/W3010775781","https://openalex.org/W3022971154","https://openalex.org/W3035105667","https://openalex.org/W3098765862","https://openalex.org/W3150893739","https://openalex.org/W4284675352","https://openalex.org/W4306317261"],"related_works":["https://openalex.org/W2096359267","https://openalex.org/W1521725692","https://openalex.org/W3008917487","https://openalex.org/W3197639690","https://openalex.org/W2026738364","https://openalex.org/W2572349046","https://openalex.org/W2901901036","https://openalex.org/W2185998359","https://openalex.org/W2381709896","https://openalex.org/W2160726690"],"abstract_inverted_index":{"In":[0,189],"order":[1],"to":[2,10,106,111,126,203],"determine":[3],"the":[4,54,73,82,90,107,121,147,157,176,192,198,213],"relevance":[5],"of":[6,20,75,149,168],"a":[7,11,69,166,205],"given":[8],"item":[9,29],"query,":[12],"most":[13,55],"modern":[14],"search":[15,158,178],"ranking":[16,43],"systems":[17],"make":[18],"use":[19,148],"features":[21,46,113],"which":[22,59,72,151],"aggregate":[23],"prior":[24],"user":[25,63,187],"behavior":[26,74,108],"for":[27,84,102],"that":[28,197],"and":[30,97,135,184],"query":[31,182],"(e.g.":[32,114],"click":[33],"rate).":[34],"For":[35,53],"practical":[36],"reasons,":[37],"when":[38],"running":[39],"A/B":[40,169],"tests":[41,170],"on":[42,156,171],"systems,":[44],"these":[45],"are":[47,124],"generally":[48],"shared":[49],"between":[50],"all":[51],"treatments.":[52],"common":[56],"experiment":[57],"designs,":[58],"randomize":[60,153],"traffic":[61,154],"by":[62,65,71],"or":[64],"session,":[66],"this":[67,117],"creates":[68],"pathway":[70,118],"units":[76,85],"in":[77,86,133,175],"one":[78,122,180,185],"treatment":[79,134],"can":[80],"effect":[81,208],"outcomes":[83],"other":[87],"treatments,":[88],"violating":[89],"Stable":[91],"Unit":[92],"Treatment":[93],"Value":[94],"Assumption":[95],"(SUTVA)":[96],"biasing":[98],"measured":[99],"outcomes.":[100],"Moreover,":[101],"experiments":[103,150],"targeting":[104],"improvements":[105],"data":[109],"available":[110],"such":[112,129],"online":[115],"exploration),":[116],"is":[119,201],"precisely":[120],"we":[123,145,164,195],"trying":[125],"affect;":[127],"if":[128],"changes":[130],"occur":[131],"identically":[132],"control,":[136],"then":[137],"they":[138],"cannot":[139],"be":[140],"measured.":[141],"To":[142,160],"address":[143],"this,":[144],"propose":[146],"instead":[152],"based":[155],"query.":[159],"validate":[161],"our":[162],"approach,":[163],"perform":[165],"pair":[167],"an":[172],"explore-exploit":[173],"framework":[174],"Amazon":[177],"page:":[179],"under":[181,186],"randomization,":[183],"randomization.":[188],"line":[190],"with":[191],"theoretical":[193],"predictions,":[194],"find":[196],"query-randomized":[199],"iteration":[200,215],"able":[202],"measure":[204],"statistically":[206],"significant":[207],"(+0.66%":[209],"Purchases,":[210,219],"p=0.001)":[211],"where":[212],"user-randomized":[214],"does":[216],"not":[217],"(-0.02%":[218],"p=0.851).":[220]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}