{"id":"https://openalex.org/W3193395366","doi":"https://doi.org/10.1145/3483382.3483402","title":"Learning from user interactions with rankings: a unification of the field","display_name":"Learning from user interactions with rankings: a unification of the field","alternate_titles":[],"publication_year":2020,"publication_date":"2020-12-01","ids":{"openalex":"https://openalex.org/W3193395366","doi":"https://doi.org/10.1145/3483382.3483402","mag":"3193395366"},"language":"en","primary_location":{"id":"doi:10.1145/3483382.3483402","doi":"https://doi.org/10.1145/3483382.3483402","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3483382.3483402","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","jufo-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W3193395366"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2012.06576","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002072527","display_name":"Harrie Oosterhuis","orcid":"https://orcid.org/0000-0002-0458-9233","observed_orcids":["https://orcid.org/0000-0002-0458-9233"]},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":true,"contributor_roles":[],"raw_author_name":"Harrie Oosterhuis","raw_affiliation_strings":["Radboud University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Radboud University","institution_ids":["https://openalex.org/I145872427"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5002072527"],"corresponding_institution_ids":["https://openalex.org/I145872427"],"apc_list":null,"apc_paid":null,"fwci":1.1624,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.82092047,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":97},"biblio":{"volume":"54","issue":"2","first_page":"1","last_page":"2"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.8076000213623047,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.8076000213623047,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.1720000058412552,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.014399999752640724,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"learning to rank","score":0.9909999966621399},{"id":"https://openalex.org/keywords/click-data","display_name":"click data","score":0.5740000009536743},{"id":"https://openalex.org/keywords/e-learning","display_name":"online learning","score":0.6320000290870667},{"id":"https://openalex.org/keywords/dueling-bandits","display_name":"dueling bandits","score":0.8740000128746033},{"id":"https://openalex.org/keywords/pairwise-learning","display_name":"pairwise learning","score":0.6629999876022339},{"id":"https://openalex.org/keywords/counterfactual-evaluation","display_name":"counterfactual evaluation","score":0.8450000286102295},{"id":"https://openalex.org/keywords/logging-policy","display_name":"logging policy","score":0.6000000238418579},{"id":"https://openalex.org/keywords/top-k-ranking","display_name":"top-k ranking","score":0.7850000262260437},{"id":"https://openalex.org/keywords/tabular-models","display_name":"tabular models","score":0.6639999747276306},{"id":"https://openalex.org/keywords/feature-based-models","display_name":"feature-based models","score":0.7940000295639038},{"id":"https://openalex.org/keywords/ranking-systems","display_name":"ranking systems","score":0.4620000123977661},{"id":"https://openalex.org/keywords/user-preferences","display_name":"user preferences","score":0.38499999046325684},{"id":"https://openalex.org/keywords/search-engines","display_name":"search engines","score":0.5220000147819519},{"id":"https://openalex.org/keywords/recommender-systems","display_name":"recommender systems","score":0.9589999914169312},{"id":"https://openalex.org/keywords/ranking-metrics","display_name":"ranking metrics","score":0.3440000116825104},{"id":"https://openalex.org/keywords/variance-reduction","display_name":"variance reduction","score":0.3919999897480011},{"id":"https://openalex.org/keywords/robust-generalization","display_name":"robust generalization","score":0.49799999594688416},{"id":"https://openalex.org/keywords/expert-opinion","display_name":"expert opinion","score":0.41200000047683716}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8749980926513672},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.8405373096466064},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.7749396562576294},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.769717812538147},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7021695971488953},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5636481642723083},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.559262752532959},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5433540940284729},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5297150611877441},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.5006067752838135},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4985463619232178},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48189836740493774},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.4588225185871124},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43964725732803345},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3483382.3483402","doi":"https://doi.org/10.1145/3483382.3483402","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3483382.3483402","pdf_url":null,"source":{"id":"https://openalex.org/S6756005","display_name":"ACM SIGIR Forum","issn_l":"0163-5840","issn":["0163-5840","1558-0229"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","jufo-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGIR Forum","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2012.06576","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2012.06576","pdf_url":"https://arxiv.org/pdf/2012.06576","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2012.06576","doi":"https://doi.org/10.48550/arxiv.2012.06576","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2012.06576","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2012.06576","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2012.06576","pdf_url":"https://arxiv.org/pdf/2012.06576","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2586376837","display_name":null,"funder_award_id":"612.001.551","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3193395366.pdf","grobid_xml":"https://content.openalex.org/works/W3193395366.grobid-xml"},"referenced_works_count":145,"referenced_works":["https://openalex.org/W179016587","https://openalex.org/W1530210183","https://openalex.org/W1533861849","https://openalex.org/W1663973292","https://openalex.org/W1835538174","https://openalex.org/W1850547517","https://openalex.org/W1900560890","https://openalex.org/W1963548721","https://openalex.org/W1967787801","https://openalex.org/W1969147614","https://openalex.org/W1973435495","https://openalex.org/W1982530130","https://openalex.org/W1982889956","https://openalex.org/W1984770088","https://openalex.org/W1985554184","https://openalex.org/W1992549066","https://openalex.org/W1997136459","https://openalex.org/W1998955392","https://openalex.org/W2003473029","https://openalex.org/W2007105673","https://openalex.org/W2007529991","https://openalex.org/W2009979684","https://openalex.org/W2011285051","https://openalex.org/W2022995284","https://openalex.org/W2023599408","https://openalex.org/W2036125087","https://openalex.org/W2041576073","https://openalex.org/W2047221353","https://openalex.org/W2049633694","https://openalex.org/W2059873455","https://openalex.org/W2061170183","https://openalex.org/W2069870183","https://openalex.org/W2073248814","https://openalex.org/W2082831169","https://openalex.org/W2090589280","https://openalex.org/W2090883204","https://openalex.org/W2094790959","https://openalex.org/W2102217663","https://openalex.org/W2108862644","https://openalex.org/W2110228583","https://openalex.org/W2112508839","https://openalex.org/W2114503508","https://openalex.org/W2115584760","https://openalex.org/W2116008435","https://openalex.org/W2117600734","https://openalex.org/W2129245267","https://openalex.org/W2143257555","https://openalex.org/W2147892741","https://openalex.org/W2148931201","https://openalex.org/W2149156280","https://openalex.org/W2149427297","https://openalex.org/W2150886314","https://openalex.org/W2152314154","https://openalex.org/W2154739689","https://openalex.org/W2154884126","https://openalex.org/W2155587858","https://openalex.org/W2156541733","https://openalex.org/W2156909104","https://openalex.org/W2163996478","https://openalex.org/W2176930994","https://openalex.org/W2184294966","https://openalex.org/W2188353343","https://openalex.org/W2210543184","https://openalex.org/W2213612645","https://openalex.org/W2259177609","https://openalex.org/W2273088453","https://openalex.org/W2279176662","https://openalex.org/W2279385734","https://openalex.org/W2294145134","https://openalex.org/W2339829457","https://openalex.org/W2340526403","https://openalex.org/W2340690086","https://openalex.org/W2341805565","https://openalex.org/W2341865734","https://openalex.org/W2402441596","https://openalex.org/W2406454855","https://openalex.org/W2407295445","https://openalex.org/W2418554883","https://openalex.org/W2464171116","https://openalex.org/W2468270644","https://openalex.org/W2505872937","https://openalex.org/W2507134384","https://openalex.org/W2511039137","https://openalex.org/W2522306626","https://openalex.org/W2535047536","https://openalex.org/W2542081672","https://openalex.org/W2552231430","https://openalex.org/W2566147423","https://openalex.org/W2590822507","https://openalex.org/W2604370872","https://openalex.org/W2741497758","https://openalex.org/W2767455847","https://openalex.org/W2767527743","https://openalex.org/W2769133055","https://openalex.org/W2769473018","https://openalex.org/W2783388661","https://openalex.org/W2784672094","https://openalex.org/W2787962292","https://openalex.org/W2797400361","https://openalex.org/W2798460079","https://openalex.org/W2798625028","https://openalex.org/W2798634418","https://openalex.org/W2798835721","https://openalex.org/W2798855994","https://openalex.org/W2890022552","https://openalex.org/W2890291106","https://openalex.org/W2898073868","https://openalex.org/W2899075954","https://openalex.org/W2899259597","https://openalex.org/W2901487151","https://openalex.org/W2904119980","https://openalex.org/W2905569957","https://openalex.org/W2908332126","https://openalex.org/W2911802745","https://openalex.org/W2912255075","https://openalex.org/W2913544988","https://openalex.org/W2914526782","https://openalex.org/W2948795993","https://openalex.org/W2955421345","https://openalex.org/W2989569331","https://openalex.org/W2995091170","https://openalex.org/W3004166557","https://openalex.org/W3012600133","https://openalex.org/W3012881846","https://openalex.org/W3013164143","https://openalex.org/W3015271280","https://openalex.org/W3026200234","https://openalex.org/W3028135017","https://openalex.org/W3028722847","https://openalex.org/W3044556022","https://openalex.org/W3088936686","https://openalex.org/W3092961465","https://openalex.org/W3104881842","https://openalex.org/W3105148189","https://openalex.org/W3112511105","https://openalex.org/W3118007541","https://openalex.org/W3126910119","https://openalex.org/W3130740428","https://openalex.org/W3146247238","https://openalex.org/W4205355130","https://openalex.org/W4206530644","https://openalex.org/W4288079518","https://openalex.org/W4288280739","https://openalex.org/W4300879629","https://openalex.org/W7099004709"],"related_works":["https://openalex.org/W1996924157","https://openalex.org/W2529460324","https://openalex.org/W3165918968","https://openalex.org/W2090374349","https://openalex.org/W2531765288","https://openalex.org/W7096569191","https://openalex.org/W2023599408","https://openalex.org/W2962980921","https://openalex.org/W2004335587"],"abstract_inverted_index":{"Ranking":[0],"systems":[1,92],"form":[2],"the":[3,27,46,53,59,65,72,75,188,198,262,276,296,299,312,337,371,382,389,407,420,431,453,456,463,505,509,512,518],"basis":[4],"for":[5,18,51,255,405,555],"online":[6,207,441,481,494,510],"search":[7],"engines":[8],"and":[9,25,106,183,226,234,340,401,495,511,536,548,558],"recommendation":[10],"services.":[11],"They":[12],"process":[13],"large":[14,223],"collections":[15],"of":[16,36,56,74,191,197,201,311,315,385,392,422,433,465,521,533,569],"items,":[17],"instance":[19],"web":[20],"pages":[21],"or":[22,68,116],"e-commerce":[23],"products,":[24],"present":[26],"user":[28,44,134,149,170,181,235,564],"with":[29,52,139,217,261,388],"a":[30,37,43,81,121,240,251,378,403,416,447,469,516,525,552],"small":[31],"ordered":[32],"selection.":[33],"The":[34,137,194,247,279,308,374,395,443,488,572],"goal":[35],"ranking":[38,91,306,424],"system":[39],"is":[40,80,141,302,491],"to":[41,78,99,104,118,130,162,176,186,209,323,344,364,539,561],"help":[42],"find":[45],"items":[47,70,351],"they":[48,61,143],"are":[49,114,144],"looking":[50],"least":[54],"amount":[55],"effort.":[57],"Thus":[58],"rankings":[60,155],"produce":[62],"should":[63],"place":[64],"most":[66],"relevant":[67],"preferred":[69],"at":[71,206,320,355,399],"top":[73],"ranking.":[76],"Learning":[77],"rank":[79,100,131,177,210,324,365,562],"field":[82],"within":[83],"machine":[84],"learning":[85,98,129,175,208,256,322,363,538,560],"that":[86,127,142,204,243,259,270,295,318,360,380,411,418,451,455,501,527],"covers":[87],"methods":[88,101,123,160,178,229,366],"which":[89,212,326],"optimize":[90],"w.r.t.":[93,304],"this":[94,428,522],"goal.":[95],"Traditional":[96],"supervised":[97,362,537],"utilize":[102],"expert-judgements":[103],"evaluate":[105],"learn,":[107],"however,":[108],"in":[109,286,370,415,479,507],"many":[110,529],"situations":[111],"such":[112],"judgements":[113],"impossible":[115],"infeasible":[117],"obtain.":[119],"As":[120,468,515],"solution,":[122],"have":[124,161],"been":[125,460],"introduced":[126],"perform":[128],"based":[132,179],"on":[133,180],"clicks":[135,140,182,258],"instead.":[136],"difficulty":[138],"not":[145,231,349],"only":[146],"affected":[147],"by":[148,153,166,214],"preferences,":[150],"but":[151],"also":[152,293,358],"what":[154],"were":[156],"displayed.":[157],"Therefore,":[158],"these":[159,192,284],"prevent":[163],"being":[164],"biased":[165],"other":[167],"factors":[168],"than":[169],"preference.":[171],"This":[172],"thesis":[173,199,313,523],"concerns":[174],"specifically":[184],"aims":[185],"unify":[187],"different":[189],"families":[190],"methods.":[193,266],"first":[195,220,334],"part":[196,310,520],"consists":[200,314],"three":[202],"chapters":[203,317],"look":[205,319],"algorithms":[211,325],"learn":[213,327,473],"directly":[215],"interacting":[216],"users.":[218],"Its":[219,333],"chapter":[221,249,281,335,376,397,445],"considers":[222,452],"scale":[224],"evaluation":[225,400,439],"shows":[227,359],"existing":[228,338],"do":[230],"guarantee":[232,245],"correctness":[233],"experience,":[236],"we":[237,292],"then":[238],"introduce":[239],"novel":[241,252,448],"method":[242,254,273,379,404,429],"can":[244,352,367,436,472,485],"both.":[246],"second":[248,309,375,519],"proposes":[250,446,524],"pairwise":[253,272],"from":[257,328,563],"contrasts":[260],"previous":[263],"prevalent":[264],"dueling-bandit":[265,277,300],"Our":[267],"experiments":[268],"show":[269,294,500],"our":[271,497],"greatly":[274],"outperforms":[275],"approach.":[278],"third":[280,396],"further":[282],"confirms":[283],"findings":[285],"an":[287,480],"extensive":[288],"experimental":[289,498],"comparison,":[290],"furthermore,":[291],"theory":[297],"behind":[298],"approach":[301,339,490],"unsound":[303],"deterministic":[305],"systems.":[307],"four":[316],"counterfactual":[321,372,438,449,513],"historically":[329],"logged":[330],"click":[331,413,466],"data.":[332,467],"takes":[336],"makes":[341],"it":[342,471],"applicable":[343,557],"top-":[345],"k":[346],"settings":[347],"where":[348,483],"all":[350],"be":[353,368],"displayed":[354],"once.":[356],"It":[357,541],"state-of-the-art":[361,506],"applied":[369],"scenario.":[373,514],"introduces":[377,402],"combines":[381],"robust":[383],"generalization":[384],"feature-based":[386],"models":[387],"high-performance":[390],"specialization":[391],"tabular":[393],"models.":[394],"looks":[398],"finding":[406],"optimal":[408],"logging":[409,457],"policy":[410,458],"collects":[412],"data":[414],"way":[417],"minimizes":[419],"variance":[421],"estimated":[423],"metrics.":[425],"By":[426],"applying":[427],"during":[430,462],"gathering":[432,464],"clicks,":[434],"one":[435],"turn":[437],"into":[440,551],"evaluation.":[442],"fourth":[444],"estimator":[450],"possibility":[454],"has":[459,542],"updated":[461],"result,":[470],"much":[474],"more":[475],"efficiently":[476],"when":[477],"deployed":[478],"scenario":[482],"interventions":[484],"take":[486],"place.":[487],"resulting":[489],"thus":[492],"both":[493,508],"counterfactual,":[496,535],"results":[499],"its":[502],"performance":[503],"matches":[504],"whole,":[517],"framework":[526],"bridges":[528],"gaps":[530],"between":[531],"areas":[532],"online,":[534],"rank.":[540],"taken":[543],"approaches,":[544],"previously":[545],"considered":[546],"independent,":[547],"unified":[549],"them":[550],"single":[553],"methodology":[554],"widely":[556],"effective":[559],"clicks.":[565],"Awarded":[566],"by:":[567,575],"University":[568],"Amsterdam,":[570,571],"Netherlands.":[573],"Supervised":[574],"Maarten":[576],"de":[577],"Rijke.":[578],"Available":[579],"at:":[580],"https://hdl.handle.net/11245.1/8ff3aa38-97fb-4d2a-8127-a29a03af4d5c.":[581]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3193395366","counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":4}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2021-08-30T00:00:00"}
