{"id":"https://openalex.org/W2051744369","doi":"https://doi.org/10.1145/2766462.2767706","title":"Optimised Scheduling of Online Experiments","display_name":"Optimised Scheduling of Online Experiments","publication_year":2015,"publication_date":"2015-08-04","ids":{"openalex":"https://openalex.org/W2051744369","doi":"https://doi.org/10.1145/2766462.2767706","mag":"2051744369"},"language":"en","primary_location":{"id":"doi:10.1145/2766462.2767706","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2766462.2767706","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102924622","display_name":"Eugene Kharitonov","orcid":"https://orcid.org/0009-0000-8653-721X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Eugene Kharitonov","raw_affiliation_strings":["Yandex & University of Glasgow, Moscow, Russian Fed"],"affiliations":[{"raw_affiliation_string":"Yandex & University of Glasgow, Moscow, Russian Fed","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057643560","display_name":"Craig Macdonald","orcid":"https://orcid.org/0000-0003-3143-279X"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Craig Macdonald","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom","University of Glasgow, Glasgow, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom#TAB#","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069551961","display_name":"Pavel Serdyukov","orcid":null},"institutions":[{"id":"https://openalex.org/I58957048","display_name":"Yandex (Russia)","ror":"https://ror.org/04dbch786","country_code":"RU","type":"company","lineage":["https://openalex.org/I58957048"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Pavel Serdyukov","raw_affiliation_strings":["Yandex, Moscow, Russian Fed","Yandex, Moscow, Russian Fed.#TAB#"],"affiliations":[{"raw_affiliation_string":"Yandex, Moscow, Russian Fed","institution_ids":[]},{"raw_affiliation_string":"Yandex, Moscow, Russian Fed.#TAB#","institution_ids":["https://openalex.org/I58957048"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079046603","display_name":"Iadh Ounis","orcid":"https://orcid.org/0000-0003-4701-3223"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Iadh Ounis","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom","University of Glasgow, Glasgow, United Kingdom#TAB#"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom#TAB#","institution_ids":["https://openalex.org/I7882870"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5102924622"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.5195,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.93151588,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"453","last_page":"462"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.845613956451416},{"id":"https://openalex.org/keywords/interleaving","display_name":"Interleaving","score":0.7169325351715088},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.6456345915794373},{"id":"https://openalex.org/keywords/queue","display_name":"Queue","score":0.5557877421379089},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5383902788162231},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.528205931186676},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4804777204990387},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4670723080635071},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.1869048774242401},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.13790583610534668},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.10392773151397705},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.08643710613250732}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.845613956451416},{"id":"https://openalex.org/C28034677","wikidata":"https://www.wikidata.org/wiki/Q17092530","display_name":"Interleaving","level":2,"score":0.7169325351715088},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.6456345915794373},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.5557877421379089},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5383902788162231},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.528205931186676},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4804777204990387},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4670723080635071},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.1869048774242401},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.13790583610534668},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.10392773151397705},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.08643710613250732},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2766462.2767706","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2766462.2767706","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.gla.ac.uk:105811","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":false,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Conference Proceedings"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1501066815","https://openalex.org/W1514907448","https://openalex.org/W1975566260","https://openalex.org/W1982530130","https://openalex.org/W2011285051","https://openalex.org/W2047221353","https://openalex.org/W2069870183","https://openalex.org/W2094790959","https://openalex.org/W2099213975","https://openalex.org/W2101234009","https://openalex.org/W2105256943","https://openalex.org/W2106628583","https://openalex.org/W2108126284","https://openalex.org/W2112508839","https://openalex.org/W2113304784","https://openalex.org/W2113640060","https://openalex.org/W2121863487","https://openalex.org/W2156541733","https://openalex.org/W2160471965","https://openalex.org/W2163996478","https://openalex.org/W2169113736","https://openalex.org/W2407295445","https://openalex.org/W4234582748","https://openalex.org/W6675354045","https://openalex.org/W6676237016"],"related_works":["https://openalex.org/W1655266410","https://openalex.org/W2389051085","https://openalex.org/W2330343234","https://openalex.org/W1901012776","https://openalex.org/W2463883322","https://openalex.org/W2229382548","https://openalex.org/W2391789612","https://openalex.org/W2814468324","https://openalex.org/W2389236462","https://openalex.org/W415847890"],"abstract_inverted_index":{"Modern":[0],"search":[1,25,34,212],"engines":[2],"increasingly":[3],"rely":[4],"on":[5,251,261],"online":[6,16,110,154],"evaluation":[7,17,56,195],"methods":[8,18],"such":[9],"as":[10,221],"A/B":[11,205,265],"tests":[12],"and":[13,203,257],"interleaving.":[14],"These":[15],"make":[19],"use":[20,86],"of":[21,40,46,52,90,102,108,116,133,152,164,218,224,232,241,254,264],"interactions":[22,119],"by":[23,148,209],"the":[24,33,38,41,50,87,91,99,106,109,117,127,131,162,166,176,185,189,216,222,229,233,247,252,262],"engine's":[26],"users":[27],"to":[28,68,84,125,140,160,169,180,243,246,259],"test":[29,206],"various":[30],"changes":[31],"in":[32,184],"engine.":[35,213],"However,":[36],"since":[37,72],"number":[39,51,115,132,223],"user":[42,118,234],"sessions":[43],"per":[44],"unit":[45],"time":[47],"is":[48,58,81,136],"limited,":[49],"simultaneously":[53],"running":[54],"on-line":[55],"experiments":[57,71,135,179,226,256],"bounded.":[59],"In":[60,94,138],"an":[61,153,194],"extreme":[62],"case,":[63],"it":[64,80],"might":[65],"be":[66,181],"impossible":[67],"deploy":[69],"all":[70],"they":[73],"arrive":[74],"faster":[75],"than":[76],"are":[77],"proccessed.":[78],"Consequently,":[79],"very":[82],"important":[83],"efficiently":[85],"limited":[88,114,230],"resource":[89,231],"user's":[92],"interactions.":[93,235],"this":[95],"paper,":[96],"we":[97,123,146,158,174,192],"formulate":[98],"novel":[100],"problem":[101],"schedule":[103,143,168,220,250],"optimisation":[104,144],"for":[105,121],"queue":[107,128],"experiments:":[111],"given":[112],"a":[113,142,150,170,210,219],"available":[120],"experimentation,":[122],"want":[124],"re-order":[126],"so":[129],"that":[130],"successful":[134,225],"maximised.":[137],"order":[139],"build":[141],"algorithm,":[145],"start":[147],"formulating":[149],"model":[151],"experimentation":[155],"pipeline.":[156],"Next,":[157],"propose":[159],"reduce":[161],"task":[163],"finding":[165],"optimal":[167],"learning-to-rank":[171],"problem,":[172],"where":[173],"require":[175],"most":[177],"promising":[178],"ranked":[182],"first":[183],"schedule.":[186],"To":[187],"evaluate":[188],"proposed":[190,237],"approach,":[191],"perform":[193],"study":[196],"using":[197],"two":[198],"datasets":[199],"containing":[200],"82":[201],"interleaving":[202,255],"35":[204],"experiments,":[207],"performed":[208],"commercial":[211],"We":[214],"measure":[215],"quality":[217],"executed":[227],"under":[228],"Our":[236],"schedulers":[238],"obtain":[239],"improvements":[240],"up":[242,258],"342%":[244],"compared":[245],"un-optimised":[248],"baseline":[249],"dataset":[253,263],"43%":[260],"tests.":[266]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}