{"id":"https://openalex.org/W2897477542","doi":"https://doi.org/10.1145/3269206.3271747","title":"Trustworthy Experimentation Under Telemetry Loss","display_name":"Trustworthy Experimentation Under Telemetry Loss","publication_year":2018,"publication_date":"2018-10-17","ids":{"openalex":"https://openalex.org/W2897477542","doi":"https://doi.org/10.1145/3269206.3271747","mag":"2897477542"},"language":"en","primary_location":{"id":"doi:10.1145/3269206.3271747","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3269206.3271747","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1903.12470","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jayant Gupchup","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jayant Gupchup","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yasaman Hosseinkashi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yasaman Hosseinkashi","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Pavel Dmitriev","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pavel Dmitriev","raw_affiliation_strings":["Outreach.io, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Outreach.io, Seattle, WA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Daniel Schneider","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Schneider","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ross Cutler","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ross Cutler","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Andrei Jefremov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andrei Jefremov","raw_affiliation_strings":["Unaffiliated, Stockholm, WA, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Unaffiliated, Stockholm, WA, Sweden","institution_ids":[]}]},{"author_position":"last","author":{"id":null,"display_name":"Martin Ellis","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Martin Ellis","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.3379,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.88760716,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"387","last_page":"396"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10742","display_name":"Peer-to-Peer Network Technologies","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/upload","display_name":"Upload","score":0.7797999978065491},{"id":"https://openalex.org/keywords/telemetry","display_name":"Telemetry","score":0.7745000123977661},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.6485999822616577},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.6437000036239624},{"id":"https://openalex.org/keywords/skew","display_name":"Skew","score":0.5863000154495239},{"id":"https://openalex.org/keywords/data-loss","display_name":"Data loss","score":0.3546000123023987}],"concepts":[{"id":"https://openalex.org/C71901391","wikidata":"https://www.wikidata.org/wiki/Q7126699","display_name":"Upload","level":2,"score":0.7797999978065491},{"id":"https://openalex.org/C183121708","wikidata":"https://www.wikidata.org/wiki/Q209867","display_name":"Telemetry","level":2,"score":0.7745000123977661},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7117999792098999},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.6485999822616577},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.6437000036239624},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.5863000154495239},{"id":"https://openalex.org/C193519340","wikidata":"https://www.wikidata.org/wiki/Q891179","display_name":"Data loss","level":2,"score":0.3546000123023987},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.3483999967575073},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.3433000147342682},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2888999879360199},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2639999985694885},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2623000144958496},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2554999887943268}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3269206.3271747","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3269206.3271747","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1903.12470","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1903.12470","pdf_url":"https://arxiv.org/pdf/1903.12470","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1903.12470","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1903.12470","pdf_url":"https://arxiv.org/pdf/1903.12470","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W1975566260","https://openalex.org/W2041508368","https://openalex.org/W2070815407","https://openalex.org/W2080770071","https://openalex.org/W2099040451","https://openalex.org/W2110086534","https://openalex.org/W2112236211","https://openalex.org/W2119400430","https://openalex.org/W2126002144","https://openalex.org/W2495130909","https://openalex.org/W2517868792","https://openalex.org/W2743849295","https://openalex.org/W2745034467","https://openalex.org/W2998685964","https://openalex.org/W4244207817","https://openalex.org/W6744237412"],"related_works":[],"abstract_inverted_index":{"Failure":[0],"to":[1,11,26,29,57,82,90,93,103,178,211],"accurately":[2],"measure":[3,104,179],"the":[4,52,70,105,129,144,152,169,180,213],"outcomes":[5],"of":[6,43,54,59,72,108,131,151,171,183,196,200,216],"an":[7,112],"experiment":[8],"can":[9,50,205],"lead":[10],"bias":[12,153],"and":[13,36,62,142,174,198,218],"incorrect":[14],"conclusions.":[15,66],"Online":[16],"controlled":[17],"experiments":[18,135],"(aka":[19],"AB":[20],"tests)":[21],"are":[22],"increasingly":[23],"being":[24],"used":[25,188],"make":[27],"decisions":[28],"improve":[30,212],"websites":[31],"as":[32,34],"well":[33],"mobile":[35],"desktop":[37],"applications.":[38],"We":[39,127],"argue":[40,76],"that":[41,77,99],"loss":[42,58,110,132],"telemetry":[44,73,109,172],"data":[45],"(during":[46],"upload":[47],"or":[48,64],"post-processing)":[49],"skew":[51],"results":[53],"experiments,":[55],"leading":[56],"statistical":[60],"power":[61],"inaccurate":[63],"erroneous":[65],"By":[67],"systematically":[68],"investigating":[69],"causes":[71],"loss,":[74,173],"we":[75,97,118,161],"it":[78,100],"is":[79,101,187],"not":[80],"practical":[81],"entirely":[83],"eliminate":[84],"it.":[85],"Consequently,":[86],"experimentation":[87,113,217],"systems":[88],"need":[89],"be":[91,206],"robust":[92],"its":[94],"effects.":[95],"Furthermore,":[96],"note":[98],"nontrivial":[102],"absolute":[106,181],"level":[107],"in":[111,136],"system.":[114],"In":[115],"this":[116,125,159],"paper,":[117],"take":[119],"a":[120,148,163],"top-down":[121],"approach":[122],"towards":[123],"solving":[124],"problem.":[126],"motivate":[128],"impact":[130,170],"qualitatively":[133],"using":[134],"real":[137],"applications":[138,191],"deployed":[139],"at":[140,192],"scale,":[141],"formalize":[143],"problem":[145],"by":[146,155,189,208],"presenting":[147],"theoretical":[149],"breakdown":[150],"introduced":[154],"loss.":[156,184],"Based":[157],"on":[158],"foundation,":[160],"present":[162,175],"general":[164,203],"framework":[165,186],"for":[166],"quantitatively":[167],"evaluating":[168],"two":[176],"solutions":[177],"levels":[182],"This":[185],"well-known":[190],"Microsoft,":[193],"with":[194],"millions":[195],"users":[197],"billions":[199],"sessions.":[201],"These":[202],"principles":[204],"adopted":[207],"any":[209],"application":[210],"overall":[214],"trustworthiness":[215],"data-driven":[219],"decision":[220],"making.":[221]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":5}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2018-10-26T00:00:00"}