{"id":"https://openalex.org/W4401864323","doi":"https://doi.org/10.1145/3637528.3671543","title":"Choosing a Proxy Metric from Past Experiments","display_name":"Choosing a Proxy Metric from Past Experiments","publication_year":2024,"publication_date":"2024-08-24","ids":{"openalex":"https://openalex.org/W4401864323","doi":"https://doi.org/10.1145/3637528.3671543"},"language":"en","primary_location":{"id":"doi:10.1145/3637528.3671543","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671543","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671543","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671543","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026221871","display_name":"Nilesh Tripuraneni","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Nilesh Tripuraneni","raw_affiliation_strings":["Google DeepMind, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Google DeepMind, San Francisco, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037624493","display_name":"Lee Richardson","orcid":"https://orcid.org/0000-0002-5407-6647"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lee Richardson","raw_affiliation_strings":["Google, San Bruno, CA, USA"],"affiliations":[{"raw_affiliation_string":"Google, San Bruno, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060694111","display_name":"Alexander D\u2019Amour","orcid":"https://orcid.org/0000-0001-7984-3366"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander D'Amour","raw_affiliation_strings":["Google DeepMind, Cambridge, MA, USA"],"affiliations":[{"raw_affiliation_string":"Google DeepMind, Cambridge, MA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103860640","display_name":"Jacopo Soriano","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jacopo Soriano","raw_affiliation_strings":["Google, San Bruno, CA, USA"],"affiliations":[{"raw_affiliation_string":"Google, San Bruno, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059293780","display_name":"Steve Yadlowsky","orcid":"https://orcid.org/0000-0003-4847-7230"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Steve Yadlowsky","raw_affiliation_strings":["Google DeepMind, San Francisco, CA, USA"],"affiliations":[{"raw_affiliation_string":"Google DeepMind, San Francisco, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5026221871"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":3.7682,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.93677348,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"5803","last_page":"5812"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10243","display_name":"Statistical Methods and Bayesian Inference","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9664999842643738,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.7870388627052307},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.643477201461792},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5918962359428406},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.10790055990219116},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09148293733596802},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.06606808304786682}],"concepts":[{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.7870388627052307},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.643477201461792},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5918962359428406},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.10790055990219116},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09148293733596802},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.06606808304786682}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3637528.3671543","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671543","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671543","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3637528.3671543","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671543","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671543","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4401864323.pdf"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1536497620","https://openalex.org/W1971189973","https://openalex.org/W1994623385","https://openalex.org/W2134469398","https://openalex.org/W2150847344","https://openalex.org/W2156577211","https://openalex.org/W2509860763","https://openalex.org/W2532610538","https://openalex.org/W2607670480","https://openalex.org/W2986211311","https://openalex.org/W3123436326","https://openalex.org/W3199351510","https://openalex.org/W4224080885","https://openalex.org/W4233369492","https://openalex.org/W4241584625","https://openalex.org/W4248681815","https://openalex.org/W4290927958","https://openalex.org/W4385562609","https://openalex.org/W4401857172"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"In":[0],"many":[1],"randomized":[2,100,158,209,231],"experiments,":[3],"the":[4,8,12,57,62,76,106,125,140,145,164,171,183,200,208],"treatment":[5,128,142,166],"effect":[6],"of":[7,15,99,108,133,144,151,157,163,207,230],"long-term":[9,25,63,146],"metric":[10,64,92,112,147,186],"(i.e.":[11],"primary":[13],"outcome":[14],"interest)":[16],"is":[17,49,181,191,214],"often":[18,28],"difficult":[19],"or":[20],"infeasible":[21],"to":[22,30,32,40,50,71,84,117,160,246],"measure.":[23],"Such":[24],"metrics":[26,55,241],"are":[27,38],"slow":[29],"react":[31],"changes":[33],"and":[34,87,130,148,218,238],"sufficiently":[35],"noisy":[36],"they":[37,59,67],"challenging":[39],"faithfully":[41],"estimate":[42],"in":[43,56,75,95,113,153,170,226],"short-horizon":[44],"experiments.":[45,101],"A":[46],"common":[47],"alternative":[48],"measure":[51],"several":[52,247],"short-term":[53],"proxy":[54,91,111,185,240],"hope":[58],"closely":[60],"track":[61],"--":[65],"so":[66],"can":[68],"be":[69],"used":[70],"effectively":[72],"guide":[73],"decision-making":[74],"near-term.":[77],"We":[78,137],"introduce":[79],"a":[80,96,114,118,149,154,188,227],"new":[81],"statistical":[82],"framework":[83],"both":[85],"define":[86],"construct":[88,239],"an":[89,109,234],"optimal":[90,110,184],"for":[93,168,187,211],"use":[94,169],"homogeneous":[97],"population":[98],"Our":[102],"procedure":[103],"first":[104],"reduces":[105],"construction":[107],"given":[115,189],"experiment":[116,134,190,210],"portfolio":[119],"optimization":[120,172],"problem":[121],"which":[122,212],"depends":[123],"on":[124,199],"true":[126],"latent":[127,165],"effects":[129,143,167],"noise":[131,205],"level":[132],"under":[135],"consideration.":[136],"then":[138],"denoise":[139],"observed":[141],"set":[150],"proxies":[152],"historical":[155],"corpus":[156,229],"experiments":[159,232],"extract":[161],"estimates":[162],"problem.":[173],"One":[174],"key":[175],"insight":[176],"derived":[177],"from":[178,233],"our":[179,220,224],"approach":[180],"that":[182,242],"not":[192],"apriori":[193],"fixed;":[194],"rather":[195],"it":[196,213],"should":[197],"depend":[198],"sample":[201],"size":[202],"(or":[203],"effective":[204],"level)":[206],"deployed.":[215],"To":[216],"instantiate":[217],"evaluate":[219],"framework,":[221],"we":[222],"employ":[223],"methodology":[225],"large":[228],"industrial":[235],"recommendation":[236],"system":[237],"perform":[243],"favorably":[244],"relative":[245],"baselines.":[248]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}