{"id":"https://openalex.org/W2903440991","doi":"https://doi.org/10.1145/3299869.3314036","title":"Snorkel DryBell","display_name":"Snorkel DryBell","publication_year":2019,"publication_date":"2019-06-18","ids":{"openalex":"https://openalex.org/W2903440991","doi":"https://doi.org/10.1145/3299869.3314036","mag":"2903440991","pmid":"https://pubmed.ncbi.nlm.nih.gov/31777414"},"language":"en","primary_location":{"id":"doi:10.1145/3299869.3314036","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3299869.3314036","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3299869.3314036","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 International Conference on Management of Data","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3299869.3314036","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011117177","display_name":"Stephen H. Bach","orcid":"https://orcid.org/0000-0003-3857-3560"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephen H. Bach","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064375280","display_name":"Daniel Rodriguez Gutierrez","orcid":"https://orcid.org/0000-0003-4291-3943"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Rodriguez","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070524866","display_name":"Yintao Liu","orcid":"https://orcid.org/0009-0004-8096-3647"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yintao Liu","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005976848","display_name":"Chong Luo","orcid":"https://orcid.org/0000-0003-4654-5682"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chong Luo","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052658396","display_name":"Haidong Shao","orcid":"https://orcid.org/0000-0002-8018-1774"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haidong Shao","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068036805","display_name":"Cassandra Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cassandra Xia","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012581799","display_name":"Souvik Sen","orcid":"https://orcid.org/0000-0002-0886-2364"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Souvik Sen","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067607533","display_name":"Alex Ratner","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Ratner","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056060162","display_name":"Braden Hancock","orcid":"https://orcid.org/0000-0003-3167-1053"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Braden Hancock","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065267529","display_name":"Houman Alborzi","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Houman Alborzi","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061856100","display_name":"Rahul Kuchhal","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rahul Kuchhal","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103148807","display_name":"Chris R\u00e9","orcid":"https://orcid.org/0000-0002-4109-0108"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris R\u00e9","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102152945","display_name":"Rob Malkin","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rob Malkin","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":13,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":14.8188,"has_fulltext":true,"cited_by_count":97,"citation_normalized_percentile":{"value":0.99166667,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"2019","issue":null,"first_page":"362","last_page":"375"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8020399808883667},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6216270327568054},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.45978856086730957},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4321553707122803},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36291828751564026},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.15688538551330566}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8020399808883667},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6216270327568054},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45978856086730957},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4321553707122803},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36291828751564026},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.15688538551330566}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3299869.3314036","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3299869.3314036","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3299869.3314036","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 International Conference on Management of Data","raw_type":"proceedings-article"},{"id":"pmid:31777414","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/31777414","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings. ACM-SIGMOD International Conference on Management of Data","raw_type":null},{"id":"pmh:oai:europepmc.org:5835666","is_oa":false,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/6879379","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1145/3299869.3314036","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3299869.3314036","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3299869.3314036","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 International Conference on Management of Data","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.5,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G2589055757","display_name":null,"funder_award_id":"N000141712266","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G324214434","display_name":null,"funder_award_id":"CCF1763315","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G538509161","display_name":null,"funder_award_id":"FA86501827865","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G6065575299","display_name":null,"funder_award_id":"FA87501720095","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G8203307141","display_name":null,"funder_award_id":"CCF1563078","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320333063","display_name":"Stanford Bio-X","ror":"https://ror.org/00f54p054"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2903440991.pdf","grobid_xml":"https://content.openalex.org/works/W2903440991.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W72116547","https://openalex.org/W108763474","https://openalex.org/W398859631","https://openalex.org/W1479807131","https://openalex.org/W1541280084","https://openalex.org/W1610496399","https://openalex.org/W1821462560","https://openalex.org/W1988529147","https://openalex.org/W2048679005","https://openalex.org/W2064675550","https://openalex.org/W2074694452","https://openalex.org/W2107598941","https://openalex.org/W2111316763","https://openalex.org/W2123442489","https://openalex.org/W2165698076","https://openalex.org/W2169585110","https://openalex.org/W2252268321","https://openalex.org/W2323506184","https://openalex.org/W2404161646","https://openalex.org/W2404544029","https://openalex.org/W2440094130","https://openalex.org/W2591700809","https://openalex.org/W2592362528","https://openalex.org/W2594696540","https://openalex.org/W2734663976","https://openalex.org/W2743948853","https://openalex.org/W2769041395","https://openalex.org/W2799269579","https://openalex.org/W2804047946","https://openalex.org/W2895604144","https://openalex.org/W2953132584","https://openalex.org/W2953384591","https://openalex.org/W2959716049","https://openalex.org/W2962915384","https://openalex.org/W2963499153","https://openalex.org/W2964318358","https://openalex.org/W4299828299","https://openalex.org/W4300456194","https://openalex.org/W6636177537"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3209574120","https://openalex.org/W3107602296","https://openalex.org/W4312192474","https://openalex.org/W4210805261","https://openalex.org/W4387297750"],"abstract_inverted_index":{"Labeling":[0],"training":[1],"data":[2,133],"is":[3],"one":[4],"of":[5,47,78,102,110,112,132,137],"the":[6,66],"most":[7],"costly":[8],"bottlenecks":[9],"in":[10,35,71,135],"developing":[11],"machine":[12],"learning-based":[13],"applications.":[14],"We":[15],"present":[16],"a":[17,53],"first-of-its-kind":[18],"study":[19],"showing":[20],"how":[21],"existing":[22],"knowledge":[23],"resources":[24,118],"from":[25],"across":[26],"an":[27,45,123],"organization":[28],"can":[29],"be":[30],"used":[31],"as":[32],"weak":[33,55],"supervision":[34,56],"order":[36,46],"to":[37,105,119],"bring":[38],"development":[39],"time":[40],"and":[41,49,85,128],"cost":[42],"down":[43],"by":[44],"magnitude,":[48],"introduce":[50],"Snorkel":[51,62,67,98],"DryBell,":[52],"new":[54],"management":[57],"system":[58],"for":[59,122],"this":[60],"setting.":[61],"DryBell":[63,99],"builds":[64],"on":[65],"framework,":[68],"extending":[69],"it":[70],"three":[72,90],"critical":[73],"aspects:":[74],"flexible,":[75],"template-based":[76],"ingestion":[77],"diverse":[79],"organizational":[80,117],"knowledge,":[81],"cross-feature":[82],"production":[83],"serving,":[84],"scalable,":[86],"sampling-free":[87],"execution.":[88],"On":[89],"classification":[91],"tasks":[92],"at":[93],"Google,":[94],"we":[95],"find":[96],"that":[97],"creates":[100],"classifiers":[101],"comparable":[103],"quality":[104],"ones":[106],"trained":[107],"with":[108],"tens":[109,136],"thousands":[111],"hand-labeled":[113],"examples,":[114],"converts":[115],"non-servable":[116],"servable":[120],"models":[121],"average":[124],"52%":[125],"performance":[126],"improvement,":[127],"executes":[129],"over":[130],"millions":[131],"points":[134],"minutes.":[138]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":27},{"year":2020,"cited_by_count":19},{"year":2019,"cited_by_count":15},{"year":2018,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2018-12-11T00:00:00"}