{"id":"https://openalex.org/W2430018012","doi":"https://doi.org/10.1145/2939502.2939515","title":"Data programming with DDLite","display_name":"Data programming with DDLite","publication_year":2016,"publication_date":"2016-06-17","ids":{"openalex":"https://openalex.org/W2430018012","doi":"https://doi.org/10.1145/2939502.2939515","mag":"2430018012"},"language":"en","primary_location":{"id":"doi:10.1145/2939502.2939515","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2939502.2939515","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2939515&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Workshop on Human-In-the-Loop Data Analytics","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2939515&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079833242","display_name":"Henry R. Ehrenberg","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Henry R. Ehrenberg","raw_affiliation_strings":["Stanford University, Stanford, California"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101727253","display_name":"Jaeho Shin","orcid":"https://orcid.org/0000-0001-5280-3356"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaeho Shin","raw_affiliation_strings":["Stanford University, Stanford, California"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076215229","display_name":"Alexander Ratner","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander J. Ratner","raw_affiliation_strings":["Stanford University, Stanford, California"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028277225","display_name":"Jason Fries","orcid":"https://orcid.org/0000-0001-9316-5768"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jason A. Fries","raw_affiliation_strings":["Stanford University, Stanford, California"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103852640","display_name":"Christopher R\u00e9","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher R\u00e9","raw_affiliation_strings":["Stanford University, Stanford, California"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5079833242"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":2.1025,"has_fulltext":true,"cited_by_count":21,"citation_normalized_percentile":{"value":0.86373955,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13280","display_name":"Biomedical and Engineering Education","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13280","display_name":"Biomedical and Engineering Education","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13937","display_name":"Genetics, Bioinformatics, and Biomedical Research","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12859","display_name":"Cell Image Analysis Techniques","score":0.9628999829292297,"subfield":{"id":"https://openalex.org/subfields/1304","display_name":"Biophysics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8580799698829651},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.6341986656188965},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.5704432725906372},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.46358203887939453},{"id":"https://openalex.org/keywords/feature-engineering","display_name":"Feature engineering","score":0.4348275661468506},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4303004741668701},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.4296647012233734},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4275444746017456},{"id":"https://openalex.org/keywords/information-extraction","display_name":"Information extraction","score":0.4260568916797638},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4236254394054413},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.42117840051651},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.4128803610801697},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3878934383392334},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3816903233528137},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.37203142046928406},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3678012192249298},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.16831639409065247},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.16385528445243835}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8580799698829651},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.6341986656188965},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.5704432725906372},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.46358203887939453},{"id":"https://openalex.org/C2778827112","wikidata":"https://www.wikidata.org/wiki/Q22245680","display_name":"Feature engineering","level":3,"score":0.4348275661468506},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4303004741668701},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.4296647012233734},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4275444746017456},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.4260568916797638},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4236254394054413},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.42117840051651},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.4128803610801697},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3878934383392334},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3816903233528137},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.37203142046928406},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3678012192249298},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.16831639409065247},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.16385528445243835},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2939502.2939515","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2939502.2939515","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2939515&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Workshop on Human-In-the-Loop Data Analytics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/2939502.2939515","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2939502.2939515","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2939515&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Workshop on Human-In-the-Loop Data Analytics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5400000214576721,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G1659159352","display_name":null,"funder_award_id":"SIMPLEX","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G1869467075","display_name":null,"funder_award_id":"N000141210041","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G367213561","display_name":null,"funder_award_id":"N66001-15-C-4043","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G3858477398","display_name":null,"funder_award_id":"IIS-1353606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G618697994","display_name":null,"funder_award_id":"N000141310129, N000141210041","funder_id":"https://openalex.org/F4320338298","funder_display_name":"Office of Naval Research Global"},{"id":"https://openalex.org/G6331000141","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320306202","funder_display_name":"Gordon and Betty Moore Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G674549868","display_name":null,"funder_award_id":"N000141310129","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G6894402473","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7852225248","display_name":"CAREER:  A Scalable, Declarative, Imprecise Database Management System","funder_award_id":"1353606","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8276531783","display_name":null,"funder_award_id":"N000141310129","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306151","display_name":"Alfred P. Sloan Foundation","ror":"https://ror.org/052csg198"},{"id":"https://openalex.org/F4320306202","display_name":"Gordon and Betty Moore Foundation","ror":"https://ror.org/006wxqw41"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320338298","display_name":"Office of Naval Research Global","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2430018012.pdf","grobid_xml":"https://content.openalex.org/works/W2430018012.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W1558632383","https://openalex.org/W1788418780","https://openalex.org/W1934084512","https://openalex.org/W1964670939","https://openalex.org/W2052569640","https://openalex.org/W2063280109","https://openalex.org/W2098679902","https://openalex.org/W2099102906","https://openalex.org/W2178441628","https://openalex.org/W2257876130","https://openalex.org/W2404161646","https://openalex.org/W2406996511","https://openalex.org/W6692126448"],"related_works":["https://openalex.org/W4321442002","https://openalex.org/W2015265939","https://openalex.org/W2284072287","https://openalex.org/W2611067230","https://openalex.org/W2480201319","https://openalex.org/W2387706296","https://openalex.org/W2155788121","https://openalex.org/W4235469518","https://openalex.org/W2294325978","https://openalex.org/W2890345561"],"abstract_inverted_index":{"Populating":[0],"large-scale":[1],"structured":[2],"databases":[3],"from":[4,145,159],"unstructured":[5],"sources":[6],"is":[7,123],"a":[8,49,124,149,200],"critical":[9],"and":[10,90,108,119,175,219],"challenging":[11],"task":[12],"in":[13,35,60,163,198],"data":[14,53,62,138,230],"analytics.":[15],"As":[16],"automated":[17],"feature":[18],"engineering":[19],"methods":[20,89],"grow":[21],"increasingly":[22],"prevalent,":[23],"constructing":[24],"sufficiently":[25],"large":[26,67],"labeled":[27,95],"training":[28,71,221],"sets":[29,215],"has":[30],"become":[31],"the":[32,61,113,184,196,209],"primary":[33],"hurdle":[34],"building":[36],"machine":[37],"learning":[38],"information":[39,169],"extraction":[40,154,170],"systems.":[41],"In":[42],"light":[43],"of":[44,69,127,152,195,211,228],"this,":[45],"we":[46,97,130],"have":[47,98],"taken":[48],"new":[50],"approach":[51,83],"called":[52],"programming":[54,63,231],"[7].":[55],"Rather":[56],"than":[57],"hand-labeling":[58],"data,":[59,96],"paradigm,":[64,129],"users":[65,147],"generate":[66],"amounts":[68],"noisy":[70],"labels":[72],"by":[73],"programmatically":[74],"encoding":[75],"domain":[76],"heuristics":[77],"as":[78],"simple":[79],"rules.":[80],"Using":[81],"this":[82,128],"over":[84],"more":[85,106],"traditional":[86],"distant":[87],"supervision":[88],"fully":[91],"supervised":[92],"approaches":[93],"using":[94],"been":[99],"able":[100],"to":[101,115],"construct":[102],"knowledge":[103],"base":[104],"systems":[105],"rapidly":[107],"with":[109,183],"higher":[110],"quality.":[111],"Since":[112],"ability":[114],"quickly":[116],"prototype,":[117],"evaluate,":[118],"debug":[120],"these":[121],"rules":[122],"key":[125,206],"component":[126],"introduce":[131],"DDLite,":[132],"an":[133,189],"interactive":[134],"development":[135],"framework":[136],"for":[137,172,216],"programming.":[139],"This":[140],"paper":[141],"reports":[142],"feedback":[143],"collected":[144],"DDLite":[146,161],"across":[148],"diverse":[150,213],"set":[151],"entity":[153],"tasks.":[155],"We":[156],"share":[157],"observations":[158],"several":[160,226],"hackathons":[162],"which":[164],"10":[165,193],"biomedical":[166],"researchers":[167],"prototyped":[168],"pipelines":[171],"chemicals,":[173],"diseases,":[174],"anatomical":[176],"named":[177],"entities.":[178],"Initial":[179],"results":[180],"were":[181],"promising,":[182],"disease":[185],"tagging":[186],"team":[187],"obtaining":[188],"F1":[190],"score":[191],"within":[192],"points":[194],"state-of-the-art":[197],"only":[199],"single":[201],"day-long":[202],"hackathon's":[203],"work.":[204],"Our":[205],"insights":[207],"concern":[208],"challenges":[210],"writing":[212],"rule":[214],"generating":[217],"labels,":[218],"exploring":[220],"data.":[222],"These":[223],"findings":[224],"motivate":[225],"areas":[227],"active":[229],"research.":[232]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":6},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":2}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}