{"id":"https://openalex.org/W3160349922","doi":"https://doi.org/10.1145/3411764.3445538","title":"UMLAUT: Debugging Deep Learning Programs using Program Structure and Model Behavior","display_name":"UMLAUT: Debugging Deep Learning Programs using Program Structure and Model Behavior","publication_year":2021,"publication_date":"2021-05-06","ids":{"openalex":"https://openalex.org/W3160349922","doi":"https://doi.org/10.1145/3411764.3445538","mag":"3160349922"},"language":"en","primary_location":{"id":"doi:10.1145/3411764.3445538","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445538","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445538","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445538","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029531613","display_name":"Eldon Schoop","orcid":"https://orcid.org/0000-0001-8951-2878"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Eldon Schoop","raw_affiliation_strings":["Berkeley Institute of Design University of California, Berkeley, United States"],"affiliations":[{"raw_affiliation_string":"Berkeley Institute of Design University of California, Berkeley, United States","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039176173","display_name":"Forrest Huang","orcid":"https://orcid.org/0009-0001-3345-5373"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Forrest Huang","raw_affiliation_strings":["University of California, Berkeley, United States"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087718561","display_name":"Bjoern Hartmann","orcid":"https://orcid.org/0000-0002-0693-0829"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bjoern Hartmann","raw_affiliation_strings":["EECS UC Berkeley, United States"],"affiliations":[{"raw_affiliation_string":"EECS UC Berkeley, United States","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5029531613"],"corresponding_institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"],"apc_list":null,"apc_paid":null,"fwci":9.6758,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":{"value":0.97977328,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"16"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8871479630470276},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.8555972576141357},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.7368572354316711},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6027036905288696},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5186021327972412},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5157061219215393},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.48114335536956787},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.4405503571033478},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4218841791152954},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3774084448814392},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.07702270150184631}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8871479630470276},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.8555972576141357},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.7368572354316711},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6027036905288696},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5186021327972412},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5157061219215393},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.48114335536956787},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.4405503571033478},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4218841791152954},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3774084448814392},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.07702270150184631},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3411764.3445538","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445538","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445538","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3411764.3445538","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3411764.3445538","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3411764.3445538","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.8500000238418579,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3160349922.pdf","grobid_xml":"https://content.openalex.org/works/W3160349922.grobid-xml"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W110870703","https://openalex.org/W1677182931","https://openalex.org/W1982001334","https://openalex.org/W2002530879","https://openalex.org/W2003238113","https://openalex.org/W2020796004","https://openalex.org/W2064893403","https://openalex.org/W2099471712","https://openalex.org/W2106411961","https://openalex.org/W2108816886","https://openalex.org/W2111642396","https://openalex.org/W2137406659","https://openalex.org/W2138350998","https://openalex.org/W2139374478","https://openalex.org/W2161052636","https://openalex.org/W2165254944","https://openalex.org/W2194775991","https://openalex.org/W2197966456","https://openalex.org/W2402144811","https://openalex.org/W2432911982","https://openalex.org/W2504150216","https://openalex.org/W2535451242","https://openalex.org/W2552408584","https://openalex.org/W2624570621","https://openalex.org/W2750384547","https://openalex.org/W2766094787","https://openalex.org/W2850992922","https://openalex.org/W2884426148","https://openalex.org/W2914209329","https://openalex.org/W2919115771","https://openalex.org/W2922234936","https://openalex.org/W2923421605","https://openalex.org/W2927610422","https://openalex.org/W2941232686","https://openalex.org/W2947012480","https://openalex.org/W2947132278","https://openalex.org/W2956281901","https://openalex.org/W2963214037","https://openalex.org/W2963240573","https://openalex.org/W2963809228","https://openalex.org/W2963847595","https://openalex.org/W2964121744","https://openalex.org/W2968594320","https://openalex.org/W2972278312","https://openalex.org/W2978190445","https://openalex.org/W3005940936","https://openalex.org/W3006436762","https://openalex.org/W3029367893","https://openalex.org/W3030054017","https://openalex.org/W3049119995","https://openalex.org/W3090643686","https://openalex.org/W3098350627","https://openalex.org/W3100269046","https://openalex.org/W3101330584","https://openalex.org/W3118608800","https://openalex.org/W4245551996"],"related_works":["https://openalex.org/W2165912799","https://openalex.org/W4321442002","https://openalex.org/W2015265939","https://openalex.org/W2284072287","https://openalex.org/W2735662278","https://openalex.org/W2611067230","https://openalex.org/W2382615723","https://openalex.org/W2480201319","https://openalex.org/W4405331580","https://openalex.org/W4311804456"],"abstract_inverted_index":{"Training":[0],"deep":[1],"neural":[2],"networks":[3],"can":[4],"generate":[5],"non-descriptive":[6],"error":[7,104,115,124],"messages":[8,105,125],"or":[9],"produce":[10],"unusual":[11],"output":[12,37,112],"without":[13],"any":[14],"explicit":[15],"errors":[16,65,149],"at":[17],"all.":[18],"While":[19],"experts":[20],"rely":[21],"on":[22],"tacit":[23],"knowledge":[24],"to":[25,34,85,106,113,139,166,173],"apply":[26],"debugging":[27,50],"strategies,":[28],"non-experts":[29],"lack":[30],"the":[31,62,87],"experience":[32],"required":[33],"interpret":[35],"model":[36,97,111,120],"and":[38,52,73,96,108,122,147,158,163],"correct":[39],"Deep":[40],"Learning":[41],"(DL)":[42],"programs.":[43,153],"In":[44],"this":[45,58],"work,":[46,59],"we":[47,60],"identify":[48],"DL":[49,93,152],"heuristics":[51],"strategies":[53],"used":[54],"by":[55],"experts,":[56],"andIn":[57],"categorize":[61],"types":[63],"of":[64,89],"novices":[66],"run":[67],"into":[68],"when":[69],"writing":[70],"ML":[71],"code,":[72,119],"map":[74],"them":[75,84],"onto":[76],"opportunities":[77],"where":[78],"tools":[79],"could":[80],"help.":[81],"We":[82,130],"use":[83],"guide":[86],"design":[88],"Umlaut.":[90],"Umlaut":[91,117,132,156],"checks":[92],"program":[94],"structure":[95],"behavior":[98],"against":[99],"these":[100],"heuristics;":[101],"provides":[102],"human-readable":[103],"users;":[107],"annotates":[109],"erroneous":[110],"facilitate":[114],"correction.":[116],"links":[118],"output,":[121],"tutorial-driven":[123],"in":[126,133,143,150],"a":[127,134,174],"single":[128],"interface.":[129],"evaluated":[131],"study":[135],"with":[136],"15":[137],"participants":[138],"determine":[140],"its":[141],"effectiveness":[142],"helping":[144],"developers":[145],"find":[146],"fix":[148],"their":[151],"Participants":[154],"using":[155],"found":[157],"fixed":[159],"significantly":[160],"more":[161,170],"bugs":[162,171],"were":[164],"able":[165],"implement":[167],"fixes":[168],"for":[169],"compared":[172],"baseline":[175],"condition.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":8}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}