{"id":"https://openalex.org/W2953027140","doi":"https://doi.org/10.1609/aaai.v31i1.10633","title":"On Human Intellect and Machine Failures: Troubleshooting Integrative Machine Learning Systems","display_name":"On Human Intellect and Machine Failures: Troubleshooting Integrative Machine Learning Systems","alternate_titles":[],"publication_year":2017,"publication_date":"2017-02-12","ids":{"openalex":"https://openalex.org/W2953027140","doi":"https://doi.org/10.1609/aaai.v31i1.10633","mag":"2953027140"},"language":"en","primary_location":{"id":"pmh:oai:ojs.aaai.org:article/10633","doi":"https://doi.org/10.1609/aaai.v31i1.10633","is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/10633","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/10633/10492","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},"parent_works":[],"lineage":["https://openalex.org/W2953027140"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/10633/10492","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011998621","display_name":"Besmira Nushi","orcid":"https://orcid.org/0000-0002-7554-8586","observed_orcids":["https://orcid.org/0000-0002-7554-8586"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Nushi, Besmira","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028114802","display_name":"Ece Semiha Kamar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kamar, Ece","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043228682","display_name":"Eric Horvitz","orcid":"https://orcid.org/0000-0002-8823-0614","observed_orcids":["https://orcid.org/0000-0002-8823-0614"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Horvitz, Eric","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111504785","display_name":"Donald Kossmann","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kossmann, Donald","raw_affiliation_strings":["ETH Zurich and Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich and Microsoft Research","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.7149,"has_fulltext":true,"cited_by_count":79,"citation_normalized_percentile":{"value":0.98465658,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7749000191688538,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7749000191688538,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0843999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.04580000042915344,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"human-in-the-loop","score":0.8050000071525574},{"id":"https://openalex.org/keywords/blame-attribution","display_name":"blame attribution","score":0.7580000162124634},{"id":"https://openalex.org/keywords/error-propagation","display_name":"error propagation","score":0.9359999895095825},{"id":"https://openalex.org/keywords/component-based-systems","display_name":"component-based systems","score":0.3330000042915344},{"id":"https://openalex.org/keywords/human-computation","display_name":"human computation","score":0.722000002861023},{"id":"https://openalex.org/keywords/system-failure-diagnosis","display_name":"system failure diagnosis","score":0.6899999976158142},{"id":"https://openalex.org/keywords/design-guidelines","display_name":"design guidelines","score":0.3580000102519989}],"concepts":[{"id":"https://openalex.org/C147494362","wikidata":"https://www.wikidata.org/wiki/Q2078905","display_name":"Troubleshooting","level":2,"score":0.9594869613647461},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.753746747970581},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6902651190757751},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.6547856330871582},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.607749342918396},{"id":"https://openalex.org/C2780146897","wikidata":"https://www.wikidata.org/wiki/Q353284","display_name":"Intellect","level":2,"score":0.5896325707435608},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.5804105997085571},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5339431166648865},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4911007881164551},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.33571040630340576},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.13977006077766418},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.13197827339172363},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C27206212","wikidata":"https://www.wikidata.org/wiki/Q34178","display_name":"Theology","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:ojs.aaai.org:article/10633","doi":"https://doi.org/10.1609/aaai.v31i1.10633","is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/10633","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/10633/10492","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:ojs.aaai.org:article/10633","doi":"https://doi.org/10.1609/aaai.v31i1.10633","is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/10633","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/10633/10492","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2953027140.pdf","grobid_xml":"https://content.openalex.org/works/W2953027140.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W1697372989","https://openalex.org/W1858383477","https://openalex.org/W1861492603","https://openalex.org/W1931639407","https://openalex.org/W1956340063","https://openalex.org/W1962925619","https://openalex.org/W1981064999","https://openalex.org/W2090645352","https://openalex.org/W2094485533","https://openalex.org/W2096175520","https://openalex.org/W2101105183","https://openalex.org/W2123301721","https://openalex.org/W2125943921","https://openalex.org/W2165254944","https://openalex.org/W2166010828","https://openalex.org/W2184059664","https://openalex.org/W2189162242","https://openalex.org/W2197829151","https://openalex.org/W2251079237","https://openalex.org/W2267310606","https://openalex.org/W2310148953","https://openalex.org/W2398018337","https://openalex.org/W2524766041","https://openalex.org/W2558644883","https://openalex.org/W2735365685","https://openalex.org/W2903158431","https://openalex.org/W7098444394"],"related_works":["https://openalex.org/W2883468834","https://openalex.org/W2810918175","https://openalex.org/W2948038809","https://openalex.org/W3010264423","https://openalex.org/W4311887678","https://openalex.org/W7168406992","https://openalex.org/W2963678934","https://openalex.org/W4409348652","https://openalex.org/W2945956065"],"abstract_inverted_index":{"We":[0,53,102],"study":[1],"the":[2,37,78,82,86,100,104,107],"problem":[3],"of":[4,14,85,106],"troubleshooting":[5,63],"machine":[6],"learning":[7],"systems":[8,26],"that":[9,21,115],"rely":[10],"on":[11,109],"analytical":[12],"pipelines":[13],"distinct":[15],"components.":[16],"Understanding":[17],"and":[18,76],"fixing":[19],"errors":[20,41],"arise":[22],"in":[23,36,81],"such":[24],"integrative":[25],"is":[27],"difficult":[28],"as":[29],"failures":[30],"can":[31,42,97],"occur":[32],"at":[33],"multiple":[34],"points":[35],"execution":[38],"workflow.":[39],"Moreover,":[40],"propagate,":[43],"become":[44],"amplified":[45],"or":[46],"be":[47],"suppressed,":[48],"making":[49],"blame":[50],"assignment":[51],"difficult.":[52],"propose":[54],"a":[55],"human-in-the-loop":[56],"methodology":[57],"which":[58],"leverages":[59],"human":[60,73],"intellect":[61],"for":[62],"system":[64,114],"failures.":[65],"The":[66,88],"approach":[67,108],"simulates":[68],"potential":[69],"component":[70],"fixes":[71],"through":[72],"computation":[74],"tasks":[75],"measures":[77],"expected":[79],"improvements":[80],"holistic":[83],"behavior":[84],"system.":[87,101],"method":[89],"provides":[90],"guidance":[91],"to":[92],"designers":[93],"about":[94],"how":[95],"they":[96],"best":[98],"improve":[99],"demonstrate":[103],"effectiveness":[105],"an":[110],"automated":[111],"image":[112],"captioning":[113],"has":[116],"been":[117],"pressed":[118],"into":[119],"real-world":[120],"use.":[121]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2953027140","counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":8},{"year":2017,"cited_by_count":2}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2019-06-27T00:00:00"}
