{"id":"https://openalex.org/W3019489177","doi":"https://doi.org/10.1145/3392878","title":"Human Factors in Model Interpretability: Industry Practices, Challenges, and Needs","display_name":"Human Factors in Model Interpretability: Industry Practices, Challenges, and Needs","publication_year":2020,"publication_date":"2020-05-28","ids":{"openalex":"https://openalex.org/W3019489177","doi":"https://doi.org/10.1145/3392878","mag":"3019489177"},"language":"en","primary_location":{"id":"doi:10.1145/3392878","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3392878","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2004.11440","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059548204","display_name":"Sung-Soo Hong","orcid":"https://orcid.org/0000-0001-6050-5404"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sungsoo Ray Hong","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068008545","display_name":"Jessica Hullman","orcid":"https://orcid.org/0000-0001-6826-3550"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jessica Hullman","raw_affiliation_strings":["Northwestern University, Evanston, IL, USA"],"affiliations":[{"raw_affiliation_string":"Northwestern University, Evanston, IL, USA","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102923390","display_name":"Enrico Bertini","orcid":"https://orcid.org/0000-0002-9932-0551"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Enrico Bertini","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5059548204"],"corresponding_institution_ids":["https://openalex.org/I57206974"],"apc_list":null,"apc_paid":null,"fwci":15.4949,"has_fulltext":false,"cited_by_count":207,"citation_normalized_percentile":{"value":0.99240772,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"4","issue":"CSCW1","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9686999917030334,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13051","display_name":"Qualitative Comparative Analysis Research","score":0.9470999836921692,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.9847977161407471},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.5935073494911194},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5438871383666992},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.5216693878173828},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5119632482528687},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5053244233131409},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.4560525119304657},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3421128988265991},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1484348177909851}],"concepts":[{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.9847977161407471},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.5935073494911194},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5438871383666992},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.5216693878173828},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5119632482528687},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5053244233131409},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4560525119304657},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3421128988265991},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1484348177909851},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3392878","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3392878","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2004.11440","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.11440","pdf_url":"https://arxiv.org/pdf/2004.11440","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2004.11440","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.11440","pdf_url":"https://arxiv.org/pdf/2004.11440","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":76,"referenced_works":["https://openalex.org/W198744195","https://openalex.org/W968305609","https://openalex.org/W1514359208","https://openalex.org/W1678356000","https://openalex.org/W1996796871","https://openalex.org/W2018754013","https://openalex.org/W2044102377","https://openalex.org/W2053075547","https://openalex.org/W2065637266","https://openalex.org/W2074694452","https://openalex.org/W2076618162","https://openalex.org/W2113882472","https://openalex.org/W2118022153","https://openalex.org/W2121613990","https://openalex.org/W2126034719","https://openalex.org/W2127058057","https://openalex.org/W2149868499","https://openalex.org/W2157928966","https://openalex.org/W2159793693","https://openalex.org/W2184059664","https://openalex.org/W2210027199","https://openalex.org/W2237502368","https://openalex.org/W2248060815","https://openalex.org/W2282821441","https://openalex.org/W2292312835","https://openalex.org/W2394669110","https://openalex.org/W2395138788","https://openalex.org/W2398344594","https://openalex.org/W2439568532","https://openalex.org/W2512274390","https://openalex.org/W2551974706","https://openalex.org/W2585669920","https://openalex.org/W2594475271","https://openalex.org/W2607223307","https://openalex.org/W2611586694","https://openalex.org/W2615497679","https://openalex.org/W2750068734","https://openalex.org/W2751777857","https://openalex.org/W2753713840","https://openalex.org/W2785327160","https://openalex.org/W2788362053","https://openalex.org/W2788403449","https://openalex.org/W2806259446","https://openalex.org/W2807910285","https://openalex.org/W2808967136","https://openalex.org/W2883424428","https://openalex.org/W2886614482","https://openalex.org/W2889730816","https://openalex.org/W2900541795","https://openalex.org/W2902011388","https://openalex.org/W2902255491","https://openalex.org/W2902673180","https://openalex.org/W2905034244","https://openalex.org/W2913346170","https://openalex.org/W2914854991","https://openalex.org/W2923421605","https://openalex.org/W2942161347","https://openalex.org/W2962772482","https://openalex.org/W2963095307","https://openalex.org/W2963438392","https://openalex.org/W2963847595","https://openalex.org/W2979893369","https://openalex.org/W2987755095","https://openalex.org/W2988450373","https://openalex.org/W3003728721","https://openalex.org/W3016099278","https://openalex.org/W3029504795","https://openalex.org/W3099361686","https://openalex.org/W3122753382","https://openalex.org/W3124792046","https://openalex.org/W3125751566","https://openalex.org/W3210030168","https://openalex.org/W4281564584","https://openalex.org/W4288086169","https://openalex.org/W4288359825","https://openalex.org/W4301225272"],"related_works":["https://openalex.org/W2905433371","https://openalex.org/W4390569940","https://openalex.org/W2888392564","https://openalex.org/W4361193272","https://openalex.org/W4310278675","https://openalex.org/W4388422664","https://openalex.org/W2806259446","https://openalex.org/W2963326959","https://openalex.org/W4312407344","https://openalex.org/W2894289927"],"abstract_inverted_index":{"As":[0],"the":[1,66,192,203,216],"use":[2,120,146],"of":[3,68,74,76,110,128,147,152],"machine":[4],"learning":[5],"(ML)":[6],"models":[7,186],"in":[8,17,39,45,65,173,209,215],"product":[9],"development":[10],"and":[11,51,60,111,119,137,167,185,212],"data-driven":[12],"decision-making":[13],"processes":[14],"became":[15],"pervasive":[16],"many":[18],"domains,":[19],"people's":[20],"focus":[21],"on":[22,124],"building":[23,179],"a":[24,125],"well-performing":[25],"model":[26,34,40,162,169],"has":[27,42],"increasingly":[28],"shifted":[29],"to":[30,62,90,105],"understanding":[31,75],"how":[32,57,107],"their":[33,69,121,210],"works.":[35],"While":[36],"scholarly":[37],"interest":[38],"interpretability":[41,64,77,82,114,133,153,163,204],"grown":[43],"rapidly":[44],"research":[46,83,220],"communities":[47],"like":[48],"HCI,":[49],"ML,":[50],"beyond,":[52],"little":[53],"is":[54],"known":[55],"about":[56],"practitioners":[58,104,207],"perceive":[59],"aim":[61],"provide":[63],"context":[67],"existing":[70],"workflows.":[71],"This":[72],"lack":[73],"as":[78,139],"practiced":[79],"may":[80],"prevent":[81],"from":[84,157],"addressing":[85],"important":[86],"needs,":[87],"or":[88],"lead":[89],"unrealistic":[91],"solutions.":[92],"To":[93],"bridge":[94],"this":[95],"gap,":[96],"we":[97,131],"conducted":[98],"22":[99],"semi-structured":[100],"interviews":[101],"with":[102],"industry":[103],"understand":[106],"they":[108,116,140],"conceive":[109],"design":[112,198],"for":[113,197],"while":[115],"plan,":[117],"build,":[118],"models.":[122,149],"Based":[123],"qualitative":[126],"analysis":[127,159],"our":[129,158],"results,":[130],"differentiate":[132],"roles,":[134,175],"processes,":[135],"goals":[136],"strategies":[138],"exist":[141],"within":[142,191],"organizations":[143],"making":[144],"heavy":[145],"ML":[148],"The":[150],"characterization":[151],"work":[154],"that":[155,161,199,206,222],"emerges":[156],"suggests":[160],"frequently":[164],"involves":[165],"cooperation":[166],"mental":[168],"comparison":[170],"between":[171,183,189,202],"people":[172,184,190],"different":[174],"often":[176],"aimed":[177],"at":[178],"trust":[180],"not":[181],"only":[182],"but":[187],"also":[188],"organization.":[193],"We":[194],"present":[195],"implications":[196],"discuss":[200],"gaps":[201],"challenges":[205],"face":[208],"practice":[211],"approaches":[213],"proposed":[214],"literature,":[217],"highlighting":[218],"possible":[219],"directions":[221],"can":[223],"better":[224],"address":[225],"real-world":[226],"needs.":[227]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":44},{"year":2024,"cited_by_count":40},{"year":2023,"cited_by_count":42},{"year":2022,"cited_by_count":30},{"year":2021,"cited_by_count":37},{"year":2020,"cited_by_count":5}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}