{"id":"https://openalex.org/W2788362053","doi":"https://doi.org/10.1145/3411764.3445315","title":"Manipulating and Measuring Model Interpretability","display_name":"Manipulating and Measuring Model Interpretability","publication_year":2021,"publication_date":"2021-05-06","ids":{"openalex":"https://openalex.org/W2788362053","doi":"https://doi.org/10.1145/3411764.3445315","mag":"2788362053"},"language":"en","primary_location":{"id":"doi:10.1145/3411764.3445315","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3411764.3445315","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1802.07810","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050672404","display_name":"Forough Poursabzi-Sangdeh","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I1321014770","display_name":"Association for Computing Machinery","ror":"https://ror.org/03wsadn68","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1321014770"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Forough Poursabzi-Sangdeh","raw_affiliation_strings":["Microsoft Research, United States","Association for Computing Machinery"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, United States","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Association for Computing Machinery","institution_ids":["https://openalex.org/I1321014770"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007746083","display_name":"Daniel G. Goldstein","orcid":"https://orcid.org/0000-0002-0970-5598"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel G Goldstein","raw_affiliation_strings":["Microsoft Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081429241","display_name":"Jake M. Hofman","orcid":"https://orcid.org/0000-0002-9364-9604"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jake M Hofman","raw_affiliation_strings":["Microsoft Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043117896","display_name":"Jennifer Wortman Vaughan","orcid":"https://orcid.org/0000-0002-7807-2018"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jennifer Wortman Wortman Vaughan","raw_affiliation_strings":["Microsoft Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046348432","display_name":"Hanna Wallach","orcid":"https://orcid.org/0000-0003-3395-7186"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanna Wallach","raw_affiliation_strings":["Microsoft Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, United States","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":12.7339,"has_fulltext":true,"cited_by_count":193,"citation_normalized_percentile":{"value":0.98954288,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"52"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10315","display_name":"Decision-Making and Behavioral Economics","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/1800","display_name":"General Decision Sciences"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/counterintuitive","display_name":"Counterintuitive","score":0.896472692489624},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.8923184871673584},{"id":"https://openalex.org/keywords/mistake","display_name":"Mistake","score":0.7164826989173889},{"id":"https://openalex.org/keywords/intuition","display_name":"Intuition","score":0.6287024021148682},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5888861417770386},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5820247530937195},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5810654163360596},{"id":"https://openalex.org/keywords/multiple-models","display_name":"Multiple Models","score":0.5211710333824158},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.4630427956581116},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.37399527430534363},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.28626567125320435},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.16973057389259338},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.09832942485809326}],"concepts":[{"id":"https://openalex.org/C101097943","wikidata":"https://www.wikidata.org/wiki/Q5176983","display_name":"Counterintuitive","level":2,"score":0.896472692489624},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.8923184871673584},{"id":"https://openalex.org/C2777179996","wikidata":"https://www.wikidata.org/wiki/Q911222","display_name":"Mistake","level":2,"score":0.7164826989173889},{"id":"https://openalex.org/C132010649","wikidata":"https://www.wikidata.org/wiki/Q189222","display_name":"Intuition","level":2,"score":0.6287024021148682},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5888861417770386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5820247530937195},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5810654163360596},{"id":"https://openalex.org/C2779714256","wikidata":"https://www.wikidata.org/wiki/Q25305062","display_name":"Multiple Models","level":2,"score":0.5211710333824158},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.4630427956581116},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.37399527430534363},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.28626567125320435},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.16973057389259338},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.09832942485809326},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3411764.3445315","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3411764.3445315","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1802.07810","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1802.07810","pdf_url":"https://arxiv.org/pdf/1802.07810","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:2788362053","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1802.07810","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1802.07810","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1802.07810","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1802.07810","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1802.07810","pdf_url":"https://arxiv.org/pdf/1802.07810","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.7799999713897705}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2788362053.pdf","grobid_xml":"https://content.openalex.org/works/W2788362053.grobid-xml"},"referenced_works_count":118,"referenced_works":["https://openalex.org/W158727920","https://openalex.org/W1508884412","https://openalex.org/W1557060895","https://openalex.org/W1951724000","https://openalex.org/W1984763500","https://openalex.org/W1985898303","https://openalex.org/W1990229920","https://openalex.org/W1994606570","https://openalex.org/W1995598019","https://openalex.org/W1995902760","https://openalex.org/W1996796871","https://openalex.org/W2004010223","https://openalex.org/W2012218966","https://openalex.org/W2013587512","https://openalex.org/W2014358849","https://openalex.org/W2025486241","https://openalex.org/W2025544150","https://openalex.org/W2031363520","https://openalex.org/W2033861818","https://openalex.org/W2038224823","https://openalex.org/W2041545805","https://openalex.org/W2046945713","https://openalex.org/W2047021735","https://openalex.org/W2056418346","https://openalex.org/W2064288400","https://openalex.org/W2065267227","https://openalex.org/W2070125214","https://openalex.org/W2072883562","https://openalex.org/W2073394394","https://openalex.org/W2075807870","https://openalex.org/W2083780116","https://openalex.org/W2099549054","https://openalex.org/W2101234009","https://openalex.org/W2104546629","https://openalex.org/W2112195476","https://openalex.org/W2115281393","https://openalex.org/W2117342634","https://openalex.org/W2126620275","https://openalex.org/W2135880665","https://openalex.org/W2137872373","https://openalex.org/W2140553307","https://openalex.org/W2141474346","https://openalex.org/W2141708418","https://openalex.org/W2149033360","https://openalex.org/W2154157725","https://openalex.org/W2158570360","https://openalex.org/W2159071801","https://openalex.org/W2164878629","https://openalex.org/W2168123127","https://openalex.org/W2168729696","https://openalex.org/W2223821166","https://openalex.org/W2225340466","https://openalex.org/W2282821441","https://openalex.org/W2296194912","https://openalex.org/W2301613449","https://openalex.org/W2337934060","https://openalex.org/W2341535507","https://openalex.org/W2367397349","https://openalex.org/W2439568532","https://openalex.org/W2551317447","https://openalex.org/W2588168955","https://openalex.org/W2594475271","https://openalex.org/W2597603852","https://openalex.org/W2603145185","https://openalex.org/W2605807262","https://openalex.org/W2611586694","https://openalex.org/W2724999822","https://openalex.org/W2765204106","https://openalex.org/W2774522520","https://openalex.org/W2786004891","https://openalex.org/W2786242872","https://openalex.org/W2794471005","https://openalex.org/W2795636034","https://openalex.org/W2795737619","https://openalex.org/W2796133875","https://openalex.org/W2796410451","https://openalex.org/W2800068874","https://openalex.org/W2809002738","https://openalex.org/W2891340972","https://openalex.org/W2894881080","https://openalex.org/W2897955056","https://openalex.org/W2898970033","https://openalex.org/W2903145273","https://openalex.org/W2909392392","https://openalex.org/W2918497321","https://openalex.org/W2923421605","https://openalex.org/W2942073295","https://openalex.org/W2942444880","https://openalex.org/W2945976633","https://openalex.org/W2959587146","https://openalex.org/W2962862931","https://openalex.org/W2963095307","https://openalex.org/W2963326959","https://openalex.org/W2963795072","https://openalex.org/W2964031043","https://openalex.org/W2964125683","https://openalex.org/W2964134873","https://openalex.org/W2964240233","https://openalex.org/W2973631113","https://openalex.org/W2974075815","https://openalex.org/W2997560917","https://openalex.org/W2999765337","https://openalex.org/W3009986018","https://openalex.org/W3016099278","https://openalex.org/W3028689275","https://openalex.org/W3085162807","https://openalex.org/W3098054859","https://openalex.org/W3102610420","https://openalex.org/W3104310207","https://openalex.org/W3121705224","https://openalex.org/W3121921079","https://openalex.org/W3122175177","https://openalex.org/W3123895079","https://openalex.org/W3125633690","https://openalex.org/W4288414189","https://openalex.org/W4289097366","https://openalex.org/W4296978576","https://openalex.org/W6703949738"],"related_works":["https://openalex.org/W2594475271","https://openalex.org/W2282821441","https://openalex.org/W2963095307","https://openalex.org/W2962790223","https://openalex.org/W2962862931","https://openalex.org/W2962772482","https://openalex.org/W2891503716","https://openalex.org/W2963847595","https://openalex.org/W2962851944","https://openalex.org/W1996796871","https://openalex.org/W2945976633","https://openalex.org/W2942157335","https://openalex.org/W2367397349","https://openalex.org/W2901895173","https://openalex.org/W3122175177","https://openalex.org/W2785327160","https://openalex.org/W2891612330","https://openalex.org/W2795530988","https://openalex.org/W2493343568","https://openalex.org/W1849277567"],"abstract_inverted_index":{"With":[0],"machine":[1,108],"learning":[2,109],"models":[3,29,43,97,110],"being":[4],"increasingly":[5],"used":[6],"to":[7,64,70,106,175,186],"aid":[8],"decision":[9],"making":[10,50],"even":[11],"in":[12,21,90,101],"high-stakes":[13],"domains,":[14],"there":[15,33],"has":[16,75],"been":[17,31,35],"a":[18,55,73,77,81,139,166],"growing":[19],"interest":[20],"developing":[22,200],"interpretable":[23,28,201],"models.":[24,202],"Although":[25],"many":[26],"supposedly":[27],"have":[30,34],"proposed,":[32],"relatively":[36],"few":[37,143],"experimental":[38],"studies":[39],"investigating":[40],"whether":[41,126],"these":[42],"achieve":[44],"their":[45],"intended":[46],"effects,":[47],"such":[48],"as":[49],"people":[51],"more":[52,111,158],"closely":[53,159],"follow":[54],"model\u2019s":[56,149,181],"predictions":[57],"when":[58,72,199],"it":[59],"is":[60],"beneficial":[61],"for":[62,179],"them":[63,69],"do":[65],"so":[66],"or":[67,112,132],"enabling":[68],"detect":[71,176],"model":[74,124,128,141,168],"made":[76],"mistake.":[78],"We":[79],"present":[80],"sequence":[82],"of":[83,117,122,195],"pre-registered":[84],"experiments":[85],"(N":[86],"=":[87],"3,":[88],"800)":[89],"which":[91],"we":[92,152],"showed":[93],"participants":[94,136,157,165],"functionally":[95],"identical":[96],"that":[98,156,170],"varied":[99],"only":[100],"two":[102],"factors":[103],"commonly":[104],"thought":[105],"make":[107],"less":[113,173],"interpretable:":[114],"the":[115,120,123,127,148,180,193],"number":[116],"features":[118,144],"and":[119,177],"transparency":[121],"(i.e.,":[125],"internals":[129],"are":[130],"clear":[131,140,167],"black":[133],"box).":[134],"Predictably,":[135],"who":[137],"saw":[138],"with":[142],"could":[145],"better":[146],"simulate":[147],"predictions.":[150,162],"However,":[151],"did":[153],"not":[154],"find":[155],"followed":[160],"its":[161],"Furthermore,":[163],"showing":[164],"meant":[169],"they":[171],"were":[172],"able":[174],"correct":[178],"sizable":[182],"mistakes,":[183],"seemingly":[184],"due":[185],"information":[187],"overload.":[188],"These":[189],"counterintuitive":[190],"findings":[191],"emphasize":[192],"importance":[194],"testing":[196],"over":[197],"intuition":[198]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":56},{"year":2020,"cited_by_count":51},{"year":2019,"cited_by_count":37},{"year":2018,"cited_by_count":9}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}