{"id":"https://openalex.org/W4387331308","doi":"https://doi.org/10.1145/3610064","title":"Eye into AI: Evaluating the Interpretability of Explainable AI Techniques through a Game with a Purpose","display_name":"Eye into AI: Evaluating the Interpretability of Explainable AI Techniques through a Game with a Purpose","publication_year":2023,"publication_date":"2023-09-28","ids":{"openalex":"https://openalex.org/W4387331308","doi":"https://doi.org/10.1145/3610064"},"language":"en","primary_location":{"id":"doi:10.1145/3610064","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610064","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610064","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3610064","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070514885","display_name":"Katelyn Morrison","orcid":"https://orcid.org/0000-0002-2644-4422"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Katelyn Morrison","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-2644-4422","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001669701","display_name":"Milind Jain","orcid":"https://orcid.org/0009-0006-0250-2022"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mayank Jain","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0009-0006-0250-2022","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000744718","display_name":"Jessica Hammer","orcid":"https://orcid.org/0000-0003-4102-2575"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jessica Hammer","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0003-4102-2575","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053097987","display_name":"Adam Perer","orcid":"https://orcid.org/0000-0002-8369-3847"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adam Perer","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-8369-3847","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":1.4276,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.84719101,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"7","issue":"CSCW2","first_page":"1","last_page":"22"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9746999740600586,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.9487473964691162},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7804726362228394},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.592940092086792},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5812573432922363},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.5606724619865417},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4861421287059784}],"concepts":[{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.9487473964691162},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7804726362228394},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.592940092086792},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5812573432922363},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.5606724619865417},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4861421287059784},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3610064","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610064","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610064","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3610064","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610064","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610064","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5928528767","display_name":null,"funder_award_id":"R01HL164906","funder_id":"https://openalex.org/F4320337338","funder_display_name":"National Heart, Lung, and Blood Institute"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337338","display_name":"National Heart, Lung, and Blood Institute","ror":"https://ror.org/012pb6c26"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387331308.pdf","grobid_xml":"https://content.openalex.org/works/W4387331308.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W1888870946","https://openalex.org/W1993471214","https://openalex.org/W2013894622","https://openalex.org/W2035683813","https://openalex.org/W2065267227","https://openalex.org/W2077805339","https://openalex.org/W2080942732","https://openalex.org/W2108598243","https://openalex.org/W2117539524","https://openalex.org/W2155034018","https://openalex.org/W2165254944","https://openalex.org/W2170989440","https://openalex.org/W2282821441","https://openalex.org/W2321348787","https://openalex.org/W2394669110","https://openalex.org/W2398344594","https://openalex.org/W2623610890","https://openalex.org/W2764324430","https://openalex.org/W2795530988","https://openalex.org/W2890938132","https://openalex.org/W2891503716","https://openalex.org/W2896010852","https://openalex.org/W2956281901","https://openalex.org/W2963095307","https://openalex.org/W2963214037","https://openalex.org/W2988450373","https://openalex.org/W2995188596","https://openalex.org/W2999637955","https://openalex.org/W3010300896","https://openalex.org/W3031898411","https://openalex.org/W3097001800","https://openalex.org/W3103751997","https://openalex.org/W3119394424","https://openalex.org/W3156106752","https://openalex.org/W3162291149","https://openalex.org/W3162428030","https://openalex.org/W3163443091","https://openalex.org/W3170542759","https://openalex.org/W3209014031","https://openalex.org/W4224291356","https://openalex.org/W4224319919","https://openalex.org/W4312666361","https://openalex.org/W6683176162"],"related_works":["https://openalex.org/W1986582023","https://openalex.org/W2961085424","https://openalex.org/W2966829450","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694"],"abstract_inverted_index":{"Recent":[0],"developments":[1],"in":[2],"explainable":[3],"AI":[4,169,186],"(XAI)":[5],"aim":[6],"to":[7,49,59,62,69,106,155,178,189],"improve":[8],"the":[9,17,127,147,190],"transparency":[10],"of":[11,19,110,149],"black-box":[12],"models.":[13],"However,":[14,72],"empirically":[15],"evaluating":[16,91,124,144],"interpretability":[18,148],"these":[20,46,73],"XAI":[21,67,92,111,132,152,182],"techniques":[22,47,68,133,153],"is":[23,33,58,187],"still":[24],"an":[25,38,116],"open":[26],"challenge.":[27],"The":[28,163],"most":[29],"common":[30,53],"evaluation":[31,56],"method":[32,89],"algorithmic":[34],"performance,":[35],"but":[36,54],"such":[37],"approach":[39],"may":[40,80],"not":[41],"accurately":[42],"represent":[43],"how":[44,120],"interpretable":[45],"are":[48],"people.":[50],"A":[51],"less":[52],"growing":[55],"strategy":[57],"leverage":[60],"crowd-workers":[61],"provide":[63,115],"feedback":[64],"on":[65],"multiple":[66],"compare":[70,181],"them.":[71],"tasks":[74],"often":[75],"feel":[76],"like":[77],"work":[78],"and":[79,125,136,139,145,180],"limit":[81],"participation.":[82],"We":[83,114],"propose":[84],"a":[85,94,97,156],"novel,":[86],"playful,":[87],"human-centered":[88],"for":[90,160],"techniques:":[93],"Game":[95],"With":[96],"Purpose":[98],"(GWAP),":[99],"Eye":[100,167,184],"into":[101,168,185],"AI,":[102],"that":[103,173],"allows":[104],"researchers":[105],"collect":[107],"human":[108],"evaluations":[109],"at":[112],"scale.":[113],"empirical":[117],"study":[118],"demonstrating":[119],"our":[121],"GWAP":[122],"supports":[123],"comparing":[126,146],"agreement":[128],"between":[129],"three":[130,151],"popular":[131],"(LIME,":[134],"Grad-CAM,":[135],"Feature":[137],"Visualization)":[138],"humans,":[140],"as":[141,143],"well":[142],"those":[150],"applied":[154],"deep":[157],"learning":[158],"model":[159],"image":[161],"classification.":[162],"data":[164],"collected":[165],"from":[166],"offers":[170],"convincing":[171],"evidence":[172],"GWAPs":[174],"can":[175],"be":[176],"used":[177],"evaluate":[179],"techniques.":[183],"available":[188],"public:":[191],"https://dig.cmu.edu/eyeintoai/.":[192]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":4}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}