{"id":"https://openalex.org/W4400524744","doi":"https://doi.org/10.1145/3626772.3657796","title":"Evaluating Search System Explainability with Psychometrics and Crowdsourcing","display_name":"Evaluating Search System Explainability with Psychometrics and Crowdsourcing","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400524744","doi":"https://doi.org/10.1145/3626772.3657796"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3657796","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657796","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3626772.3657796","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101975391","display_name":"Catherine Chen","orcid":"https://orcid.org/0009-0009-8734-436X"},"institutions":[{"id":"https://openalex.org/I27804330","display_name":"Brown University","ror":"https://ror.org/05gq02987","country_code":"US","type":"education","lineage":["https://openalex.org/I27804330"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Catherine Chen","raw_affiliation_strings":["Brown University, Providence, RI, USA"],"raw_orcid":"https://orcid.org/0009-0009-8734-436X","affiliations":[{"raw_affiliation_string":"Brown University, Providence, RI, USA","institution_ids":["https://openalex.org/I27804330"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014921416","display_name":"Carsten Eickhoff","orcid":"https://orcid.org/0000-0001-9895-4061"},"institutions":[{"id":"https://openalex.org/I8087733","display_name":"University of T\u00fcbingen","ror":"https://ror.org/03a1kwz48","country_code":"DE","type":"education","lineage":["https://openalex.org/I8087733"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Carsten Eickhoff","raw_affiliation_strings":["University of T\u00fcbingen, T\u00fcbingen, Germany"],"raw_orcid":"https://orcid.org/0000-0001-9895-4061","affiliations":[{"raw_affiliation_string":"University of T\u00fcbingen, T\u00fcbingen, Germany","institution_ids":["https://openalex.org/I8087733"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1051","last_page":"1061"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9714999794960022,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9144022464752197},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5868902802467346},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.513465940952301},{"id":"https://openalex.org/keywords/psychometric-testing","display_name":"Psychometric testing","score":0.47007209062576294},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4206102788448334},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2751704454421997},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.15871071815490723},{"id":"https://openalex.org/keywords/clinical-psychology","display_name":"Clinical psychology","score":0.15233218669891357}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9144022464752197},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5868902802467346},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.513465940952301},{"id":"https://openalex.org/C2909622525","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometric testing","level":4,"score":0.47007209062576294},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4206102788448334},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2751704454421997},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.15871071815490723},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.15233218669891357},{"id":"https://openalex.org/C3018868096","wikidata":"https://www.wikidata.org/wiki/Q2693233","display_name":"Internal consistency","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626772.3657796","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657796","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3626772.3657796","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657796","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.4300000071525574}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W584942340","https://openalex.org/W1484864026","https://openalex.org/W1709453369","https://openalex.org/W1966404545","https://openalex.org/W1983051294","https://openalex.org/W1996422672","https://openalex.org/W2066099647","https://openalex.org/W2070561899","https://openalex.org/W2088477804","https://openalex.org/W2109752330","https://openalex.org/W2134299926","https://openalex.org/W2159466611","https://openalex.org/W2282821441","https://openalex.org/W2777647957","https://openalex.org/W2783946743","https://openalex.org/W2891482011","https://openalex.org/W2891503716","https://openalex.org/W2940853875","https://openalex.org/W2963847595","https://openalex.org/W2981731882","https://openalex.org/W2997560917","https://openalex.org/W3034265020","https://openalex.org/W3034580237","https://openalex.org/W3035403907","https://openalex.org/W3138819813","https://openalex.org/W3153557349","https://openalex.org/W3156083768","https://openalex.org/W3183398589","https://openalex.org/W4206210655","https://openalex.org/W4245990317","https://openalex.org/W4248200375","https://openalex.org/W4255674373","https://openalex.org/W4284710359","https://openalex.org/W4301131586"],"related_works":["https://openalex.org/W2597099934","https://openalex.org/W4315816496","https://openalex.org/W4316088780","https://openalex.org/W4299606256","https://openalex.org/W2011844148","https://openalex.org/W1487563096","https://openalex.org/W2977196269","https://openalex.org/W2156713118","https://openalex.org/W2341696955","https://openalex.org/W4321846512"],"abstract_inverted_index":{"As":[0],"information":[1],"retrieval":[2],"(IR)":[3],"systems,":[4,115],"such":[5],"as":[6,55,142],"search":[7,83,98],"engines":[8],"and":[9,21,29,38,73,85,113,156],"conversational":[10],"agents,":[11],"become":[12],"ubiquitous":[13],"in":[14,35,64,81,150],"various":[15],"domains,":[16],"the":[17,46,60,65],"need":[18],"for":[19,94,145],"transparent":[20],"explainable":[22,36,95,112],"systems":[23,84,118],"grows":[24],"to":[25,75,109,134],"ensure":[26],"accountability,":[27],"fairness,":[28],"unbiased":[30],"results.":[31],"Despite":[32],"recent":[33],"advances":[34],"AI":[37],"IR":[39,96],"techniques,":[40],"there":[41],"is":[42],"no":[43],"consensus":[44],"on":[45],"definition":[47,62],"of":[48,79,138,153],"explainability.":[49],"Existing":[50],"approaches":[51],"often":[52],"treat":[53],"it":[54],"a":[56,101,143],"singular":[57],"notion,":[58],"disregarding":[59],"multidimensional":[61],"postulated":[63],"literature.":[66],"In":[67,100],"this":[68,136],"paper,":[69],"we":[70,105],"use":[71],"psychometrics":[72],"crowdsourcing":[74],"identify":[76],"human-centered":[77],"factors":[78],"explainability":[80,147],"Web":[82],"introduce":[86],"SSE":[87],"(Search":[88],"System":[89],"Explainability),":[90],"an":[91],"evaluation":[92,148],"metric":[93],"(XIR)":[97],"systems.":[99],"crowdsourced":[102],"user":[103],"study,":[104],"demonstrate":[106],"SSE's":[107],"ability":[108],"distinguish":[110],"between":[111],"non-explainable":[114],"showing":[116],"that":[117,128],"with":[119],"higher":[120],"scores":[121],"indeed":[122],"indicate":[123],"greater":[124],"interpretability.":[125],"We":[126],"hope":[127],"aside":[129],"from":[130],"these":[131],"concrete":[132],"contributions":[133],"XIR,":[135],"line":[137],"work":[139],"will":[140],"serve":[141],"blueprint":[144],"similar":[146],"efforts":[149],"other":[151],"domains":[152],"machine":[154],"learning":[155],"natural":[157],"language":[158],"processing.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}