{"id":"https://openalex.org/W4396833706","doi":"https://doi.org/10.1145/3613905.3636302","title":"Human-Centered Evaluation and Auditing of Language Models","display_name":"Human-Centered Evaluation and Auditing of Language Models","publication_year":2024,"publication_date":"2024-05-11","ids":{"openalex":"https://openalex.org/W4396833706","doi":"https://doi.org/10.1145/3613905.3636302"},"language":"en","primary_location":{"id":"doi:10.1145/3613905.3636302","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613905.3636302","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613905.3636302","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3613905.3636302","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083645580","display_name":"Ziang Xiao","orcid":"https://orcid.org/0000-0003-3368-0180"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210099137","display_name":"Research Canada","ror":"https://ror.org/0103eqz62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210099137"]}],"countries":["CA","US"],"is_corresponding":false,"raw_author_name":"Ziang Xiao","raw_affiliation_strings":["Johns Hopkins University, United States and Microsoft Research, Canada"],"raw_orcid":"https://orcid.org/0000-0003-3368-0180","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, United States and Microsoft Research, Canada","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210099137","https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067411025","display_name":"Wesley Hanwen Deng","orcid":"https://orcid.org/0000-0003-3375-5285"},"institutions":[{"id":"https://openalex.org/I4210142372","display_name":"Human Media","ror":"https://ror.org/04072nk43","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210142372"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wesley Hanwen Deng","raw_affiliation_strings":["Human-Computer Interaction Institution, Carnegie Mellon University, United States"],"raw_orcid":"https://orcid.org/0000-0003-3375-5285","affiliations":[{"raw_affiliation_string":"Human-Computer Interaction Institution, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I4210142372","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101479345","display_name":"Michelle S. Lam","orcid":"https://orcid.org/0000-0002-3448-5961"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michelle S. Lam","raw_affiliation_strings":["Dept. of Computer Science, Stanford University, United States"],"raw_orcid":"https://orcid.org/0000-0002-3448-5961","affiliations":[{"raw_affiliation_string":"Dept. of Computer Science, Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054624773","display_name":"Motahhare Eslami","orcid":"https://orcid.org/0000-0002-1499-3045"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Motahhare Eslami","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, United States"],"raw_orcid":"https://orcid.org/0000-0002-1499-3045","affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, United States","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079709359","display_name":"Juho Kim","orcid":"https://orcid.org/0000-0001-6348-4127"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Juho Kim","raw_affiliation_strings":["School of Computing, KAIST, Korea, Republic of"],"raw_orcid":"https://orcid.org/0000-0001-6348-4127","affiliations":[{"raw_affiliation_string":"School of Computing, KAIST, Korea, Republic of","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008276552","display_name":"Mina Lee","orcid":"https://orcid.org/0000-0002-0428-4720"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mina Lee","raw_affiliation_strings":["Microsoft Research, United States"],"raw_orcid":"https://orcid.org/0000-0002-0428-4720","affiliations":[{"raw_affiliation_string":"Microsoft Research, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024320659","display_name":"Q. Vera Liao","orcid":"https://orcid.org/0000-0003-4543-7196"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Q. Vera Liao","raw_affiliation_strings":["Microsoft Research, Canada"],"raw_orcid":"https://orcid.org/0000-0003-4543-7196","affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada","institution_ids":["https://openalex.org/I4210153468"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.2612,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.97183916,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.8110471963882446},{"id":"https://openalex.org/keywords/engineering-ethics","display_name":"Engineering ethics","score":0.5179997086524963},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.4213069677352905},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4198695421218872},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.35500434041023254},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.3465840816497803},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.34521305561065674},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.26711195707321167},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.2252945899963379},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.1119668185710907}],"concepts":[{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.8110471963882446},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.5179997086524963},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.4213069677352905},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4198695421218872},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.35500434041023254},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.3465840816497803},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.34521305561065674},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.26711195707321167},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2252945899963379},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.1119668185710907},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3613905.3636302","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613905.3636302","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613905.3636302","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3613905.3636302","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613905.3636302","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613905.3636302","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Extended Abstracts of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.4399999976158142,"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4396833706.pdf"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W1989630473","https://openalex.org/W2052312648","https://openalex.org/W2897154134","https://openalex.org/W2922258799","https://openalex.org/W2952602496","https://openalex.org/W2999637955","https://openalex.org/W2999765337","https://openalex.org/W3001807593","https://openalex.org/W3103751997","https://openalex.org/W3104847483","https://openalex.org/W3135184438","https://openalex.org/W3204393347","https://openalex.org/W3206420877","https://openalex.org/W4220747294","https://openalex.org/W4224992683","https://openalex.org/W4225295761","https://openalex.org/W4238846128","https://openalex.org/W4280609065","https://openalex.org/W4283159495","https://openalex.org/W4283170666","https://openalex.org/W4287854631","https://openalex.org/W4288058319","https://openalex.org/W4288080268","https://openalex.org/W4288359825","https://openalex.org/W4309618902","https://openalex.org/W4320342906","https://openalex.org/W4321610533","https://openalex.org/W4366549927","https://openalex.org/W4378908626","https://openalex.org/W4380361082","https://openalex.org/W4386348070","https://openalex.org/W4387801187","https://openalex.org/W6600201367","https://openalex.org/W6804645036"],"related_works":["https://openalex.org/W2149537132","https://openalex.org/W2018871932","https://openalex.org/W641279757","https://openalex.org/W370975646","https://openalex.org/W1670566515","https://openalex.org/W4242022592","https://openalex.org/W596972243","https://openalex.org/W2404937507","https://openalex.org/W3121186197","https://openalex.org/W2373849942"],"abstract_inverted_index":{"The":[0,79],"recent":[1],"advancements":[2],"in":[3,55,139],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"have":[8],"significantly":[9],"impacted":[10],"numerous,":[11],"and":[12,27,33,40,58,64,67,73,88,92,98,103,111,120,122,142],"will":[13,81],"impact":[14],"more,":[15],"real-world":[16],"applications.":[17],"However,":[18],"these":[19,31,107],"models":[20],"also":[21],"pose":[22],"significant":[23],"risks":[24],"to":[25,49,69,105,129],"individuals":[26],"society.":[28],"To":[29],"mitigate":[30],"issues":[32],"guide":[34],"future":[35,132],"model":[36],"development,":[37],"responsible":[38],"evaluation":[39,72,91,97,141],"auditing":[41,74,93,99],"of":[42],"LLMs":[43],"are":[44],"essential.":[45],"This":[46],"workshop":[47,80,127],"aims":[48,128],"address":[50],"the":[51,137],"current":[52],"\u201cevaluation":[53],"crisis\u201d":[54],"LLM":[56,71,140],"research":[57,133],"practice":[59],"by":[60],"bringing":[61],"together":[62],"HCI":[63],"AI":[65],"researchers":[66],"practitioners":[68],"rethink":[70],"from":[75],"a":[76,131],"human-centered":[77,96],"perspective.":[78],"explore":[82],"topics":[83],"around":[84],"understanding":[85],"stakeholders\u2019":[86],"needs":[87],"goals":[89],"with":[90],"LLMs,":[94],"establishing":[95],"methods,":[100,108],"developing":[101],"tools":[102],"resources":[104],"support":[106],"building":[109],"community":[110],"fostering":[112],"collaboration.":[113],"By":[114],"soliciting":[115],"papers,":[116],"organizing":[117],"invited":[118],"keynote":[119],"panel,":[121],"facilitating":[123],"group":[124],"discussions,":[125],"this":[126],"develop":[130],"agenda":[134],"for":[135],"addressing":[136],"challenges":[138],"auditing.":[143]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":12},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}