{"id":"https://openalex.org/W4386246835","doi":"https://doi.org/10.1145/3600211.3604712","title":"Supporting Human-AI Collaboration in Auditing LLMs with LLMs","display_name":"Supporting Human-AI Collaboration in Auditing LLMs with LLMs","publication_year":2023,"publication_date":"2023-08-08","ids":{"openalex":"https://openalex.org/W4386246835","doi":"https://doi.org/10.1145/3600211.3604712"},"language":"en","primary_location":{"id":"doi:10.1145/3600211.3604712","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604712","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604712","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604712","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053419988","display_name":"Charvi Rastogi","orcid":"https://orcid.org/0000-0003-0820-4115"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Charvi Rastogi","raw_affiliation_strings":["Machine Learning Department, Carnegie Mellon University, USA"],"affiliations":[{"raw_affiliation_string":"Machine Learning Department, Carnegie Mellon University, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019726734","display_name":"Marco T\u00falio Ribeiro","orcid":"https://orcid.org/0000-0002-3301-1297"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marco Tulio Ribeiro","raw_affiliation_strings":["Microsoft Research, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101636045","display_name":"Nicholas S. P. King","orcid":"https://orcid.org/0009-0002-2126-5115"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas King","raw_affiliation_strings":["Microsoft Research, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050672320","display_name":"Harsha Nori","orcid":"https://orcid.org/0000-0002-5442-1359"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Harsha Nori","raw_affiliation_strings":["Microsoft Research, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057541630","display_name":"Saleema Amershi","orcid":"https://orcid.org/0000-0002-3294-7288"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saleema Amershi","raw_affiliation_strings":["Microsoft Research, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5053419988"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":15.7695,"has_fulltext":true,"cited_by_count":62,"citation_normalized_percentile":{"value":0.9921059,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"913","last_page":"926"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sensemaking","display_name":"Sensemaking","score":0.7935796976089478},{"id":"https://openalex.org/keywords/sociotechnical-system","display_name":"Sociotechnical system","score":0.7637569904327393},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.6995216608047485},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5960846543312073},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.5503250956535339},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.4902344346046448},{"id":"https://openalex.org/keywords/stakeholder","display_name":"Stakeholder","score":0.4842202663421631},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4675625264644623},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3871021270751953},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3716248869895935},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.19211849570274353},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.14328530430793762},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.12244796752929688},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.10964557528495789}],"concepts":[{"id":"https://openalex.org/C2780554381","wikidata":"https://www.wikidata.org/wiki/Q2063340","display_name":"Sensemaking","level":2,"score":0.7935796976089478},{"id":"https://openalex.org/C127627568","wikidata":"https://www.wikidata.org/wiki/Q1639361","display_name":"Sociotechnical system","level":2,"score":0.7637569904327393},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.6995216608047485},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5960846543312073},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5503250956535339},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.4902344346046448},{"id":"https://openalex.org/C201305675","wikidata":"https://www.wikidata.org/wiki/Q852998","display_name":"Stakeholder","level":2,"score":0.4842202663421631},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4675625264644623},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3871021270751953},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3716248869895935},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.19211849570274353},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.14328530430793762},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.12244796752929688},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.10964557528495789},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3600211.3604712","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604712","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604712","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3600211.3604712","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600211.3604712","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600211.3604712","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2492749369","display_name":"CIF: Medium: Foundations of Learning from Paired Comparisons and Direct Queries","funder_award_id":"1763734","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6906924528","display_name":null,"funder_award_id":"CIF 1763734","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386246835.pdf","grobid_xml":"https://content.openalex.org/works/W4386246835.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W2011824517","https://openalex.org/W2059216172","https://openalex.org/W2059362837","https://openalex.org/W2162409443","https://openalex.org/W2583689529","https://openalex.org/W2789415069","https://openalex.org/W2809925683","https://openalex.org/W2949858875","https://openalex.org/W2956281901","https://openalex.org/W2962059918","https://openalex.org/W3001807593","https://openalex.org/W3017863658","https://openalex.org/W3035507081","https://openalex.org/W3037831233","https://openalex.org/W3159435787","https://openalex.org/W3160887439","https://openalex.org/W3163078977","https://openalex.org/W3203321135","https://openalex.org/W3204393347","https://openalex.org/W3205290952","https://openalex.org/W3206420877","https://openalex.org/W3215475614","https://openalex.org/W4224992683","https://openalex.org/W4285113702","https://openalex.org/W4309618902","https://openalex.org/W4312788391","https://openalex.org/W4313429401","https://openalex.org/W4366548330","https://openalex.org/W4385574250","https://openalex.org/W4385894687","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W3002559787","https://openalex.org/W2100609754","https://openalex.org/W2050640900","https://openalex.org/W2049050102","https://openalex.org/W2596767525","https://openalex.org/W1886987011","https://openalex.org/W2795557596","https://openalex.org/W1980714815","https://openalex.org/W3045759591","https://openalex.org/W3133630643"],"abstract_inverted_index":{"Large":[0],"language":[1,17,45,142],"models":[2,46,120],"(LLMs)":[3],"are":[4],"increasingly":[5],"becoming":[6],"all-powerful":[7],"and":[8,58,73,75,81,109,118,146,190],"pervasive":[9],"via":[10],"deployment":[11],"in":[12,70,79,121,187],"sociotechnical":[13],"systems.":[14],"Yet":[15],"these":[16,44],"models,":[18],"be":[19,29],"it":[20],"for":[21],"classification":[22],"or":[23,55],"generation,":[24],"have":[25,184],"been":[26,185],"shown":[27,186],"to":[28,35,42,60,84,112],"biased,":[30],"behave":[31],"irresponsibly,":[32],"causing":[33],"harm":[34],"people":[36],"at":[37],"scale.":[38],"It":[39],"is":[40,93],"crucial":[41],"audit":[43],"rigorously":[47],"before":[48],"deployment.":[49],"Existing":[50],"auditing":[51,88,139],"tools":[52],"use":[53],"either":[54],"both":[56],"humans":[57,117],"AI":[59],"find":[61],"failures.":[62],"In":[63],"this":[64],"work,":[65],"we":[66,103,133],"draw":[67],"upon":[68,86],"literature":[69],"human-AI":[71,110],"collaboration":[72],"sensemaking,":[74],"interview":[76],"research":[77],"experts":[78],"safe":[80],"fair":[82],"AI,":[83],"build":[85],"the":[87,100,105,126,130],"tool:":[89],"AdaTest":[90],"[36],":[91],"which":[92],"powered":[94],"by":[95],"a":[96,171],"generative":[97,119],"LLM.":[98],"Through":[99],"design":[101],"process":[102],"highlight":[104],"importance":[106],"of":[107,116,128,173],"sensemaking":[108],"communication":[111],"leverage":[113],"complementary":[114],"strengths":[115,159],"collaborative":[122],"auditing.":[123],"To":[124],"evaluate":[125],"effectiveness":[127],"AdaTest++,":[129],"augmented":[131],"tool,":[132,168],"conduct":[134],"user":[135],"studies":[136],"with":[137,166],"participants":[138],"two":[140],"commercial":[141],"models:":[143],"OpenAI\u2019s":[144],"GPT-3":[145],"Azure\u2019s":[147],"sentiment":[148],"analysis":[149,152],"model.":[150],"Qualitative":[151],"shows":[153],"that":[154,183],"AdaTest++":[155],"effectively":[156],"leverages":[157],"human":[158],"such":[160],"as":[161],"schematization,":[162],"hypothesis":[163],"testing.":[164],"Further,":[165],"our":[167],"users":[169],"identified":[170],"variety":[172],"failures":[174],"modes,":[175],"covering":[176],"26":[177],"different":[178],"topics":[179],"over":[180],"2":[181],"tasks,":[182],"formal":[188],"audits":[189],"also":[191],"those":[192],"previously":[193],"under-reported.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":32},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":6}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}