{"id":"https://openalex.org/W4283168787","doi":"https://doi.org/10.1145/3531146.3533213","title":"Who Audits the Auditors? Recommendations from a field scan of the algorithmic auditing ecosystem","display_name":"Who Audits the Auditors? Recommendations from a field scan of the algorithmic auditing ecosystem","publication_year":2022,"publication_date":"2022-06-20","ids":{"openalex":"https://openalex.org/W4283168787","doi":"https://doi.org/10.1145/3531146.3533213"},"language":"en","primary_location":{"id":"doi:10.1145/3531146.3533213","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3531146.3533213","pdf_url":null,"source":{"id":"https://openalex.org/S4363608463","display_name":"2022 ACM Conference on Fairness, Accountability, and Transparency","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.02521","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037835991","display_name":"Sasha Costanza-Chock","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Sasha Costanza-Chock","raw_affiliation_strings":["Algorithmic Justice League, USA"],"affiliations":[{"raw_affiliation_string":"Algorithmic Justice League, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011015504","display_name":"Inioluwa Deborah Raji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Inioluwa Deborah Raji","raw_affiliation_strings":["Algorithmic Justice League, USA"],"affiliations":[{"raw_affiliation_string":"Algorithmic Justice League, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047439958","display_name":"Joy Buolamwini","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Joy Buolamwini","raw_affiliation_strings":["Algorithmic Justice League, USA"],"affiliations":[{"raw_affiliation_string":"Algorithmic Justice League, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5037835991"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":20.5636,"has_fulltext":true,"cited_by_count":152,"citation_normalized_percentile":{"value":0.9961401,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1571","last_page":"1583"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10270","display_name":"Blockchain Technology Applications and Security","score":0.9764000177383423,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10270","display_name":"Blockchain Technology Applications and Security","score":0.9764000177383423,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.9276999831199646,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.9198428988456726},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5579857230186462},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.4671083688735962},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.45247119665145874},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.43140965700149536},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.05929967761039734}],"concepts":[{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.9198428988456726},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5579857230186462},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.4671083688735962},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.45247119665145874},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.43140965700149536},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.05929967761039734},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3531146.3533213","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3531146.3533213","pdf_url":null,"source":{"id":"https://openalex.org/S4363608463","display_name":"2022 ACM Conference on Fairness, Accountability, and Transparency","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 ACM Conference on Fairness Accountability and Transparency","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.02521","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.02521","pdf_url":"https://arxiv.org/pdf/2310.02521","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dspace.mit.edu:1721.1/146405","is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/146405","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM|2022 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"http://purl.org/eprint/type/ConferencePaper"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.02521","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.02521","pdf_url":"https://arxiv.org/pdf/2310.02521","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4283168787.pdf","grobid_xml":"https://content.openalex.org/works/W4283168787.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W1819662813","https://openalex.org/W2263936035","https://openalex.org/W2771827553","https://openalex.org/W2786242872","https://openalex.org/W2788481061","https://openalex.org/W2897702578","https://openalex.org/W2956792022","https://openalex.org/W2962059918","https://openalex.org/W2981869278","https://openalex.org/W2997390712","https://openalex.org/W3004528080","https://openalex.org/W3100046612","https://openalex.org/W3127162919","https://openalex.org/W3133759846","https://openalex.org/W3134210117","https://openalex.org/W3134214835","https://openalex.org/W3139568729","https://openalex.org/W3159435787","https://openalex.org/W3163680969","https://openalex.org/W3184924454","https://openalex.org/W3200957822","https://openalex.org/W3206218807","https://openalex.org/W3209763391","https://openalex.org/W4220820301","https://openalex.org/W4252148582","https://openalex.org/W4287274268","https://openalex.org/W4287552728","https://openalex.org/W4288359825","https://openalex.org/W4308654226","https://openalex.org/W6638208828"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W1967231687","https://openalex.org/W3101499207"],"abstract_inverted_index":{"Algorithmic":[0],"audits":[1,95,140,207],"(or":[2],"\u2018AI":[3],"audits\u2019)":[4],"are":[5,50,130,217],"an":[6,35,106],"increasingly":[7],"popular":[8],"mechanism":[9],"for":[10,177,189,232],"algorithmic":[11,94,103,139,165,206,220,267,278],"accountability;":[12],"however,":[13],"they":[14,216],"remain":[15],"poorly":[16],"defined.":[17],"Without":[18],"a":[19,83],"clear":[20],"understanding":[21],"of":[22,76,85,109,155,171,199,226,229,277],"audit":[23,79,230,241,268],"practices,":[24],"let":[25],"alone":[26],"widely":[27],"used":[28],"standards":[29],"or":[30,38,47,96],"regulatory":[31],"guidance,":[32],"claims":[33],"that":[34,129],"AI":[36,78,200,263],"product":[37],"system":[39],"has":[40],"been":[41],"audited,":[42],"whether":[43],"by":[44,262],"first-,":[45],"second-,":[46],"third-party":[48],"auditors,":[49],"difficult":[51],"to":[52,102,137,149,202,219,259],"verify":[53],"and":[54,62,88,113,127,133,153,158,183,197,249,270],"may":[55],"potentially":[56],"exacerbate,":[57],"rather":[58],"than":[59],"mitigate,":[60],"bias":[61],"harm.":[63],"To":[64],"address":[65],"this":[66],"knowledge":[67],"gap,":[68],"we":[69],"provide":[70],"the":[71,77,110,151,195,240,255,266],"first":[72],"comprehensive":[73],"field":[74],"scan":[75],"ecosystem.":[80],"We":[81,118,145],"share":[82],"catalog":[84],"individuals":[86,214],"(N=438)":[87],"organizations":[89],"(N=189)":[90],"who":[91],"engage":[92,203],"in":[93,204,239,265],"whose":[97],"work":[98],"is":[99],"directly":[100,253],"relevant":[101],"audits;":[104],"conduct":[105],"anonymous":[107],"survey":[108],"group":[111],"(N=152);":[112],"interview":[114],"industry":[115],"leaders":[116],"(N=10).":[117],"identify":[119],"emerging":[120],"best":[121],"practices":[122],"as":[123,125,141,167,169,186,188],"well":[124,168,187],"methods":[126],"tools":[128],"becoming":[131],"commonplace,":[132],"enumerate":[134],"common":[135],"barriers":[136],"leveraging":[138],"effective":[142],"accountability":[143],"mechanisms.":[144],"outline":[146],"policy":[147],"recommendations":[148,174],"improve":[150],"quality":[152],"impact":[154],"these":[156],"audits,":[157],"highlight":[159],"proposals":[160],"with":[161],"wide":[162],"support":[163],"from":[164],"auditors":[166],"areas":[170],"debate.":[172],"Our":[173],"have":[175],"implications":[176],"lawmakers,":[178],"regulators,":[179],"internal":[180],"company":[181],"policymakers,":[182],"standards-setting":[184],"bodies,":[185],"auditors.":[190,279],"They":[191],"are:":[192],"1)":[193],"require":[194],"owners":[196],"operators":[198],"systems":[201,264],"independent":[205],"against":[208],"clearly":[209],"defined":[210],"standards;":[211],"2)":[212],"notify":[213],"when":[215],"subject":[218],"decision-making":[221],"systems;":[222],"3)":[223],"mandate":[224],"disclosure":[225],"key":[227],"components":[228],"findings":[231],"peer":[233],"review;":[234],"4)":[235],"consider":[236],"real-world":[237],"harm":[238,246],"process,":[242],"including":[243],"through":[244],"standardized":[245],"incident":[247],"reporting":[248],"response":[250],"mechanisms;":[251],"5)":[252],"involve":[254],"stakeholders":[256],"most":[257],"likely":[258],"be":[260],"harmed":[261],"process;":[269],"6)":[271],"formalize":[272],"evaluation":[273],"and,":[274],"potentially,":[275],"accreditation":[276]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":47},{"year":2024,"cited_by_count":52},{"year":2023,"cited_by_count":37},{"year":2022,"cited_by_count":5}],"updated_date":"2026-04-24T08:23:43.765630","created_date":"2025-10-10T00:00:00"}