{"id":"https://openalex.org/W3160887439","doi":"https://doi.org/10.1145/3479577","title":"Everyday Algorithm Auditing: Understanding the Power of Everyday Users in Surfacing Harmful Algorithmic Behaviors","display_name":"Everyday Algorithm Auditing: Understanding the Power of Everyday Users in Surfacing Harmful Algorithmic Behaviors","publication_year":2021,"publication_date":"2021-10-13","ids":{"openalex":"https://openalex.org/W3160887439","doi":"https://doi.org/10.1145/3479577","mag":"3160887439"},"language":"en","primary_location":{"id":"doi:10.1145/3479577","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479577","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479577","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3479577","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Hong Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Hong Shen","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Alicia DeVos","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alicia DeVos","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Motahhare Eslami","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Motahhare Eslami","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":null,"display_name":"Kenneth Holstein","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth Holstein","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":14.9662,"has_fulltext":true,"cited_by_count":131,"citation_normalized_percentile":{"value":0.99029034,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"5","issue":"CSCW2","first_page":"1","last_page":"29"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.632099986076355,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.632099986076355,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.053700000047683716,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.03909999877214432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/everyday-life","display_name":"Everyday life","score":0.8169000148773193},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.7868000268936157},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5830000042915344},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5349000096321106},{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.3935999870300293},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.34940001368522644},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.30320000648498535}],"concepts":[{"id":"https://openalex.org/C2779018934","wikidata":"https://www.wikidata.org/wiki/Q1129653","display_name":"Everyday life","level":2,"score":0.8169000148773193},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.7868000268936157},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6532999873161316},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5830000042915344},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5349000096321106},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4007999897003174},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.3935999870300293},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.34940001368522644},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.31540000438690186},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.30320000648498535},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2985999882221222},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29420000314712524},{"id":"https://openalex.org/C195818886","wikidata":"https://www.wikidata.org/wiki/Q5421724","display_name":"Expressive power","level":2,"score":0.2937999963760376},{"id":"https://openalex.org/C2985583900","wikidata":"https://www.wikidata.org/wiki/Q722617","display_name":"Formal description","level":2,"score":0.2892000079154968},{"id":"https://openalex.org/C75606506","wikidata":"https://www.wikidata.org/wiki/Q1049183","display_name":"Formal methods","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C172195944","wikidata":"https://www.wikidata.org/wiki/Q541265","display_name":"Ubiquitous computing","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C186528591","wikidata":"https://www.wikidata.org/wiki/Q619544","display_name":"Body of knowledge","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3479577","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479577","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479577","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2105.02980","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.02980","pdf_url":"https://arxiv.org/pdf/2105.02980","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3479577","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3479577","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3479577","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1725833531","display_name":null,"funder_award_id":"IIS-2040942","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4723327940","display_name":null,"funder_award_id":"2040942","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307791","display_name":"Cisco Systems","ror":"https://ror.org/03yt1ez60"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3160887439.pdf","grobid_xml":"https://content.openalex.org/works/W3160887439.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W37678429","https://openalex.org/W1604857681","https://openalex.org/W1996434443","https://openalex.org/W2006447892","https://openalex.org/W2054551600","https://openalex.org/W2059633625","https://openalex.org/W2086853153","https://openalex.org/W2108960318","https://openalex.org/W2127492100","https://openalex.org/W2147459640","https://openalex.org/W2151496550","https://openalex.org/W2159513729","https://openalex.org/W2161392727","https://openalex.org/W2168765606","https://openalex.org/W2290102723","https://openalex.org/W2334489881","https://openalex.org/W2405896853","https://openalex.org/W2583689529","https://openalex.org/W2611145892","https://openalex.org/W2621141770","https://openalex.org/W2766415924","https://openalex.org/W2767269108","https://openalex.org/W2786242872","https://openalex.org/W2788428929","https://openalex.org/W2796239588","https://openalex.org/W2796402109","https://openalex.org/W2810705771","https://openalex.org/W2883147591","https://openalex.org/W2897154134","https://openalex.org/W2898147565","https://openalex.org/W2898883664","https://openalex.org/W2922258799","https://openalex.org/W2940680165","https://openalex.org/W2962059918","https://openalex.org/W2969884768","https://openalex.org/W3001807593","https://openalex.org/W3012736183","https://openalex.org/W3014972121","https://openalex.org/W3029552071","https://openalex.org/W3039540048","https://openalex.org/W3083206818","https://openalex.org/W3092846560","https://openalex.org/W3135757521","https://openalex.org/W3162866579","https://openalex.org/W3204393347","https://openalex.org/W3207549372","https://openalex.org/W4288359825","https://openalex.org/W4300539796","https://openalex.org/W6835209979"],"related_works":[],"abstract_inverted_index":{"A":[0],"growing":[1],"body":[2],"of":[3,39,56,72,103,146,149,161,173,208],"literature":[4],"has":[5,85],"proposed":[6],"formal":[7,19,195],"approaches":[8,21,197],"to":[9,80,88],"audit":[10],"algorithmic":[11,57,124,209],"systems":[12,43,58],"for":[13,170],"biased":[14],"and":[15,60,99,114,176,198],"harmful":[16,64],"behaviors.":[17,182],"While":[18],"auditing":[20,92,181,196,201],"have":[22,48],"been":[23,86],"greatly":[24],"impactful,":[25],"they":[26,67],"often":[27],"suffer":[28],"major":[29],"blindspots,":[30],"with":[31,76,123],"critical":[32],"issues":[33],"surfacing":[34,134],"only":[35],"in":[36,52,69,109,133,205],"the":[37,70,101,153,171,192,199],"context":[38],"everyday":[40,54,74,104,129,162,206],"use":[41,207],"once":[42],"are":[44,131],"deployed.":[45],"Recent":[46],"years":[47],"seen":[49],"many":[50],"cases":[51,160,169],"which":[53,110],"users":[55,111,130],"detect":[59],"raise":[61],"awareness":[62],"about":[63,152],"behaviors":[65,118,137,202],"that":[66,128,138,178,187,203],"encounter":[68],"course":[71],"their":[73,120],"interactions":[75,122],"these":[77,89,168],"systems.":[78,125,210],"However,":[79],"date":[81],"little":[82],"academic":[83],"attention":[84],"granted":[87],"bottom-up,":[90],"user-driven":[91],"processes.":[93],"In":[94],"this":[95],"paper,":[96],"we":[97,184],"propose":[98],"explore":[100],"concept":[102],"algorithm":[105,163],"auditing,":[106,147,164],"a":[107],"process":[108],"detect,":[112],"understand,":[113],"interrogate":[115],"problematic":[116,135],"machine":[117,136],"via":[119,142],"day-to-day":[121],"We":[126,156],"argue":[127],"powerful":[132],"may":[139],"elude":[140],"detection":[141],"more":[143],"centrally-organized":[144],"forms":[145],"regardless":[148],"users'":[150],"knowledge":[151],"underlying":[154],"algorithms.":[155],"analyze":[157],"several":[158],"real-world":[159],"drawing":[165],"lessons":[166],"from":[167],"design":[172],"future":[174],"platforms":[175],"tools":[177],"facilitate":[179],"such":[180],"Finally,":[183],"discuss":[185],"work":[186],"lies":[188],"ahead,":[189],"toward":[190],"bridging":[191],"gaps":[193],"between":[194],"organic":[200],"emerge":[204]},"counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":49},{"year":2024,"cited_by_count":23},{"year":2023,"cited_by_count":28},{"year":2022,"cited_by_count":15},{"year":2021,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2021-05-24T00:00:00"}