{"id":"https://openalex.org/W3159435787","doi":"https://doi.org/10.1145/3449148","title":"Problematic Machine Behavior","display_name":"Problematic Machine Behavior","publication_year":2021,"publication_date":"2021-04-13","ids":{"openalex":"https://openalex.org/W3159435787","doi":"https://doi.org/10.1145/3449148","mag":"3159435787"},"language":"en","primary_location":{"id":"doi:10.1145/3449148","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3449148","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007590421","display_name":"Jack Bandy","orcid":"https://orcid.org/0000-0003-3584-2999"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jack Bandy","raw_affiliation_strings":["Northwestern University, Evanston, IL, USA"],"affiliations":[{"raw_affiliation_string":"Northwestern University, Evanston, IL, USA","institution_ids":["https://openalex.org/I111979921"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5007590421"],"corresponding_institution_ids":["https://openalex.org/I111979921"],"apc_list":null,"apc_paid":null,"fwci":22.781,"has_fulltext":false,"cited_by_count":166,"citation_normalized_percentile":{"value":0.99538227,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"5","issue":"CSCW1","first_page":"1","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11045","display_name":"Privacy, Security, and Data Protection","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.8707413077354431},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6419124603271484},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.5919595956802368},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.560787558555603},{"id":"https://openalex.org/keywords/distortion","display_name":"Distortion (music)","score":0.4337146580219269},{"id":"https://openalex.org/keywords/economic-justice","display_name":"Economic Justice","score":0.4200809597969055},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.41018903255462646},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3578530251979828},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.34043270349502563},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2717679738998413},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.24238860607147217},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.196031391620636},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.11408492922782898}],"concepts":[{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.8707413077354431},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6419124603271484},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.5919595956802368},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.560787558555603},{"id":"https://openalex.org/C126780896","wikidata":"https://www.wikidata.org/wiki/Q899871","display_name":"Distortion (music)","level":4,"score":0.4337146580219269},{"id":"https://openalex.org/C139621336","wikidata":"https://www.wikidata.org/wiki/Q3190382","display_name":"Economic Justice","level":2,"score":0.4200809597969055},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.41018903255462646},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3578530251979828},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34043270349502563},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2717679738998413},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.24238860607147217},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.196031391620636},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.11408492922782898},{"id":"https://openalex.org/C194257627","wikidata":"https://www.wikidata.org/wiki/Q211554","display_name":"Amplifier","level":3,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3449148","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3449148","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.6100000143051147,"id":"https://metadata.un.org/sdg/16"},{"display_name":"Reduced inequalities","score":0.4000000059604645,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":131,"referenced_works":["https://openalex.org/W1515490832","https://openalex.org/W1648303880","https://openalex.org/W1686065478","https://openalex.org/W1974133381","https://openalex.org/W1979290264","https://openalex.org/W1996730470","https://openalex.org/W2005501262","https://openalex.org/W2018881137","https://openalex.org/W2041287014","https://openalex.org/W2062548491","https://openalex.org/W2073689153","https://openalex.org/W2079973150","https://openalex.org/W2090655302","https://openalex.org/W2097001944","https://openalex.org/W2100047082","https://openalex.org/W2109882182","https://openalex.org/W2120109270","https://openalex.org/W2127523455","https://openalex.org/W2132248112","https://openalex.org/W2149252982","https://openalex.org/W2151592910","https://openalex.org/W2151791150","https://openalex.org/W2152065333","https://openalex.org/W2161392727","https://openalex.org/W2164005910","https://openalex.org/W2170989440","https://openalex.org/W2210543184","https://openalex.org/W2212762088","https://openalex.org/W2214169095","https://openalex.org/W2297259217","https://openalex.org/W2337553400","https://openalex.org/W2339485801","https://openalex.org/W2340135383","https://openalex.org/W2361256855","https://openalex.org/W2402661097","https://openalex.org/W2405932006","https://openalex.org/W2475162543","https://openalex.org/W2530875690","https://openalex.org/W2550925836","https://openalex.org/W2557671501","https://openalex.org/W2587944871","https://openalex.org/W2588008725","https://openalex.org/W2588439030","https://openalex.org/W2588799281","https://openalex.org/W2593962681","https://openalex.org/W2611078990","https://openalex.org/W2611739737","https://openalex.org/W2727940568","https://openalex.org/W2730390070","https://openalex.org/W2732757946","https://openalex.org/W2754525251","https://openalex.org/W2766288874","https://openalex.org/W2767192735","https://openalex.org/W2771827553","https://openalex.org/W2774114807","https://openalex.org/W2777603469","https://openalex.org/W2783918171","https://openalex.org/W2788481061","https://openalex.org/W2790980374","https://openalex.org/W2791099280","https://openalex.org/W2793294490","https://openalex.org/W2795530988","https://openalex.org/W2795975316","https://openalex.org/W2796402109","https://openalex.org/W2886498471","https://openalex.org/W2888099522","https://openalex.org/W2888160501","https://openalex.org/W2888305260","https://openalex.org/W2889315866","https://openalex.org/W2892918999","https://openalex.org/W2893191427","https://openalex.org/W2897702578","https://openalex.org/W2898322995","https://openalex.org/W2898883664","https://openalex.org/W2900099998","https://openalex.org/W2900336955","https://openalex.org/W2902567760","https://openalex.org/W2903353271","https://openalex.org/W2908599433","https://openalex.org/W2909211136","https://openalex.org/W2909212904","https://openalex.org/W2912845403","https://openalex.org/W2913823566","https://openalex.org/W2915988983","https://openalex.org/W2917166626","https://openalex.org/W2920950073","https://openalex.org/W2921290281","https://openalex.org/W2937369374","https://openalex.org/W2941228000","https://openalex.org/W2942730552","https://openalex.org/W2945417260","https://openalex.org/W2945776595","https://openalex.org/W2946939925","https://openalex.org/W2950173087","https://openalex.org/W2953487715","https://openalex.org/W2955792826","https://openalex.org/W2959130780","https://openalex.org/W2962059918","https://openalex.org/W2963524705","https://openalex.org/W2964031043","https://openalex.org/W2969884768","https://openalex.org/W2986189785","https://openalex.org/W2987248894","https://openalex.org/W2987834530","https://openalex.org/W2988826827","https://openalex.org/W2992319600","https://openalex.org/W2992494379","https://openalex.org/W2995382404","https://openalex.org/W3001221685","https://openalex.org/W3001363766","https://openalex.org/W3001807593","https://openalex.org/W3003162384","https://openalex.org/W3012588689","https://openalex.org/W3013082216","https://openalex.org/W3013549950","https://openalex.org/W3013571594","https://openalex.org/W3029464893","https://openalex.org/W3032474305","https://openalex.org/W3041968715","https://openalex.org/W3087185627","https://openalex.org/W3099917429","https://openalex.org/W3103891807","https://openalex.org/W3105226597","https://openalex.org/W3119880662","https://openalex.org/W3123098919","https://openalex.org/W3125817989","https://openalex.org/W4211017739","https://openalex.org/W4235186750","https://openalex.org/W4300539796","https://openalex.org/W4310936740","https://openalex.org/W6945867317"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"While":[0],"algorithm":[1,51,173],"audits":[2],"are":[3,56],"growing":[4],"rapidly":[5],"in":[6,25,40,175],"commonality":[7],"and":[8,21,58,66,85,116,148,171],"public":[9],"importance,":[10],"relatively":[11],"little":[12],"scholarly":[13],"work":[14,20],"has":[15],"gone":[16],"toward":[17,182],"synthesizing":[18],"prior":[19],"strategizing":[22],"future":[23,157],"research":[24,180],"the":[26,124,135,165,176],"area.":[27],"This":[28],"systematic":[29],"literature":[30],"review":[31,42,96],"aims":[32],"to":[33],"do":[34],"just":[35],"that,":[36],"following":[37],"PRISMA":[38],"guidelines":[39],"a":[41],"of":[43,114,120,137,168,178],"over":[44],"500":[45],"English":[46],"articles":[47],"that":[48,154],"yielded":[49],"62":[50],"audit":[52,86,100,158],"studies.":[53],"The":[54,95,160],"studies":[55,101,125],"synthesized":[57],"organized":[59],"primarily":[60],"by":[61,163],"behavior":[62],"(discrimination,":[63],"distortion,":[64],"exploitation,":[65],"misjudgement),":[67],"with":[68],"codes":[69],"also":[70,128],"provided":[71],"for":[72,156],"domain":[73],"(e.g.":[74,80,88,132,141,145,150],"search,":[75],"vision,":[76],"advertising,":[77],"etc.),":[78,84],"organization":[79],"Google,":[81],"Facebook,":[82],"Amazon,":[83],"method":[87],"sock":[89],"puppet,":[90],"direct":[91],"scrape,":[92],"crowdsourcing,":[93],"etc.).":[94],"shows":[97],"how":[98],"previous":[99],"have":[102],"exposed":[103],"public-facing":[104],"algorithms":[105,112,118],"exhibiting":[106],"problematic":[107],"behavior,":[108],"such":[109],"as":[110],"search":[111],"culpable":[113,119],"distortion":[115],"advertising":[117,142],"discrimination.":[121],"Based":[122],"on":[123,134],"reviewed,":[126],"it":[127],"suggests":[129],"some":[130],"behaviors":[131],"discrimination":[133],"basis":[136],"intersectional":[138],"identities),":[139],"domains":[140],"algorithms),":[143],"methods":[144],"code":[146],"auditing),":[147],"organizations":[149],"Twitter,":[151],"TikTok,":[152],"LinkedIn)":[153],"call":[155],"attention.":[159],"paper":[161],"concludes":[162],"offering":[164],"common":[166],"ingredients":[167],"successful":[169],"audits,":[170],"discussing":[172],"auditing":[174],"context":[177],"broader":[179],"working":[181],"algorithmic":[183],"justice.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":60},{"year":2024,"cited_by_count":45},{"year":2023,"cited_by_count":32},{"year":2022,"cited_by_count":18},{"year":2021,"cited_by_count":7}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}