{"id":"https://openalex.org/W4386348070","doi":"https://doi.org/10.1145/3610209","title":"Sociotechnical Audits: Broadening the Algorithm Auditing Lens to Investigate Targeted Advertising","display_name":"Sociotechnical Audits: Broadening the Algorithm Auditing Lens to Investigate Targeted Advertising","publication_year":2023,"publication_date":"2023-09-28","ids":{"openalex":"https://openalex.org/W4386348070","doi":"https://doi.org/10.1145/3610209"},"language":"en","primary_location":{"id":"doi:10.1145/3610209","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610209","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610209","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3610209","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101479345","display_name":"Michelle S. Lam","orcid":"https://orcid.org/0000-0002-3448-5961"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michelle S. Lam","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-3448-5961","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101989712","display_name":"Ayush Pandit","orcid":"https://orcid.org/0000-0003-2630-8356"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ayush Pandit","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2630-8356","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025278282","display_name":"Colin H. Kalicki","orcid":"https://orcid.org/0009-0002-6353-8492"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Colin H. Kalicki","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0009-0002-6353-8492","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102822915","display_name":"Rachit Gupta","orcid":"https://orcid.org/0000-0001-9705-9057"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rachit Gupta","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0001-9705-9057","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5105546166","display_name":"P. Sahoo","orcid":"https://orcid.org/0000-0002-9121-5378"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Poonam Sahoo","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9121-5378","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086524212","display_name":"Dana\u00eb Metaxa","orcid":"https://orcid.org/0000-0001-9359-6090"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dana\u00eb Metaxa","raw_affiliation_strings":["University of Pennsylvania, Philadelphia, PA, USA"],"raw_orcid":"https://orcid.org/0000-0001-9359-6090","affiliations":[{"raw_affiliation_string":"University of Pennsylvania, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I79576946"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.4322,"has_fulltext":true,"cited_by_count":36,"citation_normalized_percentile":{"value":0.98049915,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"7","issue":"CSCW2","first_page":"1","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9646000266075134,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sociotechnical-system","display_name":"Sociotechnical system","score":0.9653362035751343},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.7705150842666626},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7112152576446533},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4088703989982605},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.25253885984420776},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.10399094223976135},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.07862129807472229}],"concepts":[{"id":"https://openalex.org/C127627568","wikidata":"https://www.wikidata.org/wiki/Q1639361","display_name":"Sociotechnical system","level":2,"score":0.9653362035751343},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.7705150842666626},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7112152576446533},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4088703989982605},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.25253885984420776},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.10399094223976135},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.07862129807472229}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3610209","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610209","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610209","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2308.15768","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.15768","pdf_url":"https://arxiv.org/pdf/2308.15768","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3610209","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3610209","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3610209","source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310572","display_name":"Brown Institute for Media Innovation","ror":"https://ror.org/02awzbn54"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386348070.pdf","grobid_xml":"https://content.openalex.org/works/W4386348070.grobid-xml"},"referenced_works_count":66,"referenced_works":["https://openalex.org/W1984346461","https://openalex.org/W2043167816","https://openalex.org/W2048694465","https://openalex.org/W2056309232","https://openalex.org/W2132447236","https://openalex.org/W2149252982","https://openalex.org/W2194321411","https://openalex.org/W2394957087","https://openalex.org/W2550925836","https://openalex.org/W2594153394","https://openalex.org/W2737117411","https://openalex.org/W2744538883","https://openalex.org/W2786672974","https://openalex.org/W2788481061","https://openalex.org/W2794342582","https://openalex.org/W2796402109","https://openalex.org/W2796410451","https://openalex.org/W2798815817","https://openalex.org/W2889476181","https://openalex.org/W2898883664","https://openalex.org/W2913860539","https://openalex.org/W2921563810","https://openalex.org/W2943551158","https://openalex.org/W2962059918","https://openalex.org/W2969884768","https://openalex.org/W2981869278","https://openalex.org/W2987834530","https://openalex.org/W3000010701","https://openalex.org/W3001807593","https://openalex.org/W3005139125","https://openalex.org/W3013549950","https://openalex.org/W3023008627","https://openalex.org/W3083206818","https://openalex.org/W3094135831","https://openalex.org/W3116326248","https://openalex.org/W3123029532","https://openalex.org/W3125964423","https://openalex.org/W3133759846","https://openalex.org/W3135322202","https://openalex.org/W3144750446","https://openalex.org/W3155422163","https://openalex.org/W3155476465","https://openalex.org/W3159059608","https://openalex.org/W3159435787","https://openalex.org/W3160887439","https://openalex.org/W3163730107","https://openalex.org/W3166396011","https://openalex.org/W3199439625","https://openalex.org/W3203606693","https://openalex.org/W3205290952","https://openalex.org/W4233995128","https://openalex.org/W4244809407","https://openalex.org/W4283168787","https://openalex.org/W4285669763","https://openalex.org/W4287993192","https://openalex.org/W4288086175","https://openalex.org/W4289328458","https://openalex.org/W4294145056","https://openalex.org/W4294623627","https://openalex.org/W4298013486","https://openalex.org/W4307020327","https://openalex.org/W4307020338","https://openalex.org/W4309618902","https://openalex.org/W4362512695","https://openalex.org/W4376117752","https://openalex.org/W4384924878"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1980714815","https://openalex.org/W3133630643","https://openalex.org/W608490485","https://openalex.org/W2481729736","https://openalex.org/W4389636114","https://openalex.org/W2236208621","https://openalex.org/W2967475239"],"abstract_inverted_index":{"Algorithm":[0],"audits":[1,82,128,307],"are":[2],"powerful":[3],"tools":[4],"for":[5,123],"studying":[6],"black-box":[7],"systems":[8,61,310],"without":[9],"direct":[10],"knowledge":[11],"of":[12,27,43,53,115,166,231,246,280,287,314],"their":[13,107,220,315],"inner":[14],"workings.":[15],"While":[16],"very":[17],"effective":[18,183],"in":[19,152,161,201,271],"examining":[20],"technical":[21,299,316],"components,":[22,300],"the":[23,44,51,63,68,77,135,174,187,202,278,285,312],"method":[24],"stops":[25],"short":[26],"a":[28,92,121,155,162,244,273],"sociotechnical":[29,54,64,93,127,164,306,309],"frame,":[30],"which":[31],"would":[32],"also":[33,142,262],"consider":[34],"users":[35,73,138,150,264],"themselves":[36],"as":[37,74,80],"an":[38,84,113,206],"integral":[39],"and":[40,72,89,105,110,141,192,200,217,229,233,239,317],"dynamic":[41],"part":[42],"system.":[45],"Addressing":[46],"this":[47,116],"limitation,":[48],"we":[49,118,158,190,204],"propose":[50],"concept":[52],"auditing:":[55],"auditing":[56],"methods":[57,295],"that":[58,177,209,253,263,296],"evaluate":[59,308],"algorithmic":[60,103,136],"at":[62],"level,":[65],"focusing":[66],"on":[67,184,277,304],"interplay":[69,313],"between":[70],"algorithms":[71],"each":[75],"impacts":[76],"other.":[78],"Just":[79],"algorithm":[81,85],"probe":[83],"with":[86,129,243,259,292],"varied":[87],"inputs":[88],"observe":[90,191],"outputs,":[91],"audit":[94,165],"(STA)":[95],"additionally":[96],"probes":[97],"users,":[98,199,260,305],"exposing":[99],"them":[100],"to":[101,147,172,198,266,268],"different":[102,269],"behavior":[104],"measuring":[106],"resulting":[108],"attitudes":[109],"behaviors.":[111],"As":[112,154],"example":[114],"method,":[117],"develop":[119],"Intervenr,":[120],"platform":[122],"conducting":[124],"browser-based,":[125],"longitudinal":[126],"consenting,":[130],"compensated":[131],"participants.":[132],"Intervenr":[133,160],"investigates":[134],"content":[137],"encounter":[139],"online,":[140],"coordinates":[143],"systematic":[144],"client-side":[145],"interventions":[146],"understand":[148],"how":[149],"change":[151],"response.":[153],"case":[156],"study,":[157],"deploy":[159,205],"two-week":[163],"online":[167],"advertising":[168],"(N":[169],"=":[170],"244)":[171],"investigate":[173],"central":[175],"premise":[176],"personalized":[178,281],"ad":[179,227,282],"targeting":[180,212,283],"is":[181],"more":[182],"users.":[185],"In":[186,290],"first":[188],"week,":[189,274],"collect":[193,223],"all":[194,219],"browser":[195],"ads":[196,255,270],"delivered":[197],"second,":[203],"ablation-style":[207],"intervention":[208],"disrupts":[210],"normal":[211],"by":[213],"randomly":[214],"pairing":[215],"participants":[216],"swapping":[218],"ads.":[221,249],"We":[222],"user-oriented":[224],"metrics":[225,235],"(self-reported":[226],"interest":[228],"feeling":[230],"representation)":[232],"advertiser-oriented":[234],"(ad":[236],"views,":[237],"clicks,":[238],"recognition)":[240],"throughout,":[241],"along":[242],"total":[245],"over":[247],"500,000":[248],"Our":[250],"STA":[251],"finds":[252],"targeted":[254],"indeed":[256],"perform":[257],"better":[258],"but":[261],"begin":[265],"acclimate":[267],"only":[272,297,302],"casting":[275],"doubt":[276],"primacy":[279],"given":[284],"impact":[286],"repeated":[288],"exposure.":[289],"comparison":[291],"other":[293],"evaluation":[294],"study":[298],"or":[301],"experiment":[303],"through":[311],"human":[318],"components.":[319]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":1}],"updated_date":"2026-06-24T13:16:06.693445","created_date":"2023-09-02T00:00:00"}