{"id":"https://openalex.org/W3014972121","doi":"https://doi.org/10.1145/3313831.3376445","title":"Co-Designing Checklists to Understand Organizational Challenges and Opportunities around Fairness in AI","display_name":"Co-Designing Checklists to Understand Organizational Challenges and Opportunities around Fairness in AI","publication_year":2020,"publication_date":"2020-04-21","ids":{"openalex":"https://openalex.org/W3014972121","doi":"https://doi.org/10.1145/3313831.3376445","mag":"3014972121"},"language":"en","primary_location":{"id":"doi:10.1145/3313831.3376445","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3313831.3376445","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376445","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376445","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033757090","display_name":"Michael Madaio","orcid":"https://orcid.org/0000-0001-5772-0488"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael A. Madaio","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055485772","display_name":"Luke Stark","orcid":"https://orcid.org/0000-0002-2537-846X"},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]},{"id":"https://openalex.org/I4402554038","display_name":"Microsoft Research Montr\u00e9al (Canada)","ror":"https://ror.org/05xdft911","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4402554038"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Luke Stark","raw_affiliation_strings":["Microsoft Research, Montreal, PQ, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Montreal, PQ, Canada","institution_ids":["https://openalex.org/I4210153468","https://openalex.org/I4402554038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043117896","display_name":"Jennifer Wortman Vaughan","orcid":"https://orcid.org/0000-0002-7807-2018"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jennifer Wortman Vaughan","raw_affiliation_strings":["Microsoft Research, New York, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, New York, NY, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046348432","display_name":"Hanna Wallach","orcid":"https://orcid.org/0000-0003-3395-7186"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanna Wallach","raw_affiliation_strings":["Microsoft Research, New York City, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, New York City, NY, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":61.5245,"has_fulltext":true,"cited_by_count":404,"citation_normalized_percentile":{"value":0.99940225,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"14"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11953","display_name":"Neuroethics, Human Enhancement, Biomedical Innovations","score":0.9693999886512756,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11058","display_name":"Ethics in Business and Education","score":0.9355999827384949,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/operationalization","display_name":"Operationalization","score":0.8748160600662231},{"id":"https://openalex.org/keywords/checklist","display_name":"Checklist","score":0.7266010642051697},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.568231463432312},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5670447945594788},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5279920101165771},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.521690309047699},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.41804155707359314},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.4067124128341675},{"id":"https://openalex.org/keywords/engineering-ethics","display_name":"Engineering ethics","score":0.3826060891151428},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3387892246246338},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.15879645943641663},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.14667150378227234},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.11128649115562439},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10700109601020813}],"concepts":[{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.8748160600662231},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.7266010642051697},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.568231463432312},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5670447945594788},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5279920101165771},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.521690309047699},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.41804155707359314},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.4067124128341675},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.3826060891151428},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3387892246246338},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.15879645943641663},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.14667150378227234},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.11128649115562439},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10700109601020813},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3313831.3376445","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3313831.3376445","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376445","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3313831.3376445","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3313831.3376445","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3313831.3376445","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.5299999713897705}],"awards":[],"funders":[{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3014972121.pdf","grobid_xml":"https://content.openalex.org/works/W3014972121.grobid-xml"},"referenced_works_count":82,"referenced_works":["https://openalex.org/W604960715","https://openalex.org/W1006997171","https://openalex.org/W1491887163","https://openalex.org/W1525067466","https://openalex.org/W1859579244","https://openalex.org/W1921009626","https://openalex.org/W1976129032","https://openalex.org/W1993979945","https://openalex.org/W1996333597","https://openalex.org/W2000117884","https://openalex.org/W2001541484","https://openalex.org/W2006447892","https://openalex.org/W2016782401","https://openalex.org/W2042360766","https://openalex.org/W2051339604","https://openalex.org/W2073653117","https://openalex.org/W2100960835","https://openalex.org/W2118183099","https://openalex.org/W2119472226","https://openalex.org/W2126847279","https://openalex.org/W2129241093","https://openalex.org/W2137080312","https://openalex.org/W2138806549","https://openalex.org/W2139834614","https://openalex.org/W2142013321","https://openalex.org/W2144663895","https://openalex.org/W2148781751","https://openalex.org/W2160343841","https://openalex.org/W2162448023","https://openalex.org/W2163102864","https://openalex.org/W2277509373","https://openalex.org/W2337932387","https://openalex.org/W2475843659","https://openalex.org/W2519750805","https://openalex.org/W2557671501","https://openalex.org/W2599025709","https://openalex.org/W2606411279","https://openalex.org/W2611748211","https://openalex.org/W2742049669","https://openalex.org/W2781306982","https://openalex.org/W2786242872","https://openalex.org/W2790628304","https://openalex.org/W2795038878","https://openalex.org/W2802642435","https://openalex.org/W2808450727","https://openalex.org/W2884972792","https://openalex.org/W2887489621","https://openalex.org/W2895484796","https://openalex.org/W2897042519","https://openalex.org/W2897154134","https://openalex.org/W2897702578","https://openalex.org/W2898911770","https://openalex.org/W2899367046","https://openalex.org/W2900572965","https://openalex.org/W2905125924","https://openalex.org/W2911227954","https://openalex.org/W2912732633","https://openalex.org/W2916904544","https://openalex.org/W2929995480","https://openalex.org/W2940545533","https://openalex.org/W2940823115","https://openalex.org/W2941090159","https://openalex.org/W2942659792","https://openalex.org/W2942721282","https://openalex.org/W2952203301","https://openalex.org/W2952696298","https://openalex.org/W2953522645","https://openalex.org/W2964276035","https://openalex.org/W2973345269","https://openalex.org/W3094025600","https://openalex.org/W3099361686","https://openalex.org/W3100046612","https://openalex.org/W3100279624","https://openalex.org/W3121189393","https://openalex.org/W3122075665","https://openalex.org/W3122189619","https://openalex.org/W3133874049","https://openalex.org/W4205923466","https://openalex.org/W4239726674","https://openalex.org/W4288359825","https://openalex.org/W4289293239","https://openalex.org/W4300869287"],"related_works":["https://openalex.org/W2944277854","https://openalex.org/W2258359646","https://openalex.org/W2493324121","https://openalex.org/W2483613126","https://openalex.org/W4234402960","https://openalex.org/W4210958110","https://openalex.org/W4210994448","https://openalex.org/W4386824558","https://openalex.org/W204135935","https://openalex.org/W2097838441"],"abstract_inverted_index":{"Many":[0],"organizations":[1,26],"have":[2,27],"published":[3],"principles":[4],"intended":[5],"to":[6,23,46],"guide":[7],"the":[8,63,126],"ethical":[9],"development":[10],"and":[11,88,91,113,132],"deployment":[12],"of":[13,65,120,128],"AI":[14,30,47,68,85,94,102,129],"systems;":[15],"however,":[16],"their":[17],"abstract":[18],"nature":[19],"makes":[20],"them":[21],"difficult":[22],"operationalize.":[24],"Some":[25],"therefore":[28],"produced":[29],"ethics":[31],"checklists,":[32,131],"as":[33,35,42,44],"well":[34],"checklists":[36,51,66,96,104],"for":[37,93,109],"more":[38],"specific":[39],"concepts,":[40],"such":[41],"fairness,":[43],"applied":[45],"systems.":[48],"But":[49],"unless":[50],"are":[52],"grounded":[53],"in":[54,67,97],"practitioners'":[55],"needs,":[56],"they":[57],"may":[58,124],"be":[59],"misused.":[60],"To":[61],"understand":[62],"role":[64],"ethics,":[69],"we":[70],"conducted":[71],"an":[72,84],"iterative":[73],"co-design":[74],"process":[75],"with":[76],"48":[77],"practitioners,":[78],"focusing":[79],"on":[80],"fairness.":[81],"We":[82,99,117],"co-designed":[83],"fairness":[86,95,103,130],"checklist":[87],"identified":[89],"desiderata":[90],"concerns":[92],"general.":[98],"found":[100],"that":[101,123],"could":[105],"provide":[106],"organizational":[107,121],"infrastructure":[108],"formalizing":[110],"ad-hoc":[111],"processes":[112],"empowering":[114],"individual":[115],"advocates.":[116],"highlight":[118],"aspects":[119],"culture":[122],"impact":[125],"efficacy":[127],"suggest":[133],"future":[134],"design":[135],"directions.":[136]},"counts_by_year":[{"year":2026,"cited_by_count":34},{"year":2025,"cited_by_count":70},{"year":2024,"cited_by_count":68},{"year":2023,"cited_by_count":90},{"year":2022,"cited_by_count":61},{"year":2021,"cited_by_count":61},{"year":2020,"cited_by_count":20}],"updated_date":"2026-06-20T22:02:38.213706","created_date":"2025-10-10T00:00:00"}