{"id":"https://openalex.org/W4380357773","doi":"https://doi.org/10.1145/3593013.3594019","title":"Representation in AI Evaluations","display_name":"Representation in AI Evaluations","publication_year":2023,"publication_date":"2023-06-12","ids":{"openalex":"https://openalex.org/W4380357773","doi":"https://doi.org/10.1145/3593013.3594019"},"language":"en","primary_location":{"id":"doi:10.1145/3593013.3594019","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3593013.3594019","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031719232","display_name":"A. S. Bergman","orcid":"https://orcid.org/0000-0002-4331-1357"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"A. Stevie Bergman","raw_affiliation_strings":["DeepMind, USA"],"affiliations":[{"raw_affiliation_string":"DeepMind, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020758501","display_name":"Lisa Anne Hendricks","orcid":"https://orcid.org/0000-0001-9340-5143"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lisa Anne Hendricks","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080160770","display_name":"Maribeth Rauh","orcid":null},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maribeth Rauh","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063588445","display_name":"Boxi W\u00fa","orcid":"https://orcid.org/0009-0007-0641-6065"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Boxi Wu","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011791808","display_name":"William Agnew","orcid":"https://orcid.org/0000-0002-1362-554X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"William Agnew","raw_affiliation_strings":["University of Washington, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Washington, United Kingdom","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043562314","display_name":"Markus Kunesch","orcid":"https://orcid.org/0000-0003-3818-7897"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Markus Kunesch","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092146268","display_name":"Isabella Duan","orcid":"https://orcid.org/0009-0001-9766-0886"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Isabella Duan","raw_affiliation_strings":["University of Chicago, USA"],"affiliations":[{"raw_affiliation_string":"University of Chicago, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032814084","display_name":"Iason Gabriel","orcid":"https://orcid.org/0000-0002-7552-4576"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Iason Gabriel","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048272174","display_name":"William Isaac","orcid":"https://orcid.org/0000-0002-1297-5409"},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"William Isaac","raw_affiliation_strings":["DeepMind, United Kingdom"],"affiliations":[{"raw_affiliation_string":"DeepMind, United Kingdom","institution_ids":["https://openalex.org/I4210090411"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5031719232"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.6183,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.94185654,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"519","last_page":"533"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9488999843597412,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7010295391082764},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.511017918586731},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4769510328769684},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.32416248321533203}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7010295391082764},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.511017918586731},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4769510328769684},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32416248321533203},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3593013.3594019","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3593013.3594019","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},{"id":"pmh:oai:ora.ox.ac.uk:uuid:b8d1d195-8a8e-4dc9-93dd-5cbe1b4839d1","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306402636","display_name":"Oxford University Research Archive (ORA) (University of Oxford)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I40120149","host_organization_name":"University of Oxford","host_organization_lineage":["https://openalex.org/I40120149"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Symplectic Elements","raw_type":"http://purl.org/coar/resource_type/c_5794"}],"best_oa_location":{"id":"doi:10.1145/3593013.3594019","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3593013.3594019","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3593013.3594019","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 ACM Conference on Fairness, Accountability, and Transparency","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4380357773.pdf","grobid_xml":"https://content.openalex.org/works/W4380357773.grobid-xml"},"referenced_works_count":62,"referenced_works":["https://openalex.org/W1582047095","https://openalex.org/W1792712577","https://openalex.org/W1993953087","https://openalex.org/W2108598243","https://openalex.org/W2149252982","https://openalex.org/W2153066044","https://openalex.org/W2194566175","https://openalex.org/W2560674852","https://openalex.org/W2807975560","https://openalex.org/W2897154134","https://openalex.org/W2913593702","https://openalex.org/W2962059918","https://openalex.org/W2990751682","https://openalex.org/W2992319600","https://openalex.org/W3005222337","https://openalex.org/W3010093963","https://openalex.org/W3013571594","https://openalex.org/W3035507081","https://openalex.org/W3037831233","https://openalex.org/W3084611909","https://openalex.org/W3098075008","https://openalex.org/W3101662419","https://openalex.org/W3118813946","https://openalex.org/W3119719070","https://openalex.org/W3121336520","https://openalex.org/W3126557831","https://openalex.org/W3133893548","https://openalex.org/W3134889244","https://openalex.org/W3135371071","https://openalex.org/W3135514117","https://openalex.org/W3159003756","https://openalex.org/W3172415559","https://openalex.org/W3172707490","https://openalex.org/W3174220540","https://openalex.org/W3177219738","https://openalex.org/W3181414820","https://openalex.org/W3185376810","https://openalex.org/W3185692625","https://openalex.org/W3202415077","https://openalex.org/W3204130547","https://openalex.org/W3207830467","https://openalex.org/W3209540659","https://openalex.org/W3212368439","https://openalex.org/W3213556453","https://openalex.org/W4220820301","https://openalex.org/W4221138850","https://openalex.org/W4233063040","https://openalex.org/W4249922884","https://openalex.org/W4281999718","https://openalex.org/W4283166422","https://openalex.org/W4283170908","https://openalex.org/W4285289154","https://openalex.org/W4287073009","https://openalex.org/W4287239856","https://openalex.org/W4287332702","https://openalex.org/W4296405185","https://openalex.org/W4302071234","https://openalex.org/W4308789076","https://openalex.org/W4388161807","https://openalex.org/W4390006064","https://openalex.org/W6749489559","https://openalex.org/W6798247767"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857","https://openalex.org/W2350741829","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Calls":[0],"for":[1,42,143,153],"representation":[2,77,88,158,166],"in":[3,89,159,167],"artificial":[4],"intelligence":[5],"(AI)":[6],"and":[7,24,32,61,122,135,146,172],"machine":[8],"learning":[9],"(ML)":[10],"are":[11],"widespread,":[12],"with":[13,71],"\"representation\"":[14],"or":[15,101],"\"representativeness\"":[16],"generally":[17],"understood":[18],"to":[19,34,46,73,92,96],"be":[20,47],"both":[21],"an":[22,29,43,98],"instrumentally":[23,125,164],"intrinsically":[25],"beneficial":[26],"quality":[27],"of":[28,51,76,87,106,124,132,140],"AI":[30,44,53,90,99],"system,":[31,65],"central":[33],"fairness":[35],"concerns.":[36],"But":[37],"what":[38,74],"does":[39],"it":[40],"mean":[41],"system":[45],"\"representative\"?":[48],"Each":[49],"element":[50],"the":[52,64,85,104,120,130,151],"lifecycle":[54],"is":[55,78,112],"geared":[56],"towards":[57,103,169],"its":[58,68],"own":[59,69],"goals":[60],"effect":[62],"on":[63,157],"therefore":[66],"requiring":[67],"analyses":[70],"regard":[72],"kind":[75],"best.":[79],"In":[80],"this":[81],"work":[82,149],"we":[83],"untangle":[84],"benefits":[86],"evaluations":[91,168],"develop":[93],"a":[94,114,154],"framework":[95],"guide":[97],"practitioner":[100],"auditor":[102],"creation":[105],"representative":[107,126],"ML":[108],"evaluations.":[109],"Representation,":[110],"however,":[111],"not":[113],"panacea.":[115],"We":[116],"further":[117],"lay":[118],"out":[119],"limitations":[121],"tensions":[123],"datasets,":[127],"such":[128],"as":[129],"necessity":[131],"data":[133],"existence":[134],"access,":[136],"surveillance":[137],"vs":[138],"expectations":[139],"privacy,":[141],"implications":[142],"foundation":[144],"models":[145],"power.":[147],"This":[148],"sets":[150],"stage":[152],"research":[155],"agenda":[156],"AI,":[160],"which":[161],"extends":[162],"beyond":[163],"valuable":[165],"refocusing":[170],"on,":[171],"empowering,":[173],"impacted":[174],"communities.":[175]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":2}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}