{"id":"https://openalex.org/W4225006216","doi":"https://doi.org/10.1145/3491102.3502004","title":"Jury Learning: Integrating Dissenting Voices into Machine Learning Models","display_name":"Jury Learning: Integrating Dissenting Voices into Machine Learning Models","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W4225006216","doi":"https://doi.org/10.1145/3491102.3502004"},"language":"en","primary_location":{"id":"doi:10.1145/3491102.3502004","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3491102.3502004","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3502004","source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3502004","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026198819","display_name":"Mitchell Gordon","orcid":"https://orcid.org/0000-0003-1008-2321"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mitchell L. Gordon","raw_affiliation_strings":["Computer Science Department, Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101479345","display_name":"Michelle S. Lam","orcid":"https://orcid.org/0000-0002-3448-5961"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michelle S. Lam","raw_affiliation_strings":["Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101836529","display_name":"Joon Sung Park","orcid":"https://orcid.org/0000-0001-5910-411X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joon Sung Park","raw_affiliation_strings":["Computer Science Department, Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011076157","display_name":"Kayur Patel","orcid":"https://orcid.org/0009-0009-5294-0727"},"institutions":[{"id":"https://openalex.org/I4210153776","display_name":"Apple (United States)","ror":"https://ror.org/059hsda18","country_code":"US","type":"company","lineage":["https://openalex.org/I4210153776"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kayur Patel","raw_affiliation_strings":["Apple Inc, United States"],"affiliations":[{"raw_affiliation_string":"Apple Inc, United States","institution_ids":["https://openalex.org/I4210153776"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045528962","display_name":"Jeffrey T. Hancock","orcid":"https://orcid.org/0000-0001-5367-2677"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeff Hancock","raw_affiliation_strings":["Department of Communication, Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Department of Communication, Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015518638","display_name":"Tatsunori Hashimoto","orcid":"https://orcid.org/0000-0003-0521-5855"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tatsunori Hashimoto","raw_affiliation_strings":["Computer Science Department, Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076189854","display_name":"Michael S. Bernstein","orcid":"https://orcid.org/0000-0001-8020-9434"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael S. Bernstein","raw_affiliation_strings":["Computer Science, Stanford University, United States"],"affiliations":[{"raw_affiliation_string":"Computer Science, Stanford University, United States","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5026198819"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":11.3894,"has_fulltext":true,"cited_by_count":119,"citation_normalized_percentile":{"value":0.99100719,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"19"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dissenting-opinion","display_name":"Dissenting opinion","score":0.969117283821106},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6548281311988831},{"id":"https://openalex.org/keywords/jury","display_name":"Jury","score":0.5428149700164795},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4986453056335449},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.11134475469589233},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.09248003363609314}],"concepts":[{"id":"https://openalex.org/C56617239","wikidata":"https://www.wikidata.org/wiki/Q1092720","display_name":"Dissenting opinion","level":2,"score":0.969117283821106},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6548281311988831},{"id":"https://openalex.org/C2776119841","wikidata":"https://www.wikidata.org/wiki/Q837675","display_name":"Jury","level":2,"score":0.5428149700164795},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4986453056335449},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.11134475469589233},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.09248003363609314}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3491102.3502004","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3491102.3502004","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3502004","source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2202.02950","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2202.02950","pdf_url":"https://arxiv.org/pdf/2202.02950","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3491102.3502004","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3491102.3502004","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3502004","source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/5","score":0.4699999988079071,"display_name":"Gender equality"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4225006216.pdf","grobid_xml":"https://content.openalex.org/works/W4225006216.grobid-xml"},"referenced_works_count":64,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W1517046895","https://openalex.org/W1702329404","https://openalex.org/W1819662813","https://openalex.org/W1989630473","https://openalex.org/W2003238113","https://openalex.org/W2100960835","https://openalex.org/W2108598243","https://openalex.org/W2124910891","https://openalex.org/W2125943921","https://openalex.org/W2145554190","https://openalex.org/W2290009368","https://openalex.org/W2473418344","https://openalex.org/W2607311634","https://openalex.org/W2738428139","https://openalex.org/W2767546953","https://openalex.org/W2773809728","https://openalex.org/W2774180115","https://openalex.org/W2796239588","https://openalex.org/W2896881735","https://openalex.org/W2898911770","https://openalex.org/W2899134503","https://openalex.org/W2914164379","https://openalex.org/W2916904544","https://openalex.org/W2922234936","https://openalex.org/W2936750472","https://openalex.org/W2941985495","https://openalex.org/W2945599316","https://openalex.org/W2954992865","https://openalex.org/W2957654274","https://openalex.org/W2963748066","https://openalex.org/W2985347336","https://openalex.org/W2988571156","https://openalex.org/W2991138171","https://openalex.org/W2995382404","https://openalex.org/W2998862821","https://openalex.org/W3003162384","https://openalex.org/W3005295611","https://openalex.org/W3010140118","https://openalex.org/W3031586727","https://openalex.org/W3031679542","https://openalex.org/W3040025862","https://openalex.org/W3047049674","https://openalex.org/W3047791797","https://openalex.org/W3080883390","https://openalex.org/W3093945404","https://openalex.org/W3104186312","https://openalex.org/W3105435131","https://openalex.org/W3105994008","https://openalex.org/W3125569121","https://openalex.org/W3153687269","https://openalex.org/W3160042174","https://openalex.org/W3162683674","https://openalex.org/W3163078977","https://openalex.org/W3163469193","https://openalex.org/W3181414820","https://openalex.org/W3189849087","https://openalex.org/W3196190653","https://openalex.org/W3207830467","https://openalex.org/W3214105842","https://openalex.org/W4245823246","https://openalex.org/W4248403502","https://openalex.org/W4300003134","https://openalex.org/W6638208828"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3207284869","https://openalex.org/W2055935989","https://openalex.org/W2883557297","https://openalex.org/W4230163270","https://openalex.org/W2621908761","https://openalex.org/W2254253365","https://openalex.org/W2378567312"],"abstract_inverted_index":{"Whose":[0],"labels":[1],"should":[2],"a":[3,57,70,86,113,122],"machine":[4],"learning":[5,88,115],"(ML)":[6],"algorithm":[7],"learn":[8],"to":[9,19,22,128,135],"emulate?":[10],"For":[11,84],"ML":[12,38,59],"tasks":[13],"ranging":[14],"from":[15,125],"online":[16,91,105],"comment":[17],"toxicity":[18,92],"misinformation":[20],"detection":[21],"medical":[23],"diagnosis,":[24],"different":[25],"groups":[26],"in":[27,77,121],"society":[28],"may":[29],"have":[30],"irreconcilable":[31],"disagreements":[32,43,64],"about":[33],"ground":[34],"truth":[35],"labels.":[36,52],"Supervised":[37],"today":[39],"resolves":[40,62],"these":[41,63],"label":[42],"implicitly":[44],"using":[45],"majority":[46],"vote,":[47],"which":[48,73],"overrides":[49],"minority":[50],"groups\u2019":[51],"We":[53],"introduce":[54],"jury":[55,87,109],"learning,":[56,110],"supervised":[58],"approach":[60],"that":[61,117,141,155,160],"explicitly":[65],"through":[66],"the":[67,81,130],"metaphor":[68],"of":[69,104,163],"jury:":[71],"defining":[72],"people":[74],"or":[75],"groups,":[76],"what":[78],"proportion,":[79],"determine":[80],"classifier\u2019s":[82],"prediction.":[83],"example,":[85],"model":[89],"for":[90],"might":[93],"centrally":[94],"feature":[95],"women":[96],"and":[97,148],"Black":[98],"jurors,":[99],"who":[100],"are":[101],"commonly":[102],"targets":[103],"harassment.":[106],"To":[107],"enable":[108],"we":[111],"contribute":[112],"deep":[114],"architecture":[116,138],"models":[118,127],"every":[119],"annotator":[120],"dataset,":[123],"samples":[124],"annotators\u2019":[126],"populate":[129],"jury,":[131],"then":[132],"runs":[133],"inference":[134],"classify.":[136],"Our":[137],"enables":[139],"juries":[140,159],"dynamically":[142],"adapt":[143],"their":[144],"composition,":[145],"explore":[146],"counterfactuals,":[147],"visualize":[149],"dissent.":[150],"A":[151],"field":[152],"evaluation":[153],"finds":[154],"practitioners":[156],"construct":[157],"diverse":[158],"alter":[161],"14%":[162],"classification":[164],"outcomes.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":30},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":47},{"year":2022,"cited_by_count":14}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}