{"id":"https://openalex.org/W3131911651","doi":"https://doi.org/10.1145/3461702.3462516","title":"Towards Unbiased and Accurate Deferral to Multiple Experts","display_name":"Towards Unbiased and Accurate Deferral to Multiple Experts","publication_year":2021,"publication_date":"2021-07-21","ids":{"openalex":"https://openalex.org/W3131911651","doi":"https://doi.org/10.1145/3461702.3462516","mag":"3131911651"},"language":"en","primary_location":{"id":"doi:10.1145/3461702.3462516","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3461702.3462516","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2102.13004","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080439623","display_name":"Vijay Keswani","orcid":"https://orcid.org/0000-0002-0729-7332"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Vijay Keswani","raw_affiliation_strings":["Yale University, New Haven, CT, USA","[Yale Univ., New Haven, CT, USA]"],"affiliations":[{"raw_affiliation_string":"Yale University, New Haven, CT, USA","institution_ids":["https://openalex.org/I32971472"]},{"raw_affiliation_string":"[Yale Univ., New Haven, CT, USA]","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023397430","display_name":"Matthew Lease","orcid":"https://orcid.org/0000-0002-0056-2834"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Lease","raw_affiliation_strings":["University of Texas at Austin, Austin, TX, USA","University of Texas at Austin, Austin, TX, USA;"],"affiliations":[{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, USA","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"University of Texas at Austin, Austin, TX, USA;","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5002843568","display_name":"Krishnaram Kenthapadi","orcid":"https://orcid.org/0000-0003-1237-087X"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Krishnaram Kenthapadi","raw_affiliation_strings":["Amazon AWS AI, East Palo Alto, CA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon AWS AI, East Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1311688040"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5080439623"],"corresponding_institution_ids":["https://openalex.org/I32971472"],"apc_list":null,"apc_paid":null,"fwci":0.84662145,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.77244969,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"154","last_page":"165"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8081341981887817},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.7536423206329346},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6534984111785889},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6453447341918945},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5858540534973145},{"id":"https://openalex.org/keywords/deferral","display_name":"Deferral","score":0.5788918137550354},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5219511389732361},{"id":"https://openalex.org/keywords/subject-matter-expert","display_name":"Subject-matter expert","score":0.4544648230075836},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.43590080738067627},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"Domain knowledge","score":0.4271978735923767},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3689461946487427},{"id":"https://openalex.org/keywords/expert-system","display_name":"Expert system","score":0.3381870687007904},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.09199666976928711}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8081341981887817},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.7536423206329346},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6534984111785889},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6453447341918945},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5858540534973145},{"id":"https://openalex.org/C2778281098","wikidata":"https://www.wikidata.org/wiki/Q885383","display_name":"Deferral","level":2,"score":0.5788918137550354},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5219511389732361},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.4544648230075836},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.43590080738067627},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.4271978735923767},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3689461946487427},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.3381870687007904},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.09199666976928711},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3461702.3462516","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3461702.3462516","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2021 AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2102.13004","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.13004","pdf_url":"https://arxiv.org/pdf/2102.13004","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:3131911651","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2102.13004.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2102.13004","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2102.13004","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2102.13004","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.13004","pdf_url":"https://arxiv.org/pdf/2102.13004","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4099999964237213}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3131911651.pdf","grobid_xml":"https://content.openalex.org/works/W3131911651.grobid-xml"},"referenced_works_count":79,"referenced_works":["https://openalex.org/W1944672","https://openalex.org/W1482468045","https://openalex.org/W1539473175","https://openalex.org/W1946594383","https://openalex.org/W2018186689","https://openalex.org/W2040825624","https://openalex.org/W2070230130","https://openalex.org/W2101946573","https://openalex.org/W2119004707","https://openalex.org/W2124002192","https://openalex.org/W2141649520","https://openalex.org/W2146502635","https://openalex.org/W2149252982","https://openalex.org/W2157928966","https://openalex.org/W2174985112","https://openalex.org/W2180771322","https://openalex.org/W2181558882","https://openalex.org/W2201853392","https://openalex.org/W2250539671","https://openalex.org/W2281090488","https://openalex.org/W2296319761","https://openalex.org/W2488247662","https://openalex.org/W2522957395","https://openalex.org/W2530395818","https://openalex.org/W2533800767","https://openalex.org/W2555956183","https://openalex.org/W2586547910","https://openalex.org/W2595653137","https://openalex.org/W2618631885","https://openalex.org/W2734639922","https://openalex.org/W2757188127","https://openalex.org/W2779326814","https://openalex.org/W2785327160","https://openalex.org/W2788481061","https://openalex.org/W2790628304","https://openalex.org/W2891018693","https://openalex.org/W2891947597","https://openalex.org/W2898970033","https://openalex.org/W2899302063","https://openalex.org/W2901481055","https://openalex.org/W2913410345","https://openalex.org/W2931108066","https://openalex.org/W2949678053","https://openalex.org/W2962753953","https://openalex.org/W2962755741","https://openalex.org/W2962922829","https://openalex.org/W2962977603","https://openalex.org/W2963100392","https://openalex.org/W2963178340","https://openalex.org/W2963302453","https://openalex.org/W2963772355","https://openalex.org/W2963917042","https://openalex.org/W2963917099","https://openalex.org/W2964006987","https://openalex.org/W2964031043","https://openalex.org/W2970236443","https://openalex.org/W2973205938","https://openalex.org/W2983697263","https://openalex.org/W2992012242","https://openalex.org/W2996936464","https://openalex.org/W2999637955","https://openalex.org/W3001754608","https://openalex.org/W3004542466","https://openalex.org/W3007910004","https://openalex.org/W3021926608","https://openalex.org/W3021934530","https://openalex.org/W3034220715","https://openalex.org/W3034854469","https://openalex.org/W3034879135","https://openalex.org/W3036540646","https://openalex.org/W3037276488","https://openalex.org/W3103146244","https://openalex.org/W3103751997","https://openalex.org/W3104831984","https://openalex.org/W3181414820","https://openalex.org/W3185350140","https://openalex.org/W4240224338","https://openalex.org/W4300071243","https://openalex.org/W6730583873"],"related_works":["https://openalex.org/W3185727840","https://openalex.org/W2134974509","https://openalex.org/W3122662941","https://openalex.org/W2167566733","https://openalex.org/W3125177298","https://openalex.org/W3153243574","https://openalex.org/W2766524544","https://openalex.org/W2021246439","https://openalex.org/W3008176646","https://openalex.org/W3156628016","https://openalex.org/W1624661769","https://openalex.org/W2400456651","https://openalex.org/W2886693943","https://openalex.org/W2106375048","https://openalex.org/W2577818566","https://openalex.org/W3046921440","https://openalex.org/W2532280165","https://openalex.org/W2792853973","https://openalex.org/W3016981248","https://openalex.org/W3172031346"],"abstract_inverted_index":{"Machine":[0],"learning":[1,53,222],"models":[2],"are":[3,111],"often":[4],"implemented":[5],"in":[6,10,25,32,46,65,150],"cohort":[7],"with":[8,13,76,113,137,173],"humans":[9],"the":[11,14,71,84,138,155,183,188,193,201,211],"pipeline,":[12],"model":[15,54,96],"having":[16,116],"an":[17,94],"option":[18],"to":[19,21,38,82,91,106,142,144,192,205],"defer":[20,143],"a":[22,74,77,98,126,131,134,165,169,207],"domain":[23,57,119],"expert":[24,58,79,90,115],"cases":[26,151],"where":[27,108,154],"it":[28,180],"has":[29,68,157],"low":[30,158],"confidence":[31],"its":[33],"inference.":[34],"Our":[35,101],"goal":[36],"is":[37],"design":[39],"mechanisms":[40],"for":[41,200,210],"ensuring":[42],"accuracy":[43,184],"and":[44,56,80,86,97,122,133,168,177,185,217],"fairness":[45,186],"such":[47],"prediction":[48],"systems":[49],"that":[50,128,179,219],"combine":[51],"machine":[52],"inferences":[55],"predictions.":[59],"Prior":[60],"work":[61,102],"on":[62,70,164,226],"\"deferral":[63],"systems\"":[64],"classification":[66],"settings":[67,107],"focused":[69],"setting":[72],"of":[73,88,120,152,187,213],"pipeline":[75],"single":[78],"aimed":[81],"accommodate":[83],"inaccuracies":[85],"biases":[87],"this":[89,104,227],"simultaneously":[92,129],"learn":[93],"inference":[95],"deferral":[99,135,139],"system.":[100],"extends":[103],"framework":[105,127,163,223],"multiple":[109],"experts":[110,149],"available,":[112],"each":[114],"their":[117],"own":[118],"expertise":[121],"biases.":[123],"We":[124,160,195],"propose":[125],"learns":[130],"classifier":[132,156],"system,":[136],"system":[140],"choosing":[141],"one":[145],"or":[146],"more":[147],"human":[148],"input":[153],"confidence.":[159],"test":[161],"our":[162,220],"synthetic":[166,175],"dataset":[167,172,209,229],"content":[170,202],"moderation":[171,203],"biased":[174],"experts,":[176],"show":[178,218],"significantly":[181],"improves":[182],"final":[189],"predictions,":[190],"compared":[191],"baselines.":[194],"also":[196],"collect":[197],"crowdsourced":[198],"labels":[199],"task":[204],"construct":[206],"real-world":[208,228],"evaluation":[212],"hybrid":[214],"machine-human":[215],"frameworks":[216],"proposed":[221],"outperforms":[224],"baselines":[225],"as":[230],"well.":[231]},"counts_by_year":[{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2026-02-09T09:26:11.010843","created_date":"2025-10-10T00:00:00"}