{"id":"https://openalex.org/W4224940987","doi":"https://doi.org/10.1145/3491102.3501999","title":"Human-AI Collaboration via Conditional Delegation: A Case Study of Content Moderation","display_name":"Human-AI Collaboration via Conditional Delegation: A Case Study of Content Moderation","publication_year":2022,"publication_date":"2022-04-27","ids":{"openalex":"https://openalex.org/W4224940987","doi":"https://doi.org/10.1145/3491102.3501999"},"language":"en","primary_location":{"id":"doi:10.1145/3491102.3501999","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3491102.3501999","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3501999","source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3501999","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000544794","display_name":"Vivian Lai","orcid":"https://orcid.org/0000-0003-3165-6136"},"institutions":[{"id":"https://openalex.org/I2802236040","display_name":"University of Colorado System","ror":"https://ror.org/00jc20583","country_code":"US","type":"education","lineage":["https://openalex.org/I2802236040"]},{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Vivian Lai","raw_affiliation_strings":["University of Colorado Boulder, United States"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, United States","institution_ids":["https://openalex.org/I2802236040","https://openalex.org/I188538660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086610894","display_name":"Samuel Carton","orcid":"https://orcid.org/0000-0001-7520-0400"},"institutions":[{"id":"https://openalex.org/I188538660","display_name":"University of Colorado Boulder","ror":"https://ror.org/02ttsq026","country_code":"US","type":"education","lineage":["https://openalex.org/I188538660"]},{"id":"https://openalex.org/I2802236040","display_name":"University of Colorado System","ror":"https://ror.org/00jc20583","country_code":"US","type":"education","lineage":["https://openalex.org/I2802236040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samuel Carton","raw_affiliation_strings":["University of Colorado Boulder, United States"],"affiliations":[{"raw_affiliation_string":"University of Colorado Boulder, United States","institution_ids":["https://openalex.org/I2802236040","https://openalex.org/I188538660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082740748","display_name":"Rajat Bhatnagar","orcid":"https://orcid.org/0000-0002-3676-0293"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rajat Bhatnagar","raw_affiliation_strings":["Amazon, United States"],"affiliations":[{"raw_affiliation_string":"Amazon, United States","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053648887","display_name":"Q. Vera Liao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Q. Vera Liao","raw_affiliation_strings":["Microsoft Research, Canada"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Canada","institution_ids":["https://openalex.org/I4210153468"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100410998","display_name":"Yunfeng Zhang","orcid":"https://orcid.org/0000-0002-1237-6035"},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yunfeng Zhang","raw_affiliation_strings":["Twitter Inc., United States"],"affiliations":[{"raw_affiliation_string":"Twitter Inc., United States","institution_ids":["https://openalex.org/I113979032"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079270249","display_name":"Chenhao Tan","orcid":"https://orcid.org/0000-0002-3981-2116"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenhao Tan","raw_affiliation_strings":["University of Chicago, United States"],"affiliations":[{"raw_affiliation_string":"University of Chicago, United States","institution_ids":["https://openalex.org/I40347166"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5000544794"],"corresponding_institution_ids":["https://openalex.org/I188538660","https://openalex.org/I2802236040"],"apc_list":null,"apc_paid":null,"fwci":11.4786,"has_fulltext":true,"cited_by_count":114,"citation_normalized_percentile":{"value":0.99046105,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7628898024559021},{"id":"https://openalex.org/keywords/delegation","display_name":"Delegation","score":0.759416937828064},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.7105739712715149},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6115283370018005},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.586226761341095},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5387559533119202},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.40630078315734863},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.3483344316482544},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34468013048171997},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3290707468986511},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.21827256679534912}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7628898024559021},{"id":"https://openalex.org/C86532276","wikidata":"https://www.wikidata.org/wiki/Q1184065","display_name":"Delegation","level":2,"score":0.759416937828064},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.7105739712715149},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6115283370018005},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.586226761341095},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5387559533119202},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.40630078315734863},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3483344316482544},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34468013048171997},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3290707468986511},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.21827256679534912},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3491102.3501999","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3491102.3501999","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3501999","source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3491102.3501999","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3491102.3501999","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3491102.3501999","source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7269281490","display_name":"CRII: CHS: Harnessing Machine Learning to Improve Human Decision Making: A Case Study on Deceptive Detection","funder_award_id":"2125113","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G960943717","display_name":"AI-DCL: EAGER: Explanations through Diverse, Feasible, and Interactive Counterfactuals","funder_award_id":"2125116","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4224940987.pdf","grobid_xml":"https://content.openalex.org/works/W4224940987.grobid-xml"},"referenced_works_count":68,"referenced_works":["https://openalex.org/W1185009508","https://openalex.org/W1501005121","https://openalex.org/W1677182931","https://openalex.org/W2003238113","https://openalex.org/W2065267227","https://openalex.org/W2123179704","https://openalex.org/W2134584261","https://openalex.org/W2154157725","https://openalex.org/W2292070666","https://openalex.org/W2340954483","https://openalex.org/W2540646130","https://openalex.org/W2588098783","https://openalex.org/W2774215862","https://openalex.org/W2775336929","https://openalex.org/W2794079986","https://openalex.org/W2795530988","https://openalex.org/W2796133875","https://openalex.org/W2891503716","https://openalex.org/W2895993994","https://openalex.org/W2899027170","https://openalex.org/W2901481055","https://openalex.org/W2901895173","https://openalex.org/W2902141984","https://openalex.org/W2905034244","https://openalex.org/W2918341242","https://openalex.org/W2941349974","https://openalex.org/W2942073295","https://openalex.org/W2949678053","https://openalex.org/W2951286828","https://openalex.org/W2962990575","https://openalex.org/W2963233086","https://openalex.org/W2963847595","https://openalex.org/W2963969878","https://openalex.org/W2964034671","https://openalex.org/W2970019270","https://openalex.org/W2970837303","https://openalex.org/W2979826702","https://openalex.org/W2979893369","https://openalex.org/W2988207158","https://openalex.org/W2998862821","https://openalex.org/W2999637955","https://openalex.org/W2999765337","https://openalex.org/W3001062618","https://openalex.org/W3004483087","https://openalex.org/W3008694996","https://openalex.org/W3012851754","https://openalex.org/W3032816739","https://openalex.org/W3034854469","https://openalex.org/W3093660685","https://openalex.org/W3099742594","https://openalex.org/W3101792976","https://openalex.org/W3103751997","https://openalex.org/W3104847483","https://openalex.org/W3105435131","https://openalex.org/W3156106752","https://openalex.org/W3159250634","https://openalex.org/W3163128657","https://openalex.org/W3163411042","https://openalex.org/W3163443091","https://openalex.org/W3174333417","https://openalex.org/W3185727840","https://openalex.org/W4239019441","https://openalex.org/W4245990317","https://openalex.org/W4288086168","https://openalex.org/W4288086169","https://openalex.org/W4288086174","https://openalex.org/W4288414189","https://openalex.org/W4297823417"],"related_works":["https://openalex.org/W2883256816","https://openalex.org/W2171408034","https://openalex.org/W3003320923","https://openalex.org/W2106140982","https://openalex.org/W2152313554","https://openalex.org/W3048672182","https://openalex.org/W1509300825","https://openalex.org/W2054620577","https://openalex.org/W3092582874","https://openalex.org/W2065450024"],"abstract_inverted_index":{"Despite":[0],"impressive":[1],"performance":[2,131],"in":[3,30,101,128],"many":[4],"benchmark":[5],"datasets,":[6],"AI":[7,39,145],"models":[8,25],"can":[9,26],"still":[10],"make":[11,44],"mistakes,":[12],"especially":[13],"among":[14],"out-of-distribution":[15,118],"examples.":[16],"It":[17],"remains":[18],"an":[19,71],"open":[20],"question":[21],"how":[22],"such":[23],"imperfect":[24],"be":[27],"used":[28],"effectively":[29],"collaboration":[31,76],"with":[32,111],"humans.":[33],"Prior":[34],"work":[35],"has":[36],"focused":[37],"on":[38],"assistance":[40],"that":[41],"helps":[42],"people":[43],"individual":[45],"high-stakes":[46],"decisions,":[47,59],"which":[48],"is":[49],"not":[50],"scalable":[51],"for":[52,74,137],"a":[53,86,92,108],"large":[54],"amount":[55],"of":[56,85,125,144],"relatively":[57],"low-stakes":[58],"e.g.,":[60],"moderating":[61],"social":[62],"media":[63],"comments.":[64],"Instead,":[65],"we":[66,94],"propose":[67],"conditional":[68,103,126],"delegation":[69,104,127],"as":[70,91],"alternative":[72],"paradigm":[73],"human-AI":[75],"where":[77],"humans":[78,100],"create":[79],"rules":[80,105],"to":[81,98,114],"indicate":[82],"trustworthy":[83],"regions":[84],"model.":[87],"Using":[88],"content":[89],"moderation":[90],"testbed,":[93],"develop":[95],"novel":[96,139],"interfaces":[97],"assist":[99],"creating":[102],"and":[106,117,132],"conduct":[107],"randomized":[109],"experiment":[110],"two":[112],"datasets":[113],"simulate":[115],"in-distribution":[116],"scenarios.":[119],"Our":[120],"study":[121],"demonstrates":[122],"the":[123,142],"promise":[124],"improving":[129],"model":[130],"provides":[133],"insights":[134],"into":[135],"design":[136],"this":[138],"paradigm,":[140],"including":[141],"effect":[143],"explanations.":[146]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":39},{"year":2024,"cited_by_count":38},{"year":2023,"cited_by_count":31},{"year":2022,"cited_by_count":2}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}