{"id":"https://openalex.org/W4308765676","doi":"https://doi.org/10.1145/3555212","title":"Reliance and Automation for Human-AI Collaborative Data Labeling Conflict Resolution","display_name":"Reliance and Automation for Human-AI Collaborative Data Labeling Conflict Resolution","publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4308765676","doi":"https://doi.org/10.1145/3555212"},"language":"en","primary_location":{"id":"doi:10.1145/3555212","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3555212","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040101822","display_name":"Michelle Brachman","orcid":"https://orcid.org/0000-0001-8152-441X"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Michelle Brachman","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003440919","display_name":"Zahra Ashktorab","orcid":"https://orcid.org/0000-0002-0686-7911"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zahra Ashktorab","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056038276","display_name":"Michael Desmond","orcid":"https://orcid.org/0000-0002-1796-1161"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Desmond","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029055674","display_name":"Evelyn Duesterwald","orcid":"https://orcid.org/0000-0003-0884-4470"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evelyn Duesterwald","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082995130","display_name":"Casey Dugan","orcid":"https://orcid.org/0000-0002-1508-2091"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Casey Dugan","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039106227","display_name":"Narendra Nath Joshi","orcid":null},"institutions":[{"id":"https://openalex.org/I1306409833","display_name":"Adobe Systems (United States)","ror":"https://ror.org/059tvcg64","country_code":"US","type":"company","lineage":["https://openalex.org/I1306409833"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Narendra Nath Joshi","raw_affiliation_strings":["Adobe, San Jose, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Adobe, San Jose, CA, USA","institution_ids":["https://openalex.org/I1306409833"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064662859","display_name":"Qian Pan","orcid":"https://orcid.org/0000-0002-0437-1736"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qian Pan","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071548974","display_name":"Aabhas Sharma","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aabhas Sharma","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5040101822"],"corresponding_institution_ids":["https://openalex.org/I4210087032"],"apc_list":null,"apc_paid":null,"fwci":2.2098,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.878341,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"6","issue":"CSCW2","first_page":"1","last_page":"27"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.963100016117096,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10809","display_name":"Occupational Health and Safety Research","score":0.940500020980835,"subfield":{"id":"https://openalex.org/subfields/3614","display_name":"Radiological and Ultrasound Technology"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.8441483378410339},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7864607572555542},{"id":"https://openalex.org/keywords/resolution","display_name":"Resolution (logic)","score":0.6840124130249023},{"id":"https://openalex.org/keywords/conflict-resolution","display_name":"Conflict resolution","score":0.6501009464263916},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5686225891113281},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37539181113243103},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3431149125099182},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.07463440299034119},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.07273542881011963}],"concepts":[{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.8441483378410339},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7864607572555542},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.6840124130249023},{"id":"https://openalex.org/C21711469","wikidata":"https://www.wikidata.org/wiki/Q1194317","display_name":"Conflict resolution","level":2,"score":0.6501009464263916},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5686225891113281},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37539181113243103},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3431149125099182},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.07463440299034119},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.07273542881011963},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3555212","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3555212","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.47999998927116394,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W190293641","https://openalex.org/W1815588791","https://openalex.org/W1971810609","https://openalex.org/W2003497265","https://openalex.org/W2025544150","https://openalex.org/W2027103861","https://openalex.org/W2036131512","https://openalex.org/W2131587748","https://openalex.org/W2143539737","https://openalex.org/W2157018954","https://openalex.org/W2162409443","https://openalex.org/W2237502368","https://openalex.org/W2290009368","https://openalex.org/W2295680652","https://openalex.org/W2403343111","https://openalex.org/W2467897964","https://openalex.org/W2474104141","https://openalex.org/W2512274390","https://openalex.org/W2570668993","https://openalex.org/W2594164625","https://openalex.org/W2599032673","https://openalex.org/W2607311634","https://openalex.org/W2620896605","https://openalex.org/W2750714283","https://openalex.org/W2756188544","https://openalex.org/W2792932412","https://openalex.org/W2795548522","https://openalex.org/W2795783386","https://openalex.org/W2798706400","https://openalex.org/W2898349875","https://openalex.org/W2899134503","https://openalex.org/W2905034244","https://openalex.org/W2916904544","https://openalex.org/W2918027211","https://openalex.org/W2941411956","https://openalex.org/W2942157335","https://openalex.org/W2942161347","https://openalex.org/W2943189559","https://openalex.org/W2979893369","https://openalex.org/W2984353433","https://openalex.org/W2987755095","https://openalex.org/W2988571156","https://openalex.org/W2990435805","https://openalex.org/W2999637955","https://openalex.org/W3003278301","https://openalex.org/W3007702218","https://openalex.org/W3012753431","https://openalex.org/W3030109276","https://openalex.org/W3030790076","https://openalex.org/W3031679542","https://openalex.org/W3032875465","https://openalex.org/W3040472729","https://openalex.org/W3088366746","https://openalex.org/W3093660685","https://openalex.org/W3103751997","https://openalex.org/W3114710097","https://openalex.org/W3124026294","https://openalex.org/W3163469193","https://openalex.org/W4234188104","https://openalex.org/W4243342770","https://openalex.org/W4245383073","https://openalex.org/W4288086169"],"related_works":["https://openalex.org/W2218202131","https://openalex.org/W84108837","https://openalex.org/W2519676117","https://openalex.org/W2155740880","https://openalex.org/W4253249845","https://openalex.org/W2148444631","https://openalex.org/W1826068234","https://openalex.org/W2125452230","https://openalex.org/W2185138819","https://openalex.org/W574916204"],"abstract_inverted_index":{"Human":[0],"data":[1],"labeling":[2],"with":[3,154],"multiple":[4],"labelers":[5,85],"and":[6,26,57,108,128,135,141,144,187,190],"the":[7,12,79,102,177],"resulting":[8],"conflict":[9,39,48,65,76,148,169],"resolution":[10,21,49,170],"remains":[11],"norm":[13],"for":[14,75,113],"many":[15],"enterprise":[16],"machine":[17],"learning":[18],"pipelines.":[19],"Conflict":[20],"can":[22,37],"be":[23],"a":[24,91,152],"time-intensive":[25],"costly":[27],"process.":[28],"Our":[29],"goal":[30],"was":[31,106,171,179],"to":[32,44,121],"study":[33,153],"how":[34,58,134],"human-AI":[35],"collaboration":[36],"improve":[38],"resolution,":[40,77],"by":[41,83],"enabling":[42],"users":[43,137],"automate":[45],"groups":[46,116],"of":[47,117,167,175],"tasks.":[50],"However,":[51],"little":[52],"is":[53,86,111],"known":[54],"about":[55],"whether":[56,123,176],"people":[59],"will":[60],"rely":[61,138],"on":[62,139,145,181,198],"automation":[63,68,110,162,178],"during":[64],"resolution.":[66,149],"Currently,":[67],"commonly":[69],"uses":[70],"labelers'":[71],"majority":[72,104],"vote":[73,105],"labels":[74,193],"as":[78],"top":[80],"chosen":[81],"label":[82],"most":[84],"often":[87],"correct.":[88],"We":[89,150,159],"envisioned":[90],"system":[92],"where":[93,101,109],"an":[94],"AI":[95,129,142,184,191],"would":[96],"assist":[97],"in":[98],"finding":[99],"cases":[100],"labeler":[103,127,140,182,189],"wrong":[107],"supported":[112],"batches":[114],"or":[115,183],"conflicts.":[118],"In":[119],"order":[120],"understand":[122],"humans":[124],"could":[125],"use":[126,166],"information":[130,143],"effectively,":[131],"we":[132],"investigated":[133],"when":[136],"automated":[146,168],"group":[147],"ran":[151],"144":[155],"Mechanical":[156],"Turk":[157],"workers.":[158],"found":[160],"that":[161],"increased":[163],"users'":[164],"accuracy/time,":[165],"relatively":[172],"similar":[173],"regardless":[174],"based":[180],"selected":[185,192],"labels,":[186],"providing":[188],"may":[194],"reduce":[195],"inappropriate":[196],"reliance":[197],"automation.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}