{"id":"https://openalex.org/W3158839228","doi":"https://doi.org/10.1145/3449163","title":"AI-Assisted Human Labeling","display_name":"AI-Assisted Human Labeling","publication_year":2021,"publication_date":"2021-04-13","ids":{"openalex":"https://openalex.org/W3158839228","doi":"https://doi.org/10.1145/3449163","mag":"3158839228"},"language":"en","primary_location":{"id":"doi:10.1145/3449163","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3449163","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003440919","display_name":"Zahra Ashktorab","orcid":"https://orcid.org/0000-0002-0686-7911"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zahra Ashktorab","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056038276","display_name":"Michael Desmond","orcid":"https://orcid.org/0000-0002-1796-1161"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Desmond","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086240513","display_name":"Josh Andr\u00e9s","orcid":"https://orcid.org/0000-0001-5882-3139"},"institutions":[{"id":"https://openalex.org/I4210120068","display_name":"IBM Research - Australia","ror":"https://ror.org/027r3nx49","country_code":"AU","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210120068"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Josh Andres","raw_affiliation_strings":["IBM Research Australia, Melbourne, VIC, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research Australia, Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I4210120068"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026028381","display_name":"Michael M\u00fcller","orcid":"https://orcid.org/0000-0001-7860-163X"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Muller","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039106227","display_name":"Narendra Nath Joshi","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Narendra Nath Joshi","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040101822","display_name":"Michelle Brachman","orcid":"https://orcid.org/0000-0001-8152-441X"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michelle Brachman","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071548974","display_name":"Aabhas Sharma","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aabhas Sharma","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067037429","display_name":"Kristina Brimijoin","orcid":"https://orcid.org/0000-0001-5616-9567"},"institutions":[{"id":"https://openalex.org/I1308814778","display_name":"Hastings Center","ror":"https://ror.org/02pmr4c75","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1308814778"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kristina Brimijoin","raw_affiliation_strings":["IBM Research, Hastings on Hudson, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Hastings on Hudson, NY, USA","institution_ids":["https://openalex.org/I1308814778"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064662859","display_name":"Qian Pan","orcid":"https://orcid.org/0000-0002-0437-1736"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qian Pan","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101998484","display_name":"Christine T. Wolf","orcid":"https://orcid.org/0000-0003-3525-7204"},"institutions":[{"id":"https://openalex.org/I4210121988","display_name":"Film Independent","ror":"https://ror.org/036cy3843","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210121988"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christine T. Wolf","raw_affiliation_strings":["Independent Researcher, San Jose, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent Researcher, San Jose, CA, USA","institution_ids":["https://openalex.org/I4210121988"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029055674","display_name":"Evelyn Duesterwald","orcid":"https://orcid.org/0000-0003-0884-4470"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Evelyn Duesterwald","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082995130","display_name":"Casey Dugan","orcid":"https://orcid.org/0000-0002-1508-2091"},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Casey Dugan","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Werner Geyer","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087032","display_name":"Cambridge Scientific (United States)","ror":"https://ror.org/001s4dh65","country_code":"US","type":"company","lineage":["https://openalex.org/I4210087032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Werner Geyer","raw_affiliation_strings":["IBM Research, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Cambridge, MA, USA","institution_ids":["https://openalex.org/I4210087032"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003669963","display_name":"Darrell Reimer","orcid":null},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Darrell Reimer","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":14,"corresponding_author_ids":["https://openalex.org/A5003440919"],"corresponding_institution_ids":["https://openalex.org/I1341412227"],"apc_list":null,"apc_paid":null,"fwci":5.872,"has_fulltext":false,"cited_by_count":41,"citation_normalized_percentile":{"value":0.96134663,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"5","issue":"CSCW1","first_page":"1","last_page":"27"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9781000018119812,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8342501521110535},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.5679221153259277},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5421188473701477},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5197436809539795},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.485617071390152},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4686260521411896},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.45666319131851196},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42593914270401},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.34688887000083923},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.08713236451148987}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8342501521110535},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.5679221153259277},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5421188473701477},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5197436809539795},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.485617071390152},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4686260521411896},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.45666319131851196},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42593914270401},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.34688887000083923},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.08713236451148987},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3449163","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3449163","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W95067517","https://openalex.org/W216499477","https://openalex.org/W1532325895","https://openalex.org/W1673310716","https://openalex.org/W1965463297","https://openalex.org/W1996728935","https://openalex.org/W2007567868","https://openalex.org/W2036431079","https://openalex.org/W2059216172","https://openalex.org/W2070517604","https://openalex.org/W2100934737","https://openalex.org/W2110151287","https://openalex.org/W2123326341","https://openalex.org/W2182687482","https://openalex.org/W2258849743","https://openalex.org/W2264788300","https://openalex.org/W2281090488","https://openalex.org/W2299467264","https://openalex.org/W2517502945","https://openalex.org/W2529518929","https://openalex.org/W2531638282","https://openalex.org/W2602896535","https://openalex.org/W2618735189","https://openalex.org/W2756029264","https://openalex.org/W2789589462","https://openalex.org/W2794079986","https://openalex.org/W2911396267","https://openalex.org/W2921524648","https://openalex.org/W2941411956","https://openalex.org/W2942073295","https://openalex.org/W2942103387","https://openalex.org/W2942157335","https://openalex.org/W2942465698","https://openalex.org/W2970837303","https://openalex.org/W2982773700","https://openalex.org/W2995482541","https://openalex.org/W3005126002","https://openalex.org/W3009312460","https://openalex.org/W3013160799","https://openalex.org/W3029504795","https://openalex.org/W3032086959","https://openalex.org/W3099545035","https://openalex.org/W3106497370","https://openalex.org/W3122753382","https://openalex.org/W3137218931","https://openalex.org/W3157973799","https://openalex.org/W4214907476","https://openalex.org/W4235216760","https://openalex.org/W4247891045","https://openalex.org/W4288086169","https://openalex.org/W4389049946"],"related_works":["https://openalex.org/W2130553454","https://openalex.org/W3022007134","https://openalex.org/W4317548404","https://openalex.org/W2087783760","https://openalex.org/W1509924131","https://openalex.org/W3163689946","https://openalex.org/W4287890939","https://openalex.org/W2033364610","https://openalex.org/W2797776314","https://openalex.org/W2153927146"],"abstract_inverted_index":{"Human":[0],"labeling":[1,33,69,72,82,120],"of":[2,11,61,80,92,118],"training":[3],"data":[4,27],"is":[5],"often":[6],"a":[7,31,42,67],"time-consuming,":[8],"expensive":[9],"part":[10],"machine":[12],"learning.":[13],"In":[14],"this":[15],"paper,":[16],"we":[17],"study":[18,45],"\"batch":[19],"labeling\",":[20],"an":[21],"AI-assisted":[22],"UX":[23],"paradigm,":[24],"that":[25],"aids":[26],"labelers":[28],"by":[29],"allowing":[30],"single":[32],"action":[34],"to":[35,37,52,66],"apply":[36],"multiple":[38],"records.":[39],"We":[40,57,87],"ran":[41],"large":[43],"scale":[44],"on":[46,83,99,127],"Mechanical":[47],"Turk":[48],"with":[49],"156":[50],"participants":[51],"investigate":[53,58,89],"labeler-AI-batching":[54,128],"system":[55,63,129],"interaction.":[56,130],"the":[59,62,78,90,100,116],"efficacy":[60],"when":[64],"compared":[65],"single-item":[68],"interface":[70],"(i.e.,":[71],"one":[73],"record":[74],"at-a-time),":[75],"and":[76,85,96,122],"evaluate":[77],"impact":[79,91],"batch":[81,119],"accuracy":[84],"time.":[86],"further":[88],"AI":[93],"algorithm":[94],"quality":[95],"its":[97],"effects":[98],"labelers'":[101],"overreliance,":[102],"as":[103,105],"well":[104],"potential":[106],"mechanisms":[107],"for":[108,115,123],"mitigating":[109],"it.":[110],"Our":[111],"work":[112,124],"offers":[113],"implications":[114],"design":[117],"systems":[121],"practices":[125],"focusing":[126]},"counts_by_year":[{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":4}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}