{"id":"https://openalex.org/W4392929788","doi":"https://doi.org/10.1145/3613904.3642089","title":"LabelAId: Just-in-time AI Interventions for Improving Human Labeling Quality and Domain Knowledge in Crowdsourcing Systems","display_name":"LabelAId: Just-in-time AI Interventions for Improving Human Labeling Quality and Domain Knowledge in Crowdsourcing Systems","publication_year":2024,"publication_date":"2024-05-11","ids":{"openalex":"https://openalex.org/W4392929788","doi":"https://doi.org/10.1145/3613904.3642089"},"language":"en","primary_location":{"id":"doi:10.1145/3613904.3642089","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642089","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642089","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642089","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100675408","display_name":"Chu Li","orcid":"https://orcid.org/0009-0003-7612-6224"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chu Li","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0009-0003-7612-6224","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101798380","display_name":"Zhihan Zhang","orcid":"https://orcid.org/0000-0001-7394-5409"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhihan Zhang","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0001-7394-5409","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037421188","display_name":"Michael Saugstad","orcid":"https://orcid.org/0000-0002-1117-4095"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Saugstad","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0002-1117-4095","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077528185","display_name":"Esteban Safranchik","orcid":"https://orcid.org/0000-0003-3226-816X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Esteban Safranchik","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0003-3226-816X","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011626439","display_name":"Minchu Kulkarni","orcid":"https://orcid.org/0009-0000-9443-953X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chaitanyashareef Kulkarni","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0009-0000-9443-953X","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102025052","display_name":"Xiaoyu Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaoyu Huang","raw_affiliation_strings":["Department of Mechanical Engineering, University of California, Berkeley, United States"],"raw_orcid":"https://orcid.org/0009-0005-0714-3711","affiliations":[{"raw_affiliation_string":"Department of Mechanical Engineering, University of California, Berkeley, United States","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039879761","display_name":"Shwetak Patel","orcid":"https://orcid.org/0000-0002-6300-4389"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shwetak Patel","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0002-6300-4389","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080282367","display_name":"Vikram Iyer","orcid":"https://orcid.org/0000-0002-3025-7953"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vikram Iyer","raw_affiliation_strings":["Paul G. Allen School of Computer Science and Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0002-3025-7953","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science and Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033639139","display_name":"Tim Althoff","orcid":"https://orcid.org/0000-0003-4793-2289"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tim Althoff","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0003-4793-2289","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016828530","display_name":"Jon E. Froehlich","orcid":"https://orcid.org/0000-0001-8291-3353"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jon E. Froehlich","raw_affiliation_strings":["Paul G. Allen School of Computer Science & Engineering, University of Washington, United States"],"raw_orcid":"https://orcid.org/0000-0001-8291-3353","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington, United States","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5100675408"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":3.5142,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.93684084,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"21"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9843999743461609,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9796000123023987,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.957961916923523},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7577136754989624},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7365373969078064},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.6133677363395691},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.58591628074646},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.583777666091919},{"id":"https://openalex.org/keywords/mistake","display_name":"Mistake","score":0.4973814785480499},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4575883150100708},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.44980043172836304},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4355441927909851},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3777152895927429},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3281010389328003},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.19215965270996094}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.957961916923523},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7577136754989624},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7365373969078064},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.6133677363395691},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.58591628074646},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.583777666091919},{"id":"https://openalex.org/C2777179996","wikidata":"https://www.wikidata.org/wiki/Q911222","display_name":"Mistake","level":2,"score":0.4973814785480499},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4575883150100708},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.44980043172836304},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4355441927909851},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3777152895927429},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3281010389328003},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.19215965270996094},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3613904.3642089","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642089","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642089","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2403.09810","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.09810","pdf_url":"https://arxiv.org/pdf/2403.09810","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3613904.3642089","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3613904.3642089","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3613904.3642089","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/11","display_name":"Sustainable cities and communities","score":0.75}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392929788.pdf"},"referenced_works_count":81,"referenced_works":["https://openalex.org/W194327752","https://openalex.org/W618423493","https://openalex.org/W1846955904","https://openalex.org/W1983336515","https://openalex.org/W1985095406","https://openalex.org/W2014100104","https://openalex.org/W2048989977","https://openalex.org/W2073338313","https://openalex.org/W2095030488","https://openalex.org/W2104749423","https://openalex.org/W2108598243","https://openalex.org/W2111298664","https://openalex.org/W2124994029","https://openalex.org/W2127008633","https://openalex.org/W2129410131","https://openalex.org/W2131723483","https://openalex.org/W2141613588","https://openalex.org/W2151401338","https://openalex.org/W2167913131","https://openalex.org/W2168765606","https://openalex.org/W2171965515","https://openalex.org/W2295598076","https://openalex.org/W2295675274","https://openalex.org/W2296739333","https://openalex.org/W2314911310","https://openalex.org/W2397110394","https://openalex.org/W2400269077","https://openalex.org/W2537561668","https://openalex.org/W2603194309","https://openalex.org/W2607311634","https://openalex.org/W2769041395","https://openalex.org/W2801371430","https://openalex.org/W2804462692","https://openalex.org/W2810116858","https://openalex.org/W2889496369","https://openalex.org/W2895604144","https://openalex.org/W2898349875","https://openalex.org/W2899483097","https://openalex.org/W2901895173","https://openalex.org/W2912311590","https://openalex.org/W2916904544","https://openalex.org/W2941798514","https://openalex.org/W2948636975","https://openalex.org/W2959716049","https://openalex.org/W2979893369","https://openalex.org/W2982243661","https://openalex.org/W2987705220","https://openalex.org/W2997309766","https://openalex.org/W3001062618","https://openalex.org/W3009731884","https://openalex.org/W3010911662","https://openalex.org/W3013160799","https://openalex.org/W3034563230","https://openalex.org/W3099742594","https://openalex.org/W3101792976","https://openalex.org/W3102960713","https://openalex.org/W3120468046","https://openalex.org/W3121257585","https://openalex.org/W3121522380","https://openalex.org/W3124258878","https://openalex.org/W3128621548","https://openalex.org/W3158509300","https://openalex.org/W3159250634","https://openalex.org/W3163443091","https://openalex.org/W3183520462","https://openalex.org/W3188564437","https://openalex.org/W3205934028","https://openalex.org/W3211401540","https://openalex.org/W4226250826","https://openalex.org/W4230939871","https://openalex.org/W4243391013","https://openalex.org/W4281988409","https://openalex.org/W4283776467","https://openalex.org/W4300723004","https://openalex.org/W4307475380","https://openalex.org/W4313047369","https://openalex.org/W4322765902","https://openalex.org/W4380365974","https://openalex.org/W4385577819","https://openalex.org/W4386270037","https://openalex.org/W4396827169"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W1590719878","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662"],"abstract_inverted_index":{"Crowdsourcing":[0],"platforms":[1],"have":[2],"transformed":[3],"distributed":[4],"problem-solving,":[5],"yet":[6],"quality":[7,14,41],"control":[8,15],"remains":[9],"a":[10],"persistent":[11],"challenge.":[12],"Traditional":[13],"measures,":[16],"such":[17],"as":[18],"prescreening":[19],"workers":[20],"and":[21,42,69,136],"refining":[22],"instructions,":[23],"often":[24],"focus":[25],"solely":[26],"on":[27,66],"optimizing":[28],"economic":[29],"output.":[30],"This":[31],"paper":[32],"explores":[33],"just-in-time":[34],"AI":[35],"interventions":[36],"to":[37,61,139],"enhance":[38],"both":[39],"labeling":[40],"domain-specific":[43],"knowledge":[44],"among":[45],"crowdworkers.":[46],"We":[47,95,130],"introduce":[48],"LabelAId,":[49],"an":[50,102],"advanced":[51],"inference":[52,87],"model":[53],"combining":[54],"Programmatic":[55],"Weak":[56],"Supervision":[57],"(PWS)":[58],"with":[59,91,112],"FT-Transformers":[60],"infer":[62],"label":[63,120],"correctness":[64],"based":[65],"user":[67],"behavior":[68],"domain":[70],"knowledge.":[71],"Our":[72],"technical":[73],"evaluation":[74],"shows":[75],"that":[76,116],"our":[77],"LabelAId":[78,98,117],"pipeline":[79],"consistently":[80],"outperforms":[81],"state-of-the-art":[82],"ML":[83],"baselines,":[84],"improving":[85],"mistake":[86],"accuracy":[88],"by":[89],"36.7%":[90],"50":[92],"downstream":[93],"samples.":[94],"then":[96],"implemented":[97],"into":[99],"Project":[100],"Sidewalk,":[101],"open-source":[103],"crowdsourcing":[104],"platform":[105],"for":[106],"urban":[107],"accessibility.":[108],"A":[109],"between-subjects":[110],"study":[111],"34":[113],"participants":[114],"demonstrates":[115],"significantly":[118],"enhances":[119],"precision":[121],"without":[122],"compromising":[123],"efficiency":[124],"while":[125],"also":[126],"increasing":[127],"labeler":[128],"confidence.":[129],"discuss":[131],"LabelAId\u2019s":[132],"success":[133],"factors,":[134],"limitations,":[135],"its":[137],"generalizability":[138],"other":[140],"crowdsourced":[141],"science":[142],"domains.":[143]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2024-03-19T00:00:00"}