{"id":"https://openalex.org/W4283460058","doi":"https://doi.org/10.1145/3501247.3531580","title":"HAEM: Obtaining Higher-Quality Classification Task Results with AI Workers","display_name":"HAEM: Obtaining Higher-Quality Classification Task Results with AI Workers","publication_year":2022,"publication_date":"2022-06-24","ids":{"openalex":"https://openalex.org/W4283460058","doi":"https://doi.org/10.1145/3501247.3531580"},"language":"en","primary_location":{"id":"doi:10.1145/3501247.3531580","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3501247.3531580","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"14th ACM Web Science Conference 2022","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104077313","display_name":"Yu Yamashita","orcid":"https://orcid.org/0009-0009-1551-7268"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Yu Yamashita","raw_affiliation_strings":["University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053260657","display_name":"Hiroyoshi Ito","orcid":"https://orcid.org/0000-0002-3265-7029"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hiroyoshi Ito","raw_affiliation_strings":["University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100887101","display_name":"Kei Wakabayashi","orcid":null},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kei Wakabayashi","raw_affiliation_strings":["University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025747344","display_name":"Masaki Kobayashi","orcid":"https://orcid.org/0000-0002-4453-0615"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masaki Kobayashi","raw_affiliation_strings":["University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071533497","display_name":"Atsuyuki Morishima","orcid":"https://orcid.org/0000-0003-4606-9065"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Atsuyuki Morishima","raw_affiliation_strings":["University of Tsukuba, Japan"],"affiliations":[{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5104077313"],"corresponding_institution_ids":["https://openalex.org/I146399215"],"apc_list":null,"apc_paid":null,"fwci":0.2176,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.61280973,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"118","last_page":"128"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.987500011920929,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7680017948150635},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7329503893852234},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6569465398788452},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6400505304336548},{"id":"https://openalex.org/keywords/confusion","display_name":"Confusion","score":0.6224414110183716},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.6205259561538696},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6202402710914612},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5559769868850708},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5506540536880493},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5352215766906738},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4452402591705322}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7680017948150635},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7329503893852234},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6569465398788452},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6400505304336548},{"id":"https://openalex.org/C2781140086","wikidata":"https://www.wikidata.org/wiki/Q557945","display_name":"Confusion","level":2,"score":0.6224414110183716},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.6205259561538696},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6202402710914612},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5559769868850708},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5506540536880493},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5352215766906738},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4452402591705322},{"id":"https://openalex.org/C127716648","wikidata":"https://www.wikidata.org/wiki/Q104053","display_name":"Phenotype","level":3,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C188082640","wikidata":"https://www.wikidata.org/wiki/Q1780899","display_name":"Complementation","level":4,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C11171543","wikidata":"https://www.wikidata.org/wiki/Q41630","display_name":"Psychoanalysis","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3501247.3531580","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3501247.3531580","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"14th ACM Web Science Conference 2022","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.5400000214576721}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W2012668444","https://openalex.org/W2064677871","https://openalex.org/W2073068515","https://openalex.org/W2098865355","https://openalex.org/W2168747479","https://openalex.org/W2262342046","https://openalex.org/W2367397349","https://openalex.org/W2471138382","https://openalex.org/W2518897583","https://openalex.org/W2562389319","https://openalex.org/W2743596952","https://openalex.org/W2789833952","https://openalex.org/W2798820905","https://openalex.org/W2899828811","https://openalex.org/W2901639336","https://openalex.org/W2918341242","https://openalex.org/W2941411956","https://openalex.org/W2977942577","https://openalex.org/W2988966271","https://openalex.org/W2996060033","https://openalex.org/W3035371891","https://openalex.org/W3119689140","https://openalex.org/W3121921079","https://openalex.org/W3156669901","https://openalex.org/W3160343688","https://openalex.org/W3164008977","https://openalex.org/W3206428286","https://openalex.org/W4211173503","https://openalex.org/W4287854312","https://openalex.org/W4288414189","https://openalex.org/W4293409613","https://openalex.org/W4296978576"],"related_works":["https://openalex.org/W2794206341","https://openalex.org/W2178171640","https://openalex.org/W1472755691","https://openalex.org/W2055733113","https://openalex.org/W2739352035","https://openalex.org/W2109094787","https://openalex.org/W2896200027","https://openalex.org/W4285197194","https://openalex.org/W3102460643","https://openalex.org/W2522741018"],"abstract_inverted_index":{"Obtaining":[0],"high-quality":[1],"results":[2,78,104],"for":[3,67,82],"a":[4,11,15,63,83,89,99,112],"fixed":[5,84],"set":[6,148],"of":[7,22,50,71,79,86,105,149],"classification":[8],"tasks":[9,87],"with":[10,88,94,154,190],"limited":[12,90],"budget":[13],"is":[14],"critical":[16],"issue":[17],"in":[18,47,111,175],"crowdsourcing.":[19],"The":[20,115],"introduction":[21],"AI":[23,54,73,96,109,129,198],"models":[24,55,130],"to":[25,38,52,74,139],"complement":[26],"the":[27,41,48,69,76,103,123,128,172,180,186],"process":[28],"should":[29],"be":[30],"explored.":[31],"However,":[32],"there":[33],"are":[34,204],"few":[35],"existing":[36],"approaches":[37],"directly":[39],"address":[40],"problem,":[42],"which":[43,121],"have":[44,215],"been":[45],"proposed":[46,116],"context":[49],"how":[51],"train":[53],"using":[56],"noisy":[57],"crowdsourced":[58],"data.":[59],"This":[60],"paper":[61],"presents":[62],"more":[64],"direct":[65],"approach":[66],"solving":[68],"problem":[70],"introducing":[72],"improve":[75],"task":[77],"human":[80,107,192],"workers":[81,110,132,199],"number":[85],"budget;":[91],"we":[92],"deal":[93],"an":[95,146],"model":[97,189],"as":[98,131],"worker":[100],"and":[101,108,133,151,159,178],"aggregates":[102],"both":[106],"symmetric":[113],"manner.":[114],"\u201cHuman-AI":[117],"EM\u201d":[118],"(HAEM)":[119],"algorithm,":[120],"extends":[122],"Dawid":[124,160,187],"Skene":[125,161,188],"model,":[126],"treats":[127],"explicitly":[134],"computes":[135],"their":[136],"confusion":[137],"matrices":[138],"derive":[140],"higher-quality":[141],"aggregation":[142],"results.":[143],"We":[144,163,194],"conducted":[145],"extensive":[147],"experiments":[150],"compared":[152],"HAEM":[153,167,181],"two":[155,173],"other":[156],"methods":[157,174],"(MBEM":[158],"model).":[162],"found":[164,196],"that":[165,179,197],"AI-powered":[166],"showed":[168],"better":[169,184],"performance":[170],"than":[171,185],"most":[176],"cases":[177],"often":[182],"performed":[183],"additional":[191],"workers.":[193],"also":[195],"work":[200],"well":[201],"when":[202],"they":[203,212],"good":[205,217],"at":[206],"identifying":[207],"particular":[208],"classes":[209],"even":[210],"if":[211],"do":[213],"not":[214],"very":[216],"overall":[218],"accuracy.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}