{"id":"https://openalex.org/W4320024129","doi":"https://doi.org/10.1109/bigdata55660.2022.10020844","title":"Efficient Evaluation of AI Workers for the Human+AI Crowd Task Assignment","display_name":"Efficient Evaluation of AI Workers for the Human+AI Crowd Task Assignment","publication_year":2022,"publication_date":"2022-12-17","ids":{"openalex":"https://openalex.org/W4320024129","doi":"https://doi.org/10.1109/bigdata55660.2022.10020844"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata55660.2022.10020844","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata55660.2022.10020844","pdf_url":null,"source":{"id":"https://openalex.org/S4363607709","display_name":"2022 IEEE International Conference on Big Data (Big Data)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081226856","display_name":"Tomoya Kanda","orcid":"https://orcid.org/0000-0003-1210-3069"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tomoya Kanda","raw_affiliation_strings":["University of Tsukuba,Japan","University of Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tsukuba,Japan","institution_ids":["https://openalex.org/I146399215"]},{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053260657","display_name":"Hiroyoshi Ito","orcid":"https://orcid.org/0000-0002-3265-7029"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hiroyoshi Ito","raw_affiliation_strings":["University of Tsukuba,Japan","University of Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tsukuba,Japan","institution_ids":["https://openalex.org/I146399215"]},{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071533497","display_name":"Atsuyuki Morishima","orcid":"https://orcid.org/0000-0003-4606-9065"},"institutions":[{"id":"https://openalex.org/I146399215","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07","country_code":"JP","type":"education","lineage":["https://openalex.org/I146399215"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Atsuyuki Morishima","raw_affiliation_strings":["University of Tsukuba,Japan","University of Tsukuba, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Tsukuba,Japan","institution_ids":["https://openalex.org/I146399215"]},{"raw_affiliation_string":"University of Tsukuba, Japan","institution_ids":["https://openalex.org/I146399215"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I146399215"],"apc_list":null,"apc_paid":null,"fwci":0.9817,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.77619152,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"3995","last_page":"4001"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.794808030128479},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7537060976028442},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5731551647186279},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5489476323127747},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.4883436858654022},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4869123697280884},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4665738344192505},{"id":"https://openalex.org/keywords/knowledge-worker","display_name":"Knowledge worker","score":0.42651987075805664},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4256677031517029},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.42448121309280396},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.12760791182518005},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10944604873657227}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.794808030128479},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7537060976028442},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5731551647186279},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5489476323127747},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.4883436858654022},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4869123697280884},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4665738344192505},{"id":"https://openalex.org/C2779089590","wikidata":"https://www.wikidata.org/wiki/Q1914017","display_name":"Knowledge worker","level":3,"score":0.42651987075805664},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4256677031517029},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.42448121309280396},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.12760791182518005},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10944604873657227},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata55660.2022.10020844","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata55660.2022.10020844","pdf_url":null,"source":{"id":"https://openalex.org/S4363607709","display_name":"2022 IEEE International Conference on Big Data (Big Data)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/8","score":0.7200000286102295,"display_name":"Decent work and economic growth"}],"awards":[{"id":"https://openalex.org/G8526979979","display_name":"Computational Division of Labor: Theoretical and Constructive Research with Meta Labor Platforms","funder_award_id":"22H00508","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320323954","display_name":"University of Tsukuba","ror":"https://ror.org/02956yf07"},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W2144642230","https://openalex.org/W2902986194","https://openalex.org/W3006601794","https://openalex.org/W3008458444","https://openalex.org/W3046659224","https://openalex.org/W3113223172","https://openalex.org/W3136016591","https://openalex.org/W3211711472","https://openalex.org/W4226379115","https://openalex.org/W6681372996","https://openalex.org/W6781748751","https://openalex.org/W6791384607","https://openalex.org/W6803979183","https://openalex.org/W6811262654"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4387860379"],"abstract_inverted_index":{"Nowadays,":[0],"it":[1],"is":[2,42,47,135],"a":[3,75,96,105],"common":[4],"practice":[5],"for":[6,74],"crowd":[7,21,23],"workers":[8,46,52,86],"to":[9,111,121,137,146,170],"develop":[10,25],"ML":[11],"models":[12],"that":[13,33,54,152],"classify":[14],"data":[15],"items.":[16],"We":[17,142],"envision":[18],"the":[19,67,82,91,117,129,133,139,155,165],"Human+AI":[20],"where":[22],"programmers":[24],"\"AI":[26],"workers,\"":[27],"which":[28],"are":[29,104],"black-box":[30],"software":[31],"agents":[32],"work":[34,80],"among":[35],"other":[36],"human":[37,51],"workers.":[38,172],"The":[39],"problem":[40,118],"here":[41],"evaluating":[43,50],"such":[44,95],"AI":[45,85,124,130,134,171],"different":[48],"from":[49,84],"in":[53],"they":[55,62,89],"may":[56],"not":[57,100],"be":[58,112],"spam":[59],"workers,":[60],"although":[61],"have":[63],"low":[64],"accuracy":[65],"at":[66],"beginning":[68],"of":[69,108,119,157,161,167],"their":[70],"learning":[71],"process":[72],"or":[73],"particular":[76],"label.":[77],"Therefore,":[78],"existing":[79],"evaluates":[81],"output":[83,90],"every":[87],"time":[88],"task":[92,109,168],"results.":[93],"Obviously,":[94],"naive":[97],"evaluation":[98,131],"does":[99],"scale":[101],"because":[102],"there":[103],"tremendous":[106],"number":[107,156,166],"results":[110],"evaluated.":[113],"This":[114],"paper":[115],"addresses":[116],"how":[120],"efficiently":[122],"evaluate":[123],"worker":[125],"outputs":[126],"by":[127,159],"skipping":[128],"when":[132],"unlikely":[136],"satisfy":[138],"expected":[140],"accuracy.":[141],"conducted":[143],"an":[144],"experiment":[145],"compare":[147],"two":[148],"strategies":[149],"and":[150],"found":[151],"both":[153],"reduce":[154],"evaluations":[158],"orders":[160],"magnitude":[162],"while":[163],"keeping":[164],"assignments":[169]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}