{"id":"https://openalex.org/W4377238720","doi":"https://doi.org/10.1145/3597201","title":"How Many Crowd Workers Do I Need? On Statistical Power when Crowdsourcing Relevance Judgments","display_name":"How Many Crowd Workers Do I Need? On Statistical Power when Crowdsourcing Relevance Judgments","publication_year":2023,"publication_date":"2023-05-22","ids":{"openalex":"https://openalex.org/W4377238720","doi":"https://doi.org/10.1145/3597201"},"language":"en","primary_location":{"id":"doi:10.1145/3597201","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597201","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597201","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3597201","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050287516","display_name":"Kevin Roitero","orcid":"https://orcid.org/0000-0002-9191-3280"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Kevin Roitero","raw_affiliation_strings":["University of Udine, Italy"],"raw_orcid":"https://orcid.org/0000-0002-9191-3280","affiliations":[{"raw_affiliation_string":"University of Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078352413","display_name":"David La Barbera","orcid":"https://orcid.org/0000-0002-8215-5502"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"David La Barbera","raw_affiliation_strings":["University of Udine, Italy"],"raw_orcid":"https://orcid.org/0000-0002-8215-5502","affiliations":[{"raw_affiliation_string":"University of Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028918939","display_name":"Michael Soprano","orcid":"https://orcid.org/0000-0002-7337-7592"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Michael Soprano","raw_affiliation_strings":["University of Udine, Italy"],"raw_orcid":"https://orcid.org/0000-0002-7337-7592","affiliations":[{"raw_affiliation_string":"University of Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["The University of Queensland, Australia"],"raw_orcid":"https://orcid.org/0000-0002-7311-3693","affiliations":[{"raw_affiliation_string":"The University of Queensland, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005575421","display_name":"Stefano Mizzaro","orcid":"https://orcid.org/0000-0002-2852-168X"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Stefano Mizzaro","raw_affiliation_strings":["University of Udine, Italy"],"raw_orcid":"https://orcid.org/0000-0002-2852-168X","affiliations":[{"raw_affiliation_string":"University of Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6720-963X","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6116,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.87436478,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"42","issue":"1","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.930635929107666},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8339141011238098},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8219445943832397},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.7262951731681824},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.622226893901825},{"id":"https://openalex.org/keywords/statistical-power","display_name":"Statistical power","score":0.5983060002326965},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.5689902901649475},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5341433882713318},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4811869263648987},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.4430619478225708},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.4374202787876129},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.41499030590057373},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4056600332260132},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36277541518211365},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2674345374107361},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.20913812518119812},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09955668449401855},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.08913993835449219}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.930635929107666},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8339141011238098},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8219445943832397},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.7262951731681824},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.622226893901825},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.5983060002326965},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.5689902901649475},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5341433882713318},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4811869263648987},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.4430619478225708},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.4374202787876129},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.41499030590057373},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4056600332260132},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36277541518211365},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2674345374107361},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.20913812518119812},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09955668449401855},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.08913993835449219},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3597201","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597201","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597201","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3597201","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597201","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597201","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.5899999737739563}],"awards":[{"id":"https://openalex.org/G8820337412","display_name":"ARC Training Centre for Information Resilience","funder_award_id":"IC200100022","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4377238720.pdf","grobid_xml":"https://content.openalex.org/works/W4377238720.grobid-xml"},"referenced_works_count":59,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W578092267","https://openalex.org/W1533667328","https://openalex.org/W1824124613","https://openalex.org/W1976076261","https://openalex.org/W1990190154","https://openalex.org/W2004918107","https://openalex.org/W2010181950","https://openalex.org/W2022995284","https://openalex.org/W2023217403","https://openalex.org/W2024166834","https://openalex.org/W2027359109","https://openalex.org/W2039017419","https://openalex.org/W2045997031","https://openalex.org/W2053528567","https://openalex.org/W2073722401","https://openalex.org/W2075893676","https://openalex.org/W2086676398","https://openalex.org/W2088901214","https://openalex.org/W2091524854","https://openalex.org/W2101496176","https://openalex.org/W2106568252","https://openalex.org/W2124504084","https://openalex.org/W2126105956","https://openalex.org/W2133084664","https://openalex.org/W2136606893","https://openalex.org/W2151401338","https://openalex.org/W2160473997","https://openalex.org/W2235125461","https://openalex.org/W2338477879","https://openalex.org/W2516238915","https://openalex.org/W2558888286","https://openalex.org/W2569307494","https://openalex.org/W2589404650","https://openalex.org/W2605232020","https://openalex.org/W2769709274","https://openalex.org/W2783946743","https://openalex.org/W2798782484","https://openalex.org/W2809759799","https://openalex.org/W2894621306","https://openalex.org/W2903058329","https://openalex.org/W2923053417","https://openalex.org/W2934118928","https://openalex.org/W2944053130","https://openalex.org/W2963196014","https://openalex.org/W2991191531","https://openalex.org/W3023252837","https://openalex.org/W3083137311","https://openalex.org/W3093924713","https://openalex.org/W3158689111","https://openalex.org/W3159852056","https://openalex.org/W3188970555","https://openalex.org/W3196879927","https://openalex.org/W4205544037","https://openalex.org/W4213171314","https://openalex.org/W4224685053","https://openalex.org/W4237909588","https://openalex.org/W4284701291","https://openalex.org/W4300869678"],"related_works":["https://openalex.org/W3003558595","https://openalex.org/W2035798565","https://openalex.org/W2316908264","https://openalex.org/W2115979595","https://openalex.org/W2585981346","https://openalex.org/W1963493069","https://openalex.org/W2067995684","https://openalex.org/W2001107817","https://openalex.org/W2259063491","https://openalex.org/W3142061843"],"abstract_inverted_index":{"To":[0],"scale":[1,151],"the":[2,41,72,99,126,143,146,149,183,193],"size":[3,88],"of":[4,49,74,107,129,148,173,195],"Information":[5],"Retrieval":[6],"collections,":[7],"crowdsourcing":[8],"has":[9],"become":[10],"a":[11,28,34,67,79,104,171],"common":[12],"way":[13,70],"to":[14,61,64,102,132,181],"collect":[15],"relevance":[16,73,150],"judgments":[17],"at":[18,97],"scale.":[19],"Crowdsourcing":[20],"experiments":[21],"usually":[22],"employ":[23],"100\u201310,000":[24],"workers,":[25],"but":[26],"such":[27,138,158,165],"number":[29,128],"is":[30,39,140,152,160],"often":[31],"decided":[32],"in":[33,66],"heuristic":[35],"way.":[36],"The":[37],"downside":[38],"that":[40,121,137,157,177],"resulting":[42,100],"dataset":[43,101],"does":[44],"not":[45],"have":[46,57],"any":[47],"guarantee":[48],"meeting":[50],"predefined":[51,105],"statistical":[52,59,108,134],"requirements":[53],"as,":[54],"for":[55],"example,":[56],"enough":[58],"power":[60],"be":[62,179],"able":[63],"distinguish":[65],"statistically":[68],"significant":[69],"between":[71],"two":[75],"documents.":[76],"We":[77,110],"propose":[78],"methodology":[80],"adapted":[81],"from":[82],"literature":[83],"on":[84,91,114,142,162,192],"sound":[85],"topic":[86],"set":[87,106,172,185,197],"design,":[89],"based":[90],"t-test":[92],"and":[93,136,187],"ANOVA,":[94],"which":[95],"aims":[96],"guaranteeing":[98],"meet":[103],"requirements.":[109],"validate":[111],"our":[112],"approach":[113],"several":[115],"public":[116],"datasets.":[117],"Our":[118],"results":[119,191],"show":[120],"we":[122,155,169,188],"can":[123,178],"reliably":[124],"estimate":[125,182],"recommended":[127],"workers":[130],"needed":[131],"achieve":[133],"power,":[135],"estimation":[139,159,175,194],"dependent":[141,161],"topic,":[144],"while":[145],"effect":[147],"limited.":[153],"Furthermore,":[154],"found":[156],"worker":[163,184],"features":[164],"as":[166],"agreement.":[167],"Finally,":[168],"describe":[170],"practical":[174],"strategies":[176],"used":[180],"size,":[186],"also":[189],"provide":[190],"document":[196],"sizes.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}