{"id":"https://openalex.org/W3184384643","doi":"https://doi.org/10.1145/3479531","title":"On the State of Reporting in Crowdsourcing Experiments and a Checklist to Aid Current Practices","display_name":"On the State of Reporting in Crowdsourcing Experiments and a Checklist to Aid Current Practices","publication_year":2021,"publication_date":"2021-10-13","ids":{"openalex":"https://openalex.org/W3184384643","doi":"https://doi.org/10.1145/3479531","mag":"3184384643"},"language":"en","primary_location":{"id":"doi:10.1145/3479531","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3479531","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2107.13519","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100688218","display_name":"Jorge Ram\u00edrez","orcid":"https://orcid.org/0000-0003-0678-8962"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Jorge Ram\u00edrez","raw_affiliation_strings":["University of Trento, Trento, Italy","University of Trento"],"affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy","institution_ids":["https://openalex.org/I193223587"]},{"raw_affiliation_string":"University of Trento","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090706427","display_name":"Burcu Sayin","orcid":"https://orcid.org/0000-0001-6804-127X"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Burcu Sayin","raw_affiliation_strings":["University of Trento, Trento, Italy","University of Trento"],"affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy","institution_ids":["https://openalex.org/I193223587"]},{"raw_affiliation_string":"University of Trento","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026694449","display_name":"Marcos B\u00e1ez","orcid":"https://orcid.org/0000-0003-1666-2474"},"institutions":[{"id":"https://openalex.org/I100532134","display_name":"Universit\u00e9 Claude Bernard Lyon 1","ror":"https://ror.org/029brtt94","country_code":"FR","type":"education","lineage":["https://openalex.org/I100532134","https://openalex.org/I203339264"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Marcos Baez","raw_affiliation_strings":["Universit\u00e9 Claude Bernard Lyon 1, Villeurbanne, France","[Claude Bernard University Lyon 1]"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Claude Bernard Lyon 1, Villeurbanne, France","institution_ids":["https://openalex.org/I100532134"]},{"raw_affiliation_string":"[Claude Bernard University Lyon 1]","institution_ids":["https://openalex.org/I100532134"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086585292","display_name":"Fabio Casati","orcid":"https://orcid.org/0000-0001-7591-9562"},"institutions":[{"id":"https://openalex.org/I4210167039","display_name":"ServiceNow (United States)","ror":"https://ror.org/05xr0bc04","country_code":"US","type":"company","lineage":["https://openalex.org/I4210167039"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fabio Casati","raw_affiliation_strings":["Servicenow, Santa Clara, CA, USA"],"affiliations":[{"raw_affiliation_string":"Servicenow, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I4210167039"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034166026","display_name":"Luca Cernuzzi","orcid":"https://orcid.org/0000-0001-7803-1067"},"institutions":[{"id":"https://openalex.org/I322348216","display_name":"Universidad Cat\u00f3lica Nuestra Se\u00f1ora de la Asunci\u00f3n","ror":"https://ror.org/04jjna482","country_code":"PY","type":"education","lineage":["https://openalex.org/I322348216"]}],"countries":["PY"],"is_corresponding":false,"raw_author_name":"Luca Cernuzzi","raw_affiliation_strings":["Catholic University of Asuncion, Asuncion, Paraguay","Catholic University of Asunci\u00f3n, Asunci\u00f3n Paraguay#TAB#"],"affiliations":[{"raw_affiliation_string":"Catholic University of Asuncion, Asuncion, Paraguay","institution_ids":["https://openalex.org/I322348216"]},{"raw_affiliation_string":"Catholic University of Asunci\u00f3n, Asunci\u00f3n Paraguay#TAB#","institution_ids":["https://openalex.org/I322348216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042705320","display_name":"Boualem Benatallah","orcid":"https://orcid.org/0000-0002-8805-1130"},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Boualem Benatallah","raw_affiliation_strings":["University of New South Wales, Sydney, NSW, Australia","University of New South Wales,#TAB#"],"affiliations":[{"raw_affiliation_string":"University of New South Wales, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I31746571"]},{"raw_affiliation_string":"University of New South Wales,#TAB#","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["University of Queensland, Brisbane, QLD, Australia","University of Queensland"],"affiliations":[{"raw_affiliation_string":"University of Queensland, Brisbane, QLD, Australia","institution_ids":["https://openalex.org/I165143802"]},{"raw_affiliation_string":"University of Queensland","institution_ids":["https://openalex.org/I165143802"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5100688218"],"corresponding_institution_ids":["https://openalex.org/I193223587"],"apc_list":null,"apc_paid":null,"fwci":1.43635963,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.79751842,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"5","issue":"CSCW2","first_page":"1","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10841","display_name":"Economic and Environmental Valuation","score":0.9175000190734863,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9943579435348511},{"id":"https://openalex.org/keywords/checklist","display_name":"Checklist","score":0.7756271958351135},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6280285120010376},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.569689929485321},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5492409467697144},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5345980525016785},{"id":"https://openalex.org/keywords/experience-sampling-method","display_name":"Experience sampling method","score":0.5087504982948303},{"id":"https://openalex.org/keywords/citizen-science","display_name":"Citizen science","score":0.48428839445114136},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.40929320454597473},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.24941405653953552},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.14519962668418884},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.10092228651046753},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.08507204055786133}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9943579435348511},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.7756271958351135},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6280285120010376},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.569689929485321},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5492409467697144},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5345980525016785},{"id":"https://openalex.org/C65499552","wikidata":"https://www.wikidata.org/wiki/Q5421061","display_name":"Experience sampling method","level":2,"score":0.5087504982948303},{"id":"https://openalex.org/C197352329","wikidata":"https://www.wikidata.org/wiki/Q1093434","display_name":"Citizen science","level":2,"score":0.48428839445114136},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.40929320454597473},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24941405653953552},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.14519962668418884},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.10092228651046753},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.08507204055786133},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3479531","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3479531","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2107.13519","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2107.13519","pdf_url":"https://arxiv.org/pdf/2107.13519","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2107.13519","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2107.13519","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"},{"id":"mag:3184384643","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2107.13519","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2107.13519","pdf_url":"https://arxiv.org/pdf/2107.13519","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3184384643.pdf","grobid_xml":"https://content.openalex.org/works/W3184384643.grobid-xml"},"referenced_works_count":86,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W148615303","https://openalex.org/W601857102","https://openalex.org/W614149592","https://openalex.org/W1482974529","https://openalex.org/W1563968750","https://openalex.org/W1608752663","https://openalex.org/W1730782591","https://openalex.org/W1786164014","https://openalex.org/W1964157254","https://openalex.org/W1969110305","https://openalex.org/W1970381522","https://openalex.org/W1972174150","https://openalex.org/W1975450585","https://openalex.org/W1985095406","https://openalex.org/W2007018772","https://openalex.org/W2007233978","https://openalex.org/W2041411104","https://openalex.org/W2065049201","https://openalex.org/W2080211902","https://openalex.org/W2094993172","https://openalex.org/W2106568252","https://openalex.org/W2108696783","https://openalex.org/W2117986881","https://openalex.org/W2133774931","https://openalex.org/W2142518823","https://openalex.org/W2146436037","https://openalex.org/W2149489787","https://openalex.org/W2151401338","https://openalex.org/W2158784126","https://openalex.org/W2164445379","https://openalex.org/W2168765606","https://openalex.org/W2169205464","https://openalex.org/W2169800142","https://openalex.org/W2170493558","https://openalex.org/W2182992846","https://openalex.org/W2202529936","https://openalex.org/W2279346661","https://openalex.org/W2339322788","https://openalex.org/W2397166854","https://openalex.org/W2401574992","https://openalex.org/W2469104253","https://openalex.org/W2563948597","https://openalex.org/W2593021276","https://openalex.org/W2599032673","https://openalex.org/W2605650008","https://openalex.org/W2618043980","https://openalex.org/W2726137762","https://openalex.org/W2728712790","https://openalex.org/W2735460460","https://openalex.org/W2754799758","https://openalex.org/W2756079620","https://openalex.org/W2777647957","https://openalex.org/W2783946743","https://openalex.org/W2789816182","https://openalex.org/W2792499629","https://openalex.org/W2794219599","https://openalex.org/W2794871313","https://openalex.org/W2795038878","https://openalex.org/W2807760453","https://openalex.org/W2884313467","https://openalex.org/W2897042519","https://openalex.org/W2899134503","https://openalex.org/W2903257353","https://openalex.org/W2911227954","https://openalex.org/W2941486823","https://openalex.org/W2941985495","https://openalex.org/W2950390700","https://openalex.org/W2954350093","https://openalex.org/W2963208067","https://openalex.org/W2971564107","https://openalex.org/W2974071289","https://openalex.org/W2994369157","https://openalex.org/W2994747757","https://openalex.org/W2998390377","https://openalex.org/W2999071598","https://openalex.org/W3013688454","https://openalex.org/W3030678554","https://openalex.org/W3095615755","https://openalex.org/W3104998832","https://openalex.org/W3121257585","https://openalex.org/W3122383744","https://openalex.org/W3123895079","https://openalex.org/W3124878131","https://openalex.org/W3125321342","https://openalex.org/W4292341621"],"related_works":["https://openalex.org/W3205769026","https://openalex.org/W3095615755","https://openalex.org/W3206003143","https://openalex.org/W3175057742","https://openalex.org/W646999627","https://openalex.org/W2789642357","https://openalex.org/W2984490367","https://openalex.org/W112024736","https://openalex.org/W2589038172","https://openalex.org/W2789043772","https://openalex.org/W2968498314","https://openalex.org/W2548342535","https://openalex.org/W2993359537","https://openalex.org/W3088618910","https://openalex.org/W2005917549","https://openalex.org/W2188598420","https://openalex.org/W1595637775","https://openalex.org/W2173800084","https://openalex.org/W2782152208","https://openalex.org/W3090947281"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1],"being":[2],"increasingly":[3],"adopted":[4],"as":[5],"a":[6,15,85,143],"platform":[7],"to":[8,23,42,61,107,127],"run":[9],"studies":[10],"with":[11,125],"human":[12],"subjects.":[13],"Running":[14],"crowdsourcing":[16,64,102,135,147],"experiment":[17],"involves":[18],"several":[19,49],"choices":[20,73],"and":[21,59,83,104,123],"strategies":[22],"successfully":[24],"port":[25],"an":[26,30],"experimental":[27,40],"design":[28],"into":[29],"otherwise":[31],"uncontrolled":[32],"research":[33,89],"environment,":[34],"e.g.,":[35],"sampling":[36],"crowd":[37],"workers,":[38],"mapping":[39],"conditions":[41],"micro-tasks,":[43],"or":[44],"ensure":[45],"quality":[46],"contributions.":[47],"While":[48],"guidelines":[50],"inform":[51],"researchers":[52],"in":[53],"these":[54],"choices,":[55,118],"guidance":[56,106],"of":[57,88,99,101,133],"how":[58],"what":[60],"report":[62],"from":[63],"experiments":[65,103],"has":[66],"been":[67],"largely":[68],"overlooked.":[69],"If":[70],"under-reported,":[71],"implementation":[72,117],"constitute":[74],"variability":[75],"sources":[76],"that":[77],"can":[78],"affect":[79],"the":[80,96,131],"experiment's":[81],"reproducibility":[82],"prevent":[84],"fair":[86],"assessment":[87],"outcomes.":[90],"In":[91],"this":[92,139],"paper,":[93],"we":[94,141],"examine":[95],"current":[97],"state":[98],"reporting":[100,110,132,146],"offer":[105],"address":[108],"associated":[109],"issues.":[111],"We":[112],"start":[113],"by":[114,138],"identifying":[115],"sensible":[116],"relying":[119],"on":[120],"existing":[121],"literature":[122],"interviews":[124],"experts,":[126],"then":[128],"extensively":[129],"analyze":[130],"171":[134],"experiments.":[136,148],"Informed":[137],"process,":[140],"propose":[142],"checklist":[144],"for":[145]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-02-09T09:26:11.010843","created_date":"2025-10-10T00:00:00"}