{"id":"https://openalex.org/W2941505948","doi":"https://doi.org/10.1145/3290605.3300522","title":"Crowdsourcing Multi-label Audio Annotation Tasks with Citizen Scientists","display_name":"Crowdsourcing Multi-label Audio Annotation Tasks with Citizen Scientists","publication_year":2019,"publication_date":"2019-04-29","ids":{"openalex":"https://openalex.org/W2941505948","doi":"https://doi.org/10.1145/3290605.3300522","mag":"2941505948"},"language":"en","primary_location":{"id":"doi:10.1145/3290605.3300522","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3290605.3300522","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056532548","display_name":"Mark Cartwright","orcid":"https://orcid.org/0000-0002-5908-390X"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mark Cartwright","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011864657","display_name":"Graham Dove","orcid":"https://orcid.org/0000-0002-3551-0209"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Graham Dove","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040801874","display_name":"Ana Elisa M\u00e9ndez M\u00e9ndez","orcid":"https://orcid.org/0000-0002-4861-5616"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ana Elisa M\u00e9ndez M\u00e9ndez","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031398497","display_name":"Juan Pablo Bello","orcid":"https://orcid.org/0000-0001-8561-5204"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan P. Bello","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007172071","display_name":"Oded Nov","orcid":"https://orcid.org/0000-0001-6410-2995"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oded Nov","raw_affiliation_strings":["New York University, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"New York University, New York, NY, USA","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5056532548"],"corresponding_institution_ids":["https://openalex.org/I57206974"],"apc_list":null,"apc_paid":null,"fwci":4.644,"has_fulltext":false,"cited_by_count":46,"citation_normalized_percentile":{"value":0.95710077,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11665","display_name":"Animal Vocal Communication and Behavior","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1309","display_name":"Developmental Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.89505535364151},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.821734607219696},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7921996712684631},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5490867495536804},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5291693210601807},{"id":"https://openalex.org/keywords/automatic-image-annotation","display_name":"Automatic image annotation","score":0.5018656253814697},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4956892728805542},{"id":"https://openalex.org/keywords/citizen-science","display_name":"Citizen science","score":0.47770312428474426},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4535059928894043},{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.4221551716327667},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36595618724823},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34021157026290894},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33535873889923096},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.31210070848464966},{"id":"https://openalex.org/keywords/image-retrieval","display_name":"Image retrieval","score":0.23476934432983398},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.12904411554336548}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.89505535364151},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.821734607219696},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7921996712684631},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5490867495536804},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5291693210601807},{"id":"https://openalex.org/C199579030","wikidata":"https://www.wikidata.org/wiki/Q2851778","display_name":"Automatic image annotation","level":4,"score":0.5018656253814697},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4956892728805542},{"id":"https://openalex.org/C197352329","wikidata":"https://www.wikidata.org/wiki/Q1093434","display_name":"Citizen science","level":2,"score":0.47770312428474426},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4535059928894043},{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.4221551716327667},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36595618724823},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34021157026290894},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33535873889923096},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.31210070848464966},{"id":"https://openalex.org/C1667742","wikidata":"https://www.wikidata.org/wiki/Q10927554","display_name":"Image retrieval","level":3,"score":0.23476934432983398},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.12904411554336548},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3290605.3300522","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3290605.3300522","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/11","score":0.8100000023841858,"display_name":"Sustainable cities and communities"}],"awards":[{"id":"https://openalex.org/G3697821359","display_name":null,"funder_award_id":"1544753","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":61,"referenced_works":["https://openalex.org/W159136832","https://openalex.org/W1506491340","https://openalex.org/W1564436296","https://openalex.org/W1565732937","https://openalex.org/W1573900212","https://openalex.org/W1579825274","https://openalex.org/W1861492603","https://openalex.org/W1966360794","https://openalex.org/W1967246308","https://openalex.org/W1983336515","https://openalex.org/W1990938720","https://openalex.org/W1994927185","https://openalex.org/W2004172681","https://openalex.org/W2005639687","https://openalex.org/W2016113989","https://openalex.org/W2022120424","https://openalex.org/W2033354470","https://openalex.org/W2041411104","https://openalex.org/W2046754846","https://openalex.org/W2058179030","https://openalex.org/W2058578276","https://openalex.org/W2060132316","https://openalex.org/W2064610985","https://openalex.org/W2068671681","https://openalex.org/W2072286734","https://openalex.org/W2092435486","https://openalex.org/W2095581768","https://openalex.org/W2098485216","https://openalex.org/W2108598243","https://openalex.org/W2111114823","https://openalex.org/W2112193291","https://openalex.org/W2115647291","https://openalex.org/W2120396827","https://openalex.org/W2143890915","https://openalex.org/W2162132784","https://openalex.org/W2171965515","https://openalex.org/W2182992846","https://openalex.org/W2183104792","https://openalex.org/W2219156715","https://openalex.org/W2279346661","https://openalex.org/W2288352304","https://openalex.org/W2413367505","https://openalex.org/W2487099897","https://openalex.org/W2509111931","https://openalex.org/W2510317136","https://openalex.org/W2531522074","https://openalex.org/W2547251973","https://openalex.org/W2548165352","https://openalex.org/W2593116425","https://openalex.org/W2595120084","https://openalex.org/W2732026016","https://openalex.org/W2774116887","https://openalex.org/W2798350598","https://openalex.org/W2803114169","https://openalex.org/W2903137797","https://openalex.org/W2912056372","https://openalex.org/W2922355292","https://openalex.org/W3099722888","https://openalex.org/W3104998832","https://openalex.org/W4230939871","https://openalex.org/W4300723004"],"related_works":["https://openalex.org/W2555391182","https://openalex.org/W2376984068","https://openalex.org/W2566406229","https://openalex.org/W2119028572","https://openalex.org/W2152482390","https://openalex.org/W2365617273","https://openalex.org/W2061851981","https://openalex.org/W2506386910","https://openalex.org/W2804095050","https://openalex.org/W2111235226"],"abstract_inverted_index":{"Annotating":[0],"rich":[1],"audio":[2,50,116],"data":[3,76,103],"is":[4,38],"an":[5],"essential":[6],"aspect":[7],"of":[8,22,114],"training":[9],"and":[10,40,127],"evaluating":[11],"machine":[12],"listening":[13],"systems.":[14],"We":[15,119,135],"approach":[16,59],"this":[17,36,58],"task":[18],"in":[19,95],"the":[20,106,112],"context":[21],"temporally-complex":[23],"urban":[24],"soundscapes,":[25],"which":[26,139],"require":[27],"multiple":[28,66,78,92],"labels":[29],"to":[30,63,74,146],"identify":[31],"overlapping":[32],"sound":[33],"sources.":[34],"Typically":[35],"work":[37],"crowdsourced,":[39],"previous":[41],"studies":[42],"have":[43],"shown":[44],"that":[45],"workers":[46],"can":[47,60],"quickly":[48],"label":[49,91],"with":[51,68,77,144],"binary":[52,122],"annotation":[53,88,117],"for":[54],"single":[55,97],"classes.":[56],"However,":[57],"be":[61],"difficult":[62],"scale":[64],"when":[65],"passes":[67],"different":[69,115],"focus":[70],"classes":[71,93],"are":[72,85],"required":[73],"annotate":[75],"labels.":[79],"In":[80],"citizen":[81,108,148],"science,":[82],"where":[83],"tasks":[84],"often":[86],"image-based,":[87],"efforts":[89],"typically":[90],"simultaneously":[94],"a":[96,128],"pass.":[98],"This":[99],"paper":[100],"describes":[101],"our":[102,137],"collection":[104],"on":[105],"Zooniverse":[107],"science":[109],"platform,":[110],"comparing":[111],"efficiencies":[113],"strategies.":[118],"compared":[120],"multiple-pass":[121],"annotation,":[123,126,143],"single-pass":[124],"multi-label":[125,133,142],"hybrid":[129],"approach:":[130],"hierarchical":[131],"multi-pass":[132],"annotation.":[134],"discuss":[136],"findings,":[138],"support":[140],"using":[141],"reference":[145],"volunteer":[147],"scientists'":[149],"motivations.":[150]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}