{"id":"https://openalex.org/W3162331882","doi":"https://doi.org/10.23919/eusipco54536.2021.9616087","title":"What is the ground truth? Reliability of multi-annotator data for audio tagging","display_name":"What is the ground truth? Reliability of multi-annotator data for audio tagging","publication_year":2021,"publication_date":"2021-08-23","ids":{"openalex":"https://openalex.org/W3162331882","doi":"https://doi.org/10.23919/eusipco54536.2021.9616087","mag":"3162331882"},"language":"en","primary_location":{"id":"doi:10.23919/eusipco54536.2021.9616087","is_oa":false,"landing_page_url":"https://doi.org/10.23919/eusipco54536.2021.9616087","pdf_url":null,"source":{"id":"https://openalex.org/S4363607854","display_name":"2021 29th European Signal Processing Conference (EUSIPCO)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 29th European Signal Processing Conference (EUSIPCO)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041615273","display_name":"Irene Mart\u00edn-Morat\u00f3","orcid":"https://orcid.org/0000-0002-2115-0193"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Irene Martin-Morato","raw_affiliation_strings":["Computing Sciences, Tampere University, Tampere, FINLAND"],"affiliations":[{"raw_affiliation_string":"Computing Sciences, Tampere University, Tampere, FINLAND","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079981416","display_name":"Annamaria Mesaros","orcid":"https://orcid.org/0000-0002-6640-9752"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Annamaria Mesaros","raw_affiliation_strings":["Computing Sciences, Tampere University, Tampere, FINLAND"],"affiliations":[{"raw_affiliation_string":"Computing Sciences, Tampere University, Tampere, FINLAND","institution_ids":["https://openalex.org/I166825849"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5041615273"],"corresponding_institution_ids":["https://openalex.org/I166825849"],"apc_list":null,"apc_paid":null,"fwci":2.7545,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.95485175,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"76","last_page":"80"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11665","display_name":"Animal Vocal Communication and Behavior","score":0.9054999947547913,"subfield":{"id":"https://openalex.org/subfields/1309","display_name":"Developmental Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.8594532012939453},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8250899910926819},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8239723443984985},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5556304454803467},{"id":"https://openalex.org/keywords/competence","display_name":"Competence (human resources)","score":0.5130757093429565},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43283113837242126},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.40733247995376587},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11723968386650085}],"concepts":[{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.8594532012939453},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8250899910926819},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8239723443984985},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5556304454803467},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.5130757093429565},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43283113837242126},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40733247995376587},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11723968386650085},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.23919/eusipco54536.2021.9616087","is_oa":false,"landing_page_url":"https://doi.org/10.23919/eusipco54536.2021.9616087","pdf_url":null,"source":{"id":"https://openalex.org/S4363607854","display_name":"2021 29th European Signal Processing Conference (EUSIPCO)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 29th European Signal Processing Conference (EUSIPCO)","raw_type":"proceedings-article"},{"id":"pmh:oai:trepo.tuni.fi:10024/220300","is_oa":false,"landing_page_url":"https://trepo.tuni.fi/handle/10024/220300","pdf_url":null,"source":{"id":"https://openalex.org/S7407055260","display_name":"Trepo - Institutional Repository of Tampere University","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2360061192","display_name":"Teaching machines to listen","funder_award_id":"332063","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G4923903852","display_name":null,"funder_award_id":"332063","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"}],"funders":[{"id":"https://openalex.org/F4320321108","display_name":"Academy of Finland","ror":"https://ror.org/05k73zm37"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W86738361","https://openalex.org/W1559121794","https://openalex.org/W1565931986","https://openalex.org/W2013378101","https://openalex.org/W2086384421","https://openalex.org/W2148347694","https://openalex.org/W2170538746","https://openalex.org/W2176625348","https://openalex.org/W2251818274","https://openalex.org/W2593116425","https://openalex.org/W2758074877","https://openalex.org/W2758358029","https://openalex.org/W2774116887","https://openalex.org/W2903137797","https://openalex.org/W2941505948","https://openalex.org/W2963859210","https://openalex.org/W2982346427","https://openalex.org/W2995055746","https://openalex.org/W3021910945","https://openalex.org/W3104211335","https://openalex.org/W3104805957","https://openalex.org/W3122150008","https://openalex.org/W3137857706","https://openalex.org/W4210764005","https://openalex.org/W4287991150","https://openalex.org/W6691296735","https://openalex.org/W6756597712","https://openalex.org/W6771994162","https://openalex.org/W6786091203"],"related_works":["https://openalex.org/W2949551779","https://openalex.org/W2795704878","https://openalex.org/W2997174683","https://openalex.org/W2803765572","https://openalex.org/W296976497","https://openalex.org/W2029202094","https://openalex.org/W2885323543","https://openalex.org/W2751762560","https://openalex.org/W2128502391","https://openalex.org/W2945416483"],"abstract_inverted_index":{"Crowdsourcing":[0],"has":[1,13],"become":[2],"a":[3,18,28,75,99,112],"common":[4],"approach":[5],"for":[6,83,98],"annotating":[7],"large":[8,19,23],"amounts":[9,24],"of":[10,16,25,36,47,56,62,77,90,125],"data.":[11],"It":[12],"the":[14,34,45,53,60,69,78,88],"advantage":[15],"harnessing":[17],"workforce":[20],"to":[21,52,58,67,110],"produce":[22],"data":[26,48,101],"in":[27,50,65],"short":[29],"time,":[30],"but":[31],"comes":[32],"with":[33,40,122],"disadvantage":[35],"employing":[37],"non-expert":[38,120],"annotators":[39,64],"different":[41,123],"backgrounds.":[42],"This":[43,72],"raises":[44],"problem":[46],"reliability,":[49],"addition":[51],"general":[54],"question":[55],"how":[57,105],"combine":[59],"opinions":[61],"multiple":[63],"order":[66],"estimate":[68,111],"ground":[70,114],"truth.":[71],"paper":[73],"presents":[74],"study":[76],"annotations":[79,118],"and":[80,93,103,127],"annotators'":[81],"reliability":[82],"audio":[84],"tagging.":[85],"We":[86],"adapt":[87],"use":[89],"Krippendorf's":[91],"alpha":[92],"multi-annotator":[94],"competence":[95],"estimation":[96],"(MACE)":[97],"multi-labeled":[100],"scenario,":[102],"present":[104],"MACE":[106],"can":[107],"be":[108],"used":[109],"candidate":[113],"truth":[115],"based":[116],"on":[117],"from":[119],"users":[121],"levels":[124],"expertise":[126],"competence.":[128]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-04-23T09:07:50.710637","created_date":"2025-10-10T00:00:00"}