{"id":"https://openalex.org/W1970792136","doi":"https://doi.org/10.1109/icdmw.2012.48","title":"Learning from Multiple Annotators: When Data is Hard and Annotators are Unreliable","display_name":"Learning from Multiple Annotators: When Data is Hard and Annotators are Unreliable","alternate_titles":[],"publication_year":2012,"publication_date":"2012-12-01","ids":{"openalex":"https://openalex.org/W1970792136","doi":"https://doi.org/10.1109/icdmw.2012.48","mag":"1970792136"},"language":"en","primary_location":{"id":"doi:10.1109/icdmw.2012.48","doi":"https://doi.org/10.1109/icdmw.2012.48","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdmw.2012.48","pdf_url":null,"source":{"id":"https://openalex.org/S4393918563","display_name":"IEEE ... International Conference on Data Mining workshops","issn_l":"2375-9259","issn":["2375-9259"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 IEEE 12th International Conference on Data Mining Workshops","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W1970792136"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071074197","display_name":"Chirine Wolley","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I21491767","display_name":"Aix-Marseille Universit\u00e9","ror":"https://ror.org/035xkbk20","country_code":"FR","type":"education","lineage":["https://openalex.org/I21491767"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Chirine Wolley","raw_affiliation_strings":["Aix-Marseille University, Marseille, France","LSIS, Aix Marseille University, Marseille, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aix-Marseille University, Marseille, France","institution_ids":["https://openalex.org/I21491767"]},{"raw_affiliation_string":"LSIS, Aix Marseille University, Marseille, France","institution_ids":["https://openalex.org/I21491767"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062055724","display_name":"Mohamed Quafafou","orcid":"https://orcid.org/0000-0003-0046-7092","observed_orcids":["https://orcid.org/0000-0003-0046-7092"]},"institutions":[{"id":"https://openalex.org/I21491767","display_name":"Aix-Marseille Universit\u00e9","ror":"https://ror.org/035xkbk20","country_code":"FR","type":"education","lineage":["https://openalex.org/I21491767"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Mohamed Quafafou","raw_affiliation_strings":["Aix-Marseille University, Marseille, France","LSIS, Aix Marseille University, Marseille, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aix-Marseille University, Marseille, France","institution_ids":["https://openalex.org/I21491767"]},{"raw_affiliation_string":"LSIS, Aix Marseille University, Marseille, France","institution_ids":["https://openalex.org/I21491767"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I21491767"],"apc_list":null,"apc_paid":null,"fwci":0.5042,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.51967973,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"514","last_page":"521"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.36469998955726624,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.36469998955726624,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.3393999934196472,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.23909999430179596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.734000027179718},{"id":"https://openalex.org/keywords/noisy-labels","display_name":"label noise","score":0.5820000171661377},{"id":"https://openalex.org/keywords/annotator-reliability","display_name":"annotator reliability","score":0.7760000228881836},{"id":"https://openalex.org/keywords/bayesian-model","display_name":"Bayesian model","score":0.35600000619888306},{"id":"https://openalex.org/keywords/probabilistic-learning","display_name":"probabilistic learning","score":0.4819999933242798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8638627529144287},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6404505372047424},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6157029271125793},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6128444671630859},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5777240991592407},{"id":"https://openalex.org/C2778732403","wikidata":"https://www.wikidata.org/wiki/Q815577","display_name":"Ignorance","level":2,"score":0.5641891956329346},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.48849618434906006},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.433799684047699},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icdmw.2012.48","doi":"https://doi.org/10.1109/icdmw.2012.48","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdmw.2012.48","pdf_url":null,"source":{"id":"https://openalex.org/S4393918563","display_name":"IEEE ... International Conference on Data Mining workshops","issn_l":"2375-9259","issn":["2375-9259"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 IEEE 12th International Conference on Data Mining Workshops","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W342297144","https://openalex.org/W1541505110","https://openalex.org/W1566045017","https://openalex.org/W1570770495","https://openalex.org/W1970381522","https://openalex.org/W1984897211","https://openalex.org/W1998839399","https://openalex.org/W2006681603","https://openalex.org/W2011953530","https://openalex.org/W2100358124","https://openalex.org/W2108598243","https://openalex.org/W2112876447","https://openalex.org/W2134305421","https://openalex.org/W2142518823","https://openalex.org/W2144372981","https://openalex.org/W2144660879","https://openalex.org/W2149489787","https://openalex.org/W2154096410","https://openalex.org/W2166775458","https://openalex.org/W2331182131","https://openalex.org/W2333269907","https://openalex.org/W2764433274","https://openalex.org/W3083113686","https://openalex.org/W3120740533","https://openalex.org/W3197494818","https://openalex.org/W4240017161","https://openalex.org/W6633761756"],"related_works":["https://openalex.org/W2150612552","https://openalex.org/W7096652153","https://openalex.org/W86096423","https://openalex.org/W7097890969","https://openalex.org/W2154096410","https://openalex.org/W111317371","https://openalex.org/W2698783112","https://openalex.org/W7147460594","https://openalex.org/W7096538321","https://openalex.org/W7015526338"],"abstract_inverted_index":{"The":[0],"crowd":[1],"sourcing":[2],"services":[3],"became":[4],"popular":[5],"making":[6],"it":[7],"easy":[8],"and":[9,65,97],"fast":[10],"to":[11,19,45,63,76,87,121],"label":[12,64,88],"datasets":[13,112],"by":[14],"multiple":[15,79],"annotators":[16,28,84,100],"in":[17,25,51],"order":[18],"achieve":[20],"supervised":[21],"learning":[22,124],"tasks.":[23],"Unfortunately,":[24],"this":[26,69],"context,":[27],"are":[29,93,101],"not":[30],"reliable":[31],"as":[32],"they":[33,92],"may":[34,48,85],"have":[35],"different":[36],"levels":[37],"of":[38,54,99],"experience":[39],"or":[40],"knowledge.":[41],"Furthermore,":[42],"the":[43,105],"data":[44,62],"be":[46],"labeled":[47],"also":[49],"vary":[50],"their":[52],"level":[53],"difficulty.":[55],"How":[56],"do":[57],"we":[58,71],"deal":[59],"with":[60],"hard":[61],"unreliable":[66],"annotators?":[67],"In":[68],"paper,":[70],"present":[72],"a":[73],"probabilistic":[74],"model":[75],"learn":[77],"from":[78],"naive":[80],"annotators,":[81],"considering":[82],"that":[83,114],"decline":[86],"an":[89],"instance":[90],"when":[91],"unsure.":[94],"Both":[95],"errors":[96],"ignorance":[98],"integrated":[102],"separately":[103],"into":[104],"proposed":[106],"Bayesian":[107],"model.":[108],"Experiments":[109],"on":[110],"several":[111],"show":[113],"our":[115],"method":[116],"achieves":[117],"superior":[118],"performance":[119],"compared":[120],"other":[122],"efficient":[123],"algorithms.":[125]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W1970792136","counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":1}],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2016-06-24T00:00:00"}
