{"id":"https://openalex.org/W1985555206","doi":"https://doi.org/10.1109/allerton.2014.7028481","title":"Active learning from uncertain crowd annotations","display_name":"Active learning from uncertain crowd annotations","publication_year":2014,"publication_date":"2014-09-01","ids":{"openalex":"https://openalex.org/W1985555206","doi":"https://doi.org/10.1109/allerton.2014.7028481","mag":"1985555206"},"language":"en","primary_location":{"id":"doi:10.1109/allerton.2014.7028481","is_oa":false,"landing_page_url":"https://doi.org/10.1109/allerton.2014.7028481","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2014 52nd Annual Allerton Conference on Communication, Control, and Computing (Allerton)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100395035","display_name":"Yan Yan","orcid":"https://orcid.org/0000-0001-9108-6767"},"institutions":[{"id":"https://openalex.org/I4210134091","display_name":"Yahoo (United States)","ror":"https://ror.org/040dkzz12","country_code":"US","type":"company","lineage":["https://openalex.org/I4210134091"]},{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yan Yan","raw_affiliation_strings":["Northeastern University, Yahoo! Labs, Sunnyvale, CA","Northeastern University and is now at Yahoo! Labs, Sunnyvale, CA"],"affiliations":[{"raw_affiliation_string":"Northeastern University, Yahoo! Labs, Sunnyvale, CA","institution_ids":["https://openalex.org/I4210134091"]},{"raw_affiliation_string":"Northeastern University and is now at Yahoo! Labs, Sunnyvale, CA","institution_ids":["https://openalex.org/I4210134091","https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111998240","display_name":"R\u00f3mer Rosales","orcid":null},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Romer Rosales","raw_affiliation_strings":["LinkedIn Corp., Mountain View, CA"],"affiliations":[{"raw_affiliation_string":"LinkedIn Corp., Mountain View, CA","institution_ids":["https://openalex.org/I1316064682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111408590","display_name":"Glenn Fung","orcid":null},"institutions":[{"id":"https://openalex.org/I1316064682","display_name":"LinkedIn (United States)","ror":"https://ror.org/02fyxhe35","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I1316064682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Glenn Fung","raw_affiliation_strings":["LinkedIn Corp., Mountain View, CA, USA","LinkedIn Corporation, Mountain View, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"LinkedIn Corp., Mountain View, CA, USA","institution_ids":["https://openalex.org/I1316064682"]},{"raw_affiliation_string":"LinkedIn Corporation, Mountain View, CA, USA#TAB#","institution_ids":["https://openalex.org/I1316064682"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042038501","display_name":"Jennifer Dy","orcid":"https://orcid.org/0000-0002-8430-134X"},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jennifer Dy","raw_affiliation_strings":["Electrical and Computer Engineering Department, Northeastern University, Boston, MA","Electrical and Computer Engineering Department, Northeastern University, Boston, MA#TAB#"],"affiliations":[{"raw_affiliation_string":"Electrical and Computer Engineering Department, Northeastern University, Boston, MA","institution_ids":["https://openalex.org/I12912129"]},{"raw_affiliation_string":"Electrical and Computer Engineering Department, Northeastern University, Boston, MA#TAB#","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100395035"],"corresponding_institution_ids":["https://openalex.org/I12912129","https://openalex.org/I4210134091"],"apc_list":null,"apc_paid":null,"fwci":0.818,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.7971581,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":"4","issue":null,"first_page":"385","last_page":"392"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.920526385307312},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8722465634346008},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6938682794570923},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6596527099609375},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.659421443939209},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6281949281692505},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.5355944633483887},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.4864499270915985},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.48264700174331665},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.41374269127845764},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11891093850135803}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.920526385307312},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8722465634346008},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6938682794570923},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6596527099609375},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.659421443939209},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6281949281692505},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.5355944633483887},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.4864499270915985},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.48264700174331665},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.41374269127845764},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11891093850135803},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/allerton.2014.7028481","is_oa":false,"landing_page_url":"https://doi.org/10.1109/allerton.2014.7028481","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2014 52nd Annual Allerton Conference on Communication, Control, and Computing (Allerton)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1484084878","https://openalex.org/W1525088415","https://openalex.org/W1528361845","https://openalex.org/W1970381522","https://openalex.org/W1984897211","https://openalex.org/W1992419399","https://openalex.org/W2010135967","https://openalex.org/W2022363284","https://openalex.org/W2026246169","https://openalex.org/W2050611670","https://openalex.org/W2076580309","https://openalex.org/W2080021732","https://openalex.org/W2085989833","https://openalex.org/W2092824627","https://openalex.org/W2099111195","https://openalex.org/W2115305054","https://openalex.org/W2149273804","https://openalex.org/W2154096410","https://openalex.org/W2181558882","https://openalex.org/W2185789379","https://openalex.org/W2333269907","https://openalex.org/W2570764145","https://openalex.org/W2949071206","https://openalex.org/W2951911250","https://openalex.org/W3029645440","https://openalex.org/W3203127690","https://openalex.org/W4240017161","https://openalex.org/W6628826841","https://openalex.org/W6682171051","https://openalex.org/W6682471470","https://openalex.org/W6685936606","https://openalex.org/W6686485044"],"related_works":["https://openalex.org/W4292388283","https://openalex.org/W3204418343","https://openalex.org/W2949671220","https://openalex.org/W1560624709","https://openalex.org/W3166286441","https://openalex.org/W3214142563","https://openalex.org/W3111760155","https://openalex.org/W1492505081","https://openalex.org/W4389374014","https://openalex.org/W3170999895"],"abstract_inverted_index":{"Supervised":[0],"learning":[1,31,127,196,219,231],"means":[2],"there":[3],"is":[4,15,50,92,159,183],"a":[5,40,123,128],"teacher":[6],"providing":[7],"labels":[8,19],"given":[9],"data":[10,63,118,146],"samples,":[11],"and":[12,67,144,164],"the":[13,18,45,117,134,137,142,145,169,221,233],"goal":[14],"to":[16,56,84,155,162,194,206,216],"predict":[17],"of":[20,88,136,172,177],"unseen":[21],"instances.":[22],"In":[23],"general,":[24],"these":[25],"labelers":[26,83],"may":[27,98,106,113,139,152],"make":[28],"mistakes.":[29],"Typical":[30],"methods":[32],"rely":[33],"on":[34,116],"an":[35],"often":[36],"overlooked":[37],"assumption":[38],"that":[39,147],"single":[41],"expert":[42],"can":[43,64],"provide":[44],"required":[46],"supervision;":[47],"however,":[48],"it":[49,75,158],"becoming":[51],"more":[52,100],"common":[53],"for":[54,77,126,181,228],"supervision":[55,89],"be":[57,65,99,107,214],"available":[58],"in":[59,203,232],"many":[60,82,156],"forms":[61,87],"as":[62,94,185],"shared":[66],"processed":[68],"by":[69],"increasingly":[70],"larger":[71],"audiences.":[72],"This":[73,223],"makes":[74],"possible":[76],"not":[78,165,198],"just":[79],"one":[80],"but":[81,209],"offer":[85],"some":[86],"(this":[90],"phenomena":[91],"coined":[93],"crowdsourcing).":[95],"Some":[96],"annotators":[97,138,167],"reliable":[101],"than":[102],"others,":[103],"malicious,":[104],"or":[105],"correlated":[108],"with":[109,141],"others.":[110],"Annotator":[111],"effectiveness":[112],"vary":[114,140],"depending":[115],"instance":[119],"presented.":[120],"We":[121],"utilize":[122],"probabilistic":[124],"model":[125,220],"classifier":[129],"from":[130],"multiple":[131],"annotators,":[132,157],"where":[133],"reliability":[135],"annotator":[143,212],"they":[148],"observe.":[149],"Although":[150],"we":[151,201],"have":[153,168],"access":[154],"still":[160],"expensive":[161],"label":[163,207],"all":[166],"same":[170],"level":[171],"expertise.":[173],"The":[174,188],"general":[175],"problem":[176],"intelligently":[178],"choosing":[179],"instances":[180],"labeling":[182],"known":[184],"active":[186,195,230],"learning.":[187],"crowdsourcing":[189,234],"paradigm":[190],"posits":[191],"new":[192],"challenges":[193],"-":[197],"only":[199],"are":[200],"interested":[202],"which":[204,211],"sample":[205],"next":[208],"also":[210],"should":[213],"queried":[215],"benefit":[217],"our":[218],"most.":[222],"paper":[224],"presents":[225],"different":[226],"approaches":[227],"performing":[229],"setting.":[235]},"counts_by_year":[{"year":2017,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}