{"id":"https://openalex.org/W2160780144","doi":"https://doi.org/10.1186/1471-2105-14-s12-s5","title":"Learning by aggregating experts and filtering novices: a solution to crowdsourcing problems in bioinformatics","display_name":"Learning by aggregating experts and filtering novices: a solution to crowdsourcing problems in bioinformatics","publication_year":2013,"publication_date":"2013-09-01","ids":{"openalex":"https://openalex.org/W2160780144","doi":"https://doi.org/10.1186/1471-2105-14-s12-s5","mag":"2160780144","pmid":"https://pubmed.ncbi.nlm.nih.gov/24268030"},"language":"en","primary_location":{"id":"doi:10.1186/1471-2105-14-s12-s5","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-14-s12-s5","pdf_url":"https://bmcbioinformatics.biomedcentral.com/track/pdf/10.1186/1471-2105-14-S12-S5","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://bmcbioinformatics.biomedcentral.com/track/pdf/10.1186/1471-2105-14-S12-S5","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050360222","display_name":"Ping Zhang","orcid":"https://orcid.org/0000-0002-4601-0779"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ping Zhang","raw_affiliation_strings":["Healthcare Analytics Research, IBM T.J. Watson Research Center, Yorktown Heights, NY, 10598, USA"],"affiliations":[{"raw_affiliation_string":"Healthcare Analytics Research, IBM T.J. Watson Research Center, Yorktown Heights, NY, 10598, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009453487","display_name":"Weidan Cao","orcid":"https://orcid.org/0000-0001-5417-2121"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weidan Cao","raw_affiliation_strings":["School of Media and Communication, Temple University, Philadelphia, PA, 19122, USA"],"affiliations":[{"raw_affiliation_string":"School of Media and Communication, Temple University, Philadelphia, PA, 19122, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044038055","display_name":"Zoran Obradovi\u0107","orcid":"https://orcid.org/0000-0002-2051-0142"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zoran Obradovic","raw_affiliation_strings":["Center for Data Analytics and Biomedical Informatics, Temple University, Philadelphia, PA, 19122, USA"],"affiliations":[{"raw_affiliation_string":"Center for Data Analytics and Biomedical Informatics, Temple University, Philadelphia, PA, 19122, USA","institution_ids":["https://openalex.org/I84392919"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5044038055"],"corresponding_institution_ids":["https://openalex.org/I84392919"],"apc_list":{"value":1690,"currency":"GBP","value_usd":2072},"apc_paid":{"value":1690,"currency":"GBP","value_usd":2072},"fwci":1.5044,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.88545592,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"14","issue":"S12","first_page":"S5","last_page":"S5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8333704471588135},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.817995548248291},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.6249855756759644},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5810858011245728},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5787274241447449},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5606167912483215},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5453469753265381},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4458829462528229},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.43030983209609985}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8333704471588135},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.817995548248291},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6249855756759644},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5810858011245728},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5787274241447449},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5606167912483215},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5453469753265381},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4458829462528229},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.43030983209609985},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011506","descriptor_name":"Proteins","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D011506","descriptor_name":"Proteins","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D011506","descriptor_name":"Proteins","qualifier_ui":"Q000145","qualifier_name":"classification","is_major_topic":false},{"descriptor_ui":"D011506","descriptor_name":"Proteins","qualifier_ui":"Q000737","qualifier_name":"chemistry","is_major_topic":false},{"descriptor_ui":"D011506","descriptor_name":"Proteins","qualifier_ui":"Q000737","qualifier_name":"chemistry","is_major_topic":false},{"descriptor_ui":"D011506","descriptor_name":"Proteins","qualifier_ui":"Q000737","qualifier_name":"chemistry","is_major_topic":false},{"descriptor_ui":"D016015","descriptor_name":"Logistic Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016015","descriptor_name":"Logistic Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016015","descriptor_name":"Logistic Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D063045","descriptor_name":"Crowdsourcing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1186/1471-2105-14-s12-s5","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-14-s12-s5","pdf_url":"https://bmcbioinformatics.biomedcentral.com/track/pdf/10.1186/1471-2105-14-S12-S5","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},{"id":"pmid:24268030","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/24268030","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC bioinformatics","raw_type":null},{"id":"pmh:oai:europepmc.org:2841446","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/3848820","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1186/1471-2105-14-s12-s5","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-14-s12-s5","pdf_url":"https://bmcbioinformatics.biomedcentral.com/track/pdf/10.1186/1471-2105-14-S12-S5","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320307773","display_name":"GlaxoSmithKline","ror":"https://ror.org/01xsqw823"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2160780144.pdf","grobid_xml":"https://content.openalex.org/works/W2160780144.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W82089429","https://openalex.org/W82775311","https://openalex.org/W96765277","https://openalex.org/W111317371","https://openalex.org/W154869794","https://openalex.org/W191327111","https://openalex.org/W203292285","https://openalex.org/W1839790115","https://openalex.org/W1963596489","https://openalex.org/W1964989311","https://openalex.org/W1970381522","https://openalex.org/W1981276685","https://openalex.org/W1983046379","https://openalex.org/W1987308257","https://openalex.org/W1989235034","https://openalex.org/W2010135967","https://openalex.org/W2022363284","https://openalex.org/W2050611670","https://openalex.org/W2053820259","https://openalex.org/W2070126374","https://openalex.org/W2112921341","https://openalex.org/W2118658029","https://openalex.org/W2119983891","https://openalex.org/W2129345386","https://openalex.org/W2131618209","https://openalex.org/W2138039051","https://openalex.org/W2142518823","https://openalex.org/W2149273804","https://openalex.org/W2152009989","https://openalex.org/W2154096410","https://openalex.org/W2154501851","https://openalex.org/W2158470315","https://openalex.org/W2159394270","https://openalex.org/W2159938554","https://openalex.org/W2169802764","https://openalex.org/W2221430606","https://openalex.org/W2467426667","https://openalex.org/W2596585349","https://openalex.org/W2597289420","https://openalex.org/W2604272474","https://openalex.org/W3090888339","https://openalex.org/W4238914075"],"related_works":["https://openalex.org/W2832644133","https://openalex.org/W2029202094","https://openalex.org/W1676177962","https://openalex.org/W296976497","https://openalex.org/W3130354799","https://openalex.org/W2599286794","https://openalex.org/W2902855764","https://openalex.org/W2098316015","https://openalex.org/W4288474950","https://openalex.org/W2971142271"],"abstract_inverted_index":{"The":[0],"proposed":[1,33],"method":[2,34],"is":[3,19],"applicable":[4],"for":[5,56],"meta-learning":[6],"from":[7],"multiple":[8],"existing":[9],"classification":[10],"models":[11],"and":[12],"noisy":[13],"annotations":[14,24],"obtained":[15,26],"by":[16,27,50],"humans.":[17],"It":[18],"particularly":[20],"beneficial":[21],"when":[22],"many":[23],"are":[25],"novice":[28],"labellers.":[29],"In":[30],"addition,":[31],"the":[32,52],"can":[35,43],"provide":[36],"further":[37],"characterization":[38],"of":[39],"each":[40,57],"annotator":[41],"that":[42],"help":[44],"in":[45],"developing":[46],"more":[47],"accurate":[48],"classifiers":[49],"identifying":[51],"most":[53],"competent":[54],"annotators":[55],"data":[58],"instance.":[59]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}