{"id":"https://openalex.org/W1964989311","doi":"https://doi.org/10.1109/bibm.2012.6392657","title":"Integration of multiple annotators by aggregating experts and filtering novices","display_name":"Integration of multiple annotators by aggregating experts and filtering novices","publication_year":2012,"publication_date":"2012-10-01","ids":{"openalex":"https://openalex.org/W1964989311","doi":"https://doi.org/10.1109/bibm.2012.6392657","mag":"1964989311"},"language":"en","primary_location":{"id":"doi:10.1109/bibm.2012.6392657","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bibm.2012.6392657","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 IEEE International Conference on Bioinformatics and Biomedicine","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050360222","display_name":"Ping Zhang","orcid":"https://orcid.org/0000-0002-4601-0779"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ping Zhang","raw_affiliation_strings":["Center for Data Analytics and Biomedical Informatics, Temple University, Philadelphia, PA, USA","Center for Data Analytics and Biomedicai Informatics, Temple University, Philadelphia, PA 19122, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Center for Data Analytics and Biomedical Informatics, Temple University, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Center for Data Analytics and Biomedicai Informatics, Temple University, Philadelphia, PA 19122, USA#TAB#","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044038055","display_name":"Zoran Obradovi\u0107","orcid":"https://orcid.org/0000-0002-2051-0142"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zoran Obradovic","raw_affiliation_strings":["Center for Data Analytics and Biomedical Informatics, Temple University, Philadelphia, PA, USA","Center for Data Analytics and Biomedicai Informatics, Temple University, Philadelphia, PA 19122, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Center for Data Analytics and Biomedical Informatics, Temple University, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I84392919"]},{"raw_affiliation_string":"Center for Data Analytics and Biomedicai Informatics, Temple University, Philadelphia, PA 19122, USA#TAB#","institution_ids":["https://openalex.org/I84392919"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5050360222"],"corresponding_institution_ids":["https://openalex.org/I84392919"],"apc_list":null,"apc_paid":null,"fwci":1.9998,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.89692798,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9817000031471252,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8545129895210266},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.666415810585022},{"id":"https://openalex.org/keywords/biomedicine","display_name":"Biomedicine","score":0.6339568495750427},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.6280587911605835},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5791598558425903},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5653608441352844},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5565659999847412},{"id":"https://openalex.org/keywords/bioinformatics","display_name":"Bioinformatics","score":0.11951395869255066}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8545129895210266},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.666415810585022},{"id":"https://openalex.org/C66782513","wikidata":"https://www.wikidata.org/wiki/Q864601","display_name":"Biomedicine","level":2,"score":0.6339568495750427},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.6280587911605835},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5791598558425903},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5653608441352844},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5565659999847412},{"id":"https://openalex.org/C60644358","wikidata":"https://www.wikidata.org/wiki/Q128570","display_name":"Bioinformatics","level":1,"score":0.11951395869255066},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/bibm.2012.6392657","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bibm.2012.6392657","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2012 IEEE International Conference on Bioinformatics and Biomedicine","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.348.4772","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.348.4772","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.dabi.temple.edu/%7Ezoran/papers/B314_4099.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.423.3677","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.423.3677","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://astro.temple.edu/~tua87106/BIBM_AEFN.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7099999785423279,"display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320307773","display_name":"GlaxoSmithKline","ror":"https://ror.org/01xsqw823"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W82089429","https://openalex.org/W82775311","https://openalex.org/W154869794","https://openalex.org/W1970381522","https://openalex.org/W1978394996","https://openalex.org/W1987308257","https://openalex.org/W2010135967","https://openalex.org/W2050611670","https://openalex.org/W2055601365","https://openalex.org/W2065500018","https://openalex.org/W2112921341","https://openalex.org/W2116664070","https://openalex.org/W2118658029","https://openalex.org/W2125943921","https://openalex.org/W2138039051","https://openalex.org/W2154096410","https://openalex.org/W2154501851","https://openalex.org/W2169802764","https://openalex.org/W2221430606","https://openalex.org/W3090888339","https://openalex.org/W6603267975","https://openalex.org/W6603353687","https://openalex.org/W6677516641","https://openalex.org/W6682471470"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4224009465","https://openalex.org/W4286629047","https://openalex.org/W4306321456","https://openalex.org/W4285260836","https://openalex.org/W3046775127","https://openalex.org/W3170094116","https://openalex.org/W4205958290","https://openalex.org/W4205814101"],"abstract_inverted_index":{"Learning":[0],"from":[1,5],"noisy":[2],"labels":[3,14,40],"obtained":[4],"multiple":[6],"annotators":[7,35,92,103],"and":[8,22,36,56,75,118],"without":[9],"access":[10],"to":[11],"any":[12],"true":[13],"is":[15,29,68,87],"an":[16],"increasingly":[17],"important":[18],"problem":[19],"in":[20],"bioinformatics":[21,125],"biomedicine.":[23],"In":[24],"our":[25],"method,":[26],"this":[27],"challenge":[28],"addressed":[30],"by":[31],"iteratively":[32],"filtering":[33],"low-quality":[34,91],"estimating":[37],"the":[38,44,65,72,84,95,100,109,113],"consensus":[39],"based":[41],"only":[42],"on":[43,52],"remaining":[45],"experts":[46],"that":[47,64],"provide":[48,62],"higher-quality":[49],"annotations.":[50],"Experiments":[51],"biomedical":[53],"text":[54],"classification":[55],"CASP9":[57],"protein":[58],"disorder":[59],"prediction":[60],"tasks":[61],"evidence":[63],"proposed":[66],"algorithm":[67,97],"more":[69,120],"accurate":[70],"than":[71],"majority":[73],"voting":[74],"previously":[76],"developed":[77],"multi-annotator":[78],"approaches.":[79],"The":[80],"benefit":[81],"of":[82,115],"using":[83],"new":[85,96],"method":[86],"particularly":[88],"large":[89],"when":[90],"dominate.":[93],"Moreover,":[94],"also":[98],"suggests":[99],"most":[101],"relevant":[102],"for":[104,111,124],"each":[105,116],"instance,":[106],"thus":[107],"paving":[108],"way":[110],"understanding":[112],"behaviors":[114],"annotator":[117],"building":[119],"reliable":[121],"predictive":[122],"models":[123],"applications.":[126]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2013,"cited_by_count":3}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}