{"id":"https://openalex.org/W2156418292","doi":"https://doi.org/10.1186/1471-2105-10-s9-s10","title":"Evaluation of a large-scale biomedical data annotation initiative","display_name":"Evaluation of a large-scale biomedical data annotation initiative","publication_year":2009,"publication_date":"2009-09-01","ids":{"openalex":"https://openalex.org/W2156418292","doi":"https://doi.org/10.1186/1471-2105-10-s9-s10","mag":"2156418292","pmid":"https://pubmed.ncbi.nlm.nih.gov/19761564"},"language":"en","primary_location":{"id":"doi:10.1186/1471-2105-10-s9-s10","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-10-s9-s10","pdf_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-10-S9-S10","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-10-S9-S10","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054080485","display_name":"Ronilda Lacson","orcid":"https://orcid.org/0000-0003-3300-1176"},"institutions":[{"id":"https://openalex.org/I1283280774","display_name":"Brigham and Women's Hospital","ror":"https://ror.org/04b6nzv94","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283280774","https://openalex.org/I48633490"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210143601","display_name":"Decision Systems (United States)","ror":"https://ror.org/0434dpa13","country_code":"US","type":"company","lineage":["https://openalex.org/I4210143601"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ronilda Lacson","raw_affiliation_strings":["Departments of Biomedical & Health Informatics, University of Washington, Seattle, WA, USA. rlacson@dsg.harvard.edu","Decision Systems Group, Brigham & Women's Hospital, Harvard Medical School, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Departments of Biomedical & Health Informatics, University of Washington, Seattle, WA, USA. rlacson@dsg.harvard.edu","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"Decision Systems Group, Brigham & Women's Hospital, Harvard Medical School, Boston, MA, USA","institution_ids":["https://openalex.org/I4210143601","https://openalex.org/I1283280774"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019027996","display_name":"Erik Pitzer","orcid":"https://orcid.org/0000-0002-3056-7716"},"institutions":[{"id":"https://openalex.org/I4210136249","display_name":"University of Applied Sciences Upper Austria","ror":"https://ror.org/03jqp6d56","country_code":"AT","type":"education","lineage":["https://openalex.org/I4210136249"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Erik Pitzer","raw_affiliation_strings":["Upper Austria University of Applied Sciences, Hagenberg, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Upper Austria University of Applied Sciences, Hagenberg, Austria","institution_ids":["https://openalex.org/I4210136249"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008928704","display_name":"Ludwig Christian Hinske","orcid":"https://orcid.org/0000-0001-7273-5899"},"institutions":[{"id":"https://openalex.org/I1283280774","display_name":"Brigham and Women's Hospital","ror":"https://ror.org/04b6nzv94","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283280774","https://openalex.org/I48633490"]},{"id":"https://openalex.org/I4210143601","display_name":"Decision Systems (United States)","ror":"https://ror.org/0434dpa13","country_code":"US","type":"company","lineage":["https://openalex.org/I4210143601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christian Hinske","raw_affiliation_strings":["Decision Systems Group, Brigham & Women's Hospital, Harvard Medical School, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Decision Systems Group, Brigham & Women's Hospital, Harvard Medical School, Boston, MA, USA","institution_ids":["https://openalex.org/I4210143601","https://openalex.org/I1283280774"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067274615","display_name":"Pedro A. F. Galante","orcid":"https://orcid.org/0000-0002-4820-4155"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pedro Galante","raw_affiliation_strings":["Ludwig Institute for Cancer Research, Sao Paolo Branch, Sao Paulo, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ludwig Institute for Cancer Research, Sao Paolo Branch, Sao Paulo, Brazil","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084415954","display_name":"Lucila Ohno\u2010Machado","orcid":"https://orcid.org/0000-0002-8005-7327"},"institutions":[{"id":"https://openalex.org/I1283280774","display_name":"Brigham and Women's Hospital","ror":"https://ror.org/04b6nzv94","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283280774","https://openalex.org/I48633490"]},{"id":"https://openalex.org/I4210143601","display_name":"Decision Systems (United States)","ror":"https://ror.org/0434dpa13","country_code":"US","type":"company","lineage":["https://openalex.org/I4210143601"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lucila Ohno-Machado","raw_affiliation_strings":["Decision Systems Group, Brigham & Women's Hospital, Harvard Medical School, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Decision Systems Group, Brigham & Women's Hospital, Harvard Medical School, Boston, MA, USA","institution_ids":["https://openalex.org/I4210143601","https://openalex.org/I1283280774"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5054080485"],"corresponding_institution_ids":["https://openalex.org/I1283280774","https://openalex.org/I201448701","https://openalex.org/I4210143601"],"apc_list":{"value":1690,"currency":"GBP","value_usd":2072},"apc_paid":{"value":1690,"currency":"GBP","value_usd":2072},"fwci":0.6847,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.70675082,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"10","issue":"S9","first_page":"S10","last_page":"S10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10885","display_name":"Gene expression and cancer classification","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10887","display_name":"Bioinformatics and Genomic Networks","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8052232265472412},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5785328149795532},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.447068452835083},{"id":"https://openalex.org/keywords/disease","display_name":"Disease","score":0.4323877692222595},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.33218979835510254},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2475990355014801},{"id":"https://openalex.org/keywords/pathology","display_name":"Pathology","score":0.20643466711044312}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8052232265472412},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5785328149795532},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.447068452835083},{"id":"https://openalex.org/C2779134260","wikidata":"https://www.wikidata.org/wiki/Q12136","display_name":"Disease","level":2,"score":0.4323877692222595},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.33218979835510254},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2475990355014801},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.20643466711044312}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016247","descriptor_name":"Information Storage and Retrieval","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D020869","descriptor_name":"Gene Expression Profiling","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D020869","descriptor_name":"Gene Expression Profiling","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D020869","descriptor_name":"Gene Expression Profiling","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D020869","descriptor_name":"Gene Expression Profiling","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D030541","descriptor_name":"Databases, Genetic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D030541","descriptor_name":"Databases, Genetic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D030541","descriptor_name":"Databases, Genetic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D030541","descriptor_name":"Databases, Genetic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":4,"locations":[{"id":"doi:10.1186/1471-2105-10-s9-s10","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-10-s9-s10","pdf_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-10-S9-S10","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},{"id":"pmid:19761564","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/19761564","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC bioinformatics","raw_type":null},{"id":"pmh:oai:dash.harvard.edu:1/4931095","is_oa":true,"landing_page_url":"http://nrs.harvard.edu/urn-3:HUL.InstRepos:4931095","pdf_url":null,"source":{"id":"https://openalex.org/S4306401540","display_name":"Digital Access to Scholarship at Harvard (DASH) (Harvard University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I136199984","host_organization_name":"Harvard University","host_organization_lineage":["https://openalex.org/I136199984"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"},{"id":"pmh:oai:pubmedcentral.nih.gov:2745681","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/2745681","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"BMC Bioinformatics","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1186/1471-2105-10-s9-s10","is_oa":true,"landing_page_url":"https://doi.org/10.1186/1471-2105-10-s9-s10","pdf_url":"https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/1471-2105-10-S9-S10","source":{"id":"https://openalex.org/S19032547","display_name":"BMC Bioinformatics","issn_l":"1471-2105","issn":["1471-2105"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320256","host_organization_name":"BioMed Central","host_organization_lineage":["https://openalex.org/P4310320256","https://openalex.org/P4310319965"],"host_organization_lineage_names":["BioMed Central","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"BMC Bioinformatics","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.6299999952316284}],"awards":[],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2156418292.pdf","grobid_xml":"https://content.openalex.org/works/W2156418292.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W153781075","https://openalex.org/W1480287196","https://openalex.org/W1519027168","https://openalex.org/W1551684349","https://openalex.org/W1986601486","https://openalex.org/W2032872152","https://openalex.org/W2083737836","https://openalex.org/W2091978351","https://openalex.org/W2100471807","https://openalex.org/W2105713982","https://openalex.org/W2117722273","https://openalex.org/W2145543651","https://openalex.org/W2159083595","https://openalex.org/W2162311237","https://openalex.org/W2167980479","https://openalex.org/W2168546553","https://openalex.org/W2604748391","https://openalex.org/W4230872509"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W3031052312","https://openalex.org/W4389568370","https://openalex.org/W3032375762","https://openalex.org/W1995515455","https://openalex.org/W2080531066","https://openalex.org/W3108674512","https://openalex.org/W1506200166","https://openalex.org/W1489783725"],"abstract_inverted_index":{"BACKGROUND:":[0],"This":[1],"study":[2],"describes":[3],"a":[4,146,150],"large-scale":[5],"manual":[6,143],"re-annotation":[7,144],"of":[8,70,108,145],"data":[9],"samples":[10,62],"in":[11,68,149],"the":[12,23],"Gene":[13],"Expression":[14],"Omnibus":[15],"(GEO),":[16],"using":[17,128],"variables":[18],"and":[19,44,54,90,115,124,130],"values":[20,104],"derived":[21],"from":[22],"National":[24],"Cancer":[25],"Institute":[26],"thesaurus.":[27],"A":[28],"framework":[29],"is":[30,41,49,139],"described":[31],"for":[32,37,105],"creating":[33],"an":[34],"annotation":[35,47],"scheme":[36],"various":[38],"diseases":[39],"that":[40,137],"flexible,":[42],"comprehensive,":[43],"scalable.":[45],"The":[46,96],"structure":[48],"evaluated":[50],"by":[51],"measuring":[52],"coverage":[53],"agreement":[55,123,126],"between":[56],"annotators.":[57],"RESULTS:":[58],"There":[59,118],"were":[60],"12,500":[61],"annotated":[63],"with":[64,102],"approximately":[65],"30":[66],"variables,":[67],"each":[69],"six":[71],"disease":[72,81,112],"categories":[73],"-":[74],"breast":[75],"cancer,":[76,78],"colon":[77],"inflammatory":[79],"bowel":[80],"(IBD),":[82],"rheumatoid":[83],"arthritis":[84],"(RA),":[85],"systemic":[86],"lupus":[87],"erythematosus":[88],"(SLE),":[89],"Type":[91],"1":[92],"diabetes":[93],"mellitus":[94],"(DM).":[95],"annotators":[97],"provided":[98],"excellent":[99],"variable":[100],"coverage,":[101],"known":[103],"over":[106],"98%":[107],"three":[109],"critical":[110],"variables:":[111],"state,":[113],"tissue,":[114],"sample":[116],"type.":[117],"was":[119],"89%":[120],"strict":[121],"inter-annotator":[122],"92%":[125],"when":[127],"semantic":[129],"partial":[131],"similarity":[132],"measures.":[133],"CONCLUSION:":[134],"We":[135],"show":[136],"it":[138],"possible":[140],"to":[141],"perform":[142],"large":[147],"repository":[148],"reliable":[151],"manner.":[152]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2013,"cited_by_count":3}],"updated_date":"2026-06-19T17:40:00.097472","created_date":"2016-06-24T00:00:00"}