{"id":"https://openalex.org/W4401200607","doi":"https://doi.org/10.1093/jamia/ocae197","title":"Utilizing active learning strategies in machine-assisted annotation for clinical named entity recognition: a comprehensive analysis considering annotation costs and target effectiveness","display_name":"Utilizing active learning strategies in machine-assisted annotation for clinical named entity recognition: a comprehensive analysis considering annotation costs and target effectiveness","publication_year":2024,"publication_date":"2024-07-31","ids":{"openalex":"https://openalex.org/W4401200607","doi":"https://doi.org/10.1093/jamia/ocae197","pmid":"https://pubmed.ncbi.nlm.nih.gov/39081233"},"language":"en","primary_location":{"id":"doi:10.1093/jamia/ocae197","is_oa":false,"landing_page_url":"https://doi.org/10.1093/jamia/ocae197","pdf_url":null,"source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC11491619/pdf/ocae197.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089429883","display_name":"Jiaxing Liu","orcid":"https://orcid.org/0000-0002-5184-4313"},"institutions":[{"id":"https://openalex.org/I158934434","display_name":"Zhongnan University of Economics and Law","ror":"https://ror.org/04yqxxq63","country_code":"CN","type":"education","lineage":["https://openalex.org/I158934434"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaxing Liu","raw_affiliation_strings":["School of Statistics and Mathematics, Zhongnan University of Economics and Law , Wuhan, Hubei 430073,","School of Statistics and Mathematics, Zhongnan University of Economics and Law, Wuhan, Hubei 430073, China"],"raw_orcid":"https://orcid.org/0000-0002-5184-4313","affiliations":[{"raw_affiliation_string":"School of Statistics and Mathematics, Zhongnan University of Economics and Law , Wuhan, Hubei 430073,","institution_ids":["https://openalex.org/I158934434"]},{"raw_affiliation_string":"School of Statistics and Mathematics, Zhongnan University of Economics and Law, Wuhan, Hubei 430073, China","institution_ids":["https://openalex.org/I158934434"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024939512","display_name":"Zoie Shui-Yee Wong","orcid":"https://orcid.org/0000-0003-4499-9779"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]},{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]},{"id":"https://openalex.org/I4210092806","display_name":"St. Luke's International University","ror":"https://ror.org/00e5yzw53","country_code":"JP","type":"education","lineage":["https://openalex.org/I4210092806"]},{"id":"https://openalex.org/I4210116964","display_name":"New South Wales Institute of Psychiatry","ror":"https://ror.org/02fjxcw19","country_code":"AU","type":"education","lineage":["https://openalex.org/I4210116964"]}],"countries":["AU","JP"],"is_corresponding":true,"raw_author_name":"Zoie S Y Wong","raw_affiliation_strings":["Graduate School of Public Health, St Luke\u2019s International University , OMURA Susumu & Mieko Memorial St Luke\u2019s Center for Clinical Academia , Chuo-ku, Tokyo 104-0045,","School of Medical Sciences, The Unviersity of Sydney , Camperdown, NSW 2050,","The Kirby Institute, University of New South Wales , Sydney, NSW 2052,","Graduate School of Public Health, St Luke's International University, OMURA Susumu & Mieko Memorial St Luke's Center for Clinical Academia, Chuo-ku, Tokyo 104-0045, Japan","School of Medical Sciences, The Unviersity of Sydney, Camperdown, NSW 2050, Australia","The Kirby Institute, University of New South Wales, Sydney, NSW 2052, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Graduate School of Public Health, St Luke\u2019s International University , OMURA Susumu & Mieko Memorial St Luke\u2019s Center for Clinical Academia , Chuo-ku, Tokyo 104-0045,","institution_ids":["https://openalex.org/I4210092806"]},{"raw_affiliation_string":"School of Medical Sciences, The Unviersity of Sydney , Camperdown, NSW 2050,","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"The Kirby Institute, University of New South Wales , Sydney, NSW 2052,","institution_ids":["https://openalex.org/I4210116964"]},{"raw_affiliation_string":"Graduate School of Public Health, St Luke's International University, OMURA Susumu & Mieko Memorial St Luke's Center for Clinical Academia, Chuo-ku, Tokyo 104-0045, Japan","institution_ids":["https://openalex.org/I4210092806"]},{"raw_affiliation_string":"School of Medical Sciences, The Unviersity of Sydney, Camperdown, NSW 2050, Australia","institution_ids":["https://openalex.org/I129604602"]},{"raw_affiliation_string":"The Kirby Institute, University of New South Wales, Sydney, NSW 2052, Australia","institution_ids":["https://openalex.org/I31746571"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5024939512"],"corresponding_institution_ids":["https://openalex.org/I129604602","https://openalex.org/I31746571","https://openalex.org/I4210092806","https://openalex.org/I4210116964"],"apc_list":{"value":3967,"currency":"USD","value_usd":3967},"apc_paid":null,"fwci":2.113,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.88900304,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"31","issue":"11","first_page":"2632","last_page":"2640"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.22849999368190765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.22849999368190765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.218299999833107,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1738000065088272,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.917750358581543},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7698807120323181},{"id":"https://openalex.org/keywords/named-entity-recognition","display_name":"Named-entity recognition","score":0.5980705618858337},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5832816958427429},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5240164995193481},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.43697088956832886},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38663002848625183},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.33571934700012207},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07527637481689453}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.917750358581543},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7698807120323181},{"id":"https://openalex.org/C2779135771","wikidata":"https://www.wikidata.org/wiki/Q403574","display_name":"Named-entity recognition","level":3,"score":0.5980705618858337},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5832816958427429},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5240164995193481},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.43697088956832886},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38663002848625183},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33571934700012207},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07527637481689453},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000069553","descriptor_name":"Supervised Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D066264","descriptor_name":"Datasets as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1093/jamia/ocae197","is_oa":false,"landing_page_url":"https://doi.org/10.1093/jamia/ocae197","pdf_url":null,"source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},{"id":"pmid:39081233","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39081233","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association : JAMIA","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:11491619","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11491619","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC11491619/pdf/ocae197.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"J Am Med Inform Assoc","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:11491619","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11491619","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC11491619/pdf/ocae197.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"J Am Med Inform Assoc","raw_type":"Text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G501835962","display_name":null,"funder_award_id":"2023AFB414","funder_id":"https://openalex.org/F4320322186","funder_display_name":"Natural Science Foundation of Hubei Province"},{"id":"https://openalex.org/G7301810964","display_name":null,"funder_award_id":"2722023BQ053","funder_id":"https://openalex.org/F4320310114","funder_display_name":"Zhongnan University of Economics and Law"},{"id":"https://openalex.org/G7355680185","display_name":null,"funder_award_id":"18H03336","funder_id":"https://openalex.org/F4320334764","funder_display_name":"Japan Society for the Promotion of Science"}],"funders":[{"id":"https://openalex.org/F4320310114","display_name":"Zhongnan University of Economics and Law","ror":"https://ror.org/04yqxxq63"},{"id":"https://openalex.org/F4320322186","display_name":"Natural Science Foundation of Hubei Province","ror":null},{"id":"https://openalex.org/F4320334764","display_name":"Japan Society for the Promotion of Science","ror":"https://ror.org/00hhkn466"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null},{"id":"https://openalex.org/F4320337238","display_name":"Tr\u01b0\u1eddng \u0110\u1ea1i h\u1ecdc Kinh t\u1ebf - Lu\u1eadt, \u0110\u1ea1i h\u1ecdc Qu\u1ed1c gia Th\u00e0nh ph\u1ed1 H\u1ed3 Ch\u00ed Minh","ror":null}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4401200607.pdf"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W96276655","https://openalex.org/W1647671624","https://openalex.org/W1866463160","https://openalex.org/W1956471287","https://openalex.org/W2114039834","https://openalex.org/W2144245357","https://openalex.org/W2153081307","https://openalex.org/W2278381402","https://openalex.org/W2727926256","https://openalex.org/W2745019517","https://openalex.org/W2755561459","https://openalex.org/W2908840510","https://openalex.org/W2959026463","https://openalex.org/W2979250794","https://openalex.org/W2993873509","https://openalex.org/W3043747042","https://openalex.org/W3121137019","https://openalex.org/W3202190760","https://openalex.org/W3205626500","https://openalex.org/W4206797693","https://openalex.org/W4382467393","https://openalex.org/W4384662964","https://openalex.org/W4386530347","https://openalex.org/W4391292768","https://openalex.org/W6636915900","https://openalex.org/W6853757165"],"related_works":["https://openalex.org/W4377865234","https://openalex.org/W3005759282","https://openalex.org/W2078793151","https://openalex.org/W3128216712","https://openalex.org/W3017222382","https://openalex.org/W3136915866","https://openalex.org/W4390279576","https://openalex.org/W2886890203","https://openalex.org/W4313535650","https://openalex.org/W2287770975"],"abstract_inverted_index":{"OBJECTIVES:":[0],"Active":[1],"learning":[2,231],"(AL)":[3],"has":[4],"rarely":[5],"integrated":[6],"diversity-based":[7,58,75],"and":[8,64,69,103,114,116,175,233],"uncertainty-based":[9,77,112],"strategies":[10,39,67,110],"into":[11],"a":[12,57,135,181],"dynamic":[13,37,66,225],"sampling":[14],"framework":[15],"for":[16,27,45,147,161],"clinical":[17,46,93],"named":[18],"entity":[19],"recognition":[20],"(NER).":[21],"Machine-assisted":[22],"annotation":[23,43,235],"is":[24],"becoming":[25],"popular":[26],"creating":[28],"gold-standard":[29],"labels.":[30],"This":[31],"study":[32],"investigated":[33],"the":[34,82,108,125,131,156,185,217,223,242,246],"effectiveness":[35,138,219,248],"of":[36,72,127],"AL":[38,55],"under":[40,184],"simulated":[41],"machine-assisted":[42,238],"scenarios":[44],"NER.":[47],"MATERIALS":[48],"AND":[49,214],"METHODS:":[50],"We":[51,106],"proposed":[52,109,224],"3":[53,91],"new":[54],"strategies:":[56],"strategy":[59,226],"(CLUSTER)":[60],"based":[61],"on":[62,90,140,152],"Sentence-BERT":[63],"2":[65],"(CLC":[68],"CNBSE)":[70],"capable":[71],"switching":[73],"from":[74],"to":[76,133,199],"strategies.":[78,119],"Using":[79],"BioClinicalBERT":[80],"as":[81],"foundational":[83],"NER":[84,94],"model,":[85],"we":[86],"conducted":[87],"simulation":[88,187],"experiments":[89],"medication-related":[92],"datasets":[95],"independently:":[96],"i2b2":[97],"2009,":[98],"n2c2":[99],"2018":[100],"(Track":[101],"2),":[102],"MADE":[104],"1.0.":[105],"compared":[107],"with":[111],"(LC":[113],"NBSE)":[115],"passive-learning":[117],"(RANDOM)":[118],"Performance":[120],"was":[121,220,249],"primarily":[122],"measured":[123],"by":[124,130],"number":[126],"edits":[128,170,196,244],"made":[129],"annotators":[132],"achieve":[134,179,200],"desired":[136],"target":[137,150,164,183,202,211,218,247],"evaluated":[139],"independent":[141],"test":[142],"sets.":[143],"RESULTS:":[144],"When":[145,159,216],"aiming":[146,160],"98%":[148],"overall":[149,163],"effectiveness,":[151,165,203],"average,":[153],"CLUSTER":[154,174,206,240],"required":[155,167,193,241],"fewest":[157,243],"edits.":[158],"99%":[162,201],"CNBSE":[166,192,227],"20.4%":[168],"fewer":[169,195],"than":[171,197],"NBSE":[172,198],"did.":[173],"RANDOM":[176,208],"could":[177],"not":[178],"such":[180],"high":[182],"pool-based":[186],"experiment.":[188],"For":[189],"high-difficulty":[190],"entities,":[191],"22.5%":[194],"whereas":[204],"neither":[205],"nor":[207],"achieved":[209],"93%":[210],"effectiveness.":[212],"DISCUSSION":[213],"CONCLUSION:":[215],"set":[221,250],"high,":[222],"exhibited":[228],"both":[229],"strong":[230],"capabilities":[232],"low":[234],"costs":[236],"in":[237],"annotation.":[239],"when":[245],"low.":[251]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-16T07:32:37.131356","created_date":"2025-10-10T00:00:00"}