{"id":"https://openalex.org/W3112410503","doi":"https://doi.org/10.1093/jamia/ocaa261","title":"Can reproducibility be improved in clinical natural language processing? A study of 7 clinical NLP suites","display_name":"Can reproducibility be improved in clinical natural language processing? A study of 7 clinical NLP suites","publication_year":2020,"publication_date":"2020-11-03","ids":{"openalex":"https://openalex.org/W3112410503","doi":"https://doi.org/10.1093/jamia/ocaa261","mag":"3112410503","pmid":"https://pubmed.ncbi.nlm.nih.gov/33319904"},"language":"en","primary_location":{"id":"doi:10.1093/jamia/ocaa261","is_oa":true,"landing_page_url":"https://doi.org/10.1093/jamia/ocaa261","pdf_url":"https://academic.oup.com/jamia/article-pdf/28/3/504/36428698/ocaa261.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://academic.oup.com/jamia/article-pdf/28/3/504/36428698/ocaa261.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029363661","display_name":"William Digan","orcid":"https://orcid.org/0000-0001-7290-3282"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I204730241","display_name":"Universit\u00e9 Paris Cit\u00e9","ror":"https://ror.org/05f82e368","country_code":"FR","type":"education","lineage":["https://openalex.org/I204730241"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210091437","display_name":"Sorbonne Paris Cit\u00e9","ror":"https://ror.org/001z21q04","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210091437"]},{"id":"https://openalex.org/I4210092322","display_name":"Centre de Recherche des Cordeliers","ror":"https://ror.org/00dmms154","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210092322"]},{"id":"https://openalex.org/I4210102325","display_name":"H\u00f4pital Europ\u00e9en Georges-Pompidou","ror":"https://ror.org/016vx5156","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159","https://openalex.org/I4210102325","https://openalex.org/I4210120235"]},{"id":"https://openalex.org/I4210109141","display_name":"H\u00f4pital Europ\u00e9en","ror":"https://ror.org/01tfhsg94","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210109141"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"William Digan","raw_affiliation_strings":["Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210109141","https://openalex.org/I4210102325"]},{"raw_affiliation_string":"INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France","institution_ids":["https://openalex.org/I4210092322","https://openalex.org/I204730241","https://openalex.org/I154526488","https://openalex.org/I4210091437","https://openalex.org/I39804081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005499870","display_name":"Aur\u00e9lie N\u00e9v\u00e9ol","orcid":"https://orcid.org/0000-0002-1846-9144"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I4210115485","display_name":"Laboratoire d'Informatique pour la M\u00e9canique et les Sciences de l'Ing\u00e9nieur","ror":"https://ror.org/01raq4x89","country_code":"FR","type":"facility","lineage":["https://openalex.org/I102197404","https://openalex.org/I1294671590","https://openalex.org/I4210115485","https://openalex.org/I4210159245"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Aur\u00e9lie N\u00e9v\u00e9ol","raw_affiliation_strings":["Universit\u00e9 Paris Saclay, CNRS, LIMSI, Orsay, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Paris Saclay, CNRS, LIMSI, Orsay, France","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I4210115485","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002976747","display_name":"Antoine Neuraz","orcid":"https://orcid.org/0000-0001-7142-6728"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I204730241","display_name":"Universit\u00e9 Paris Cit\u00e9","ror":"https://ror.org/05f82e368","country_code":"FR","type":"education","lineage":["https://openalex.org/I204730241"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210091437","display_name":"Sorbonne Paris Cit\u00e9","ror":"https://ror.org/001z21q04","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210091437"]},{"id":"https://openalex.org/I4210092322","display_name":"Centre de Recherche des Cordeliers","ror":"https://ror.org/00dmms154","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210092322"]},{"id":"https://openalex.org/I4210097159","display_name":"Assistance Publique \u2013 H\u00f4pitaux de Paris","ror":"https://ror.org/00pg5jh14","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Antoine Neuraz","raw_affiliation_strings":["Department of Medical Informatics, Necker Children\u2019s Hospital, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, Necker Children\u2019s Hospital, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210097159"]},{"raw_affiliation_string":"INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France","institution_ids":["https://openalex.org/I4210092322","https://openalex.org/I204730241","https://openalex.org/I154526488","https://openalex.org/I4210091437","https://openalex.org/I39804081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087418731","display_name":"Maxime Wack","orcid":"https://orcid.org/0000-0002-4989-5533"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I204730241","display_name":"Universit\u00e9 Paris Cit\u00e9","ror":"https://ror.org/05f82e368","country_code":"FR","type":"education","lineage":["https://openalex.org/I204730241"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210091437","display_name":"Sorbonne Paris Cit\u00e9","ror":"https://ror.org/001z21q04","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210091437"]},{"id":"https://openalex.org/I4210092322","display_name":"Centre de Recherche des Cordeliers","ror":"https://ror.org/00dmms154","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210092322"]},{"id":"https://openalex.org/I4210102325","display_name":"H\u00f4pital Europ\u00e9en Georges-Pompidou","ror":"https://ror.org/016vx5156","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159","https://openalex.org/I4210102325","https://openalex.org/I4210120235"]},{"id":"https://openalex.org/I4210109141","display_name":"H\u00f4pital Europ\u00e9en","ror":"https://ror.org/01tfhsg94","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210109141"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Maxime Wack","raw_affiliation_strings":["Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210109141","https://openalex.org/I4210102325"]},{"raw_affiliation_string":"INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France","institution_ids":["https://openalex.org/I4210092322","https://openalex.org/I204730241","https://openalex.org/I154526488","https://openalex.org/I4210091437","https://openalex.org/I39804081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083800526","display_name":"David Baudoin","orcid":"https://orcid.org/0000-0002-1552-8356"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I204730241","display_name":"Universit\u00e9 Paris Cit\u00e9","ror":"https://ror.org/05f82e368","country_code":"FR","type":"education","lineage":["https://openalex.org/I204730241"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210091437","display_name":"Sorbonne Paris Cit\u00e9","ror":"https://ror.org/001z21q04","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210091437"]},{"id":"https://openalex.org/I4210092322","display_name":"Centre de Recherche des Cordeliers","ror":"https://ror.org/00dmms154","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210092322"]},{"id":"https://openalex.org/I4210102325","display_name":"H\u00f4pital Europ\u00e9en Georges-Pompidou","ror":"https://ror.org/016vx5156","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159","https://openalex.org/I4210102325","https://openalex.org/I4210120235"]},{"id":"https://openalex.org/I4210109141","display_name":"H\u00f4pital Europ\u00e9en","ror":"https://ror.org/01tfhsg94","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210109141"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"David Baudoin","raw_affiliation_strings":["Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210109141","https://openalex.org/I4210102325"]},{"raw_affiliation_string":"INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France","institution_ids":["https://openalex.org/I4210092322","https://openalex.org/I204730241","https://openalex.org/I154526488","https://openalex.org/I4210091437","https://openalex.org/I39804081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037397269","display_name":"Anita Burgun","orcid":"https://orcid.org/0000-0001-6855-4366"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I204730241","display_name":"Universit\u00e9 Paris Cit\u00e9","ror":"https://ror.org/05f82e368","country_code":"FR","type":"education","lineage":["https://openalex.org/I204730241"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210091437","display_name":"Sorbonne Paris Cit\u00e9","ror":"https://ror.org/001z21q04","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210091437"]},{"id":"https://openalex.org/I4210092322","display_name":"Centre de Recherche des Cordeliers","ror":"https://ror.org/00dmms154","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210092322"]},{"id":"https://openalex.org/I4210097159","display_name":"Assistance Publique \u2013 H\u00f4pitaux de Paris","ror":"https://ror.org/00pg5jh14","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159"]},{"id":"https://openalex.org/I4210102325","display_name":"H\u00f4pital Europ\u00e9en Georges-Pompidou","ror":"https://ror.org/016vx5156","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159","https://openalex.org/I4210102325","https://openalex.org/I4210120235"]},{"id":"https://openalex.org/I4210109141","display_name":"H\u00f4pital Europ\u00e9en","ror":"https://ror.org/01tfhsg94","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210109141"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Anita Burgun","raw_affiliation_strings":["Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","Department of Medical Informatics, Necker Children\u2019s Hospital, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210109141","https://openalex.org/I4210102325"]},{"raw_affiliation_string":"Department of Medical Informatics, Necker Children\u2019s Hospital, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210097159"]},{"raw_affiliation_string":"INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France","institution_ids":["https://openalex.org/I4210092322","https://openalex.org/I204730241","https://openalex.org/I154526488","https://openalex.org/I4210091437","https://openalex.org/I39804081"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039181362","display_name":"Bastien Rance","orcid":"https://orcid.org/0000-0003-4417-1197"},"institutions":[{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"government","lineage":["https://openalex.org/I154526488"]},{"id":"https://openalex.org/I204730241","display_name":"Universit\u00e9 Paris Cit\u00e9","ror":"https://ror.org/05f82e368","country_code":"FR","type":"education","lineage":["https://openalex.org/I204730241"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210091437","display_name":"Sorbonne Paris Cit\u00e9","ror":"https://ror.org/001z21q04","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210091437"]},{"id":"https://openalex.org/I4210092322","display_name":"Centre de Recherche des Cordeliers","ror":"https://ror.org/00dmms154","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210092322"]},{"id":"https://openalex.org/I4210102325","display_name":"H\u00f4pital Europ\u00e9en Georges-Pompidou","ror":"https://ror.org/016vx5156","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210097159","https://openalex.org/I4210102325","https://openalex.org/I4210120235"]},{"id":"https://openalex.org/I4210109141","display_name":"H\u00f4pital Europ\u00e9en","ror":"https://ror.org/01tfhsg94","country_code":"FR","type":"healthcare","lineage":["https://openalex.org/I4210109141"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Bastien Rance","raw_affiliation_strings":["Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Medical Informatics, H\u00f4pital Europ\u00e9en Georges Pompidou, Assistance publique\u2013H\u00f4pitaux de Paris, Paris, France","institution_ids":["https://openalex.org/I4210109141","https://openalex.org/I4210102325"]},{"raw_affiliation_string":"INSERM, Centre de Recherche des Cordeliers, UMRS 1138, Universit\u00e9 de Paris, Universit\u00e9 Sorbonne Paris Cit\u00e9, Paris, France","institution_ids":["https://openalex.org/I4210092322","https://openalex.org/I204730241","https://openalex.org/I154526488","https://openalex.org/I4210091437","https://openalex.org/I39804081"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5029363661"],"corresponding_institution_ids":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I39804081","https://openalex.org/I4210091437","https://openalex.org/I4210092322","https://openalex.org/I4210102325","https://openalex.org/I4210109141"],"apc_list":{"value":3967,"currency":"USD","value_usd":3967},"apc_paid":{"value":3967,"currency":"USD","value_usd":3967},"fwci":2.1672,"has_fulltext":true,"cited_by_count":35,"citation_normalized_percentile":{"value":0.88508883,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"28","issue":"3","first_page":"504","last_page":"515"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.49549999833106995,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.49549999833106995,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.36079999804496765,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.0681999996304512,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.7188290357589722},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.664546012878418},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.6368362903594971},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6019253730773926}],"concepts":[{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.7188290357589722},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.664546012878418},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.6368362903594971},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6019253730773926},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D003628","descriptor_name":"Database Management Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003628","descriptor_name":"Database Management Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003628","descriptor_name":"Database Management Systems","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008490","descriptor_name":"Medical Informatics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008490","descriptor_name":"Medical Informatics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008490","descriptor_name":"Medical Informatics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D019295","descriptor_name":"Computational Biology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":5,"locations":[{"id":"doi:10.1093/jamia/ocaa261","is_oa":true,"landing_page_url":"https://doi.org/10.1093/jamia/ocaa261","pdf_url":"https://academic.oup.com/jamia/article-pdf/28/3/504/36428698/ocaa261.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},{"id":"pmid:33319904","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33319904","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association : JAMIA","raw_type":null},{"id":"pmh:oai:HAL:hal-03118455v1","is_oa":true,"landing_page_url":"https://hal.science/hal-03118455","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of the American Medical Informatics Association, 2020, 28 (3), pp.504-515. ⟨10.1093/jamia/ocaa261⟩","raw_type":"Journal articles"},{"id":"pmh:oai:HAL:hal-03887189v1","is_oa":false,"landing_page_url":"https://inria.hal.science/hal-03887189","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of the American Medical Informatics Association, 2021, 28 (3), pp.504-515. ⟨10.1093/jamia/ocaa261⟩","raw_type":"Journal articles"},{"id":"pmh:oai:pubmedcentral.nih.gov:7936396","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7936396","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"J Am Med Inform Assoc","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1093/jamia/ocaa261","is_oa":true,"landing_page_url":"https://doi.org/10.1093/jamia/ocaa261","pdf_url":"https://academic.oup.com/jamia/article-pdf/28/3/504/36428698/ocaa261.pdf","source":{"id":"https://openalex.org/S129839026","display_name":"Journal of the American Medical Informatics Association","issn_l":"1067-5027","issn":["1067-5027","1527-974X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of the American Medical Informatics Association","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4099999964237213}],"awards":[{"id":"https://openalex.org/G3776455082","display_name":null,"funder_award_id":"ANR-15-CE23-0028","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3112410503.pdf","grobid_xml":"https://content.openalex.org/works/W3112410503.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W1483822252","https://openalex.org/W1490312738","https://openalex.org/W1550258693","https://openalex.org/W1607983314","https://openalex.org/W1975004953","https://openalex.org/W2027979924","https://openalex.org/W2036318837","https://openalex.org/W2040298842","https://openalex.org/W2096252540","https://openalex.org/W2096797897","https://openalex.org/W2104148262","https://openalex.org/W2110417468","https://openalex.org/W2123442489","https://openalex.org/W2140503409","https://openalex.org/W2141541659","https://openalex.org/W2146089916","https://openalex.org/W2233154023","https://openalex.org/W2283438976","https://openalex.org/W2302501749","https://openalex.org/W2346143774","https://openalex.org/W2394812591","https://openalex.org/W2396881363","https://openalex.org/W2406493898","https://openalex.org/W2534699084","https://openalex.org/W2572857085","https://openalex.org/W2573536041","https://openalex.org/W2604622795","https://openalex.org/W2605897695","https://openalex.org/W2606769376","https://openalex.org/W2613409207","https://openalex.org/W2750943285","https://openalex.org/W2755388963","https://openalex.org/W2769851464","https://openalex.org/W2773787581","https://openalex.org/W2785607065","https://openalex.org/W2806901780","https://openalex.org/W2891378911","https://openalex.org/W2895296143","https://openalex.org/W2898155085","https://openalex.org/W2899085690","https://openalex.org/W2903593834","https://openalex.org/W2913154181","https://openalex.org/W2949129058","https://openalex.org/W2955573656","https://openalex.org/W3006653136","https://openalex.org/W3037706039","https://openalex.org/W3102863256","https://openalex.org/W4229930240","https://openalex.org/W4289360595","https://openalex.org/W6632766574","https://openalex.org/W6748083584","https://openalex.org/W6751752195"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W3134060498","https://openalex.org/W3204019825"],"abstract_inverted_index":{"BACKGROUND:":[0],"The":[1],"increasing":[2],"complexity":[3],"of":[4,32,62,95,120,145,224,231],"data":[5],"streams":[6],"and":[7,54,79,90,167,173,227],"computational":[8],"processes":[9],"in":[10,42,239],"modern":[11],"clinical":[12,63,80,162,241],"health":[13],"information":[14],"systems":[15,36],"makes":[16],"reproducibility":[17,47,61,88,113,131,238],"challenging.":[18],"Clinical":[19],"natural":[20],"language":[21],"processing":[22],"(NLP)":[23],"pipelines":[24],"are":[25,178],"routinely":[26],"leveraged":[27],"for":[28,149,154,160,170,233],"the":[29,46,60,71,109,118,125,207,218,237],"secondary":[30],"use":[31,230],"data.":[33],"Workflow":[34],"management":[35],"(WMS)":[37],"have":[38],"been":[39],"widely":[40],"used":[41],"bioinformatics":[43,56,78,219],"to":[44,99,111,124,157,235],"handle":[45],"bottleneck.":[48],"OBJECTIVE:":[49],"To":[50],"evaluate":[51],"if":[52],"WMS":[53,140,201],"other":[55],"practices":[57,89],"could":[58,212],"impact":[59],"NLP":[64,122,163,195,210],"frameworks.":[65],"MATERIALS":[66],"AND":[67],"METHODS:":[68],"Based":[69],"on":[70,139,200],"literature":[72,110],"across":[73],"multiple":[74],"researcho":[75],"fields":[76],"(NLP,":[77],"informatics)":[81],"we":[82,116],"selected":[83,135],"articles":[84,184],"which":[85],"(1)":[86],"review":[87],"(2)":[91],"highlight":[92],"a":[93,203,240],"set":[94],"rules":[96],"or":[97,102,229],"guidelines":[98],"ensure":[100],"tool":[101],"pipeline":[103],"reproducibility.":[104],"We":[105,128],"aggregate":[106],"insight":[107],"from":[108,133,185,214,217],"define":[112],"recommendations.":[114,126],"Finally,":[115],"assess":[117],"compliance":[119,205],"7":[121],"frameworks":[123,198,211],"RESULTS:":[127],"identified":[129],"40":[130],"features":[132,146,148,153,159,169,190],"8":[134],"articles.":[136],"Frameworks":[137],"based":[138,199],"match":[141],"more":[142],"than":[143],"50%":[144],"(26":[147],"LAPPS":[150],"Grid,":[151],"22":[152],"OpenMinted)":[155],"compared":[156],"18":[158],"current":[161],"framework":[164],"(cTakes,":[165],"CLAMP)":[166],"17":[168],"GATE,":[171],"ScispaCy,":[172],"Textflows.":[174],"DISCUSSION:":[175],"34":[176],"recommendations":[177],"endorsed":[179],"by":[180,193],"at":[181],"least":[182],"2":[183],"our":[186],"selection.":[187],"Overall,":[188],"15":[189],"were":[191],"adopted":[192],"every":[194],"Framework.":[196],"Nevertheless,":[197],"had":[202],"better":[204],"with":[206],"features.":[208],"CONCLUSION:":[209],"benefit":[213],"lessons":[215],"learned":[216],"field":[220],"(eg,":[221],"public":[222],"repositories":[223],"curated":[225],"tools":[226],"workflows":[228],"containers":[232],"shareability)":[234],"enhance":[236],"setting.":[242]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":5}],"updated_date":"2026-05-27T09:02:27.158192","created_date":"2025-10-10T00:00:00"}