{"id":"https://openalex.org/W2567649635","doi":"https://doi.org/10.18653/v1/w16-6110","title":"Replicability of Research in Biomedical Natural Language Processing: a pilot evaluation for a coding task","display_name":"Replicability of Research in Biomedical Natural Language Processing: a pilot evaluation for a coding task","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2567649635","doi":"https://doi.org/10.18653/v1/w16-6110","mag":"2567649635"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w16-6110","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-6110","pdf_url":"https://www.aclweb.org/anthology/W16-6110.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Seventh International Workshop on Health Text\n          Mining and Information Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W16-6110.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005499870","display_name":"Aur\u00e9lie N\u00e9v\u00e9ol","orcid":"https://orcid.org/0000-0002-1846-9144"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Aurelie Neveol","raw_affiliation_strings":["CNRS, Universit\u00e9 Paris-Saclay, F-91405 Orsay"],"affiliations":[{"raw_affiliation_string":"CNRS, Universit\u00e9 Paris-Saclay, F-91405 Orsay","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062221063","display_name":"Kevin Bretonnel Cohen","orcid":"https://orcid.org/0000-0003-1749-8290"},"institutions":[{"id":"https://openalex.org/I2802236040","display_name":"University of Colorado System","ror":"https://ror.org/00jc20583","country_code":"US","type":"education","lineage":["https://openalex.org/I2802236040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Cohen","raw_affiliation_strings":["University of Colorado, USA"],"affiliations":[{"raw_affiliation_string":"University of Colorado, USA","institution_ids":["https://openalex.org/I2802236040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047002517","display_name":"Cyril Grouin","orcid":"https://orcid.org/0000-0001-5809-188X"},"institutions":[{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1294671590"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Cyril Grouin","raw_affiliation_strings":["CNRS, Universit\u00e9 Paris-Saclay, F-91405 Orsay"],"affiliations":[{"raw_affiliation_string":"CNRS, Universit\u00e9 Paris-Saclay, F-91405 Orsay","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I1294671590"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109709674","display_name":"Aude Robert","orcid":null},"institutions":[{"id":"https://openalex.org/I4210131199","display_name":"Paris Cardiovascular Research Center","ror":"https://ror.org/03gvnh520","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I204730241","https://openalex.org/I4210131199"]},{"id":"https://openalex.org/I4210128283","display_name":"Centre d'\u00c9pid\u00e9miologie sur les Causes M\u00e9dicales de D\u00e9c\u00e8s","ror":"https://ror.org/02bn1dj67","country_code":"FR","type":"facility","lineage":["https://openalex.org/I154526488","https://openalex.org/I4210128283","https://openalex.org/I4210160774"]},{"id":"https://openalex.org/I154526488","display_name":"Inserm","ror":"https://ror.org/02vjkv261","country_code":"FR","type":"funder","lineage":["https://openalex.org/I154526488"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Aude Robert","raw_affiliation_strings":["INSERM-CpiDC, Paris, France","INSERM-C\u00e9piDC, Paris, France"],"affiliations":[{"raw_affiliation_string":"INSERM-CpiDC, Paris, France","institution_ids":["https://openalex.org/I154526488","https://openalex.org/I4210131199"]},{"raw_affiliation_string":"INSERM-C\u00e9piDC, Paris, France","institution_ids":["https://openalex.org/I4210128283","https://openalex.org/I154526488"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5005499870"],"corresponding_institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I277688954"],"apc_list":null,"apc_paid":null,"fwci":6.2039,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.96338941,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"78","last_page":"84"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/clef","display_name":"Clef","score":0.7927699089050293},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7903646230697632},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.5860081315040588},{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.5487726926803589},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5247713923454285},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4635879695415497},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.4137359857559204},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36960500478744507},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3623161017894745},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3292636573314667}],"concepts":[{"id":"https://openalex.org/C107763842","wikidata":"https://www.wikidata.org/wiki/Q181040","display_name":"Clef","level":3,"score":0.7927699089050293},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7903646230697632},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.5860081315040588},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.5487726926803589},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5247713923454285},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4635879695415497},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.4137359857559204},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36960500478744507},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3623161017894745},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3292636573314667},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/w16-6110","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-6110","pdf_url":"https://www.aclweb.org/anthology/W16-6110.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Seventh International Workshop on Health Text\n          Mining and Information Analysis","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/w16-6110","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-6110","pdf_url":"https://www.aclweb.org/anthology/W16-6110.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Seventh International Workshop on Health Text\n          Mining and Information Analysis","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6600000262260437}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2567649635.pdf","grobid_xml":"https://content.openalex.org/works/W2567649635.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W997324903","https://openalex.org/W1975004953","https://openalex.org/W2003844967","https://openalex.org/W2027383523","https://openalex.org/W2046766973","https://openalex.org/W2138392083","https://openalex.org/W2187364461","https://openalex.org/W2191243614","https://openalex.org/W2251964473","https://openalex.org/W2406493898","https://openalex.org/W2414283705","https://openalex.org/W2576491980","https://openalex.org/W2915704452","https://openalex.org/W3024839386"],"related_works":["https://openalex.org/W1839123017","https://openalex.org/W4232115401","https://openalex.org/W1592594506","https://openalex.org/W2970828463","https://openalex.org/W3138512925","https://openalex.org/W117735687","https://openalex.org/W2917705549","https://openalex.org/W2915106852","https://openalex.org/W2970434686","https://openalex.org/W2918134408"],"abstract_inverted_index":{"The":[0],"scientific":[1],"community":[2,131,138],"is":[3,91],"facing":[4],"raising":[5],"concerns":[6],"about":[7],"the":[8,23,34,45,51,56,67,107,116,120,127,130,133],"reproducibility":[9],"of":[10,44,80,87,109,111,118,129,135,140],"research":[11],"in":[12,19],"many":[13],"fields.":[14],"To":[15],"address":[16],"this":[17],"issue":[18],"Natural":[20],"Language":[21],"Processing,":[22],"CLEF":[24],"eHealth":[25],"2016":[26],"lab":[27],"offered":[28],"a":[29,95],"replication":[30,46,136],"track":[31],"together":[32],"with":[33,50],"Clinical":[35],"Information":[36],"Extraction":[37],"task.":[38],"Herein,":[39],"we":[40,64],"report":[41],"detailed":[42],"results":[43,60],"experiments":[47,123],"carried":[48],"out":[49],"three":[52],"systems":[53,68],"submitted":[54],"to":[55,105,125],"track.":[57],"While":[58],"all":[59],"were":[61,69],"ultimately":[62],"replicated,":[63],"found":[65],"that":[66],"poorly":[70],"rated":[71],"by":[72,102],"analysts":[73],"on":[74],"documentation":[75],"aspects":[76],"such":[77],"as":[78],"\"ease":[79],"understanding":[81],"system":[82,90,103],"requirements\"":[83],"(33%)":[84],"and":[85,137],"\"provision":[86],"information":[88],"while":[89],"running\"":[92],"(33%).":[93],"As":[94],"result,":[96],"simple":[97],"steps":[98],"could":[99],"be":[100],"taken":[101],"authors":[104],"increase":[106],"ease":[108,117],"replicability":[110],"their":[112],"work,":[113],"thereby":[114],"increasing":[115],"re-using":[119],"systems.":[121,142],"Our":[122],"aim":[124],"raise":[126],"awareness":[128],"towards":[132],"challenges":[134],"sharing":[139],"NLP":[141]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}