{"id":"https://openalex.org/W4399992129","doi":null,"title":"An evaluation of current benchmarking strategies for French biomedical language models","display_name":"An evaluation of current benchmarking strategies for French biomedical language models","alternate_titles":[],"publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4399992129"},"language":"fr","primary_location":{"id":"pmh:oai:HAL:hal-04622983v1","doi":null,"is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04622983","pdf_url":"https://inria.hal.science/hal-04622983/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"35\u00e8mes Journ\u00e9es d'\u00c9tudes sur la Parole (JEP) 31\u00e8me Conf\u00e9rence sur le Traitement Automatique des Langues Naturelles (TALN) 26\u00e8me Rencontre des \u00c9tudiants Chercheurs en Informatique pour le Traitement Automatique des Langues (RECITAL), 2024, Toulouse, France. pp.1-16","raw_type":"info:eu-repo/semantics/conferenceObject"},"parent_works":[],"lineage":["https://openalex.org/W4399992129"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://inria.hal.science/hal-04622983/document","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5099096330","display_name":"Felix E. Herron","orcid":"https://orcid.org/0009-0005-9771-4994","observed_orcids":["https://orcid.org/0009-0005-9771-4994"]},"institutions":[{"id":"https://openalex.org/I2746051580","display_name":"Universit\u00e9 Paris Sciences et Lettres","ror":"https://ror.org/013cjyk83","country_code":"FR","type":"education","lineage":["https://openalex.org/I2746051580"]},{"id":"https://openalex.org/I4210104430","display_name":"Laboratoire d'Informatique de Grenoble","ror":"https://ror.org/01c8rcg82","country_code":"FR","type":"facility","lineage":["https://openalex.org/I106785703","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I4210104430","https://openalex.org/I4210159245","https://openalex.org/I899635006"]},{"id":"https://openalex.org/I4210137929","display_name":"Lamsade","ror":"https://ror.org/03pnp1a74","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I2746051580","https://openalex.org/I4210137929","https://openalex.org/I4210159245","https://openalex.org/I56435720"]},{"id":"https://openalex.org/I56435720","display_name":"Universit\u00e9 Paris Dauphine-PSL","ror":"https://ror.org/052bz7812","country_code":"FR","type":"education","lineage":["https://openalex.org/I2746051580","https://openalex.org/I56435720"]},{"id":"https://openalex.org/I899635006","display_name":"Universit\u00e9 Grenoble Alpes","ror":"https://ror.org/02rx3b187","country_code":"FR","type":"education","lineage":["https://openalex.org/I899635006"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Herron, Felix","raw_affiliation_strings":["LAMSADE - Laboratoire d'analyse et mod\u00e9lisation de syst\u00e8mes pour l'aide \u00e0 la d\u00e9cision (Place de Lattre de Tassigny 75775 PARIS CEDEX 16 - France)","UGA - Universit\u00e9 Grenoble Alpes (Adresse CS 40700 - 38058 Grenoble cedex - France)","Universit\u00e9 Paris Dauphine-PSL (Place du Mar\u00e9chal de Lattre de Tassigny\r\n75775 PARIS Cedex 16 - France)","LIG - Laboratoire d'Informatique de Grenoble (UMR 5217 - Laboratoire LIG - B\u00e2timent IMAG - 700 avenue Centrale - Domaine Universitaire de Saint-Martin-d\u2019H\u00e8res \r\nAdresse postale : CS 40700 - 38058 Grenoble cedex 9\r\nT\u00e9l. : 04 57 42 14 00 - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LAMSADE - Laboratoire d'analyse et mod\u00e9lisation de syst\u00e8mes pour l'aide \u00e0 la d\u00e9cision (Place de Lattre de Tassigny 75775 PARIS CEDEX 16 - France)","institution_ids":["https://openalex.org/I4210137929"]},{"raw_affiliation_string":"UGA - Universit\u00e9 Grenoble Alpes (Adresse CS 40700 - 38058 Grenoble cedex - France)","institution_ids":["https://openalex.org/I899635006"]},{"raw_affiliation_string":"Universit\u00e9 Paris Dauphine-PSL (Place du Mar\u00e9chal de Lattre de Tassigny\r\n75775 PARIS Cedex 16 - France)","institution_ids":["https://openalex.org/I2746051580","https://openalex.org/I56435720"]},{"raw_affiliation_string":"LIG - Laboratoire d'Informatique de Grenoble (UMR 5217 - Laboratoire LIG - B\u00e2timent IMAG - 700 avenue Centrale - Domaine Universitaire de Saint-Martin-d\u2019H\u00e8res \r\nAdresse postale : CS 40700 - 38058 Grenoble cedex 9\r\nT\u00e9l. : 04 57 42 14 00 - France)","institution_ids":["https://openalex.org/I4210104430"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5099096330"],"corresponding_institution_ids":["https://openalex.org/I2746051580","https://openalex.org/I4210104430","https://openalex.org/I4210137929","https://openalex.org/I56435720","https://openalex.org/I899635006"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"16"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.7821999788284302,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.7821999788284302,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1559000015258789,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.025200000032782555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/biomedical-language-models","display_name":"biomedical language models","score":0.6669999957084656},{"id":"https://openalex.org/keywords/french-language","display_name":"French language","score":0.7129999995231628},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.8479999899864197},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"natural language understanding","score":0.7620000243186951},{"id":"https://openalex.org/keywords/clinical-natural-language-processing","display_name":"clinical natural language processing","score":0.6100000143051147},{"id":"https://openalex.org/keywords/domain-specific-pre-training","display_name":"domain-specific pre-training","score":0.7799999713897705},{"id":"https://openalex.org/keywords/stakeholder-collaboration","display_name":"stakeholder collaboration","score":0.6200000047683716},{"id":"https://openalex.org/keywords/benchmark-evaluation","display_name":"benchmark evaluation","score":0.3889999985694885}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8449479937553406},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6536027193069458},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.6218284368515015},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37515145540237427},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.37057411670684814},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19162753224372864},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.09269323945045471},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.07097974419593811},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:HAL:hal-04622983v1","doi":null,"is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04622983","pdf_url":"https://inria.hal.science/hal-04622983/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"35\u00e8mes Journ\u00e9es d'\u00c9tudes sur la Parole (JEP) 31\u00e8me Conf\u00e9rence sur le Traitement Automatique des Langues Naturelles (TALN) 26\u00e8me Rencontre des \u00c9tudiants Chercheurs en Informatique pour le Traitement Automatique des Langues (RECITAL), 2024, Toulouse, France. pp.1-16","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-04622983v1","doi":null,"is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04622983","pdf_url":"https://inria.hal.science/hal-04622983/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"35\u00e8mes Journ\u00e9es d'\u00c9tudes sur la Parole (JEP) 31\u00e8me Conf\u00e9rence sur le Traitement Automatique des Langues Naturelles (TALN) 26\u00e8me Rencontre des \u00c9tudiants Chercheurs en Informatique pour le Traitement Automatique des Langues (RECITAL), 2024, Toulouse, France. pp.1-16","raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399992129.pdf","grobid_xml":"https://content.openalex.org/works/W4399992129.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W2134167315","https://openalex.org/W2159024459","https://openalex.org/W2396881363","https://openalex.org/W2559662148","https://openalex.org/W2596602780","https://openalex.org/W2607299704","https://openalex.org/W2923014074","https://openalex.org/W2937845937","https://openalex.org/W2963341956","https://openalex.org/W2963748441","https://openalex.org/W2971258845","https://openalex.org/W2973363064","https://openalex.org/W2986154550","https://openalex.org/W3012501605","https://openalex.org/W3088728443","https://openalex.org/W3089187463","https://openalex.org/W3093198074","https://openalex.org/W3123459983","https://openalex.org/W3152258780","https://openalex.org/W3159493748","https://openalex.org/W3169341408","https://openalex.org/W3170111948","https://openalex.org/W3194676777","https://openalex.org/W4221161239","https://openalex.org/W4281631307","https://openalex.org/W4362603292","https://openalex.org/W4377041488","https://openalex.org/W4385227045","https://openalex.org/W4399552820","https://openalex.org/W4400018477","https://openalex.org/W7150900196"],"related_works":["https://openalex.org/W7131791961","https://openalex.org/W4362598818","https://openalex.org/W4393404004","https://openalex.org/W3046375318","https://openalex.org/W4401823782","https://openalex.org/W3100452049","https://openalex.org/W7161965099","https://openalex.org/W3206153991","https://openalex.org/W4385565282","https://openalex.org/W6980772874"],"abstract_inverted_index":{"We":[0,15,60,119],"describe":[1],"the":[2,92,106],"current":[3],"state":[4],"of":[5,41,91,101,108],"benchmarking":[6],"for":[7,113,139],"French":[8,95,124,147],"language":[9,12,115,126],"biomedical":[10,21,26,43,51,58,64,79,96,125,134,148],"natural":[11,114],"processing":[13],"(NLP).":[14],"note":[16,61],"two":[17],"important":[18,133],"criteria":[19],"in":[20,34,55,67,142],"benchmarking:":[22],"first,":[23],"that":[24,49,62,121],"a":[25,30,38,42,50],"benchmark":[27,52],"clearly":[28],"simulate":[29],"specific":[31],"use":[32],"cases,":[33],"order":[35],"to":[36,72,84],"offer":[37],"useful":[39],"evaluation":[40],"model's":[44],"real":[45],"life":[46],"applicability.":[47],"Second:":[48],"be":[53],"created":[54],"collaboration":[56],"with":[57],"professionals.":[59],"many":[63],"benchmarks,":[65],"particularly":[66],"French,":[68],"do":[69],"not":[70],"adhere":[71,82],"these":[73],"criteria;":[74],"however,":[75],"we":[76,88,137],"highlight":[77],"other":[78],"benchmarks":[80,97],"which":[81],"better":[83],"those":[85],"criteria.":[86],"Furthermore,":[87],"evaluate":[89],"some":[90,122],"most":[93],"common":[94],"on":[98,132],"an":[99,140],"array":[100],"models":[102,127],"and":[103,110],"empirically":[104],"support":[105],"necessity":[107],"domain-specific":[109],"language-specific":[111],"pre-training":[112],"understanding":[116],"(NLU)":[117],"tasks.":[118,135],"show":[120],"popular":[123],"perform":[128],"poorly":[129],"and/or":[130],"inconsistently":[131],"Finally,":[136],"advocate":[138],"increase":[141],"publicly":[143],"available,":[144],"clinically":[145],"targeted":[146],"NLU":[149],"benchmarks.":[150]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4399992129","counts_by_year":[],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2024-06-26T00:00:00"}
