{"id":"https://openalex.org/W1664649366","doi":null,"title":"Principles of Evaluation in Natural Language Processing","display_name":"Principles of Evaluation in Natural Language Processing","publication_year":2007,"publication_date":"2007-05-15","ids":{"openalex":"https://openalex.org/W1664649366","mag":"1664649366"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.377.378","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.377.378","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://atala.org/IMG/pdf/intro_lh_sc_pap.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hal.science/hal-00502700","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047797750","display_name":"Patrick Paroubek","orcid":"https://orcid.org/0000-0002-4302-1894","observed_orcids":["https://orcid.org/0000-0002-4302-1894"]},"institutions":[{"id":"https://openalex.org/I102197404","display_name":"Universit\u00e9 Paris-Sud","ror":"https://ror.org/028rypz17","country_code":"FR","type":"education","lineage":["https://openalex.org/I102197404","https://openalex.org/I277688954"]},{"id":"https://openalex.org/I4210115485","display_name":"Laboratoire d'Informatique pour la M\u00e9canique et les Sciences de l'Ing\u00e9nieur","ror":"https://ror.org/01raq4x89","country_code":"FR","type":"facility","lineage":["https://openalex.org/I102197404","https://openalex.org/I1294671590","https://openalex.org/I277688954","https://openalex.org/I4210115485","https://openalex.org/I4210159245"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Patrick Paroubek","raw_affiliation_strings":["LIMSI - Laboratoire d'Informatique pour la M\u00e9canique et les Sciences de l'Ing\u00e9nieur (Universit\u00e9 Paris-Sud B\u00e2t. 507 - Rue du Belv\u00e9d\u00e8re -91405 ORSAY CEDEX - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LIMSI - Laboratoire d'Informatique pour la M\u00e9canique et les Sciences de l'Ing\u00e9nieur (Universit\u00e9 Paris-Sud B\u00e2t. 507 - Rue du Belv\u00e9d\u00e8re -91405 ORSAY CEDEX - France)","institution_ids":["https://openalex.org/I102197404","https://openalex.org/I4210115485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054103649","display_name":"St\u00e9phane Chaudiron","orcid":"https://orcid.org/0000-0001-5142-4768","observed_orcids":["https://orcid.org/0000-0001-5142-4768"]},"institutions":[{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]},{"id":"https://openalex.org/I4387156027","display_name":"Groupe d'Etudes et de Recherche Interdisciplinaire en Information et Communication","ror":"https://ror.org/02tznxc41","country_code":"FR","type":"facility","lineage":["https://openalex.org/I2279609970","https://openalex.org/I4387156027"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"St\u00e9phane Chaudiron","raw_affiliation_strings":["GERIICO - Groupe d'\u00c9tudes et de Recherche Interdisciplinaire en Information et COmmunication - ULR 4073 (Universit\u00e9 de Lille - Campus Pont de Bois - B\u00e2t B1 niveau 1 - BP 60149 - 59653 VILLENEUVE D'ASCQ CEDEX - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"GERIICO - Groupe d'\u00c9tudes et de Recherche Interdisciplinaire en Information et COmmunication - ULR 4073 (Universit\u00e9 de Lille - Campus Pont de Bois - B\u00e2t B1 niveau 1 - BP 60149 - 59653 VILLENEUVE D'ASCQ CEDEX - France)","institution_ids":["https://openalex.org/I2279609970","https://openalex.org/I4387156027"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068115280","display_name":"Lynette Hirschman","orcid":"https://orcid.org/0000-0001-8442-0720","observed_orcids":["https://orcid.org/0000-0001-8442-0720"]},"institutions":[{"id":"https://openalex.org/I44896327","display_name":"Mitre (United States)","ror":"https://ror.org/03ks2a131","country_code":"US","type":"company","lineage":["https://openalex.org/I44896327"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lynette Hirschman","raw_affiliation_strings":["The MITRE corporation (202 Burlington Rd., Bedford, MA - United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The MITRE corporation (202 Burlington Rd., Bedford, MA - United States)","institution_ids":["https://openalex.org/I44896327"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":"48","issue":"1","first_page":"7","last_page":"31"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.002099999925121665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language-processing-evaluation","display_name":"natural language processing evaluation","score":0.8669999837875366},{"id":"https://openalex.org/keywords/gold-standard","display_name":"gold standard","score":0.7590000033378601},{"id":"https://openalex.org/keywords/annotation-quality","display_name":"annotation quality","score":0.968999981880188},{"id":"https://openalex.org/keywords/technology-evaluation","display_name":"technology evaluation","score":0.7350000143051147},{"id":"https://openalex.org/keywords/usage-evaluation","display_name":"usage evaluation","score":0.9900000095367432},{"id":"https://openalex.org/keywords/evaluation-standards","display_name":"evaluation standards","score":0.9070000052452087}],"concepts":[{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.5147898197174072},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49162203073501587},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3953676223754883},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.17233994603157043},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.06270569562911987}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.377.378","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.377.378","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://atala.org/IMG/pdf/intro_lh_sc_pap.pdf","raw_type":"text"},{"id":"pmh:oai:HAL:hal-00502700v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-00502700","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Revue TAL : traitement automatique des langues, 2007, 48 (1), pp.7-31","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:lilloa.univ-lille.fr:20.500.12210/71867","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.12210/71867","pdf_url":null,"source":{"id":"https://openalex.org/S4306402203","display_name":"LillOA (Universit\u00e9 de Lille (University Of Lille))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210123514","host_organization_name":"Centre d'\u00c9tudes en Civilisations, Langues et Lettres \u00c9trang\u00e8res","host_organization_lineage":["https://openalex.org/I4210123514"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-00502700v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-00502700","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Revue TAL : traitement automatique des langues, 2007, 48 (1), pp.7-31","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W7591206","https://openalex.org/W12836875","https://openalex.org/W32753084","https://openalex.org/W90118760","https://openalex.org/W139048851","https://openalex.org/W147552783","https://openalex.org/W147838145","https://openalex.org/W161537933","https://openalex.org/W657583424","https://openalex.org/W1488341298","https://openalex.org/W1511332196","https://openalex.org/W1512002606","https://openalex.org/W1550436572","https://openalex.org/W1552513054","https://openalex.org/W1554394657","https://openalex.org/W1579838312","https://openalex.org/W1592736283","https://openalex.org/W1632114991","https://openalex.org/W1651372031","https://openalex.org/W1865928303","https://openalex.org/W1970298864","https://openalex.org/W1979562318","https://openalex.org/W1980460629","https://openalex.org/W1982149803","https://openalex.org/W1983405110","https://openalex.org/W2002919652","https://openalex.org/W2037563745","https://openalex.org/W2038660571","https://openalex.org/W2045474093","https://openalex.org/W2058455545","https://openalex.org/W2068882115","https://openalex.org/W2076498403","https://openalex.org/W2080887205","https://openalex.org/W2081445117","https://openalex.org/W2092339162","https://openalex.org/W2093555081","https://openalex.org/W2096145771","https://openalex.org/W2101105183","https://openalex.org/W2114924751","https://openalex.org/W2138547388","https://openalex.org/W2144845931","https://openalex.org/W2151664495","https://openalex.org/W2153166272","https://openalex.org/W2157963512","https://openalex.org/W2160643434","https://openalex.org/W2161383849","https://openalex.org/W2164181714","https://openalex.org/W2185566774","https://openalex.org/W2249819665","https://openalex.org/W2291598608","https://openalex.org/W2295407839","https://openalex.org/W2394825261","https://openalex.org/W2502247839","https://openalex.org/W2586241512","https://openalex.org/W2587282545","https://openalex.org/W2772882249","https://openalex.org/W4285719527","https://openalex.org/W6903428514"],"related_works":["https://openalex.org/W2406764702","https://openalex.org/W7062678974","https://openalex.org/W2342766632","https://openalex.org/W3155563385","https://openalex.org/W1552513054","https://openalex.org/W4243233316","https://openalex.org/W130277965","https://openalex.org/W6941302338","https://openalex.org/W1599512667","https://openalex.org/W7166836052"],"abstract_inverted_index":{"ABSTRACT.":[0],"In":[1],"this":[2,94],"special":[3,95],"issue":[4,96],"of":[5,24,31,44,53,61,64,67,69,78,82,90],"TAL,":[6],"we":[7,48],"look":[8],"at":[9],"the":[10,29,51,65,70,91],"fundamental":[11],"principles":[12],"underlying":[13],"evaluation":[14,34,74],"in":[15,93],"natural":[16,58],"language":[17,59],"processing.":[18],"We":[19],"adopt":[20],"a":[21,32,37,41,54,87],"global":[22],"point":[23],"view":[25],"that":[26],"goes":[27],"beyond":[28],"horizon":[30],"single":[33],"campaign":[35],"or":[36],"particular":[38],"protocol.":[39],"After":[40],"brief":[42],"review":[43],"history":[45],"and":[46,75,81,97],"terminology,":[47],"will":[49],"address":[50],"topic":[52],"gold":[55],"standard":[56],"for":[57],"processing,":[60],"annotation":[62],"quality,":[63],"amount":[66],"data,":[68],"difference":[71],"between":[72],"technology":[73],"usage":[76],"evaluation,":[77],"dialog":[79],"systems,":[80],"standards,":[83],"before":[84],"concluding":[85],"with":[86],"short":[88],"discussion":[89],"articles":[92,199],"some":[98],"prospective":[99],"remarks.":[100],"R\u00c9SUM\u00c9.":[101],"Dans":[102],"ce":[103],"num\u00e9ro":[104,201],"sp\u00e9cial":[105],"de":[106,127,135,155,164,169,178,183,187,192],"TAL":[107],"nous":[108,109,125,151],"int\u00e9ressons":[110],"aux":[111],"principes":[112],"fondamentaux":[113],"qui":[114],"sous-tendent":[115],"l\u2019\u00e9valuation":[116,184],"pour":[117,158],"le":[118,153,159],"traitement":[119,160],"automatique":[120],"du":[121,161,200],"langage":[122,162],"naturel,":[123,163],"que":[124],"abordons":[126],"mani\u00e8re":[128],"globale,":[129],"c\u2019est":[130],"\u00e0":[131],"dire":[132],"au":[133],"del\u00e0":[134],"l\u2019horizon":[136],"d\u2019une":[137],"seule":[138],"campagne":[139],"d\u2019\u00e9valuation":[140],"ou":[141],"d\u2019un":[142],"protocole":[143],"particulier.":[144],"Apr\u00e8s":[145],"un":[146],"rappel":[147],"historique":[148],"et":[149,180,202],"terminologique,":[150],"aborderons":[152],"sujet":[154],"la":[156,165,170],"r\u00e9f\u00e9rence":[157],"qualit\u00e9":[166],"des":[167,172,174,185,189,198],"annotations,":[168],"quantit\u00e9":[171],"donn\u00e9es,":[173],"diff\u00e9rence":[175],"entre":[176],"\u00e9valuation":[177,181],"technologie":[179],"d\u2019usage,":[182],"syst\u00e8mes":[186],"dialogue,":[188],"standards":[190],"avant":[191],"conclure":[193],"sur":[194],"une":[195],"bref":[196],"pr\u00e9sentation":[197],"quelques":[203],"remarques":[204],"prospectives.":[205]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":6},{"year":2016,"cited_by_count":1},{"year":2013,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}