{"id":"https://openalex.org/W646753399","doi":"https://doi.org/10.63317/2gnpsbdnmnos","title":"Analyzing the Impact of Prevalence on the Evaluation of a Manual Annotation Campaign","display_name":"Analyzing the Impact of Prevalence on the Evaluation of a Manual Annotation Campaign","publication_year":2012,"publication_date":"2012-05-01","ids":{"openalex":"https://openalex.org/W646753399","doi":"https://doi.org/10.63317/2gnpsbdnmnos","mag":"646753399"},"language":"en","primary_location":{"id":"doi:10.63317/2gnpsbdnmnos","doi":"https://doi.org/10.63317/2gnpsbdnmnos","is_oa":true,"landing_page_url":"https://doi.org/10.63317/2gnpsbdnmnos","pdf_url":"http://www.lrec-conf.org/proceedings/lrec2012/pdf/549_Paper.pdf","source":{"id":"https://openalex.org/S7407088301","display_name":"Language Resources and Evaluation Conference (LREC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"http://www.lrec-conf.org/proceedings/lrec2012/pdf/549_Paper.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074102447","display_name":"Kar\u00ebn Fort","orcid":"https://orcid.org/0000-0002-0723-8850","observed_orcids":["https://orcid.org/0000-0002-0723-8850"]},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I1295345811","display_name":"Institut de l'Information Scientifique et Technique","ror":"https://ror.org/02mn0vt57","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1295345811","https://openalex.org/I4210140053"]},{"id":"https://openalex.org/I4210091279","display_name":"Universit\u00e9 Sorbonne Paris Nord","ror":"https://ror.org/0199hds37","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210091279"]},{"id":"https://openalex.org/I4210156583","display_name":"Laboratoire d'Informatique de Paris-Nord","ror":"https://ror.org/05g1zjw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583","https://openalex.org/I4210159245"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Kar\u00ebn Fort","raw_affiliation_strings":["INIST -CNRS 2 alle de Brabois , 54500 Vandoeuvre-ls-Nancy , France","LIPN - Laboratoire d'Informatique de Paris-Nord (Institut Galil\u00e9e, Universit\u00e9 Paris 13, 99 avenue Jean-Baptiste Cl\u00e9ment, F-93430, Villetaneuse - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"INIST -CNRS 2 alle de Brabois , 54500 Vandoeuvre-ls-Nancy , France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I1295345811"]},{"raw_affiliation_string":"LIPN - Laboratoire d'Informatique de Paris-Nord (Institut Galil\u00e9e, Universit\u00e9 Paris 13, 99 avenue Jean-Baptiste Cl\u00e9ment, F-93430, Villetaneuse - France)","institution_ids":["https://openalex.org/I4210091279","https://openalex.org/I4210156583"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104075145","display_name":"Claire Fran\u00e7ois","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1295345811","display_name":"Institut de l'Information Scientifique et Technique","ror":"https://ror.org/02mn0vt57","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1295345811","https://openalex.org/I4210140053"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Claire Fran\u00e7ois","raw_affiliation_strings":["INIST - Institut de l'information scientifique et technique (2 rue Jean Zay CS 10310 F-54519 Vandoeuvre-l\u00e8s-Nancy - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"INIST - Institut de l'information scientifique et technique (2 rue Jean Zay CS 10310 F-54519 Vandoeuvre-l\u00e8s-Nancy - France)","institution_ids":["https://openalex.org/I1295345811"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036405594","display_name":"Olivier Galibert","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I124158823","display_name":"Conservatoire National des Arts et M\u00e9tiers","ror":"https://ror.org/0175hh227","country_code":"FR","type":"education","lineage":["https://openalex.org/I124158823","https://openalex.org/I4210134562"]},{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I4210091279","display_name":"Universit\u00e9 Sorbonne Paris Nord","ror":"https://ror.org/0199hds37","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210091279"]},{"id":"https://openalex.org/I4210107682","display_name":"Laboratoire National de M\u00e9trologie et d'Essais","ror":"https://ror.org/01ph39d13","country_code":"FR","type":"other","lineage":["https://openalex.org/I4210107682"]},{"id":"https://openalex.org/I4210156583","display_name":"Laboratoire d'Informatique de Paris-Nord","ror":"https://ror.org/05g1zjw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583","https://openalex.org/I4210159245"]},{"id":"https://openalex.org/I4405258865","display_name":"Laboratoire commun de m\u00e9trologie","ror":"https://ror.org/01atqpz53","country_code":"FR","type":"facility","lineage":["https://openalex.org/I124158823","https://openalex.org/I4210107682","https://openalex.org/I4210134562","https://openalex.org/I4405258865"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Olivier Galibert","raw_affiliation_strings":["LIPN , Universit\u00e9 Paris 13 & CNRS 99 av. J.B. Cl\u00e9ment , 93430 Villetaneuse , France","LNE 29 avenue Roger Hennequin , 78190 Trappes , France","LCM - Laboratoire commun de m\u00e9trologie LNE-CNAM (CNAM Case courrier 2D7P30 rue du Landy\r\n93210 LA PLAINE SAINT DENIS - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LIPN , Universit\u00e9 Paris 13 & CNRS 99 av. J.B. Cl\u00e9ment , 93430 Villetaneuse , France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583"]},{"raw_affiliation_string":"LNE 29 avenue Roger Hennequin , 78190 Trappes , France","institution_ids":["https://openalex.org/I4210107682"]},{"raw_affiliation_string":"LCM - Laboratoire commun de m\u00e9trologie LNE-CNAM (CNAM Case courrier 2D7P30 rue du Landy\r\n93210 LA PLAINE SAINT DENIS - France)","institution_ids":["https://openalex.org/I124158823","https://openalex.org/I4405258865"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004623395","display_name":"Maha Ghribi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I1295345811","display_name":"Institut de l'Information Scientifique et Technique","ror":"https://ror.org/02mn0vt57","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1295345811","https://openalex.org/I4210140053"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Maha Ghribi","raw_affiliation_strings":["INIST -CNRS 2 alle de Brabois , 54500 Vandoeuvre-ls-Nancy , France","INIST - Institut de l'information scientifique et technique (2 rue Jean Zay CS 10310 F-54519 Vandoeuvre-l\u00e8s-Nancy - France)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"INIST -CNRS 2 alle de Brabois , 54500 Vandoeuvre-ls-Nancy , France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I1295345811"]},{"raw_affiliation_string":"INIST - Institut de l'information scientifique et technique (2 rue Jean Zay CS 10310 F-54519 Vandoeuvre-l\u00e8s-Nancy - France)","institution_ids":["https://openalex.org/I1295345811"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.4478,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.97393048,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1474","last_page":"1480"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.5508000254631042,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.5508000254631042,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.3668999969959259,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10015","display_name":"Genomics and Phylogenetic Studies","score":0.00930000003427267,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-annotator-agreement","display_name":"inter-annotator agreement","score":0.8040000200271606},{"id":"https://openalex.org/keywords/prevalence-bias","display_name":"prevalence bias","score":0.7120000123977661},{"id":"https://openalex.org/keywords/scientific-abstracts","display_name":"scientific abstracts","score":0.9240000247955322},{"id":"https://openalex.org/keywords/sparse-annotations","display_name":"sparse annotations","score":0.6940000057220459},{"id":"https://openalex.org/keywords/contingency-tables","display_name":"contingency tables","score":0.6079999804496765},{"id":"https://openalex.org/keywords/weighted-kappa","display_name":"weighted kappa","score":0.5720000267028809},{"id":"https://openalex.org/keywords/f-measure","display_name":"F-measure","score":0.7009999752044678},{"id":"https://openalex.org/keywords/annotation-evaluation","display_name":"annotation evaluation","score":0.3089999854564667}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8782036304473877},{"id":"https://openalex.org/C91998498","wikidata":"https://www.wikidata.org/wiki/Q1541178","display_name":"Contingency table","level":2,"score":0.8164442777633667},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7366827726364136},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.6963919997215271},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5840112566947937},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5448295474052429},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5225631594657898},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.45484575629234314},{"id":"https://openalex.org/C97508593","wikidata":"https://www.wikidata.org/wiki/Q1359069","display_name":"Contingency","level":2,"score":0.4421299993991852},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.391010046005249},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2610677480697632},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.25097012519836426},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.63317/2gnpsbdnmnos","doi":"https://doi.org/10.63317/2gnpsbdnmnos","is_oa":true,"landing_page_url":"https://doi.org/10.63317/2gnpsbdnmnos","pdf_url":"http://www.lrec-conf.org/proceedings/lrec2012/pdf/549_Paper.pdf","source":{"id":"https://openalex.org/S7407088301","display_name":"Language Resources and Evaluation Conference (LREC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},{"id":"pmh:oai:HAL:hal-00709174v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-00709174","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.lrec-conf.org/proceedings/lrec2012/pdf/549_Paper.pdf","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.63317/2gnpsbdnmnos","doi":"https://doi.org/10.63317/2gnpsbdnmnos","is_oa":true,"landing_page_url":"https://doi.org/10.63317/2gnpsbdnmnos","pdf_url":"http://www.lrec-conf.org/proceedings/lrec2012/pdf/549_Paper.pdf","source":{"id":"https://openalex.org/S7407088301","display_name":"Language Resources and Evaluation Conference (LREC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Language Resources and Evaluation Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322892","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73"},{"id":"https://openalex.org/F4320323910","display_name":"Institut National de la Recherche Agronomique","ror":"https://ror.org/01x3gbx83"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W646753399.pdf","grobid_xml":"https://content.openalex.org/works/W646753399.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W21503661","https://openalex.org/W25733797","https://openalex.org/W79139011","https://openalex.org/W1747645360","https://openalex.org/W2002664886","https://openalex.org/W2007963708","https://openalex.org/W2037789405","https://openalex.org/W2040447587","https://openalex.org/W2053154970","https://openalex.org/W2060215531","https://openalex.org/W2091950237","https://openalex.org/W2092269560","https://openalex.org/W2133943399","https://openalex.org/W2141766660","https://openalex.org/W2153072998","https://openalex.org/W2153222072","https://openalex.org/W2153804780","https://openalex.org/W2344663006","https://openalex.org/W2607354300"],"related_works":["https://openalex.org/W7096986362","https://openalex.org/W2621793207","https://openalex.org/W2295892333","https://openalex.org/W2156418292","https://openalex.org/W7160161636","https://openalex.org/W2251089262","https://openalex.org/W2187888730","https://openalex.org/W4399985381","https://openalex.org/W7220952498","https://openalex.org/W2025672980"],"abstract_inverted_index":{"This":[0],"article":[1],"details":[2],"work":[3],"aiming":[4],"at":[5],"evaluating":[6],"the":[7,10,29,33,45,49,58,61,75,82,84,96,105],"quality":[8],"of":[9,13],"manual":[11],"annotation":[12],"gene":[14],"renaming":[15],"couples":[16],"in":[17],"scientific":[18],"abstracts,":[19],"which":[20,65],"generates":[21],"sparse":[22],"annotations.To":[23],"evaluate":[24],"these":[25],"annotations,":[26],"we":[27],"compare":[28],"results":[30,97],"obtained":[31],"using":[32],"commonly":[34],"advocated":[35],"inter-annotator":[36],"agreement":[37],"coefficients":[38,51],"such":[39],"as":[40,55,57],"S,":[41],"\u03ba":[42],"and":[43,53,60],"\u03c0,":[44],"less":[46],"known":[47],"R,":[48],"weighted":[50],"\u03ba\u03c9":[52],"\u03b1":[54],"well":[56],"F-measure":[59],"SER.We":[62],"analyze":[63],"to":[64,94],"extent":[66],"they":[67],"are":[68],"relevant":[69],"for":[70],"our":[71],"data.We":[72],"then":[73],"study":[74],"bias":[76],"introduced":[77],"by":[78,80,98],"prevalence":[79],"changing":[81],"way":[83,93],"contingency":[85],"table":[86],"is":[87],"built.We":[88],"finally":[89],"propose":[90],"an":[91],"original":[92],"synthesize":[95],"computing":[99],"distances":[100],"between":[101],"categories,":[102],"based":[103],"on":[104],"produced":[106],"annotations.":[107]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":1},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":3}],"updated_date":"2026-10-25T09:29:13.331239","created_date":"2016-06-24T00:00:00"}