{"id":"https://openalex.org/W3164499433","doi":"https://doi.org/10.3233/shti210132","title":"Inter-Rater Reliability of Unstructured Text Labeling: Artificially vs. Naturally Intelligent Approaches","display_name":"Inter-Rater Reliability of Unstructured Text Labeling: Artificially vs. Naturally Intelligent Approaches","publication_year":2021,"publication_date":"2021-05-27","ids":{"openalex":"https://openalex.org/W3164499433","doi":"https://doi.org/10.3233/shti210132","mag":"3164499433","pmid":"https://pubmed.ncbi.nlm.nih.gov/34042717"},"language":"en","primary_location":{"id":"doi:10.3233/shti210132","is_oa":true,"landing_page_url":"https://doi.org/10.3233/shti210132","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/SHTI210132","source":{"id":"https://openalex.org/S4210179765","display_name":"Studies in health technology and informatics","issn_l":"0926-9630","issn":["0926-9630","1879-8365"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in Health Technology and Informatics","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/SHTI210132","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034654432","display_name":"Gleb Danilov","orcid":"https://orcid.org/0000-0003-1442-5993"},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"Gleb Danilov","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042539278","display_name":"Alexandra Kosyrkova","orcid":"https://orcid.org/0000-0002-3019-5203"},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Alexandra Kosyrkova","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088460022","display_name":"Maria Shults","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Maria Shults","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076026952","display_name":"Semen Melchenko","orcid":"https://orcid.org/0000-0001-7060-0667"},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Semen Melchenko","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034798985","display_name":"Tatyana Tsukanova","orcid":"https://orcid.org/0000-0002-7596-3720"},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Tatyana Tsukanova","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085342552","display_name":"Michael Shifrin","orcid":"https://orcid.org/0000-0002-9606-5559"},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Michael Shifrin","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067820015","display_name":"Potapov Aa","orcid":"https://orcid.org/0000-0001-8343-3511"},"institutions":[{"id":"https://openalex.org/I4210122729","display_name":"Burdenko Neurosurgery Institute","ror":"https://ror.org/02t1t6n83","country_code":"RU","type":"healthcare","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210110862","https://openalex.org/I4210122729"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Alexander Potapov","raw_affiliation_strings":["Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation"],"affiliations":[{"raw_affiliation_string":"Laboratory of Biomedical Informatics and Artificial Intelligence, National Medical Research Center for Neurosurgery named after N.N. Burdenko, Moscow, Russian Federation","institution_ids":["https://openalex.org/I4210122729"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5034654432"],"corresponding_institution_ids":["https://openalex.org/I4210122729"],"apc_list":null,"apc_paid":null,"fwci":3.0758,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.92606791,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"281","issue":null,"first_page":"118","last_page":"122"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9858999848365784,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9858999848365784,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9629999995231628,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14374","display_name":"Statistical Methods in Epidemiology","score":0.9587000012397766,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6854802370071411},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6763418912887573},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.634341835975647},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5334988236427307},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5250603556632996},{"id":"https://openalex.org/keywords/unstructured-data","display_name":"Unstructured data","score":0.5091573596000671},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5046533346176147},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.44118812680244446},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2610611021518707},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.24873942136764526},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08269524574279785}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6854802370071411},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6763418912887573},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.634341835975647},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5334988236427307},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5250603556632996},{"id":"https://openalex.org/C2781252014","wikidata":"https://www.wikidata.org/wiki/Q1141900","display_name":"Unstructured data","level":3,"score":0.5091573596000671},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5046533346176147},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.44118812680244446},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2610611021518707},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.24873942136764526},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08269524574279785},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000069550","descriptor_name":"Machine Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012189","descriptor_name":"Retrospective Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012189","descriptor_name":"Retrospective Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012189","descriptor_name":"Retrospective Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057286","descriptor_name":"Electronic Health Records","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.3233/shti210132","is_oa":true,"landing_page_url":"https://doi.org/10.3233/shti210132","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/SHTI210132","source":{"id":"https://openalex.org/S4210179765","display_name":"Studies in health technology and informatics","issn_l":"0926-9630","issn":["0926-9630","1879-8365"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in Health Technology and Informatics","raw_type":"book-chapter"},{"id":"pmid:34042717","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/34042717","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in health technology and informatics","raw_type":null}],"best_oa_location":{"id":"doi:10.3233/shti210132","is_oa":true,"landing_page_url":"https://doi.org/10.3233/shti210132","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/SHTI210132","source":{"id":"https://openalex.org/S4210179765","display_name":"Studies in health technology and informatics","issn_l":"0926-9630","issn":["0926-9630","1879-8365"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318577","host_organization_name":"IOS Press","host_organization_lineage":["https://openalex.org/P4310318577"],"host_organization_lineage_names":["IOS Press"],"type":"book series"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Studies in Health Technology and Informatics","raw_type":"book-chapter"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.800000011920929}],"awards":[{"id":"https://openalex.org/G2934504215","display_name":null,"funder_award_id":"18-29-","funder_id":"https://openalex.org/F4320321079","funder_display_name":"Russian Foundation for Basic Research"}],"funders":[{"id":"https://openalex.org/F4320321079","display_name":"Russian Foundation for Basic Research","ror":"https://ror.org/02mh1ke95"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3164499433.pdf","grobid_xml":"https://content.openalex.org/works/W3164499433.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W1989492805","https://openalex.org/W2135046866","https://openalex.org/W2775486588","https://openalex.org/W2990724214","https://openalex.org/W3015443789","https://openalex.org/W3040684673"],"related_works":["https://openalex.org/W3184725726","https://openalex.org/W3203889067","https://openalex.org/W2378793138","https://openalex.org/W2759357633","https://openalex.org/W3169533859","https://openalex.org/W2761062406","https://openalex.org/W2594011304","https://openalex.org/W2486684331","https://openalex.org/W1541499806","https://openalex.org/W3169168948"],"abstract_inverted_index":{"Unstructured":[0],"medical":[1,24],"text":[2,120,144],"labeling":[3,121,145],"technologies":[4],"are":[5,133],"expected":[6],"to":[7,29,62],"be":[8,113,124,138],"highly":[9],"demanded":[10],"since":[11],"the":[12,23,31,38,55,58,88,95,147,160],"interest":[13],"in":[14,22,44,107],"artificial":[15],"intelligence":[16],"and":[17,53,130],"natural":[18],"language":[19],"processing":[20],"arises":[21],"domain.":[25],"Our":[26],"study":[27],"aimed":[28],"assess":[30,54],"agreement":[32,70],"between":[33,71,115],"experts":[34],"who":[35],"judged":[36],"on":[37,49,75],"fact":[39],"of":[40,57,149],"pulmonary":[41],"embolism":[42],"(PE)":[43],"neurosurgical":[45],"cases":[46,109],"retrospectively":[47],"based":[48],"electronic":[50],"health":[51],"records":[52],"utility":[56],"machine":[59,96],"learning":[60,97,136],"approach":[61],"automate":[63],"this":[64],"process.":[65],"We":[66],"observed":[67],"a":[68,139],"moderate":[69],"3":[72,116],"independent":[73,117],"raters":[74],"PE":[76],"detection":[77],"(Light's":[78],"kappa":[79],"=":[80,83,100,104],"0.568,":[81],"p":[82],"0).":[84],"Labeling":[85],"sentences":[86],"with":[87],"method":[89],"we":[90],"proposed":[91],"earlier":[92],"might":[93,123,137],"improve":[94],"results":[98],"(accuracy":[99],"0.97,":[101],"ROC":[102],"AUC":[103],"0.98)":[105],"even":[106],"those":[108],"that":[110],"could":[111],"not":[112],"agreed":[114],"raters.":[118],"Medical":[119],"techniques":[122],"more":[125],"efficient":[126],"when":[127,146],"strict":[128],"rules":[129],"semi-automated":[131],"approaches":[132],"implemented.":[134],"Machine":[135],"good":[140],"option":[141],"for":[142],"unstructured":[143],"reliability":[148],"textual":[150],"data":[151],"is":[152],"properly":[153],"addressed.":[154],"This":[155],"project":[156],"was":[157],"supported":[158],"by":[159],"RFBR":[161],"grant":[162],"18-29-22085.":[163]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}