{"id":"https://openalex.org/W4400031865","doi":"https://doi.org/10.1093/bioinformatics/btae420","title":"Assessing citation integrity in biomedical publications: corpus annotation and NLP models","display_name":"Assessing citation integrity in biomedical publications: corpus annotation and NLP models","publication_year":2024,"publication_date":"2024-06-26","ids":{"openalex":"https://openalex.org/W4400031865","doi":"https://doi.org/10.1093/bioinformatics/btae420","pmid":"https://pubmed.ncbi.nlm.nih.gov/38924508"},"language":"en","primary_location":{"id":"doi:10.1093/bioinformatics/btae420","is_oa":true,"landing_page_url":"https://doi.org/10.1093/bioinformatics/btae420","pdf_url":"https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btae420/58342072/btae420.pdf","source":{"id":"https://openalex.org/S52395412","display_name":"Bioinformatics","issn_l":"1367-4803","issn":["1367-4803","1367-4811"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bioinformatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btae420/58342072/btae420.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047546994","display_name":"M. Janina Sarol","orcid":"https://orcid.org/0000-0002-0991-4429"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maria Janina Sarol","raw_affiliation_strings":["Informatics Programs, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Informatics Programs, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086870297","display_name":"Shufan Ming","orcid":"https://orcid.org/0009-0009-7657-6018"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shufan Ming","raw_affiliation_strings":["School of Information Sciences, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information Sciences, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113852259","display_name":"Shruthan Radhakrishna","orcid":null},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shruthan Radhakrishna","raw_affiliation_strings":["Department of Computer Science, University of Illinois Urbana-Champaign , Champaign, IL 61801, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana-Champaign , Champaign, IL 61801, United States","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001716299","display_name":"Jodi Schneider","orcid":"https://orcid.org/0000-0002-5098-5667"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jodi Schneider","raw_affiliation_strings":["School of Information Sciences, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States"],"raw_orcid":"https://orcid.org/0000-0002-5098-5667","affiliations":[{"raw_affiliation_string":"School of Information Sciences, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016571803","display_name":"Halil Kilicoglu","orcid":"https://orcid.org/0000-0003-3987-9393"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Halil Kilicoglu","raw_affiliation_strings":["School of Information Sciences, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States"],"raw_orcid":"https://orcid.org/0000-0003-3987-9393","affiliations":[{"raw_affiliation_string":"School of Information Sciences, University of Illinois Urbana-Champaign , Champaign, IL 61820, United States","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5016571803"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":{"value":3618,"currency":"USD","value_usd":3618},"apc_paid":{"value":3618,"currency":"USD","value_usd":3618},"fwci":3.864,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.94092846,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"40","issue":"7","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.3953999876976013,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11492","display_name":"Academic integrity and plagiarism","score":0.3953999876976013,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13607","display_name":"Academic Publishing and Open Access","score":0.11620000004768372,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.10899999737739563,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8698723316192627},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.744022011756897},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.7030376195907593},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.6365131139755249},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6277279257774353},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4281124472618103},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11489254236221313}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8698723316192627},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.744022011756897},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.7030376195907593},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.6365131139755249},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6277279257774353},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4281124472618103},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11489254236221313}],"mesh":[{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009323","descriptor_name":"Natural Language Processing","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D011642","descriptor_name":"Publications","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011642","descriptor_name":"Publications","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011642","descriptor_name":"Publications","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011642","descriptor_name":"Publications","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011642","descriptor_name":"Publications","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011642","descriptor_name":"Publications","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D035843","descriptor_name":"Biomedical Research","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":3,"locations":[{"id":"doi:10.1093/bioinformatics/btae420","is_oa":true,"landing_page_url":"https://doi.org/10.1093/bioinformatics/btae420","pdf_url":"https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btae420/58342072/btae420.pdf","source":{"id":"https://openalex.org/S52395412","display_name":"Bioinformatics","issn_l":"1367-4803","issn":["1367-4803","1367-4811"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bioinformatics","raw_type":"journal-article"},{"id":"pmid:38924508","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38924508","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bioinformatics (Oxford, England)","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:11231046","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/11231046","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC11231046/pdf/btae420.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bioinformatics","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1093/bioinformatics/btae420","is_oa":true,"landing_page_url":"https://doi.org/10.1093/bioinformatics/btae420","pdf_url":"https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btae420/58342072/btae420.pdf","source":{"id":"https://openalex.org/S52395412","display_name":"Bioinformatics","issn_l":"1367-4803","issn":["1367-4803","1367-4811"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bioinformatics","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8299999833106995,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2943613363","display_name":"Natural Language Processing to Assess and Improve Citation Integrity in Biomedical Publications","funder_award_id":"ORIIR220073","funder_id":"https://openalex.org/F4320306085","funder_display_name":"U.S. Department of Health and Human Services"}],"funders":[{"id":"https://openalex.org/F4320306085","display_name":"U.S. Department of Health and Human Services","ror":"https://ror.org/033jnv181"},{"id":"https://openalex.org/F4320332407","display_name":"Office of the Assistant Secretary for Health","ror":"https://ror.org/023n3xx64"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4400031865.pdf"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W141719311","https://openalex.org/W767067438","https://openalex.org/W1541229271","https://openalex.org/W1799284535","https://openalex.org/W1847240671","https://openalex.org/W2009777888","https://openalex.org/W2014957631","https://openalex.org/W2019753053","https://openalex.org/W2023155726","https://openalex.org/W2069751085","https://openalex.org/W2097350739","https://openalex.org/W2112239114","https://openalex.org/W2119858271","https://openalex.org/W2136213468","https://openalex.org/W2316884669","https://openalex.org/W2343758449","https://openalex.org/W2406561316","https://openalex.org/W2618923223","https://openalex.org/W2774860949","https://openalex.org/W2913959954","https://openalex.org/W2952174148","https://openalex.org/W2975166839","https://openalex.org/W2985520515","https://openalex.org/W3046375318","https://openalex.org/W3092867125","https://openalex.org/W3131199412","https://openalex.org/W3173450924","https://openalex.org/W3189135819","https://openalex.org/W4200394820","https://openalex.org/W4221143046","https://openalex.org/W4252076394","https://openalex.org/W4292779060","https://openalex.org/W4367318167","https://openalex.org/W6601369466","https://openalex.org/W6677922754","https://openalex.org/W6733971936","https://openalex.org/W6778883912","https://openalex.org/W6809646742"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W1533177136","https://openalex.org/W4380994516","https://openalex.org/W3204019825"],"abstract_inverted_index":{"MOTIVATION:":[0],"Citations":[1],"have":[2],"a":[3,51,120,124,169],"fundamental":[4],"role":[5],"in":[6,32,64,87,94,188],"scholarly":[7],"communication":[8],"and":[9,13,41,53,77,98,168,179,208,231,240],"assessment.":[10],"Citation":[11,202],"accuracy":[12,100,114,174],"transparency":[14],"is":[15,210],"crucial":[16],"for":[17,47,164,173,195,213],"the":[18,38,88,95,99,102,158,223,238,241],"integrity":[19],"of":[20,101,106],"scientific":[21,39],"evidence.":[22],"In":[23],"this":[24],"work,":[25],"we":[26,118],"focus":[27],"on":[28],"quotation":[29,203],"errors,":[30],"errors":[31,63,204],"citation":[33,85,108,153,156,229],"content":[34],"that":[35,42],"can":[36],"distort":[37],"evidence":[40,92],"are":[43,205],"hard":[44],"to":[45,60,79,129,216,227],"detect":[46],"humans.":[48],"We":[49,68,137,236],"construct":[50],"corpus":[52,239],"propose":[54],"natural":[55],"language":[56,146],"processing":[57],"(NLP)":[58],"methods":[59],"identify":[61,217],"such":[62],"biomedical":[65,73],"publications.":[66],"RESULTS:":[67],"manually":[69],"annotated":[70,111],"100":[71],"highly-cited":[72],"publications":[74],"(reference":[75],"articles)":[76],"citations":[78,131,197],"them.":[80],"The":[81,148],"annotation":[82],"involved":[83],"labeling":[84],"context":[86],"citing":[89],"article,":[90,97],"relevant":[91],"sentences":[93,154],"reference":[96],"citation.":[103],"A":[104],"total":[105],"3063":[107],"instances":[109],"were":[110],"(39.18%":[112],"with":[113,123,143,161],"errors).":[115],"For":[116],"NLP,":[117],"combined":[119],"sentence":[121],"retriever":[122],"fine-tuned":[125,170],"claim":[126],"verification":[127],"model":[128,160,172,244],"label":[130,175],"as":[132,155],"ACCURATE,":[133],"NOT_ACCURATE,":[134],"or":[135],"IRRELEVANT.":[136],"also":[138],"explored":[139],"few-shot":[140],"in-context":[141,184],"learning":[142,185],"generative":[144],"large":[145],"models.":[147],"best":[149],"performing":[150],"model-which":[151],"uses":[152],"context,":[157],"BM25":[159],"MonoT5":[162],"reranker":[163],"retrieving":[165],"top-20":[166],"sentences,":[167],"MultiVerS":[171],"classification-yielded":[176],"0.59":[177],"micro-F1":[178],"0.52":[180],"macro-F1":[181],"score.":[182],"GPT-4":[183],"performed":[186],"better":[187],"identifying":[189],"accurate":[190],"citations,":[191],"but":[192],"it":[193,209],"lagged":[194],"erroneous":[196,218],"(0.65":[198],"micro-F1,":[199],"0.45":[200],"macro-F1).":[201],"often":[206],"subtle,":[207],"currently":[211],"challenging":[212],"NLP":[214,243],"models":[215,224],"citations.":[219],"With":[220],"further":[221],"improvements,":[222],"could":[225],"serve":[226],"improve":[228],"quality":[230],"accuracy.":[232],"AVAILABILITY":[233],"AND":[234],"IMPLEMENTATION:":[235],"make":[237],"best-performing":[242],"publicly":[245],"available":[246],"at":[247],"https://github.com/ScienceNLP-Lab/Citation-Integrity/.":[248]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-25T15:57:00.446498","created_date":"2025-10-10T00:00:00"}