{"id":"https://openalex.org/W4416534882","doi":"https://doi.org/10.1016/j.infsof.2026.108204","title":"LLM4SCREENLIT: Recommendations on assessing the performance of large language models for screening literature in systematic reviews","display_name":"LLM4SCREENLIT: Recommendations on assessing the performance of large language models for screening literature in systematic reviews","publication_year":2026,"publication_date":"2026-06-08","ids":{"openalex":"https://openalex.org/W4416534882","doi":"https://doi.org/10.1016/j.infsof.2026.108204"},"language":"en","primary_location":{"id":"doi:10.1016/j.infsof.2026.108204","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108204","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.infsof.2026.108204","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081451369","display_name":"Lech Madeyski","orcid":"https://orcid.org/0000-0003-3907-3357"},"institutions":[{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]},{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":true,"raw_author_name":"Lech Madeyski","raw_affiliation_strings":["Wroclaw University of Science and Technology, Wyb. Wyspianskiego 27, Wroclaw, 50-370, Poland"],"raw_orcid":"https://orcid.org/0000-0003-3907-3357","affiliations":[{"raw_affiliation_string":"Wroclaw University of Science and Technology, Wyb. Wyspianskiego 27, Wroclaw, 50-370, Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012102325","display_name":"Barbara Kitchenham","orcid":"https://orcid.org/0000-0002-6134-8460"},"institutions":[{"id":"https://openalex.org/I56007636","display_name":"Keele University","ror":"https://ror.org/00340yn33","country_code":"GB","type":"education","lineage":["https://openalex.org/I56007636"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Barbara Kitchenham","raw_affiliation_strings":["Keele University, UK"],"raw_orcid":"https://orcid.org/0000-0002-6134-8460","affiliations":[{"raw_affiliation_string":"Keele University, UK","institution_ids":["https://openalex.org/I56007636"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027684680","display_name":"Martin Shepperd","orcid":"https://orcid.org/0000-0003-1874-6145"},"institutions":[{"id":"https://openalex.org/I124357947","display_name":"University of London","ror":"https://ror.org/04cw6st05","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947"]},{"id":"https://openalex.org/I59433898","display_name":"Brunel University of London","ror":"https://ror.org/00dn4t376","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I59433898"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Martin Shepperd","raw_affiliation_strings":["Brunel University of London, UK"],"raw_orcid":"https://orcid.org/0000-0003-1874-6145","affiliations":[{"raw_affiliation_string":"Brunel University of London, UK","institution_ids":["https://openalex.org/I59433898","https://openalex.org/I124357947"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5081451369"],"corresponding_institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"],"apc_list":{"value":3350,"currency":"USD","value_usd":3350},"apc_paid":{"value":3350,"currency":"USD","value_usd":3350},"fwci":4.9737,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.9024478,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":"198","issue":null,"first_page":"108204","last_page":"108204"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.5893999934196472,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.5893999934196472,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.09539999812841415,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.05220000073313713,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.7566999793052673},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.5252000093460083},{"id":"https://openalex.org/keywords/confusion","display_name":"Confusion","score":0.4973999857902527},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.4964999854564667},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.4936000108718872},{"id":"https://openalex.org/keywords/false-positive-paradox","display_name":"False positive paradox","score":0.4447000026702881},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.3889000117778778},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.3264999985694885}],"concepts":[{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.7566999793052673},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5666999816894531},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.5252000093460083},{"id":"https://openalex.org/C2781140086","wikidata":"https://www.wikidata.org/wiki/Q557945","display_name":"Confusion","level":2,"score":0.4973999857902527},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.4964999854564667},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.4936000108718872},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4555000066757202},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.4447000026702881},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.39500001072883606},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.39480000734329224},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3889000117778778},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.3264999985694885},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3068000078201294},{"id":"https://openalex.org/C138602881","wikidata":"https://www.wikidata.org/wiki/Q2709591","display_name":"Confusion matrix","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C2777083192","wikidata":"https://www.wikidata.org/wiki/Q1814648","display_name":"Plain language","level":2,"score":0.2815000116825104},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26820001006126404},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25270000100135803},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1016/j.infsof.2026.108204","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108204","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2511.12635","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2511.12635","pdf_url":"https://arxiv.org/pdf/2511.12635","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:bura.brunel.ac.uk:2438/33549","is_oa":true,"landing_page_url":"http://bura.brunel.ac.uk/handle/2438/33549","pdf_url":null,"source":{"id":"https://openalex.org/S4306401473","display_name":"Brunel University Research Archive (BURA) (Brunel University London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I59433898","host_organization_name":"Brunel University of London","host_organization_lineage":["https://openalex.org/I59433898"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},{"id":"doi:10.48550/arxiv.2511.12635","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2511.12635","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1016/j.infsof.2026.108204","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.infsof.2026.108204","pdf_url":null,"source":{"id":"https://openalex.org/S205010575","display_name":"Information and Software Technology","issn_l":"0950-5849","issn":["0950-5849","1873-6025"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information and Software Technology","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Context:":[0],"Large":[1],"language":[2],"models":[3],"(LLMs)":[4],"are":[5,280],"increasingly":[6],"used":[7,20],"to":[8,21,274,292],"screen":[9],"literature":[10],"for":[11,40,46,60,245],"systematic":[12],"reviews":[13],"(SRs),":[14],"but":[15,204,300],"the":[16,27,127,141,157,167,176,185,191,201,205,250,271],"standard":[17],"confusion-matrix":[18],"metrics":[19],"evaluate":[22],"them":[23],"can":[24],"mislead":[25],"under":[26],"imbalanced,":[28],"cost-asymmetric":[29],"conditions":[30],"of":[31,156,181,196],"screening.":[32],"Objective:":[33],"We":[34,91],"develop":[35],"and":[36,45,48,86,107,154,172,239,254,278,283,295],"justify":[37],"LLM4SCREENLIT\u2014practical":[38],"recommendations":[39],"researchers":[41],"conducting":[42],"LLM-screening":[43],"evaluations":[44,234],"editors":[47],"reviewers":[49,284],"assessing":[50],"such":[51],"studies\u2014differentiated":[52],"by":[53],"study":[54,272],"type":[55],"(retrospective":[56],"benchmarking":[57],"vs.":[58],"deployment":[59],"a":[61,78,93,229],"specific":[62],"SR).":[63],"Method:":[64],"Using":[65],"Delgado-Chaves":[66],"et":[67,116,120,124],"al.":[68,117,121,125],"(2025),":[69],"an":[70],"18-LLM":[71],"benchmark":[72],"across":[73],"three":[74,111],"biomedical":[75],"SRs,":[76],"as":[77,228,258,289],"motivating":[79],"example,":[80],"we":[81],"reviewed":[82],"28":[83],"additional":[84],"papers":[85,159],"extracted":[87],"their":[88],"reported":[89,146,150],"metrics.":[90],"propose":[92],"Weighted":[94],"Matthews":[95],"Correlation":[96],"Coefficient":[97],"(WMCC)":[98],"that":[99],"integrates":[100],"MCC\u2019s":[101],"chance-correction":[102],"with":[103,267],"asymmetric":[104],"misclassification":[105],"costs,":[106],"validated":[108],"it":[109],"on":[110,175],"software-engineering":[112],"(SE)":[113],"reanalyses":[114],"(Felizardo":[115],"2024;":[118,122],"Syriani":[119],"Huotala":[123],"2025),":[126],"largest":[128,168],"covering":[129],"9":[130],"LLMs":[131],"\u00d7":[132],"24":[133],"SE":[134,169,189],"secondary":[135],"studies":[136],"(34,528":[137],"articles).":[138],"Results:":[139],"Across":[140],"29":[142],"papers,":[143],"only":[144,148,207],"10%":[145],"MCC,":[147],"24%":[149],"full":[151,251],"confusion":[152,252],"matrices,":[153],"none":[155],"five":[158],"claiming":[160],"workload":[161],"savings":[162],"priced":[163],"false-negative":[164],"cost.":[165],"In":[166],"reanalysis,":[170],"MCC":[171,244],"WMCC":[173,242],"disagree":[174],"best":[177],"LLM":[178,193],"in":[179,184],"55%":[180],"evaluable":[182],"studies;":[183],"most":[186],"striking":[187],"9695-article":[188],"study,":[190],"Accuracy-best":[192],"loses":[194],"63.3%":[195],"relevant":[197],"evidence":[198],"(Lost":[199],"Evidence),":[200],"MCC-best":[202],"43.9%,":[203],"WMCC-best":[206],"5.8%.":[208],"Sensitivity":[209],"analysis":[210],"(median":[211],"crossover":[212],"at":[213],"w":[214,225],"\u2248":[215],"2":[216],".":[217],"7":[218,222],",":[219],"all":[220],"<":[221],")":[223],"supports":[224],"=":[226],"10":[227],"conservative":[230],"default.":[231],"Conclusions:":[232],"SR-screening":[233],"should":[235,264,285],"prioritize":[236],"Lost":[237],"Evidence":[238],"use":[240],"cost-sensitive":[241],"alongside":[243],"ranking.":[246],"Reporting":[247],"must":[248],"include":[249],"matrix":[253],"treat":[255],"unclassifiable":[256],"outputs":[257],"positives":[259],"requiring":[260],"human":[261],"review.":[262],"Designs":[263],"be":[265],"leakage-aware,":[266],"non-LLM":[268],"baselines":[269],"when":[270],"aims":[273],"inform":[275],"SR":[276],"practice":[277],"labels":[279],"available.":[281],"Editors":[282],"require":[286],"these":[287],"elements":[288],"routine.":[290],"Extension":[291],"full-text":[293],"screening":[294],"data":[296],"extraction":[297],"is":[298],"principled":[299],"pending":[301],"empirical":[302],"validation.":[303]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-12T07:33:03.550515","created_date":"2025-11-19T00:00:00"}