{"id":"https://openalex.org/W4399668246","doi":"https://doi.org/10.1145/3661167.3661172","title":"The Promise and Challenges of Using LLMs to Accelerate the Screening Process of Systematic Reviews","display_name":"The Promise and Challenges of Using LLMs to Accelerate the Screening Process of Systematic Reviews","publication_year":2024,"publication_date":"2024-06-14","ids":{"openalex":"https://openalex.org/W4399668246","doi":"https://doi.org/10.1145/3661167.3661172"},"language":"en","primary_location":{"id":"doi:10.1145/3661167.3661172","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3661167.3661172","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3661167.3661172?download=true","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3661167.3661172?download=true","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001730023","display_name":"Aleksi Huotala","orcid":"https://orcid.org/0000-0002-5220-8730"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Aleksi Huotala","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Finland"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072326922","display_name":"Miikka Kuutila","orcid":"https://orcid.org/0000-0002-3695-7280"},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Miikka Kuutila","raw_affiliation_strings":["Faculty of Computer Science, Dalhousie University, Canada"],"affiliations":[{"raw_affiliation_string":"Faculty of Computer Science, Dalhousie University, Canada","institution_ids":["https://openalex.org/I129902397"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036483230","display_name":"Paul Ralph","orcid":"https://orcid.org/0000-0002-7411-0857"},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Paul Ralph","raw_affiliation_strings":["Faculty of Computer Science, Dalhousie University, Canada"],"affiliations":[{"raw_affiliation_string":"Faculty of Computer Science, Dalhousie University, Canada","institution_ids":["https://openalex.org/I129902397"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078824435","display_name":"Mika M\u00e4ntyl\u00e4","orcid":"https://orcid.org/0000-0002-2841-5879"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]},{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Mika M\u00e4ntyl\u00e4","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Finland and M3S, University of Oulu, Finland"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Finland and M3S, University of Oulu, Finland","institution_ids":["https://openalex.org/I98381234","https://openalex.org/I133731052"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5001730023"],"corresponding_institution_ids":["https://openalex.org/I133731052"],"apc_list":null,"apc_paid":null,"fwci":3.6406,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.93699741,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"262","last_page":"271"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9790999889373779,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9790999889373779,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9776999950408936,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9236999750137329,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5923168063163757},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.5315207242965698},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4755144715309143},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.39839547872543335},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.23611128330230713},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.19595551490783691},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.18066230416297913}],"concepts":[{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5923168063163757},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.5315207242965698},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4755144715309143},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.39839547872543335},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.23611128330230713},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.19595551490783691},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.18066230416297913},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3661167.3661172","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3661167.3661172","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3661167.3661172?download=true","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3661167.3661172","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3661167.3661172","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3661167.3661172?download=true","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322337","display_name":"Killam Trusts","ror":"https://ror.org/021cbtn85"},{"id":"https://openalex.org/F4320335354","display_name":"Strategic Research Council","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399668246.pdf","grobid_xml":"https://content.openalex.org/works/W4399668246.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W1481128830","https://openalex.org/W1507711477","https://openalex.org/W1760234209","https://openalex.org/W1977603982","https://openalex.org/W2519404962","https://openalex.org/W2807522649","https://openalex.org/W2895553377","https://openalex.org/W2961191798","https://openalex.org/W2999173898","https://openalex.org/W3135809019","https://openalex.org/W3143985904","https://openalex.org/W3149778443","https://openalex.org/W3173957073","https://openalex.org/W4220698983","https://openalex.org/W4308643041","https://openalex.org/W4362515116","https://openalex.org/W4385848697","https://openalex.org/W4386081573","https://openalex.org/W4387144848","https://openalex.org/W4388718056","https://openalex.org/W4390897455","https://openalex.org/W4391871418","https://openalex.org/W4392791588","https://openalex.org/W6778883912","https://openalex.org/W6967667630"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Context:":[0],"Systematic":[1],"review":[2],"(SR)":[3],"is":[4,41,208,272],"a":[5,90,203],"popular":[6],"research":[7,271],"method":[8],"in":[9,145,172,195,229,264],"software":[10],"engineering":[11],"(SE).":[12],"However,":[13],"conducting":[14],"an":[15,18,72],"SR":[16,29,277],"takes":[17],"average":[19],"of":[20,27,135,149,262,268],"67":[21],"weeks.":[22],"Thus,":[23],"automating":[24,65],"any":[25],"step":[26],"the":[28,33,44,108,132,142,146,163,169,196,230,260,265],"process":[30,232,267],"could":[31],"reduce":[32],"effort":[34],"associated":[35],"with":[36,83,95,127,283,291],"SRs.":[37],"Objective:":[38],"Our":[39,199],"objective":[40],"to":[42,46,106,153,241,279,286],"investigate":[43],"extent":[45],"which":[47],"Large":[48],"Language":[49],"Models":[50],"(LLMs)":[51],"can":[52],"accelerate":[53],"title-abstract":[54,66,78,150,243],"screening":[55,67,79,110,133,151,155,165,182,197,231,244,266,284],"by":[56,100],"(1)":[57],"simplifying":[58],"abstracts":[59,88],"for":[60,80,226],"human":[61,75,96,193,237,256],"screeners,":[62],"and":[63,86,103,125,178,186,217],"(2)":[64],"entirely.":[68],"Method:":[69],"We":[70,112,274],"performed":[71,77],"experiment":[73,94],"where":[74],"screeners":[76,97,194],"20":[81],"papers":[82],"both":[84],"original":[85],"simplified":[87],"from":[89],"prior":[91],"SR.":[92],"The":[93],"was":[98],"reproduced":[99],"instructing":[101],"GPT-3.5":[102],"GPT-4":[104],"LLMs":[105,225,240,249,263],"perform":[107,189],"same":[109],"tasks.":[111,198],"also":[113],"studied":[114,139],"whether":[115],"different":[116],"prompting":[117,219],"techniques":[118],"(Zero-shot":[119],"(ZS),":[120],"One-shot":[121,218],"(OS),":[122],"Few-shot":[123,126,216],"(FS),":[124],"Chain-of-Thought":[128],"(FS-CoT)":[129],"prompting)":[130],"improve":[131,236],"performance":[134],"LLMs.":[136],"Lastly,":[137],"we":[138],"if":[140],"redesigning":[141],"prompt":[143,187],"used":[144,171],"LLM":[147,185,206,213,292],"reproduction":[148],"leads":[152],"improved":[154],"performance.":[156,183,238],"Results:":[157],"Text":[158],"simplification":[159,228],"did":[160],"not":[161,234,251],"increase":[162],"screeners\u2019":[164],"performance,":[166],"but":[167,247],"reduced":[168],"time":[170],"screening.":[173,293],"Screeners\u2019":[174],"scientific":[175],"literacy":[176],"skills":[177],"researcher":[179],"status":[180],"predict":[181],"Some":[184],"combinations":[188],"as":[190,192],"well":[191],"results":[200],"indicate":[201],"that":[202],"more":[204,253,270,288],"recent":[205],"(GPT-4)":[207],"better":[209],"than":[210,255],"its":[211],"predecessor":[212],"(GPT-3.5).":[214],"Additionally,":[215],"outperforms":[220],"Zero-shot":[221],"prompting.":[222],"Conclusion:":[223],"Using":[224,239],"text":[227],"does":[233],"significantly":[235,252],"automate":[242],"seems":[245],"promising,":[246],"current":[248],"are":[250],"accurate":[254],"screeners.":[257],"To":[258],"recommend":[259,275],"use":[261],"SRs,":[269],"needed.":[273],"future":[276],"studies":[278],"publish":[280],"replication":[281],"packages":[282],"data":[285],"enable":[287],"conclusive":[289],"experimenting":[290]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":3}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}