{"id":"https://openalex.org/W4385285443","doi":"https://doi.org/10.1109/access.2023.3299211","title":"Reproducible Searches in Systematic Reviews: An Evaluation and Guidelines","display_name":"Reproducible Searches in Systematic Reviews: An Evaluation and Guidelines","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385285443","doi":"https://doi.org/10.1109/access.2023.3299211"},"language":"en","primary_location":{"id":"doi:10.1109/access.2023.3299211","doi":"https://doi.org/10.1109/access.2023.3299211","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3299211","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10194950.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","ki-jl-1","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"volume":null,"type":"review","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10194950.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100415036","display_name":"Zheng Li","orcid":"https://orcid.org/0000-0002-9704-7651","observed_orcids":["https://orcid.org/0000-0002-9704-7651"]},"institutions":[{"id":"https://openalex.org/I126231945","display_name":"Queen's University Belfast","ror":"https://ror.org/00hswnk62","country_code":"GB","type":"education","lineage":["https://openalex.org/I126231945"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zheng Li","raw_affiliation_strings":["School of Electronics, Electrical Engineering and Computer Science, Queen’s University Belfast, Belfast, U.K"],"raw_orcid":"https://orcid.org/0000-0002-9704-7651","affiliations":[{"raw_affiliation_string":"School of Electronics, Electrical Engineering and Computer Science, Queen’s University Belfast, Belfast, U.K","institution_ids":["https://openalex.org/I126231945"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063087518","display_name":"Austen Rainer","orcid":"https://orcid.org/0000-0001-8868-263X","observed_orcids":["https://orcid.org/0000-0001-8868-263X"]},"institutions":[{"id":"https://openalex.org/I126231945","display_name":"Queen's University Belfast","ror":"https://ror.org/00hswnk62","country_code":"GB","type":"education","lineage":["https://openalex.org/I126231945"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Austen Rainer","raw_affiliation_strings":["School of Electronics, Electrical Engineering and Computer Science, Queen’s University Belfast, Belfast, U.K"],"raw_orcid":"https://orcid.org/0000-0001-8868-263X","affiliations":[{"raw_affiliation_string":"School of Electronics, Electrical Engineering and Computer Science, Queen’s University Belfast, Belfast, U.K","institution_ids":["https://openalex.org/I126231945"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I126231945"],"apc_list":{"value":1950,"currency":"USD","value_usd":1950},"apc_paid":{"value":1950,"currency":"USD","value_usd":1950},"fwci":0.6993,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.68479263,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"11","issue":null,"first_page":"84048","last_page":"84060"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9599999785423279,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9599999785423279,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.010099999606609344,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.007199999876320362,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.8669999837875366},{"id":"https://openalex.org/keywords/automated-literature-search","display_name":"automated literature search","score":0.5860000252723694},{"id":"https://openalex.org/keywords/systematic-reviews","display_name":"systematic reviews","score":0.8230000138282776},{"id":"https://openalex.org/keywords/search-engines","display_name":"search engines","score":0.4729999899864197}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7858769297599792},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6443630456924438},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6114805340766907},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5810716152191162},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5155117511749268},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.42181020975112915},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4012308716773987},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3208090662956238},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2023.3299211","doi":"https://doi.org/10.1109/access.2023.3299211","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3299211","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10194950.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","ki-jl-1","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:pure.qub.ac.uk/portal:openaire/2b64068e-bdda-4e5c-a3f5-f98d1a334c81","doi":null,"is_oa":true,"landing_page_url":"https://pure.qub.ac.uk/en/publications/2b64068e-bdda-4e5c-a3f5-f98d1a334c81","pdf_url":"https://pureadmin.qub.ac.uk/ws/files/508441152/Searches.pdf","source":{"id":"https://openalex.org/S4306402319","display_name":"Research Portal (Queen's University Belfast)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I126231945","host_organization_name":"Queen's University Belfast","host_organization_lineage":["https://openalex.org/I126231945"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Li, Z & Rainer, A 2023, 'Reproducible searches in systematic reviews: an evaluation and guidelines', IEEE Access, vol. 11, pp. 84048 -84060. https://doi.org/10.1109/ACCESS.2023.3299211","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:7dabef544cf7466d8d8ce9f29b722420","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/7dabef544cf7466d8d8ce9f29b722420","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 11, Pp 84048-84060 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2023.3299211","doi":"https://doi.org/10.1109/access.2023.3299211","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3299211","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10194950.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-scientific","cwts-core","doaj","jufo-1","ki-jl-1","norway-1","oarl","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4043000042438507,"display_name":"Good health and well-being","id":"https://metadata.un.org/sdg/3"}],"sustainable_development_goals_aurora":[{"score":0.4300000071525574,"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G7642564126","display_name":null,"funder_award_id":"ES/V00848X/1","funder_id":"https://openalex.org/F4320334630","funder_display_name":"Economic and Social Research Council"}],"funders":[{"id":"https://openalex.org/F4320334630","display_name":"Economic and Social Research Council","ror":"https://ror.org/03n0ht308"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385285443.pdf","grobid_xml":"https://content.openalex.org/works/W4385285443.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W601857102","https://openalex.org/W659618760","https://openalex.org/W750932891","https://openalex.org/W1669992463","https://openalex.org/W1877420198","https://openalex.org/W1964067312","https://openalex.org/W1975675278","https://openalex.org/W1987265329","https://openalex.org/W1999798506","https://openalex.org/W2008411739","https://openalex.org/W2016004512","https://openalex.org/W2028836139","https://openalex.org/W2065827414","https://openalex.org/W2118328848","https://openalex.org/W2128748997","https://openalex.org/W2139380319","https://openalex.org/W2145338466","https://openalex.org/W2147471614","https://openalex.org/W2148170929","https://openalex.org/W2163621605","https://openalex.org/W2168076234","https://openalex.org/W2168484999","https://openalex.org/W2169957228","https://openalex.org/W2173536757","https://openalex.org/W2399859574","https://openalex.org/W2570018362","https://openalex.org/W2600366513","https://openalex.org/W2621774609","https://openalex.org/W2737407328","https://openalex.org/W2772141859","https://openalex.org/W2791349162","https://openalex.org/W2944196828","https://openalex.org/W2971334721","https://openalex.org/W3110069973","https://openalex.org/W3161606367","https://openalex.org/W4253518412","https://openalex.org/W4308627330"],"related_works":["https://openalex.org/W4389285887","https://openalex.org/W4376637409","https://openalex.org/W6987458011","https://openalex.org/W6887676558","https://openalex.org/W6902638935","https://openalex.org/W2791349162","https://openalex.org/W2249623838","https://openalex.org/W7214943195","https://openalex.org/W2524045858","https://openalex.org/W2409511678"],"abstract_inverted_index":{"[Context:]":[0],"The":[1],"Systematic":[2],"Review":[3],"is":[4,38],"promoted":[5],"as":[6],"a":[7,13,22],"more":[8,108,193],"reliable":[9],"way":[10],"of":[11,16,24,33,41,49,63,70,78,89,100,111,124,136,142,159,166,174,183,198,204,211,230],"producing":[12],"high-quality":[14],"review":[15],"prior":[17],"research.":[18],"But":[19],"there":[20,216],"are":[21,119,145,169,214,217],"range":[23],"threats":[25],"that":[26,222],"can":[27,188,224],"undermine":[28],"the":[29,39,47,61,68,76,84,91,98,105,112,125,137,143,157,171,199,205,209,228],"reliability":[30],"and":[31,55,74,86,133,181,196,203],"quality":[32],"such":[34],"reviews.":[35],"One":[36],"threat":[37],"reproducibility":[40,62,99,229],"automated":[42,101,151,167],"searches.":[43,64,102,232],"[Objectives:]":[44],"To":[45],"evaluate":[46],"state-of-practice":[48],"reproducible":[50],"searches":[51,69,126,138],"in":[52,178],"secondary":[53,72],"studies,":[54,107],"to":[56,59,93,96,226],"consider":[57],"ways":[58,95],"improve":[60,97,227],"[Method:]":[65],"We":[66,82,147],"re-run":[67],"621":[71,106],"studies":[73],"analyse":[75],"outcomes":[77],"those":[79],"(attempted)":[80],"re-runs.":[81],"use":[83],"outcomes,":[85],"our":[87,212],"experience":[88],"re-running":[90],"searches,":[92],"propose":[94,148],"[Results:]":[103],"With":[104],"than":[109],"50%":[110],"literal":[113],"search":[114,160,168,184,200,206],"strings":[115,201],"(ignoring":[116],"other":[117],"settings)":[118,129],"not":[120],"reusable;":[121],"about":[122],"87%":[123],"(e.g.,":[127,176],"with":[128],"cannot":[130],"be":[131,189],"repeated;":[132],"around":[134],"94%":[135],"(including":[139],"all":[140],"elements":[141],"search)":[144],"irreproducible.":[146],"guidelines":[149],"for":[150],"search,":[152],"directing":[153],"particular":[154],"attention":[155],"at":[156],"formulation":[158,195],"strings.":[161],"[Conclusion:]":[162],"While":[163,208],"some":[164],"aspects":[165,187],"beyond":[170],"direct":[172],"control":[173],"researchers":[175,223],"variations":[177],"features,":[179],"constraints":[180],"performance":[182],"engines),":[185],"many":[186,218],"effectively":[190],"managed":[191],"through":[192],"careful":[194],"execution":[197],"themselves,":[202],"settings.":[207],"results":[210],"evaluation":[213],"disappointing":[215],"simple,":[219],"concrete":[220],"steps":[221],"make":[225],"their":[231]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-07-27T00:00:00"}