{"id":"https://openalex.org/W4383376717","doi":"https://doi.org/10.31234/osf.io/c93gq","title":"The SAFE Procedure: A Practical Stopping Heuristic for Active Learning-Based Screening in Systematic Reviews and Meta-Analyses","display_name":"The SAFE Procedure: A Practical Stopping Heuristic for Active Learning-Based Screening in Systematic Reviews and Meta-Analyses","publication_year":2023,"publication_date":"2023-07-05","ids":{"openalex":"https://openalex.org/W4383376717","doi":"https://doi.org/10.31234/osf.io/c93gq"},"language":"en","primary_location":{"id":"doi:10.31234/osf.io/c93gq","doi":"https://doi.org/10.31234/osf.io/c93gq","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/c93gq","pdf_url":"https://psyarxiv.com/c93gq/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://psyarxiv.com/c93gq/download","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092414900","display_name":"Josien Boetje","orcid":"https://orcid.org/0000-0003-1542-6672","observed_orcids":["https://orcid.org/0000-0003-1542-6672"]},"institutions":[{"id":"https://openalex.org/I173063890","display_name":"University of Applied Sciences Utrecht","ror":"https://ror.org/028z9kw20","country_code":"NL","type":"education","lineage":["https://openalex.org/I173063890"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Josien Boetje","raw_affiliation_strings":["Research group Digital Ethics, Knowledge Center Learning and Innovation (LENI), Archimedes Institute, HU University of Applied Sciences Utrecht, Utrecht, the Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-1542-6672","affiliations":[{"raw_affiliation_string":"Research group Digital Ethics, Knowledge Center Learning and Innovation (LENI), Archimedes Institute, HU University of Applied Sciences Utrecht, Utrecht, the Netherlands","institution_ids":["https://openalex.org/I173063890"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041799420","display_name":"Rens van de Schoot","orcid":"https://orcid.org/0000-0001-7736-2091","observed_orcids":["https://orcid.org/0000-0001-7736-2091","https://orcid.org/0000-0002-0766-3988"]},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Rens van de Schoot","raw_affiliation_strings":["Department of Methodology and Statistics, Faculty of Social and Behavioral Sciences, Utrecht University, The Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-7736-2091","affiliations":[{"raw_affiliation_string":"Department of Methodology and Statistics, Faculty of Social and Behavioral Sciences, Utrecht University, The Netherlands","institution_ids":["https://openalex.org/I193662353"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5092414900"],"corresponding_institution_ids":["https://openalex.org/I173063890"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.00139999995008111,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/active-learning-machine-learning","display_name":"active learning (machine learning)","score":0.9580000042915344},{"id":"https://openalex.org/keywords/systematic-review-screening","display_name":"systematic review screening","score":0.6890000104904175},{"id":"https://openalex.org/keywords/asreview","display_name":"ASReview","score":0.8379999995231628},{"id":"https://openalex.org/keywords/model-uncertainty","display_name":"model uncertainty","score":0.8479999899864197}],"concepts":[{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.7891223430633545},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7247213125228882},{"id":"https://openalex.org/C5465570","wikidata":"https://www.wikidata.org/wiki/Q5326898","display_name":"Early stopping","level":3,"score":0.6222010850906372},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.6167280077934265},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5860835909843445},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5526401400566101},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.4855444133281708},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43232643604278564},{"id":"https://openalex.org/C99414536","wikidata":"https://www.wikidata.org/wiki/Q7098950","display_name":"Optimal stopping","level":2,"score":0.413776695728302},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.39299890398979187},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.349529504776001},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.14966389536857605},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07578340172767639},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.31234/osf.io/c93gq","doi":"https://doi.org/10.31234/osf.io/c93gq","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/c93gq","pdf_url":"https://psyarxiv.com/c93gq/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},{"id":"pmh:oai:dspace.library.uu.nl:1874/438338","doi":null,"is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/438338","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"pmh:oai:share.osf.io:725c7fee-9401-4a38-a348-c6638930db40","doi":null,"is_oa":true,"landing_page_url":"https://osf.io/c93gq","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"pmh:oai:share.osf.io:E013C-F1D-1D2","doi":null,"is_oa":false,"landing_page_url":"http://osf.io/c93gq/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401687","display_name":"PsyArXiv (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"preprint"}],"best_oa_location":{"id":"doi:10.31234/osf.io/c93gq","doi":"https://doi.org/10.31234/osf.io/c93gq","is_oa":true,"landing_page_url":"https://doi.org/10.31234/osf.io/c93gq","pdf_url":"https://psyarxiv.com/c93gq/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.7599999904632568,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, justice, and strong institutions"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4383376717.pdf","grobid_xml":"https://content.openalex.org/works/W4383376717.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W1024712121","https://openalex.org/W2021894617","https://openalex.org/W2043566294","https://openalex.org/W2065383075","https://openalex.org/W2086226100","https://openalex.org/W2099419573","https://openalex.org/W2114415944","https://openalex.org/W2122839539","https://openalex.org/W2132910334","https://openalex.org/W2135815793","https://openalex.org/W2341384651","https://openalex.org/W2557283755","https://openalex.org/W2560438049","https://openalex.org/W2600107025","https://openalex.org/W2621485502","https://openalex.org/W2785836523","https://openalex.org/W2797780449","https://openalex.org/W2808847453","https://openalex.org/W2884443195","https://openalex.org/W2895410484","https://openalex.org/W2999536597","https://openalex.org/W3003735349","https://openalex.org/W3010940523","https://openalex.org/W3093336514","https://openalex.org/W3111412136","https://openalex.org/W3128349626","https://openalex.org/W3143985904","https://openalex.org/W3168270921","https://openalex.org/W3178695691","https://openalex.org/W3193973034","https://openalex.org/W3195783210","https://openalex.org/W4200398992","https://openalex.org/W4225297356","https://openalex.org/W4288041857","https://openalex.org/W4293715440","https://openalex.org/W6755998472","https://openalex.org/W6920863868","https://openalex.org/W6950101330","https://openalex.org/W6958420638","https://openalex.org/W6977729465"],"related_works":["https://openalex.org/W6939709918","https://openalex.org/W6977235543","https://openalex.org/W4414583665","https://openalex.org/W6920734856","https://openalex.org/W3110318438","https://openalex.org/W4417449083","https://openalex.org/W6907528452","https://openalex.org/W4385381256","https://openalex.org/W3139098936","https://openalex.org/W2785540181"],"abstract_inverted_index":{"Active":[0],"learning":[1,21,55,109,195,226],"has":[2],"become":[3],"an":[4,135],"increasingly":[5],"popular":[6],"method":[7,262],"for":[8,102,192,249],"screening":[9,112,152,200,237],"large":[10],"amounts":[11],"of":[12,35,62,65,76,94,119,138,146,185,235,241],"data":[13],"in":[14,111,150,196,266,281],"systematic":[15,197],"reviews":[16],"and":[17,91,133,164,178,204,212,279],"meta-analyses.":[18],"The":[19,60,169,228,260],"active":[20,54,108,194,225],"process":[22,56,110],"continually":[23],"improves":[24],"its":[25],"predictions":[26],"on":[27,125,255],"the":[28,33,46,53,68,74,80,86,107,144,151,175,183,224,233,239],"remaining":[29],"unlabeled":[30],"records,":[31,244],"with":[32,238],"goal":[34],"identifying":[36,267],"all":[37],"relevant":[38,148,167,243,268],"records":[39,66,269],"as":[40,42,180,182],"early":[41,163],"possible.":[43],"However,":[44],"determining":[45,103,220],"optimal":[47],"point":[48],"at":[49],"which":[50,271],"to":[51,105,142,159,187,219,222,251,257,275],"stop":[52,258],"is":[57,122,130],"a":[58,89,99,126,202,216,246],"challenge.":[59],"cost":[61,75,184],"additional":[63],"labeling":[64],"by":[67,79],"reviewer":[69],"must":[70],"be":[71],"balanced":[72],"against":[73],"errors":[77],"made":[78],"current":[81],"model.":[82],"This":[83,190],"paper":[84,121],"introduces":[85],"SAFE":[87,154,170],"procedure,":[88],"practical":[90,203,247],"conservative":[92,217],"set":[93],"stopping":[95,128,139,161,230],"heuristics":[96,140,158],"that":[97,123,207],"offers":[98],"clear":[100],"guideline":[101],"when":[104,221,256],"end":[106,223],"software":[113],"like":[114],"ASReview.":[115],"Our":[116],"main":[117],"conclusion":[118],"this":[120],"relying":[124],"single":[127],"rule":[129],"not":[131],"sufficient":[132],"employing":[134],"eclectic":[136],"mix":[137],"helps":[141],"minimize":[143],"risk":[145,240],"missing":[147,166,242],"papers":[149],"processThe":[153],"procedure":[155,171,191],"combines":[156],"different":[157],"avoid":[160],"too":[162],"potentially":[165],"records.":[168,189],"takes":[172],"into":[173],"account":[174],"model's":[176],"accuracy":[177],"uncertainty,":[179],"well":[181],"continuing":[186],"label":[188],"using":[193],"literature":[198],"review":[199],"provides":[201],"efficient":[205],"approach":[206,218],"can":[208,263,272],"save":[209],"significant":[210],"time":[211],"resources":[213],"while":[214],"ensuring":[215],"process.":[227],"proposed":[229,261],"heuristic":[231],"balances":[232],"costs":[234],"continued":[236],"providing":[245],"solution":[248],"reviewers":[250],"make":[252],"informed":[253],"decisions":[254],"screening.":[259],"assist":[264],"researchers":[265],"early,":[270],"ultimately":[273],"lead":[274],"improved":[276],"evidence":[277],"synthesis":[278],"decision-making":[280],"many":[282],"fields.":[283]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2023-07-07T00:00:00"}