{"id":"https://openalex.org/W3209051520","doi":"https://doi.org/10.5281/zenodo.4013206","title":"Researcher-in-the-loop for systematic reviewing of text databases","display_name":"Researcher-in-the-loop for systematic reviewing of text databases","publication_year":2020,"publication_date":"2020-06-25","ids":{"openalex":"https://openalex.org/W3209051520","doi":"https://doi.org/10.5281/zenodo.4013206","mag":"3209051520"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.4013206","doi":"https://doi.org/10.5281/zenodo.4013206","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.4013206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"volume":null,"type":"other","is_traditional":false,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.4013206","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041799420","display_name":"Rens van de Schoot","orcid":"https://orcid.org/0000-0001-7736-2091","observed_orcids":["https://orcid.org/0000-0001-7736-2091","https://orcid.org/0000-0002-0766-3988"]},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Van de Schoot, Rens","raw_affiliation_strings":["Utrecht University"],"raw_orcid":"https://orcid.org/0000-0001-7736-2091","affiliations":[{"raw_affiliation_string":"Utrecht University","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027479191","display_name":"Jonathan De Bruin","orcid":"https://orcid.org/0000-0002-4297-0502","observed_orcids":["https://orcid.org/0000-0002-4297-0502"]},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"De Bruin, Jonathan","raw_affiliation_strings":["Utrecht University"],"raw_orcid":"https://orcid.org/0000-0002-4297-0502","affiliations":[{"raw_affiliation_string":"Utrecht University","institution_ids":["https://openalex.org/I193662353"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I193662353"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.3278999924659729,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.3278999924659729,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.29490000009536743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.16140000522136688,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/human-in-the-loop-machine-learning","display_name":"human-in-the-loop learning","score":0.9539999961853027},{"id":"https://openalex.org/keywords/systematic-reviews","display_name":"systematic reviews","score":0.5220000147819519},{"id":"https://openalex.org/keywords/text-databases","display_name":"text databases","score":0.6759999990463257},{"id":"https://openalex.org/keywords/covid-19-literature","display_name":"COVID-19 literature","score":0.7770000100135803},{"id":"https://openalex.org/keywords/literature-search-automation","display_name":"literature search automation","score":0.5059999823570251},{"id":"https://openalex.org/keywords/relevance-ranking","display_name":"relevance ranking","score":0.3790000081062317},{"id":"https://openalex.org/keywords/inclusion-criteria","display_name":"inclusion criteria","score":0.3659999966621399},{"id":"https://openalex.org/keywords/active-learning-machine-learning","display_name":"active learning (machine learning)","score":0.49900001287460327},{"id":"https://openalex.org/keywords/clinical-guidelines","display_name":"clinical guidelines","score":0.31200000643730164},{"id":"https://openalex.org/keywords/information-overload","display_name":"information overload","score":0.3490000069141388},{"id":"https://openalex.org/keywords/human-machine-interaction","display_name":"human-machine interaction","score":0.41100001335144043},{"id":"https://openalex.org/keywords/bias-in-systematic-reviews","display_name":"bias in systematic reviews","score":0.43700000643730164}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5478453636169434},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.5294484496116638},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.45176130533218384},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.41974133253097534},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.359543114900589},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1356046497821808},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.09560585021972656}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.5281/zenodo.4013206","doi":"https://doi.org/10.5281/zenodo.4013206","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.4013206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},{"id":"doi:10.5281/zenodo.4013207","doi":"https://doi.org/10.5281/zenodo.4013207","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.4013207","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},{"id":"mag:3209051520","doi":null,"is_oa":false,"landing_page_url":"https://zenodo.org/record/4013207","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"doi:10.5281/zenodo.4013206","doi":"https://doi.org/10.5281/zenodo.4013206","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.4013206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7109095945","https://openalex.org/W7213459036","https://openalex.org/W4226426944","https://openalex.org/W7110523907","https://openalex.org/W6990800123","https://openalex.org/W4396941516","https://openalex.org/W4221151033","https://openalex.org/W7141119996","https://openalex.org/W3192495122","https://openalex.org/W4409348652"],"abstract_inverted_index":{"Machine":[0],"learning":[1,50,56,60,104,280,321],"with":[2,108,117,199,207,241,259,361],"human":[3,46,72,110,294],"interaction":[4,107],"is":[5,88,111,126,134,227,270,345,430,464],"gaining":[6],"popularity.":[7],"Humans":[8],"are":[9,19,92,171,187,290],"intelligent":[10],"but":[11,23,139,342],"slow":[12],"and":[13,21,48,62,69,197,355,401,419,452],"not":[14,24,135,276],"very":[15],"accurate,":[16],"whereas":[17],"machines":[18],"fast":[20],"accurate":[22],"(very)":[25],"intelligent.":[26],"Combining":[27],"the":[28,37,71,76,86,106,109,132,136,140,148,169,182,208,242,249,268,274,287,296,299,329,334,365,377,388,421,476],"strengths":[29],"of":[30,39,44,121,129,143,147,180,202,214,257,267,273,298,309,318,331,336,367,379,387,431,479],"both":[31],"in":[32,54,160,189,286,376,410,426],"interactive":[33],"processes":[34],"can":[35,51],"advance":[36],"capabilities":[38],"machine":[40,49,55,65,103,279,320],"learning.":[41,66],"The":[42,264,302,461],"idea":[43],"combining":[45,448],"intelligence":[47],"be":[52,95,156,158,239],"found":[53,188],"techniques":[57,91],"like":[58,204,364],"active":[59,453],"(AL)":[61],"Human-in-the-Loop":[63],"(HITL)":[64],"In":[67,100,166,223,311],"AL":[68],"HITL,":[70],"classifies":[73],"unlabeled":[74],"items":[75,83],"model":[77,87,116,133,183],"selects":[78],"to":[79,94,113,210,238,353,434,438,483],"learn":[80],"from,":[81],"e.g.,":[82],"for":[84,97,174,458],"which":[85],"uncertain.":[89],"These":[90],"proven":[93],"effective":[96],"training":[98],"models.":[99],"most":[101],"HITL":[102,258],"applications,":[105,225],"used":[112],"train":[114],"a":[115,118,127,144,164,194,229,254,271,277,293,305,316,411,436,489],"minimum":[119],"number":[120,335],"labeling":[122,480],"tasks.":[123],"However,":[124],"there":[125,226],"type":[128],"problems":[130],"where":[131],"primary":[137,265],"output,":[138],"systematic":[141,205,231,307,394,427],"exploration":[142],"relevant":[145,288,368,389,408],"subset":[146],"given":[149],"dataset.":[150],"Usually,":[151],"these":[152,176,224,348],"data":[153,177,235,284,350,415],"points":[154,236,285],"should":[155],"all":[157,234,406],"seen,":[159],"any":[161,340],"case,":[162,168],"by":[163,292,473],"human.":[165],"this":[167,312,372,441],"algorithm(s)":[170],"interactively":[172],"optimized":[173],"finding":[175],"points,":[178],"instead":[179],"making":[181],"more":[184,485],"accurate.":[185],"Examples":[186],"applications":[190],"were":[191],"challenges":[192,363],"have":[193],"unique":[195,261],"character":[196],"deal":[198],"large":[200],"volumes":[201],"data,":[203,275],"reviewing":[206,487],"goal":[209],"create":[211],"an":[212],"overview":[213],"scientific":[215,326,337],"literature,":[216],"developing":[217],"medical":[218,381],"guidelines,":[219],"or":[220],"police":[221],"investigations.":[222],"also":[228],"strong":[230,306],"component":[232],"as":[233,253],"need":[237],"judged":[240],"same":[243],"inclusion":[244],"criteria.":[245],"Therefore,":[246,444],"we":[247,314,445],"propose":[248],"term":[250],"Researcher-In-The-Loop":[251],"(RITL)":[252],"special":[255],"case":[256],"three":[260],"components:":[262],"(1)":[263],"output":[266],"process":[269],"selection":[272,289],"trained":[278],"model,":[281],"(2)":[282],"All":[283,347],"seen":[291],"at":[295],"end":[297],"process,":[300],"(3)":[301],"use-case":[303,317],"requires":[304],"way":[308,437],"working.":[310],"presentation,":[313],"study":[315],"RITL":[319],"about":[322],"reading":[323],"COVID-19":[324],"related":[325],"papers.":[327],"With":[328],"emergence":[330],"online":[332],"publishing,":[333],"papers":[338],"on":[339],"topic,":[341],"especially":[343],"COVID-19,":[344],"skyrocketing.":[346],"textual":[349],"present":[351],"opportunities":[352],"scholars":[354,391],"practitioners,":[356],"while":[357],"simultaneously":[358],"confronting":[359],"them":[360],"new":[362],"sparsity":[366],"items.":[369],"For":[370],"example,":[371],"challenge":[373],"shows":[374],"up":[375],"development":[378],"specific":[380],"guidelines.":[382],"To":[383,423],"develop":[384,393],"comprehensive":[385],"overviews":[386],"studies,":[390,418],"often":[392],"reviews.":[395],"Such":[396],"reviews":[397],"entail":[398],"several":[399],"explicit":[400],"reproducible":[402],"steps,":[403],"including":[404],"identifying":[405],"likely":[407],"publications":[409],"standardized":[412],"way,":[413],"extracting":[414],"from":[416],"eligible":[417],"synthesizing":[420],"results.":[422],"avoid":[424],"bias":[425],"reviews,":[428],"it":[429],"crucial":[432],"importance":[433],"find":[435],"effectively":[439],"automatize":[440],"screening":[442],"process.":[443],"developed":[446],"software":[447],"natural":[449],"language":[450],"processing":[451],"learning:":[454],"ASReview:":[455],"Active":[456],"Learning":[457],"Systematic":[459],"Reviews.":[460],"source":[462,467],"code":[463],"available":[465],"open":[466],"via":[468],"GitHub.":[469],"We":[470],"show":[471],"that":[472],"using":[474],"ASReview":[475],"laborintensive":[477],"task":[478],"text":[481],"leads":[482],"far":[484],"efficient":[486],"than":[488],"full":[490],"manual":[491],"review.":[492]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2021-11-08T00:00:00"}