{"id":"https://openalex.org/W3155157729","doi":"https://doi.org/10.1145/3404835.3463013","title":"Stopping Criteria for Technology Assisted Reviews based on Counting Processes","display_name":"Stopping Criteria for Technology Assisted Reviews based on Counting Processes","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3155157729","doi":"https://doi.org/10.1145/3404835.3463013","mag":"3155157729"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3463013","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463013","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012465973","display_name":"Alison Sneyd","orcid":"https://orcid.org/0000-0001-6850-5871"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alison Sneyd","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101566333","display_name":"Mark Stevenson","orcid":"https://orcid.org/0000-0002-9483-6006"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Stevenson","raw_affiliation_strings":["University of Sheffield, Sheffield, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Sheffield, Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I91136226"],"apc_list":null,"apc_paid":null,"fwci":1.9383,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.8838835,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"2293","last_page":"2297"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12478","display_name":"Wikis in Education and Collaboration","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9803000092506409,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7585439682006836},{"id":"https://openalex.org/keywords/clef","display_name":"Clef","score":0.746820330619812},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.7180251479148865},{"id":"https://openalex.org/keywords/counting-process","display_name":"Counting process","score":0.6559440493583679},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.6071869730949402},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4342028498649597},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3595011830329895},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10840320587158203},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08727943897247314},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.07918471097946167},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07503402233123779},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.07016786932945251}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7585439682006836},{"id":"https://openalex.org/C107763842","wikidata":"https://www.wikidata.org/wiki/Q181040","display_name":"Clef","level":3,"score":0.746820330619812},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.7180251479148865},{"id":"https://openalex.org/C2781104640","wikidata":"https://www.wikidata.org/wiki/Q11827313","display_name":"Counting process","level":2,"score":0.6559440493583679},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.6071869730949402},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4342028498649597},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3595011830329895},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10840320587158203},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08727943897247314},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.07918471097946167},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07503402233123779},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.07016786932945251},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3404835.3463013","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463013","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.whiterose.ac.uk:175032","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400854","display_name":"White Rose Research Online (University of Leeds, The University of Sheffield, University of York)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2800616092","host_organization_name":"White Rose University Consortium","host_organization_lineage":["https://openalex.org/I2800616092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Proceedings Paper"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6700000166893005}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2075893676","https://openalex.org/W2097749765","https://openalex.org/W2341384651","https://openalex.org/W2474897245","https://openalex.org/W2901234380","https://openalex.org/W2905296953","https://openalex.org/W2963439330","https://openalex.org/W2986864098","https://openalex.org/W3005236729","https://openalex.org/W3089621169"],"related_works":["https://openalex.org/W1839123017","https://openalex.org/W4232115401","https://openalex.org/W1592594506","https://openalex.org/W2970828463","https://openalex.org/W3138512925","https://openalex.org/W117735687","https://openalex.org/W2917705549","https://openalex.org/W2915106852","https://openalex.org/W2611815462","https://openalex.org/W2397564744"],"abstract_inverted_index":{"Technology":[0],"Assisted":[1],"Review":[2],"(TAR)":[3],"aims":[4],"to":[5,11,25,32,43,69,117],"minimise":[6],"the":[7,56,102],"manual":[8],"judgements":[9],"required":[10],"identify":[12],"relevant":[13,60],"documents.":[14,35],"Reductions":[15],"in":[16],"workload":[17],"are":[18,51,62,115],"dependent":[19],"on":[20,53,96,101],"a":[21,39,74,92,97],"reviewer":[22],"being":[23],"able":[24],"make":[26],"an":[27],"informed":[28],"decision":[29],"about":[30],"when":[31],"stop":[33],"examining":[34],"Counting":[36],"processes":[37],"offer":[38],"theoretically":[40],"sound":[41],"approach":[42],"creating":[44],"stopping":[45],"criteria":[46],"for":[47,86],"TAR":[48,106],"approaches":[49,111],"that":[50,109,114],"based":[52,95],"analysis":[54],"of":[55,73,91],"rate":[57,93],"at":[58],"which":[59,80],"documents":[61],"observed.":[63],"This":[64],"paper":[65],"introduces":[66],"two":[67],"modifications":[68],"existing":[70],"approaches:":[71],"application":[72],"Cox":[75],"Process":[76],"(a":[77],"counting":[78],"process":[79],"has":[81],"not":[82],"previously":[83],"been":[84],"used":[85],"this":[87],"problem)":[88],"and":[89],"use":[90],"function":[94],"power":[98],"law.":[99],"Experiments":[100],"CLEF":[103],"2017":[104],"e-Health":[105],"collection":[107],"demonstrates":[108],"these":[110],"produces":[112],"results":[113],"superior":[116],"those":[118],"reported":[119],"previously.":[120]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2}],"updated_date":"2026-07-02T09:51:11.867554","created_date":"2025-10-10T00:00:00"}