{"id":"https://openalex.org/W2741023348","doi":"https://doi.org/10.1145/3077136.3080707","title":"A Test Collection for Evaluating Retrieval of Studies for Inclusion in Systematic Reviews","display_name":"A Test Collection for Evaluating Retrieval of Studies for Inclusion in Systematic Reviews","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2741023348","doi":"https://doi.org/10.1145/3077136.3080707","mag":"2741023348"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3080707","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080707","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075371460","display_name":"Harrisen Scells","orcid":"https://orcid.org/0000-0001-9578-7157"},"institutions":[{"id":"https://openalex.org/I160993911","display_name":"Queensland University of Technology","ror":"https://ror.org/03pnv4752","country_code":"AU","type":"education","lineage":["https://openalex.org/I160993911"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Harrisen Scells","raw_affiliation_strings":["Queensland University of Technology, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"Queensland University of Technology, Brisbane, Australia","institution_ids":["https://openalex.org/I160993911"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076031002","display_name":"Guido Zuccon","orcid":"https://orcid.org/0000-0003-0271-5563"},"institutions":[{"id":"https://openalex.org/I160993911","display_name":"Queensland University of Technology","ror":"https://ror.org/03pnv4752","country_code":"AU","type":"education","lineage":["https://openalex.org/I160993911"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Guido Zuccon","raw_affiliation_strings":["Queensland University of Technology, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"Queensland University of Technology, Brisbane, Australia","institution_ids":["https://openalex.org/I160993911"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087733750","display_name":"Bevan Koopman","orcid":"https://orcid.org/0000-0001-5577-3391"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"funder","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Bevan Koopman","raw_affiliation_strings":["CSIRO, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"CSIRO, Brisbane, Australia","institution_ids":["https://openalex.org/I1292875679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052381125","display_name":"Anthony Deacon","orcid":"https://orcid.org/0000-0001-5051-4817"},"institutions":[{"id":"https://openalex.org/I160993911","display_name":"Queensland University of Technology","ror":"https://ror.org/03pnv4752","country_code":"AU","type":"education","lineage":["https://openalex.org/I160993911"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Anthony Deacon","raw_affiliation_strings":["Queensland University of Technology, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"Queensland University of Technology, Brisbane, Australia","institution_ids":["https://openalex.org/I160993911"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048218964","display_name":"Leif Azzopardi","orcid":"https://orcid.org/0000-0002-6900-0557"},"institutions":[{"id":"https://openalex.org/I181647926","display_name":"University of Strathclyde","ror":"https://ror.org/00n3w3b69","country_code":"GB","type":"education","lineage":["https://openalex.org/I181647926"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Leif Azzopardi","raw_affiliation_strings":["Strathclyde University, Glasgow, Scotland UK"],"affiliations":[{"raw_affiliation_string":"Strathclyde University, Glasgow, Scotland UK","institution_ids":["https://openalex.org/I181647926"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045420626","display_name":"Shlomo Geva","orcid":"https://orcid.org/0000-0003-1340-2802"},"institutions":[{"id":"https://openalex.org/I160993911","display_name":"Queensland University of Technology","ror":"https://ror.org/03pnv4752","country_code":"AU","type":"education","lineage":["https://openalex.org/I160993911"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Shlomo Geva","raw_affiliation_strings":["Queensland University of Technology, Brisbane, Australia"],"affiliations":[{"raw_affiliation_string":"Queensland University of Technology, Brisbane, Australia","institution_ids":["https://openalex.org/I160993911"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5075371460"],"corresponding_institution_ids":["https://openalex.org/I160993911"],"apc_list":null,"apc_paid":null,"fwci":3.8332,"has_fulltext":false,"cited_by_count":58,"citation_normalized_percentile":{"value":0.94039599,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1237","last_page":"1240"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9805999994277954,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.970300018787384,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7922745943069458},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7290153503417969},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.6192486882209778},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6137973666191101},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6054573655128479},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.5692926645278931},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5493719577789307},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5417668223381042},{"id":"https://openalex.org/keywords/inclusion","display_name":"Inclusion (mineral)","score":0.47054606676101685},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.43919456005096436},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.38853806257247925},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35473188757896423},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.08170083165168762},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.07562202215194702}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7922745943069458},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7290153503417969},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.6192486882209778},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6137973666191101},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6054573655128479},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.5692926645278931},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5493719577789307},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5417668223381042},{"id":"https://openalex.org/C109359841","wikidata":"https://www.wikidata.org/wiki/Q728944","display_name":"Inclusion (mineral)","level":2,"score":0.47054606676101685},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.43919456005096436},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.38853806257247925},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35473188757896423},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.08170083165168762},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.07562202215194702},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3077136.3080707","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080707","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:strathprints.strath.ac.uk:62696","is_oa":false,"landing_page_url":"https://strathprints.strath.ac.uk/view/author/517553.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4306402226","display_name":"Strathprints: The University of Strathclyde institutional repository (University of Strathclyde)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I181647926","host_organization_name":"University of Strathclyde","host_organization_lineage":["https://openalex.org/I181647926"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"NonPeerReviewed"},{"id":"pmh:oai:eprints.qut.edu.au:107558","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306402607","display_name":"QUT ePrints (Queensland University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I160993911","host_organization_name":"Queensland University of Technology","host_organization_lineage":["https://openalex.org/I160993911"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"Chapter in Book, Report or Conference volume"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.75}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W148283619","https://openalex.org/W168362576","https://openalex.org/W168544607","https://openalex.org/W1506859583","https://openalex.org/W1576855214","https://openalex.org/W1851653583","https://openalex.org/W1888011339","https://openalex.org/W2024963031","https://openalex.org/W2079370125","https://openalex.org/W2095601341","https://openalex.org/W2111722073","https://openalex.org/W2138118601","https://openalex.org/W2147469877","https://openalex.org/W2170508680","https://openalex.org/W2484269232","https://openalex.org/W2923640488","https://openalex.org/W4313371821"],"related_works":["https://openalex.org/W3185553570","https://openalex.org/W2085384747","https://openalex.org/W2155401483","https://openalex.org/W2088166309","https://openalex.org/W3045247719","https://openalex.org/W2106071040","https://openalex.org/W2102102835","https://openalex.org/W4312133475","https://openalex.org/W2070311887","https://openalex.org/W2115099528"],"abstract_inverted_index":{"This":[0,67],"paper":[1],"introduces":[2],"a":[3,38],"test":[4,98,159],"collection":[5,99,131,147,160],"for":[6,18,37,62,94,128,135],"evaluating":[7],"the":[8,65,89,109,130,143,146],"effectiveness":[9,81],"of":[10,76,101,145],"different":[11],"methods":[12],"used":[13,134],"to":[14,51,59,148],"retrieve":[15,52],"research":[16,105],"studies":[17,28,106],"inclusion":[19,32,63],"in":[20,64],"systematic":[21,121],"reviews.":[22],"Systematic":[23,34],"reviews":[24,35],"appraise":[25],"and":[26,71,91,123,142],"synthesise":[27],"that":[29,78],"meet":[30],"specific":[31],"criteria.":[33],"intended":[36],"biomedical":[39],"science":[40],"audience":[41],"use":[42,144],"boolean":[43],"queries":[44],"with":[45],"many,":[46],"often":[47],"complex,":[48],"search":[49],"clauses":[50],"studies;":[53],"these":[54],"are":[55,140],"then":[56],"manually":[57],"screened":[58],"determine":[60],"eligibility":[61],"review.":[66],"process":[68],"is":[69,156,161],"expensive":[70],"time":[72],"consuming.":[73],"The":[74,158],"development":[75],"systems":[77],"improve":[79],"retrieval":[80,137],"will":[82],"have":[83],"an":[84],"immediate":[85],"impact":[86],"by":[87],"reducing":[88],"complexity":[90],"resources":[92],"required":[93],"this":[95],"process.":[96],"Our":[97],"consists":[100],"approximately":[102],"26":[103],"million":[104],"extracted":[107,118],"from":[108,119],"freely":[110],"available":[111,162],"MEDLINE":[112],"database,":[113],"94":[114],"review":[115],"(query)":[116],"topics":[117],"Cochrane":[120],"reviews,":[122],"corresponding":[124],"relevance":[125],"assessments.":[126],"Tasks":[127],"which":[129],"can":[132],"be":[133],"information":[136],"system":[138],"evaluation":[139],"described":[141],"evaluate":[149],"common":[150],"baselines":[151],"within":[152],"one":[153],"such":[154],"task":[155],"demonstrated.":[157],"at":[163],"https://github.com/ielab/SIGIR2017-PICO-Collection.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":3}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}