{"id":"https://openalex.org/W2811097964","doi":"https://doi.org/10.1145/3209978.3210102","title":"Technology Assisted Reviews","display_name":"Technology Assisted Reviews","publication_year":2018,"publication_date":"2018-06-27","ids":{"openalex":"https://openalex.org/W2811097964","doi":"https://doi.org/10.1145/3209978.3210102","mag":"2811097964"},"language":"en","primary_location":{"id":"doi:10.1145/3209978.3210102","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3209978.3210102","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pure.uva.nl/ws/files/37067657/p949_zou.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044839127","display_name":"Jie Zou","orcid":"https://orcid.org/0000-0003-0911-8542"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Jie Zou","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107206632","display_name":"Dan Li","orcid":"https://orcid.org/0009-0006-4079-2091"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Dan Li","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055639036","display_name":"Evangelos Kanoulas","orcid":"https://orcid.org/0000-0002-8312-0694"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]},{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Evangelos Kanoulas","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5044839127"],"corresponding_institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":2.0275,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.89888143,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"949","last_page":"952"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9811999797821045,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8196799755096436},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7112258672714233},{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.5863898396492004},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5818653702735901},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.5711964964866638},{"id":"https://openalex.org/keywords/relevance-feedback","display_name":"Relevance feedback","score":0.5056271553039551},{"id":"https://openalex.org/keywords/carry","display_name":"Carry (investment)","score":0.4973807632923126},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.48605969548225403},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.4669688045978546},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.4533424377441406},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37987834215164185},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.35680004954338074},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33744680881500244},{"id":"https://openalex.org/keywords/image-retrieval","display_name":"Image retrieval","score":0.08449956774711609}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8196799755096436},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7112258672714233},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.5863898396492004},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5818653702735901},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5711964964866638},{"id":"https://openalex.org/C2779532271","wikidata":"https://www.wikidata.org/wiki/Q445558","display_name":"Relevance feedback","level":4,"score":0.5056271553039551},{"id":"https://openalex.org/C2776299755","wikidata":"https://www.wikidata.org/wiki/Q432449","display_name":"Carry (investment)","level":2,"score":0.4973807632923126},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.48605969548225403},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.4669688045978546},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.4533424377441406},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37987834215164185},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35680004954338074},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33744680881500244},{"id":"https://openalex.org/C1667742","wikidata":"https://www.wikidata.org/wiki/Q10927554","display_name":"Image retrieval","level":3,"score":0.08449956774711609},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3209978.3210102","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3209978.3210102","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire/a793351e-d19c-4386-8e21-5f10d7707c9c","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/technology-assisted-reviews-finding-the-last-few-relevant-documents-by-asking-yesno-questions-to-reviewers(a793351e-d19c-4386-8e21-5f10d7707c9c).html","pdf_url":"https://pure.uva.nl/ws/files/37067657/p949_zou.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zou, J, Li, D & Kanoulas, E 2018, Technology assisted reviews: Finding the last few relevant documents by asking yes/no questions to reviewers. in SIGIR #41 proceedings : Ann Arbor, Michigan, USA, 08-12, July 2018. Association for Computing Machinery, New York, NY, pp. 949-952, 41st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2018, Ann Arbor, United States, 8/07/18. https://doi.org/10.1145/3209978.3210102","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:dare.uva.nl:publications/a793351e-d19c-4386-8e21-5f10d7707c9c","is_oa":true,"landing_page_url":"http://www.scopus.com/inward/record.url?scp=85051559451&partnerID=8YFLogxK","pdf_url":"http://hdl.handle.net/11245.1/a793351e-d19c-4386-8e21-5f10d7707c9c","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zou, J, Li, D & Kanoulas, E 2018, Technology assisted reviews: Finding the last few relevant documents by asking yes/no questions to reviewers. in SIGIR #41 proceedings : Ann Arbor, Michigan, USA, 08-12, July 2018. Association for Computing Machinery, New York, NY, pp. 949-952, 41st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2018, Ann Arbor, United States, 8/07/18. https://doi.org/10.1145/3209978.3210102","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:uvapub:oai:dare.uva.nl:publications/a793351e-d19c-4386-8e21-5f10d7707c9c","is_oa":true,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/technology-assisted-reviews-finding-the-last-few-relevant-documents-by-asking-yesno-questions-to-reviewers(a793351e-d19c-4386-8e21-5f10d7707c9c).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SIGIR #41 proceedings: Ann Arbor, Michigan, USA, 08-12, July 2018, 949 - 952","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"pmh:oai:dare.uva.nl:openaire/a793351e-d19c-4386-8e21-5f10d7707c9c","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/technology-assisted-reviews-finding-the-last-few-relevant-documents-by-asking-yesno-questions-to-reviewers(a793351e-d19c-4386-8e21-5f10d7707c9c).html","pdf_url":"https://pure.uva.nl/ws/files/37067657/p949_zou.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Zou, J, Li, D & Kanoulas, E 2018, Technology assisted reviews: Finding the last few relevant documents by asking yes/no questions to reviewers. in SIGIR #41 proceedings : Ann Arbor, Michigan, USA, 08-12, July 2018. Association for Computing Machinery, New York, NY, pp. 949-952, 41st Annual International ACM SIGIR Conference on Research and Development in Information Retrieval, SIGIR 2018, Ann Arbor, United States, 8/07/18. https://doi.org/10.1145/3209978.3210102","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2811097964.pdf","grobid_xml":"https://content.openalex.org/works/W2811097964.grobid-xml"},"referenced_works_count":6,"referenced_works":["https://openalex.org/W2029075138","https://openalex.org/W2123142779","https://openalex.org/W2145677303","https://openalex.org/W2165387257","https://openalex.org/W2741811485","https://openalex.org/W4313371821"],"related_works":["https://openalex.org/W2168627904","https://openalex.org/W2358294942","https://openalex.org/W2121179064","https://openalex.org/W1970945111","https://openalex.org/W2386201655","https://openalex.org/W4246004437","https://openalex.org/W4381744813","https://openalex.org/W4367460280","https://openalex.org/W4328049145","https://openalex.org/W2030220383"],"abstract_inverted_index":{"The":[0,126],"goal":[1],"of":[2,27,42,99,122],"a":[3,31,38,64,112],"technology-assisted":[4],"review":[5],"is":[6,36],"to":[7,67,124],"achieve":[8],"high":[9],"recall":[10],"with":[11],"low":[12],"human":[13],"effort.":[14,141],"Continuous":[15],"active":[16],"learning":[17],"algorithms":[18],"have":[19],"demonstrated":[20],"good":[21],"performance":[22,35,137],"in":[23,30,86,102],"locating":[24],"the":[25,48,57,79,95,103,119],"majority":[26],"relevant":[28,51],"documents":[29,52,74],"collection,":[32],"however":[33],"their":[34],"reaching":[37],"plateau":[39],"when":[40],"80%-90%":[41],"them":[43],"has":[44],"been":[45],"found.":[46],"Finding":[47],"last":[49,70],"few":[50],"typically":[53],"requires":[54],"exhaustively":[55],"reviewing":[56,140],"collection.":[58],"In":[59],"this":[60,109],"paper,":[61],"we":[62,110],"propose":[63],"novel":[65],"method":[66,77,116,133],"identify":[68],"these":[69],"few,":[71],"but":[72],"significant,":[73],"efficiently.":[75],"Our":[76],"makes":[78],"hypothesis":[80],"that":[81,89,117,130],"entities":[82],"carry":[83],"vital":[84],"information":[85],"documents,":[87],"and":[88],"reviewers":[90],"can":[91,134],"answer":[92],"questions":[93,123],"about":[94],"presence":[96],"or":[97],"absence":[98],"an":[100],"entity":[101],"missing":[104],"relevance":[105],"documents.":[106],"Based":[107],"on":[108],"devise":[111],"sequential":[113],"Bayesian":[114],"search":[115],"selects":[118],"optimal":[120],"sequence":[121],"ask.":[125],"experimental":[127],"results":[128],"show":[129],"our":[131],"proposed":[132],"greatly":[135],"improve":[136],"requiring":[138],"less":[139]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":2}],"updated_date":"2026-03-17T17:19:04.345684","created_date":"2018-07-10T00:00:00"}