{"id":"https://openalex.org/W2742025582","doi":"https://doi.org/10.1145/3077136.3080812","title":"Navigating Imprecision in Relevance Assessments on the Road to Total Recall","display_name":"Navigating Imprecision in Relevance Assessments on the Road to Total Recall","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2742025582","doi":"https://doi.org/10.1145/3077136.3080812","mag":"2742025582"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3080812","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3077136.3080812","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3080812&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=3080812&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084091807","display_name":"Gordon V. Cormack","orcid":"https://orcid.org/0000-0002-5890-0293"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Gordon V. Cormack","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083981495","display_name":"Maura R. Grossman","orcid":"https://orcid.org/0000-0003-2279-4262"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Maura R. Grossman","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5","last_page":"14"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.8040902614593506},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7945573329925537},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7367386817932129},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.684481143951416},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.6221733093261719},{"id":"https://openalex.org/keywords/relevance-feedback","display_name":"Relevance feedback","score":0.5988147258758545},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5889502763748169},{"id":"https://openalex.org/keywords/data-curation","display_name":"Data curation","score":0.434115469455719},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4039759635925293},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3660832345485687},{"id":"https://openalex.org/keywords/image-retrieval","display_name":"Image retrieval","score":0.2835366725921631},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.1766955852508545},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.13501006364822388},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.08564493060112}],"concepts":[{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.8040902614593506},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7945573329925537},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7367386817932129},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.684481143951416},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.6221733093261719},{"id":"https://openalex.org/C2779532271","wikidata":"https://www.wikidata.org/wiki/Q445558","display_name":"Relevance feedback","level":4,"score":0.5988147258758545},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5889502763748169},{"id":"https://openalex.org/C91632574","wikidata":"https://www.wikidata.org/wiki/Q15088675","display_name":"Data curation","level":2,"score":0.434115469455719},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4039759635925293},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3660832345485687},{"id":"https://openalex.org/C1667742","wikidata":"https://www.wikidata.org/wiki/Q10927554","display_name":"Image retrieval","level":3,"score":0.2835366725921631},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.1766955852508545},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.13501006364822388},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.08564493060112},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3077136.3080812","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3077136.3080812","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3080812&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3077136.3080812","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3077136.3080812","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3080812&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2742025582.pdf","grobid_xml":"https://content.openalex.org/works/W2742025582.grobid-xml"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W15201044","https://openalex.org/W1554394657","https://openalex.org/W1685426458","https://openalex.org/W1700671489","https://openalex.org/W1964488075","https://openalex.org/W1971609128","https://openalex.org/W2001947543","https://openalex.org/W2008285050","https://openalex.org/W2009954908","https://openalex.org/W2022995284","https://openalex.org/W2029075138","https://openalex.org/W2033912027","https://openalex.org/W2083605078","https://openalex.org/W2087511931","https://openalex.org/W2120308175","https://openalex.org/W2122053905","https://openalex.org/W2141527751","https://openalex.org/W2341384651","https://openalex.org/W2460936583","https://openalex.org/W2511371195","https://openalex.org/W2533861448","https://openalex.org/W2558888286","https://openalex.org/W2565695915","https://openalex.org/W2573384236","https://openalex.org/W2604321711","https://openalex.org/W2887522542","https://openalex.org/W4233471163","https://openalex.org/W4236645963","https://openalex.org/W4251188927","https://openalex.org/W6680807580"],"related_works":["https://openalex.org/W2358294942","https://openalex.org/W2121179064","https://openalex.org/W2047973478","https://openalex.org/W1970945111","https://openalex.org/W2386201655","https://openalex.org/W4246004437","https://openalex.org/W4381744813","https://openalex.org/W2032182853","https://openalex.org/W4367460280","https://openalex.org/W4328049145"],"abstract_inverted_index":{"Technology-assisted":[0],"review":[1,28,187],"(\"TAR\")":[2],"systems":[3,100,176],"seek":[4],"to":[5,11,90,111,168],"achieve":[6,40],"\"total":[7],"recall\";":[8],"that":[9,33,62,87,101,154,173,212,240,277],"is,":[10],"approach,":[12],"as":[13,15,53,93,95,116,121,130,143,145,227],"nearly":[14],"possible,":[16],"the":[17,44,54,64,78,126,138,146,160,181,199,235,249,262,282],"ideal":[18],"of":[19,80,125,134,140,148,185,208,253,261,301],"100%":[20,23,103,106],"recall":[21,46,104,244,284],"and":[22,47,77,105,183,204,216,245,258,285],"precision,":[24,107,287],"while":[25,288],"minimizing":[26],"human":[27,113,186],"effort.":[29],"The":[30,132,232],"literature":[31],"reports":[32],"TAR":[34,99,195,241,298],"methods":[35],"using":[36,196,234],"relevance":[37,149],"feedback":[38],"can":[39,177],"considerably":[41],"greater":[42],"than":[43,159,248,269,292],"65%":[45,48],"precision":[49,247],"reported":[50],"by":[51,193,218,251,266],"Voorhees":[52],"\"practical":[55],"upper":[56],"bound":[57],"on":[58,179],"retrieval":[59],"performance...":[60],"since":[61],"is":[63,109,191],"level":[65],"at":[66],"which":[67],"humans":[68],"agree":[69],"with":[70],"one":[71],"another\"":[72],"(Variations":[73],"in":[74,88,223,299],"Relevance":[75],"Judgments":[76],"Measurement":[79],"Retrieval":[81],"Effectiveness,":[82],"2000).":[83],"This":[84,189],"work":[85],"argues":[86],"order":[89],"build":[91],"-":[92,98],"well":[94,144],"to,":[96],"evaluate":[97],"approach":[102],"it":[108],"necessary":[110],"model":[112,135],"assessment,":[114],"not":[115],"absolute":[117],"ground":[118],"truth,":[119],"but":[120],"an":[122],"indirect":[123],"indicator":[124],"amorphous":[127],"property":[128],"known":[129],"\"relevance.\"":[131],"choice":[133],"impacts":[136],"both":[137,180],"evaluation":[139],"system":[141],"effectiveness,":[142],"simulation":[147],"feedback.":[150],"Models":[151],"are":[152],"presented":[153],"better":[155,286],"fit":[156],"available":[157],"data":[158,238],"infallible":[161],"ground-truth":[162],"model.":[163],"These":[164],"models":[165],"suggest":[166],"ways":[167],"improve":[169,178],"TAR-system":[170],"effectiveness":[171],"so":[172],"hybrid":[174],"human-computer":[175],"accuracy":[182],"efficiency":[184],"alone.":[188],"hypothesis":[190],"tested":[192],"simulating":[194],"two":[197,254,270],"datasets:":[198],"TREC":[200,236],"4":[201,237],"AdHoc":[202],"collection,":[203],"a":[205,219],"dataset":[206],"consisting":[207],"401,960":[209,293],"email":[210,263],"messages":[211],"were":[213],"manually":[214],"reviewed":[215],"classified":[217],"single":[220],"individual,":[221],"Roger,":[222,267],"his":[224,273],"official":[225],"capacity":[226],"Senior":[228],"State":[229],"Records":[230],"Archivist.":[231],"results":[233],"show":[239],"achieves":[242],"higher":[243,246],"assessments":[250],"either":[252],"independent":[255],"NIST":[256],"assessors,":[257],"blind":[259],"adjudication":[260],"dataset,":[264],"conducted":[265],"more":[268],"years":[271],"after":[272],"original":[274],"review,":[275],"shows":[276],"he":[278,296],"could":[279],"have":[280],"achieved":[281],"same":[283],"reviewing":[289],"substantially":[290],"fewer":[291],"emails,":[294],"had":[295],"employed":[297],"place":[300],"exhaustive":[302],"manual":[303],"review.":[304]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2022,"cited_by_count":3},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}