{"id":"https://openalex.org/W7168405860","doi":"https://doi.org/10.1145/3805712.3808629","title":"Total Recall QA: A Verifiable Evaluation Suite for Deep Research Agents","display_name":"Total Recall QA: A Verifiable Evaluation Suite for Deep Research Agents","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168405860","doi":"https://doi.org/10.1145/3805712.3808629"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3808629","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808629","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3808629","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140780711","display_name":"Mahta Rafiee","orcid":"https://orcid.org/0009-0007-1410-4897"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mahta Rafiee","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, USA"],"raw_orcid":"https://orcid.org/0009-0007-1410-4897","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140865597","display_name":"Heydar Soudani","orcid":"https://orcid.org/0000-0003-0393-8662"},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Heydar Soudani","raw_affiliation_strings":["Radboud University, Nijmegen, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-0393-8662","affiliations":[{"raw_affiliation_string":"Radboud University, Nijmegen, Netherlands","institution_ids":["https://openalex.org/I145872427"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140920049","display_name":"Zahra Abbasiantaeb","orcid":"https://orcid.org/0000-0002-4046-3419"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Zahra Abbasiantaeb","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-4046-3419","affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140930373","display_name":"Mohammad Aliannejadi","orcid":"https://orcid.org/0000-0002-9447-4172"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Mohammad Aliannejadi","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-9447-4172","affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140839677","display_name":"Faegheh Hasibi","orcid":"https://orcid.org/0009-0006-9986-482X"},"institutions":[{"id":"https://openalex.org/I145872427","display_name":"Radboud University Nijmegen","ror":"https://ror.org/016xsfp80","country_code":"NL","type":"education","lineage":["https://openalex.org/I145872427"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Faegheh Hasibi","raw_affiliation_strings":["Radboud University, Nijmegen, Netherlands"],"raw_orcid":"https://orcid.org/0009-0006-9986-482X","affiliations":[{"raw_affiliation_string":"Radboud University, Nijmegen, Netherlands","institution_ids":["https://openalex.org/I145872427"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140870383","display_name":"Hamed Zamani","orcid":"https://orcid.org/0000-0002-0800-3340"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hamed Zamani","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, USA"],"raw_orcid":"https://orcid.org/0000-0002-0800-3340","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3365","last_page":"3373"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.6610999703407288},{"id":"https://openalex.org/keywords/verifiable-secret-sharing","display_name":"Verifiable secret sharing","score":0.6305999755859375},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.39070001244544983},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.25360000133514404}],"concepts":[{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.6610999703407288},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.64410001039505},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.6305999755859375},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4049000144004822},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.39070001244544983},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.25360000133514404},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2533999979496002},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.24869999289512634},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2410999983549118},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.24089999496936798}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3808629","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808629","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805712.3808629","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808629","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W2080133951","https://openalex.org/W2740492458","https://openalex.org/W2962716111","https://openalex.org/W2971196067","https://openalex.org/W4224919569","https://openalex.org/W4252076394","https://openalex.org/W4284664419","https://openalex.org/W4310923309","https://openalex.org/W4385570281","https://openalex.org/W4388926373","https://openalex.org/W4389081052","https://openalex.org/W4392487838","https://openalex.org/W4399554477","https://openalex.org/W4400526284","https://openalex.org/W4400526908","https://openalex.org/W4402670101","https://openalex.org/W4402670856","https://openalex.org/W4403882995","https://openalex.org/W4405253355","https://openalex.org/W4409166856","https://openalex.org/W4411119806","https://openalex.org/W4414854692","https://openalex.org/W4414991262","https://openalex.org/W4415069437","https://openalex.org/W4415271421","https://openalex.org/W4416176141","https://openalex.org/W4416189150","https://openalex.org/W4417089504","https://openalex.org/W6891936057","https://openalex.org/W7077555568","https://openalex.org/W7116876396"],"related_works":[],"abstract_inverted_index":{"Deep":[0],"research":[1,45,63,79,100,144,178],"agents":[2,46,101],"have":[3],"emerged":[4],"as":[5,149],"LLM-based":[6],"systems":[7],"designed":[8],"to":[9,20,162],"perform":[10],"multi-step":[11],"information":[12,26,29],"seeking":[13],"and":[14,37,57,94,118,153,160,176,180,184],"reasoning":[15],"over":[16],"large,":[17],"open-domain":[18],"sources":[19],"answer":[21],"complex":[22],"questions":[23],"by":[24,77],"synthesizing":[25],"from":[27,122,147],"multiple":[28],"sources.":[30],"Given":[31],"the":[32,35,87,105,164,171],"complexity":[33],"of":[34,43,55,89,166],"task":[36,88],"despite":[38],"various":[39],"recent":[40],"efforts,":[41],"evaluation":[42,102,117],"deep":[44,62,99,143,177],"remains":[47],"fundamentally":[48],"challenging.":[49],"This":[50],"paper":[51],"identifies":[52],"a":[53,96,123,129,142,150,154],"list":[54],"requirements":[56],"optional":[58],"properties":[59],"for":[60,98,187],"evaluating":[61],"agents.":[64],"We":[65,169],"observe":[66],"that":[67,103],"existing":[68],"benchmarks":[69],"do":[70],"not":[71],"satisfy":[72],"all":[73],"identified":[74,106],"requirements.":[75],"Inspired":[76],"prior":[78],"on":[80],"TREC":[81],"Total":[82,90],"Recall":[83,91],"Tracks,":[84],"we":[85,139],"introduce":[86],"Question":[92],"Answering":[93],"develop":[95],"framework":[97,109],"satisfies":[104],"criteria.":[107],"Our":[108],"constructs":[110],"single-answer,":[111],"total":[112],"recall":[113],"queries":[114],"with":[115,128,173],"precise":[116],"relevance":[119],"judgments":[120],"derived":[121],"structured":[124],"knowledge":[125,158],"base":[126,159],"paired":[127],"text":[130],"corpus,":[131],"enabling":[132],"large-scale":[133],"data":[134,167],"construction.":[135],"Using":[136],"this":[137],"framework,":[138],"build":[140],"TRQA,":[141],"benchmark":[145,170],"constructed":[146],"Wikidata-Wikipedia":[148],"real-world":[151],"source":[152],"synthetically":[155],"generated":[156],"e-commerce":[157],"corpus":[161],"mitigate":[163],"effects":[165],"contamination.":[168],"collection":[172],"representative":[174],"retriever":[175],"models":[179],"establish":[181],"baseline":[182],"retrieval":[183],"end-to-end":[185],"results":[186],"future":[188],"comparative":[189],"evaluation.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-17T05:52:16.776730","created_date":"2026-07-16T00:00:00"}