{"id":"https://openalex.org/W7168373441","doi":"https://doi.org/10.1145/3805712.3808585","title":"<i>LiveRAG:</i> A Diverse Q&amp;A Dataset with Varying Difficulty Level for RAG Evaluation","display_name":"<i>LiveRAG:</i> A Diverse Q&amp;A Dataset with Varying Difficulty Level for RAG Evaluation","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168373441","doi":"https://doi.org/10.1145/3805712.3808585"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3808585","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808585","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3808585","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140885526","display_name":"David Carmel","orcid":"https://orcid.org/0000-0003-1161-7084"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"David Carmel","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0000-0003-1161-7084","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140790095","display_name":"Simone Filice","orcid":"https://orcid.org/0009-0002-6735-9950"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Simone Filice","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0009-0002-6735-9950","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140780446","display_name":"Guy Horowitz","orcid":"https://orcid.org/0009-0001-5093-7235"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Guy Horowitz","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0009-0001-5093-7235","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140843921","display_name":"Yoelle Maarek","orcid":"https://orcid.org/0000-0003-3160-7115"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Yoelle Maarek","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0000-0003-3160-7115","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140772974","display_name":"Alexander Shtoff","orcid":"https://orcid.org/0009-0000-1147-3872"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Alexander Shtoff","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0009-0000-1147-3872","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140873578","display_name":"Oren Somekh","orcid":"https://orcid.org/0009-0003-8454-5568"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Oren Somekh","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0009-0003-8454-5568","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140779628","display_name":"Ran Tavory","orcid":"https://orcid.org/0009-0000-5279-9596"},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Ran Tavory","raw_affiliation_strings":["Technology Innovation Institute (TII), Haifa, Israel"],"raw_orcid":"https://orcid.org/0009-0000-5279-9596","affiliations":[{"raw_affiliation_string":"Technology Innovation Institute (TII), Haifa, Israel","institution_ids":["https://openalex.org/I174306211"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I174306211"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3109","last_page":"3116"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.2946000099182129},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.2799000144004822},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.26159998774528503},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.2549000084400177},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.25}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5422000288963318},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4875999987125397},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.2946000099182129},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.26159998774528503},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2549000084400177},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.25},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2418999969959259},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.24040000140666962}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3808585","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808585","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805712.3808585","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808585","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1535520578","https://openalex.org/W2130900715","https://openalex.org/W2252136820","https://openalex.org/W2889787757","https://openalex.org/W2891340257","https://openalex.org/W2912924812","https://openalex.org/W2949849869","https://openalex.org/W2950681488","https://openalex.org/W2963339397","https://openalex.org/W2963430447","https://openalex.org/W2963964168","https://openalex.org/W3115947671","https://openalex.org/W3123742938","https://openalex.org/W3169283738","https://openalex.org/W3175204434","https://openalex.org/W3177468934","https://openalex.org/W4226199262","https://openalex.org/W4285220249","https://openalex.org/W4385573898","https://openalex.org/W4385740286","https://openalex.org/W4411630045","https://openalex.org/W4412886939","https://openalex.org/W4415799040"],"related_works":[],"abstract_inverted_index":{"With":[0],"Retrieval-Augmented":[1],"Generation":[2],"(RAG)":[3],"becoming":[4],"more":[5,7,162],"and":[6,36,111,138,160],"prominent":[8],"in":[9,141],"generative":[10],"AI":[11],"solutions,":[12],"there":[13],"is":[14,50,70,106],"an":[15,117],"emerging":[16],"need":[17],"for":[18,98],"systematically":[19],"evaluating":[20,99],"its":[21],"effectiveness.":[22],"We":[23],"introduce":[24],"the":[25,53,57,82,86,128,132,152],"LiveRAG":[26,59,147],"benchmark,":[27],"a":[28],"publicly":[29],"available":[30,78],"dataset":[31],"of":[32,43,135],"895":[33],"synthetic":[34,48],"questions":[35],"answers":[37],"designed":[38],"to":[39,79,122],"support":[40],"systematic":[41,158],"evaluation":[42],"RAG-based":[44],"Q&A":[45,164],"systems.":[46,165],"This":[47],"benchmark":[49,148],"derived":[51,114],"from":[52,115],"one":[54],"used":[55,97],"during":[56,81],"SIGIR'2025":[58],"challenge,":[60,83],"where":[61],"competitors":[62,80],"were":[63,96],"evaluated":[64],"under":[65],"strict":[66],"time":[67],"constraints.":[68],"It":[69],"augmented":[71],"with":[72,90,108],"information":[73],"that":[74],"was":[75],"not":[76],"made":[77],"such":[84],"as":[85],"ground-truth":[87],"answers,":[88],"together":[89],"their":[91,139],"associated":[92,107],"supporting":[93],"claims":[94],"which":[95],"competitors'":[100,123],"answers.":[101],"In":[102],"addition,":[103],"each":[104],"question":[105,130],"estimated":[109],"difficulty":[110,136],"discriminability":[112],"scores,":[113],"applying":[116],"Item":[118],"Response":[119],"Theory":[120],"model":[121],"responses.":[124],"Our":[125],"analysis":[126],"highlights":[127],"benchmark's":[129],"diversity,":[131],"wide":[133],"range":[134],"levels,":[137],"usefulness":[140],"differentiating":[142],"between":[143],"system":[144],"capabilities.":[145],"The":[146],"will":[149],"hopefully":[150],"help":[151],"community":[153],"advance":[154],"RAG":[155],"research,":[156],"conduct":[157],"evaluation,":[159],"develop":[161],"robust":[163]},"counts_by_year":[],"updated_date":"2026-07-17T05:52:16.776730","created_date":"2026-07-16T00:00:00"}