{"id":"https://openalex.org/W4403753183","doi":"https://doi.org/10.48550/arxiv.2407.11005","title":"RAGBench: Explainable Benchmark for Retrieval-Augmented Generation Systems","display_name":"RAGBench: Explainable Benchmark for Retrieval-Augmented Generation Systems","publication_year":2024,"publication_date":"2024-06-25","ids":{"openalex":"https://openalex.org/W4403753183","doi":"https://doi.org/10.48550/arxiv.2407.11005"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.11005","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.11005","pdf_url":"https://arxiv.org/pdf/2407.11005","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.11005","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5099039182","display_name":"Robert Friel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Friel, Robert","raw_affiliation_strings":["Galileo Technologies Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Galileo Technologies Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080520923","display_name":"Masha Belyi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Belyi, Masha","raw_affiliation_strings":["Galileo Technologies Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Galileo Technologies Inc","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016375024","display_name":"Atindriyo Sanyal","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sanyal, Atindriyo","raw_affiliation_strings":["Galileo Technologies Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Galileo Technologies Inc","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":180,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9473000168800354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9473000168800354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0414000004529953,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.00279999990016222,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9990000128746033},{"id":"https://openalex.org/keywords/rag-evaluation","display_name":"RAG evaluation","score":0.8199999928474426},{"id":"https://openalex.org/keywords/benchmark-dataset","display_name":"benchmark dataset","score":0.8889999985694885},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8109999895095825},{"id":"https://openalex.org/keywords/document-retrieval","display_name":"document retrieval","score":0.8050000071525574},{"id":"https://openalex.org/keywords/user-manual","display_name":"user manual","score":0.843999981880188},{"id":"https://openalex.org/keywords/roberta-fine-tuning","display_name":"RoBERTa fine-tuning","score":0.7860000133514404},{"id":"https://openalex.org/keywords/annotated-data","display_name":"annotated dataset","score":0.367000013589859}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7761251926422119},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6140425801277161},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4815412163734436},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08194932341575623},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.034458935260772705}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.11005","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.11005","pdf_url":"https://arxiv.org/pdf/2407.11005","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.11005","doi":"https://doi.org/10.48550/arxiv.2407.11005","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.11005","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.11005","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.11005","pdf_url":"https://arxiv.org/pdf/2407.11005","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403753183.pdf","grobid_xml":"https://content.openalex.org/works/W4403753183.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2789566302","https://openalex.org/W2947497897","https://openalex.org/W2970482702","https://openalex.org/W3034478991","https://openalex.org/W3083182073","https://openalex.org/W3174986053","https://openalex.org/W3176098057","https://openalex.org/W3198243861","https://openalex.org/W4317898419","https://openalex.org/W4385570384","https://openalex.org/W4389518925","https://openalex.org/W4389519460","https://openalex.org/W4389520670","https://openalex.org/W4393147129","https://openalex.org/W4398757454","https://openalex.org/W4399555009","https://openalex.org/W4401042808","https://openalex.org/W4402683739","https://openalex.org/W4415798682","https://openalex.org/W4415799040"],"related_works":["https://openalex.org/W4416249945","https://openalex.org/W4411113072","https://openalex.org/W4406892583","https://openalex.org/W4415190889","https://openalex.org/W7134291307","https://openalex.org/W7164908464","https://openalex.org/W4412931388","https://openalex.org/W4412966133","https://openalex.org/W6966545553","https://openalex.org/W4403560390"],"abstract_inverted_index":{"Retrieval-Augmented":[0],"Generation":[1],"(RAG)":[2],"has":[3],"become":[4],"a":[5,29,34,51,67,131,184],"standard":[6],"architectural":[7],"pattern":[8],"for":[9,37,121,164],"incorporating":[10],"domain-specific":[11,35],"knowledge":[12],"into":[13],"user-facing":[14],"chat":[15],"applications":[16],"powered":[17],"by":[18,27],"Large":[19],"Language":[20],"Models":[21],"(LLMs).":[22],"RAG":[23,64,89,103,137,143,159,177,190,214],"systems":[24,65],"are":[25,108],"characterized":[26],"(1)":[28],"document":[30],"retriever":[31],"that":[32,49,175],"queries":[33],"corpus":[36],"context":[38],"information":[39],"relevant":[40,120],"to":[41,70,181],"an":[42,47],"input":[43],"query,":[44],"and":[45,58,77,101,135,201],"(2)":[46],"LLM":[48],"generates":[50],"response":[52],"based":[53],"on":[54,188],"the":[55,71,85,127,147,189,203,211],"provided":[56],"query":[57],"context.":[59],"However,":[60],"comprehensive":[61],"evaluation":[62,75,129,138,157,178,191,215],"of":[63,73,92,133,158,167,205,213],"remains":[66],"challenge":[68],"due":[69],"lack":[72],"unified":[74],"criteria":[76],"annotated":[78],"datasets.":[79],"In":[80],"response,":[81],"we":[82,125,173],"introduce":[83],"RAGBench:":[84],"first":[86],"comprehensive,":[87],"large-scale":[88],"benchmark":[90],"dataset":[91,149],"100k":[93],"examples.":[94],"It":[95],"covers":[96],"five":[97],"unique":[98],"industry-specific":[99],"domains":[100],"various":[102],"task":[104],"types.":[105],"RAGBench":[106,152,206],"examples":[107],"sourced":[109],"from":[110],"industry":[111,122],"corpora":[112],"such":[113],"as":[114],"user":[115],"manuals,":[116],"making":[117],"it":[118],"particularly":[119],"applications.":[123,169],"Further,":[124],"formalize":[126],"TRACe":[128,208],"framework:":[130],"set":[132],"explainable":[134,153],"actionable":[136,162],"metrics":[139],"applicable":[140],"across":[141],"all":[142],"domains.":[144],"We":[145,193],"release":[146],"labeled":[148],"at":[150],"https://huggingface.co/datasets/rungalileo/ragbench.":[151],"labels":[154],"facilitate":[155],"holistic":[156],"systems,":[160],"enabling":[161],"feedback":[163],"continuous":[165],"improvement":[166],"production":[168],"Thorough":[170],"extensive":[171],"benchmarking,":[172],"find":[174],"LLM-based":[176],"methods":[179],"struggle":[180],"compete":[182],"with":[183,207],"finetuned":[185],"RoBERTa":[186],"model":[187],"task.":[192],"identify":[194],"areas":[195],"where":[196],"existing":[197],"approaches":[198],"fall":[199],"short":[200],"propose":[202],"adoption":[204],"towards":[209],"advancing":[210],"state":[212],"systems.":[216]},"counts_by_year":[{"year":2026,"cited_by_count":124},{"year":2025,"cited_by_count":51},{"year":2024,"cited_by_count":5}],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2024-10-26T00:00:00"}