{"id":"https://openalex.org/W7212045926","doi":"https://doi.org/10.48550/arxiv.2609.05760","title":"RAGMark: A Comprehensive Framework for Benchmarking Retrieval-Augmented Generation Systems","display_name":"RAGMark: A Comprehensive Framework for Benchmarking Retrieval-Augmented Generation Systems","publication_year":2026,"publication_date":"2026-09-04","ids":{"openalex":"https://openalex.org/W7212045926","doi":"https://doi.org/10.48550/arxiv.2609.05760"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.05760","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.05760","pdf_url":"https://arxiv.org/pdf/2609.05760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.05760","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091839586","display_name":"Zlatan Feric","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feric, Zlatan","raw_affiliation_strings":["Northeastern University , Boston , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University , Boston , MA , USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119958875","display_name":"Amir Taherin","orcid":"https://orcid.org/0000-0002-3646-0901","observed_orcids":["https://orcid.org/0000-0002-3646-0901"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Taherin, Amir","raw_affiliation_strings":["Northeastern University , Boston , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University , Boston , MA , USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148419740","display_name":"Bin Ren","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ren, Bin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100651384","display_name":"Yanzhi Wang","orcid":"https://orcid.org/0000-0002-3024-7990","observed_orcids":["https://orcid.org/0000-0002-3024-7990","https://orcid.org/0009-0009-3924-6466","https://orcid.org/0009-0008-4749-5860"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]},{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Yanzhi","raw_affiliation_strings":["College of William & Mary , VA , USA","Northeastern University , Boston , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of William & Mary , VA , USA","institution_ids":["https://openalex.org/I16285277"]},{"raw_affiliation_string":"Northeastern University , Boston , MA , USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125574748","display_name":"Jennifer Dy","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dy, Jennifer","raw_affiliation_strings":["Northeastern University , Boston , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University , Boston , MA , USA","institution_ids":["https://openalex.org/I12912129"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061128237","display_name":"David R Kaeli","orcid":"https://orcid.org/0000-0002-5692-0151","observed_orcids":["https://orcid.org/0000-0002-5692-0151"]},"institutions":[{"id":"https://openalex.org/I12912129","display_name":"Northeastern University","ror":"https://ror.org/04t5xt781","country_code":"US","type":"education","lineage":["https://openalex.org/I12912129"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kaeli, David","raw_affiliation_strings":["Northeastern University , Boston , MA , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northeastern University , Boston , MA , USA","institution_ids":["https://openalex.org/I12912129"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9258000254631042,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9258000254631042,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.03420000150799751,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.01810000091791153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9750000238418579},{"id":"https://openalex.org/keywords/benchmarking-framework","display_name":"benchmarking framework","score":0.9649999737739563},{"id":"https://openalex.org/keywords/vector-databases","display_name":"vector databases","score":0.7570000290870667},{"id":"https://openalex.org/keywords/re-ranking","display_name":"re-ranking","score":0.6679999828338623},{"id":"https://openalex.org/keywords/kv-cache-compression","display_name":"KV cache compression","score":0.6970000267028809},{"id":"https://openalex.org/keywords/context-reduction","display_name":"context reduction","score":0.6959999799728394},{"id":"https://openalex.org/keywords/energy-consumption","display_name":"energy consumption","score":0.8119999766349792},{"id":"https://openalex.org/keywords/open-domain-question-answering","display_name":"open-domain question answering","score":0.6990000009536743},{"id":"https://openalex.org/keywords/memory-bandwidth","display_name":"memory bandwidth","score":0.5299999713897705},{"id":"https://openalex.org/keywords/time-to-first-token","display_name":"time-to-first-token","score":0.6259999871253967}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8532999753952026},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5465999841690063},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.45190000534057617},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4458000063896179},{"id":"https://openalex.org/C78548338","wikidata":"https://www.wikidata.org/wiki/Q2493","display_name":"Data compression","level":2,"score":0.4429999887943268},{"id":"https://openalex.org/C52723943","wikidata":"https://www.wikidata.org/wiki/Q1127410","display_name":"Serialization","level":2,"score":0.4171999990940094},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.41290000081062317},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.4032000005245209},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.39469999074935913},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.3702999949455261},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.36489999294281006},{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.3553999960422516},{"id":"https://openalex.org/C67212190","wikidata":"https://www.wikidata.org/wiki/Q104851","display_name":"Firmware","level":2,"score":0.3546999990940094},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.34850001335144043},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.34790000319480896},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.34200000762939453},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.3391999900341034},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.33730000257492065},{"id":"https://openalex.org/C2780165032","wikidata":"https://www.wikidata.org/wiki/Q16869822","display_name":"Energy consumption","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C2742236","wikidata":"https://www.wikidata.org/wiki/Q924713","display_name":"Efficient energy use","level":2,"score":0.3125},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.30720001459121704},{"id":"https://openalex.org/C7366592","wikidata":"https://www.wikidata.org/wiki/Q1255620","display_name":"Dram","level":2,"score":0.3059000074863434},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.2953999936580658},{"id":"https://openalex.org/C140745168","wikidata":"https://www.wikidata.org/wiki/Q1210082","display_name":"Tree traversal","level":2,"score":0.28999999165534973},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2888000011444092},{"id":"https://openalex.org/C25797200","wikidata":"https://www.wikidata.org/wiki/Q828137","display_name":"Compression ratio","level":3,"score":0.28769999742507935},{"id":"https://openalex.org/C10558101","wikidata":"https://www.wikidata.org/wiki/Q689855","display_name":"Smart grid","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.28519999980926514},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.2800000011920929},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.2687999904155731},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.266400009393692},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.26170000433921814},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.2558000087738037}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.05760","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.05760","pdf_url":"https://arxiv.org/pdf/2609.05760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.05760","doi":"https://doi.org/10.48550/arxiv.2609.05760","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.05760","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.05760","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.05760","pdf_url":"https://arxiv.org/pdf/2609.05760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G7514556556","display_name":"Puerto Rico Testsite for Exploring Contamination Threats (PROTECT)","funder_award_id":"P42ES017198","funder_id":"https://openalex.org/F4320337361","funder_display_name":"National Institute of Environmental Health Sciences"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337361","display_name":"National Institute of Environmental Health Sciences","ror":"https://ror.org/00j4k1h63"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7212045926.pdf","grobid_xml":"https://content.openalex.org/works/W7212045926.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7135014373","https://openalex.org/W7214796080","https://openalex.org/W4416429862","https://openalex.org/W7162193579","https://openalex.org/W6929588129","https://openalex.org/W4414371540","https://openalex.org/W7213348079","https://openalex.org/W4415798436","https://openalex.org/W7202155810","https://openalex.org/W6910238582"],"abstract_inverted_index":{"We":[0,110,159],"present":[1],"RAGMark,":[2,87],"a":[3],"modular":[4,67],"benchmarking":[5],"framework":[6,55],"for":[7],"advanced":[8],"Retrieval-Augmented":[9],"Generation":[10],"(RAG)":[11],"systems":[12],"targeting":[13],"small-scale":[14],"multi-GPU":[15],"environments.":[16],"RAGMark":[17,180],"evaluates":[18],"diverse":[19],"RAG":[20,60,91],"components,":[21,68],"including":[22],"retrievers,":[23],"vector":[24,107],"databases,":[25],"prompt-processing":[26],"methods,":[27,105],"and":[28,51,63,69,82,106,129,133,148,176],"generator":[29],"models,":[30],"while":[31,78,113,143],"collecting":[32],"detailed":[33],"per-stage":[34],"metrics":[35],"such":[36],"as":[37],"latency,":[38,173],"GPU":[39],"utilization,":[40],"memory":[41,127,174],"consumption,":[42],"power":[43],"usage,":[44],"time":[45],"to":[46,72,157],"first":[47],"token":[48],"(TTFT),":[49],"throughput,":[50],"answer":[52],"quality.":[53],"The":[54,179],"is":[56,70,183],"highly":[57],"extensible,":[58],"separating":[59],"stages,":[61],"timing,":[62],"resource":[64],"monitoring":[65],"into":[66],"designed":[71],"efficiently":[73],"sweep":[74],"large":[75],"configuration":[76],"spaces":[77],"minimizing":[79],"repeated":[80],"model":[81,101],"database":[83,108],"initialization":[84],"overhead.":[85],"Using":[86],"we":[88],"characterize":[89],"five":[90],"workloads":[92],"on":[93],"open-domain":[94],"QA":[95],"datasets":[96],"across":[97,125],"varying":[98],"retrieval":[99],"depths,":[100],"scales,":[102],"reranking,":[103],"compression":[104,134,140],"configurations.":[109],"show":[111],"that":[112],"autoregressive":[114],"generation":[115],"dominates":[116],"latency":[117],"in":[118],"naive":[119],"pipelines,":[120],"context-reduction":[121],"techniques":[122],"shift":[123],"bottlenecks":[124],"compute,":[126],"bandwidth,":[128],"preprocessing":[130],"stages.":[131],"Reranking":[132],"produce":[135],"compounding":[136],"benefits:":[137],"reranking":[138],"reduces":[139],"workload":[141],"itself,":[142],"both":[144],"jointly":[145],"reduce":[146],"prefill":[147],"KV-cache":[149],"traversal":[150],"costs,":[151],"lowering":[152],"energy":[153,177],"consumption":[154],"by":[155],"up":[156],"66%.":[158],"further":[160],"observe":[161],"strong":[162],"cross-stage":[163],"interactions,":[164],"where":[165],"small":[166],"upstream":[167],"context":[168],"reductions":[169],"cascade":[170],"through":[171],"downstream":[172],"traffic,":[175],"consumption.":[178],"source":[181],"code":[182],"publicly":[184],"available":[185],"at:":[186],"https://github.com/zferic/RAGMark.":[187]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-10T00:00:00"}