{"id":"https://openalex.org/W4403571069","doi":"https://doi.org/10.48550/arxiv.2410.10594","title":"VisRAG: Vision-based Retrieval-augmented Generation on Multi-modality Documents","display_name":"VisRAG: Vision-based Retrieval-augmented Generation on Multi-modality Documents","publication_year":2024,"publication_date":"2024-10-14","ids":{"openalex":"https://openalex.org/W4403571069","doi":"https://doi.org/10.48550/arxiv.2410.10594"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.10594","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.10594","pdf_url":"https://arxiv.org/pdf/2410.10594","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.10594","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029226343","display_name":"Yu Shi","orcid":"https://orcid.org/0000-0002-8511-2110","observed_orcids":["https://orcid.org/0000-0002-8511-2110","https://orcid.org/0009-0005-9125-3293","https://orcid.org/0000-0002-6441-3221"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Shi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113234632","display_name":"Chaoyue Tang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Chaoyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044844079","display_name":"Bokai Xu","orcid":"https://orcid.org/0009-0008-0661-507X","observed_orcids":["https://orcid.org/0009-0008-0661-507X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Bokai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100862506","display_name":"Junbo Cui","orcid":"https://orcid.org/0009-0000-3031-5212","observed_orcids":["https://orcid.org/0009-0000-3031-5212"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Junbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111325358","display_name":"Junhao Ran","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ran, Junhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034266748","display_name":"Yukun Yan","orcid":"https://orcid.org/0000-0003-2421-7098","observed_orcids":["https://orcid.org/0000-0003-2421-7098"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan, Yukun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070196864","display_name":"Zhenghao Liu","orcid":"https://orcid.org/0000-0001-8067-1477","observed_orcids":["https://orcid.org/0000-0001-8067-1477"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zhenghao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100400179","display_name":"Shuo Wang","orcid":"https://orcid.org/0000-0002-3054-6608","observed_orcids":["https://orcid.org/0000-0002-3054-6608"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067406265","display_name":"Xu Han","orcid":"https://orcid.org/0000-0001-9861-4868","observed_orcids":["https://orcid.org/0000-0001-9861-4868"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100320723","display_name":"Zhiyuan Liu","orcid":"https://orcid.org/0000-0002-7709-2543","observed_orcids":["https://orcid.org/0000-0002-7709-2543","https://orcid.org/0009-0002-0195-7114"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Zhiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5046448314","display_name":"Maosong Sun","orcid":"https://orcid.org/0000-0002-6011-6115","observed_orcids":["https://orcid.org/0000-0002-6011-6115"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Maosong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9868000149726868,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.010499999858438969,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9909999966621399},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9639999866485596},{"id":"https://openalex.org/keywords/document-embeddings","display_name":"document embeddings","score":0.8230000138282776},{"id":"https://openalex.org/keywords/document-layout","display_name":"document layout","score":0.5630000233650208},{"id":"https://openalex.org/keywords/document-image-retrieval","display_name":"document image retrieval","score":0.734000027179718},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8180000185966492}],"concepts":[{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.8271085023880005},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5837533473968506},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5385265350341797},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3923293352127075},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.366742342710495}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.10594","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.10594","pdf_url":"https://arxiv.org/pdf/2410.10594","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.10594","doi":"https://doi.org/10.48550/arxiv.2410.10594","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.10594","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.10594","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.10594","pdf_url":"https://arxiv.org/pdf/2410.10594","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2385859805","https://openalex.org/W2530972254","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Retrieval-augmented":[0],"generation":[1,95,145,159],"(RAG)":[2],"is":[3,80,177],"an":[4,87],"effective":[5],"technique":[6],"that":[7,40,149,175],"enables":[8],"large":[9],"language":[10],"models":[11],"(LLMs)":[12],"to":[13,32,75,92,100,134],"utilize":[14,33],"external":[15],"knowledge":[16],"sources":[17],"for":[18,194],"generation.":[19],"However,":[20],"current":[21],"RAG":[22,64,153,170,195],"systems":[23],"are":[24,203],"solely":[25],"based":[26],"on":[27,196],"text,":[28,77],"rendering":[29],"it":[30,189],"impossible":[31],"vision":[34],"information":[35,113,120],"like":[36],"layout":[37],"and":[38,89,108,131,140,158,183,201],"images":[39],"play":[41],"crucial":[42],"roles":[43],"in":[44,114,138,154,179],"real-world":[45],"multi-modality":[46,197],"documents.":[47,198],"In":[48,66],"this":[49,56,67],"paper,":[50],"we":[51],"introduce":[52],"VisRAG,":[53],"which":[54],"tackles":[55],"issue":[57],"by":[58],"establishing":[59],"a":[60,84,97,142,162,191],"vision-language":[61],"model":[62],"(VLM)-based":[63],"pipeline.":[65,171],"pipeline,":[68],"instead":[69],"of":[70,96,110,144],"first":[71],"parsing":[72,125],"the":[73,78,94,106,111,115,119,124,136,156],"document":[74,79],"obtain":[76],"directly":[81],"embedded":[82],"using":[83],"VLM":[85],"as":[86,190],"image":[88],"then":[90],"retrieved":[91],"enhance":[93],"VLM.":[98],"Compared":[99],"traditional":[101,152,168],"text-based":[102,169],"RAG,":[103],"VisRAG":[104,139,150,176],"maximizes":[105],"retention":[107],"utilization":[109],"data":[112,133,182,202],"original":[116],"documents,":[117],"eliminating":[118],"loss":[121],"introduced":[122],"during":[123],"process.":[126],"We":[127],"collect":[128],"both":[129,155],"open-source":[130],"synthetic":[132],"train":[135],"retriever":[137],"explore":[141],"variety":[143],"methods.":[146],"Experiments":[147],"demonstrate":[148],"outperforms":[151],"retrieval":[157],"stages,":[160],"achieving":[161],"20--40%":[163],"end-to-end":[164],"performance":[165],"gain":[166],"over":[167],"Further":[172],"analysis":[173],"reveals":[174],"efficient":[178],"utilizing":[180],"training":[181],"demonstrates":[184],"strong":[185],"generalization":[186],"capability,":[187],"positioning":[188],"promising":[192],"solution":[193],"Our":[199],"code":[200],"available":[204],"at":[205],"https://github.com/openbmb/visrag.":[206]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3}],"updated_date":"2026-10-13T10:08:12.033832","created_date":"2025-10-10T00:00:00"}