{"id":"https://openalex.org/W4399991306","doi":"https://doi.org/10.48550/arxiv.2402.17840","title":"Follow My Instruction and Spill the Beans: Scalable Data Extraction from Retrieval-Augmented Generation Systems","display_name":"Follow My Instruction and Spill the Beans: Scalable Data Extraction from Retrieval-Augmented Generation Systems","publication_year":2024,"publication_date":"2024-02-27","ids":{"openalex":"https://openalex.org/W4399991306","doi":"https://doi.org/10.48550/arxiv.2402.17840"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.17840","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.17840","pdf_url":"https://arxiv.org/pdf/2402.17840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.17840","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055702764","display_name":"Zhenting Qi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qi, Zhenting","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101598769","display_name":"Hanlin Zhang","orcid":"https://orcid.org/0000-0001-8869-6863","observed_orcids":["https://orcid.org/0000-0001-8869-6863"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Hanlin","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009547049","display_name":"Eric P. Xing","orcid":"https://orcid.org/0009-0005-9158-4201","observed_orcids":["https://orcid.org/0009-0005-9158-4201","https://orcid.org/0000-0002-3683-4280"]},"institutions":[{"id":"https://openalex.org/I4210113480","display_name":"Mohamed bin Zayed University of Artificial Intelligence","ror":"https://ror.org/0258gkt32","country_code":"AE","type":"education","lineage":["https://openalex.org/I4210113480"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["AE","US"],"is_corresponding":false,"raw_author_name":"Xing, Eric","raw_affiliation_strings":["Carnegie Mellon University","Mohamed bin Zayed University of Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Mohamed bin Zayed University of Artificial Intelligence","institution_ids":["https://openalex.org/I4210113480"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108381794","display_name":"Sham M. Kakade","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kakade, Sham","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015520086","display_name":"Himabindu Lakkaraju","orcid":"https://orcid.org/0000-0001-7922-6544","observed_orcids":["https://orcid.org/0000-0001-7922-6544"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lakkaraju, Himabindu","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.004999999888241291,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9940000176429749},{"id":"https://openalex.org/keywords/prompt-injection","display_name":"prompt injection","score":0.902999997138977},{"id":"https://openalex.org/keywords/instruction-following","display_name":"instruction following","score":0.6100000143051147},{"id":"https://openalex.org/keywords/data-extraction-attacks","display_name":"data extraction attacks","score":0.4970000088214874},{"id":"https://openalex.org/keywords/position-bias","display_name":"position bias","score":0.6010000109672546},{"id":"https://openalex.org/keywords/instruction-tuned-language-models","display_name":"instruction-tuned language models","score":0.7900000214576721},{"id":"https://openalex.org/keywords/gpt","display_name":"GPT","score":0.5099999904632568}],"concepts":[{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.736011266708374},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6258510947227478},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.5778346061706543},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36094868183135986},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.21561625599861145},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.16738751530647278},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.10791423916816711}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.17840","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.17840","pdf_url":"https://arxiv.org/pdf/2402.17840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.17840","doi":"https://doi.org/10.48550/arxiv.2402.17840","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.17840","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.17840","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.17840","pdf_url":"https://arxiv.org/pdf/2402.17840","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4720003262","display_name":null,"funder_award_id":"N00014-22","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G826810440","display_name":"AI Institute for Societal Decision Making (AI-SDM)","funder_award_id":"2229881","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399991306.pdf","grobid_xml":"https://content.openalex.org/works/W4399991306.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4404352866","https://openalex.org/W7166854845","https://openalex.org/W7160380836","https://openalex.org/W4407308219","https://openalex.org/W7223006049","https://openalex.org/W7161783429","https://openalex.org/W4415981883","https://openalex.org/W4392425374","https://openalex.org/W7204018931","https://openalex.org/W4404782880"],"abstract_inverted_index":{"Retrieval-Augmented":[0],"Generation":[1],"(RAG)":[2],"improves":[3],"pre-trained":[4],"models":[5,144],"by":[6,133,196,205],"incorporating":[7],"external":[8],"knowledge":[9],"at":[10,167,177],"test":[11],"time":[12],"to":[13,40,107,141],"enable":[14],"customized":[15,164],"adaptation.":[16],"We":[17,30,89],"study":[18,91,140],"the":[19,47,80,84,98,198],"risk":[20],"of":[21,49,66,94,100,114,180,185,193],"datastore":[22,48,153],"leakage":[23,154],"in":[24,116],"Retrieval-In-Context":[25],"RAG":[26,50,95,143],"Language":[27],"Models":[28],"(LMs).":[29],"show":[31,125],"that":[32,69,103,126,150],"an":[33,112,148],"adversary":[34],"can":[35,110,129,151],"exploit":[36],"LMs'":[37],"instruction-following":[38],"capabilities":[39],"easily":[41],"extract":[42,173],"text":[43,174],"data":[44,109,175],"verbatim":[45,176],"from":[46,182,190],"systems":[51],"built":[52],"with":[53,155,166,200],"instruction-tuned":[54],"LMs":[55,68],"via":[56],"prompt":[57],"injection.":[58],"The":[59],"vulnerability":[60,128],"exists":[61],"for":[62,120],"a":[63,156,178,183,191],"wide":[64],"range":[65],"modern":[67,121],"span":[70],"Llama2,":[71],"Mistral/Mixtral,":[72],"Vicuna,":[73],"SOLAR,":[74],"WizardLM,":[75],"Qwen1.5,":[76],"and":[77,79,123,171,188],"Platypus2,":[78],"exploitability":[81],"exacerbates":[82],"as":[83],"model":[85],"size":[86],"scales":[87],"up.":[88],"also":[90],"multiple":[92],"effects":[93],"setup":[96],"on":[97,160],"extractability":[99],"data,":[101],"indicating":[102],"following":[104],"unexpected":[105],"instructions":[106],"regurgitate":[108],"be":[111,130],"outcome":[113],"failure":[115],"effectively":[117],"utilizing":[118],"contexts":[119],"LMs,":[122],"further":[124],"such":[127],"greatly":[131],"mitigated":[132],"position":[134],"bias":[135],"elimination":[136],"strategies.":[137],"Extending":[138],"our":[139],"production":[142],"GPTs,":[145],"we":[146,172],"design":[147],"attack":[149],"cause":[152],"100%":[157],"success":[158],"rate":[159,179],"25":[161],"randomly":[162],"selected":[163],"GPTs":[165,199],"most":[168],"2":[169],"queries,":[170],"41%":[181],"book":[184],"77,000":[186],"words":[187,195],"3%":[189],"corpus":[192],"1,569,000":[194],"prompting":[197],"only":[201],"100":[202],"queries":[203],"generated":[204],"themselves.":[206]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2024-06-25T00:00:00"}