{"id":"https://openalex.org/W4391590739","doi":"https://doi.org/10.48550/arxiv.2402.02103","title":"D\u00e9j\u00e0 Vu Memorization in Vision-Language Models","display_name":"D\u00e9j\u00e0 Vu Memorization in Vision-Language Models","publication_year":2024,"publication_date":"2024-02-03","ids":{"openalex":"https://openalex.org/W4391590739","doi":"https://doi.org/10.48550/arxiv.2402.02103"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.02103","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.02103","pdf_url":"https://arxiv.org/pdf/2402.02103","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.02103","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026609618","display_name":"Bargav Jayaraman","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Jayaraman, Bargav","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104257054","display_name":"Chuan Guo","orcid":"https://orcid.org/0000-0002-1250-7306"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Chuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5010790447","display_name":"Kamalika Chaudhuri","orcid":"https://orcid.org/0000-0001-9646-7710"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chaudhuri, Kamalika","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5026609618"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8377000093460083,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8377000093460083,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.8271999955177307,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.7087000012397766,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/d\u00e9j\u00e0-vu","display_name":"D\u00e9j\u00e0 vu","score":0.9492950439453125},{"id":"https://openalex.org/keywords/memorization","display_name":"Memorization","score":0.6728923916816711},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4388739764690399},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.40502214431762695},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34985893964767456},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3485260605812073},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.20528113842010498}],"concepts":[{"id":"https://openalex.org/C81366925","wikidata":"https://www.wikidata.org/wiki/Q158103","display_name":"D\u00e9j\u00e0 vu","level":2,"score":0.9492950439453125},{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.6728923916816711},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4388739764690399},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.40502214431762695},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34985893964767456},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3485260605812073},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.20528113842010498}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.02103","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.02103","pdf_url":"https://arxiv.org/pdf/2402.02103","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.02103","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.02103","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.02103","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.02103","pdf_url":"https://arxiv.org/pdf/2402.02103","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391590739.pdf","grobid_xml":"https://content.openalex.org/works/W4391590739.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W3092374121","https://openalex.org/W1672754555","https://openalex.org/W4385187041","https://openalex.org/W2796556676","https://openalex.org/W2905968449","https://openalex.org/W2518203517","https://openalex.org/W2796185523","https://openalex.org/W165526195","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Vision-Language":[0],"Models":[1],"(VLMs)":[2],"have":[3],"emerged":[4],"as":[5,17,109,111],"the":[6,65,74,85,128],"state-of-the-art":[7],"representation":[8],"learning":[9],"solution,":[10],"with":[11],"myriads":[12],"of":[13],"downstream":[14,130],"applications":[15],"such":[16],"image":[18,86],"classification,":[19],"retrieval":[20],"and":[21,96,99],"generation.":[22],"A":[23],"natural":[24],"question":[25],"is":[26,103],"whether":[27],"these":[28],"models":[29],"memorize":[30],"their":[31],"training":[32,75],"data,":[33],"which":[34,50],"also":[35],"has":[36],"implications":[37],"for":[38,45,105],"generalization.":[39],"We":[40,88],"propose":[41],"a":[42],"new":[43],"method":[44],"measuring":[46],"memorization":[47,92,123],"in":[48,73],"VLMs,":[49],"we":[51,62,116],"call":[52],"d\u00e9j\u00e0":[53,90],"vu":[54,91],"memorization.":[55],"For":[56],"VLMs":[57],"trained":[58,107],"on":[59,108],"image-caption":[60,113],"pairs,":[61],"show":[63,100,117],"that":[64,101,118],"model":[66],"indeed":[67],"retains":[68],"information":[69],"about":[70],"individual":[71],"objects":[72],"images":[76],"beyond":[77],"what":[78],"can":[79],"be":[80],"inferred":[81],"from":[82],"correlations":[83],"or":[84],"caption.":[87],"evaluate":[89],"at":[93],"both":[94],"sample":[95],"population":[97],"level,":[98],"it":[102],"significant":[104],"OpenCLIP":[106],"many":[110],"50M":[112],"pairs.":[114],"Finally,":[115],"text":[119],"randomization":[120],"considerably":[121],"mitigates":[122],"while":[124],"only":[125],"moderately":[126],"impacting":[127],"model's":[129],"task":[131],"performance.":[132]},"counts_by_year":[],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}