{"id":"https://openalex.org/W4415524633","doi":"https://doi.org/10.1109/mlsp62443.2025.11204323","title":"Learning or Cheating? Assessing Data Contamination in Large Vision-Language Models","display_name":"Learning or Cheating? Assessing Data Contamination in Large Vision-Language Models","publication_year":2025,"publication_date":"2025-08-31","ids":{"openalex":"https://openalex.org/W4415524633","doi":"https://doi.org/10.1109/mlsp62443.2025.11204323"},"language":null,"primary_location":{"id":"doi:10.1109/mlsp62443.2025.11204323","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mlsp62443.2025.11204323","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 35th International Workshop on Machine Learning for Signal Processing (MLSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054102478","display_name":"Ahmed Masry","orcid":null},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Ahmed Masry","raw_affiliation_strings":["York University,Department of Electrical Engineering and Computer Science,Toronto,Canada"],"affiliations":[{"raw_affiliation_string":"York University,Department of Electrical Engineering and Computer Science,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014789889","display_name":"M. Firoz Ahmed","orcid":"https://orcid.org/0000-0001-9272-0643"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mahir Ahmed","raw_affiliation_strings":["York University,Department of Electrical Engineering and Computer Science,Toronto,Canada"],"affiliations":[{"raw_affiliation_string":"York University,Department of Electrical Engineering and Computer Science,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029539188","display_name":"Ridwan Mahbub","orcid":null},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ridwan Mahbub","raw_affiliation_strings":["York University,Department of Electrical Engineering and Computer Science,Toronto,Canada"],"affiliations":[{"raw_affiliation_string":"York University,Department of Electrical Engineering and Computer Science,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5054102478"],"corresponding_institution_ids":["https://openalex.org/I192455969"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.15956111,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.7955999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.7955999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.8378999829292297},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.6362000107765198},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.4203999936580658},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.3614000082015991},{"id":"https://openalex.org/keywords/chart","display_name":"Chart","score":0.35089999437332153},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.33550000190734863},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.29820001125335693}],"concepts":[{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.8378999829292297},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6765999794006348},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6362000107765198},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.527400016784668},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5009999871253967},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4203999936580658},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.375},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3614000082015991},{"id":"https://openalex.org/C190812933","wikidata":"https://www.wikidata.org/wiki/Q28923","display_name":"Chart","level":2,"score":0.35089999437332153},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.33550000190734863},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.29820001125335693},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.28060001134872437},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.27810001373291016},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C3020493868","wikidata":"https://www.wikidata.org/wiki/Q55631277","display_name":"Real world data","level":2,"score":0.2743000090122223},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.2630000114440918},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2524999976158142}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mlsp62443.2025.11204323","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mlsp62443.2025.11204323","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE 35th International Workshop on Machine Learning for Signal Processing (MLSP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320319952","display_name":"Canada Foundation for Innovation","ror":"https://ror.org/000az4664"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2986619406","https://openalex.org/W3003484198","https://openalex.org/W3120043490","https://openalex.org/W4213213306","https://openalex.org/W4285107714","https://openalex.org/W4285255856","https://openalex.org/W4285292339","https://openalex.org/W4385573930","https://openalex.org/W4402670101","https://openalex.org/W4403791888"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Vision-Language":[1],"Models":[2],"(VLMs)":[3],"have":[4],"demonstrated":[5],"remarkable":[6],"capabilities":[7],"in":[8,164],"tasks":[9],"that":[10,113,135],"involve":[11],"both":[12,63,136],"visual":[13,70,94],"and":[14,22,45,65,78,89,105,133,138,154],"textual":[15],"understanding,":[16],"including":[17,74],"chart":[18],"interpretation,":[19],"document":[20],"comprehension,":[21],"geometric":[23],"reasoning.":[24,126],"However,":[25],"concerns":[26],"remain":[27],"about":[28],"whether":[29],"these":[30,50],"models":[31],"truly":[32],"generalize":[33],"or":[34],"if":[35],"their":[36],"performance":[37,131],"on":[38,120],"popular":[39],"benchmarks":[40,97],"is":[41],"influenced":[42],"by":[43],"overfitting":[44],"data":[46,60,143,152],"contamination.":[47,144],"To":[48],"investigate":[49],"concerns,":[51],"we":[52,108],"propose":[53],"a":[54,110],"systematic":[55],"evaluation":[56],"framework":[57],"for":[58],"assessing":[59],"contamination":[61,111],"of":[62,150],"closed-source":[64,137],"open-source":[66,139],"VLMs":[67,140,163],"using":[68],"multiple":[69],"question":[71,90,95],"answering":[72,96],"benchmarks,":[73],"ChartQA,":[75],"DocVQA,":[76,99],"InfoVQA,":[77,100],"MathVista.":[79],"Our":[80,127],"approach":[81],"applies":[82],"structured":[83],"perturbations":[84],"such":[85],"as":[86],"image":[87],"replacement":[88],"completion":[91],"to":[92,157],"four":[93],"(ChartQA,":[98],"MathVista).":[101],"By":[102],"comparing":[103],"original":[104],"perturbed":[106],"accuracies,":[107],"compute":[109],"score":[112],"measures":[114],"how":[115],"much":[116],"each":[117],"model":[118],"relies":[119],"memorized":[121],"examples":[122],"versus":[123],"genuine":[124],"multimodal":[125],"results":[128],"reveal":[129],"significant":[130],"shifts":[132],"conclude":[134],"suffer":[141],"from":[142],"These":[145],"findings":[146],"underscore":[147],"the":[148],"necessity":[149],"rigorous":[151],"filtering":[153],"independent":[155],"evaluations":[156],"ensure":[158],"robust":[159],"generalization":[160],"before":[161],"deploying":[162],"real-world":[165],"applications.":[166]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-24T00:00:00"}