{"id":"https://openalex.org/W7213486361","doi":null,"title":"Testing Large Vision-Language Models: A Systematic Review","display_name":"Testing Large Vision-Language Models: A Systematic Review","publication_year":2026,"publication_date":"2026-09-15","ids":{"openalex":"https://openalex.org/W7213486361"},"language":"en","primary_location":{"id":"pmh:oai:HAL:hal-05743141v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-05743141v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2026","raw_type":"info:eu-repo/semantics/preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hal.science/hal-05743141v1","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058935035","display_name":"Moses Openja","orcid":"https://orcid.org/0000-0002-4115-3834","observed_orcids":["https://orcid.org/0000-0002-4115-3834"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]},{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]},{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Openja, Moses","raw_affiliation_strings":["EPM - \u00c9cole Polytechnique de Montr\u00e9al (Campus de l'Universit\u00e9 de Montr\u00e9al - 2500, chemin de Polytechnique - Montr\u00e9al (Qu\u00e9bec) H3T 1J4 - Canada)","York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EPM - \u00c9cole Polytechnique de Montr\u00e9al (Campus de l'Universit\u00e9 de Montr\u00e9al - 2500, chemin de Polytechnique - Montr\u00e9al (Qu\u00e9bec) H3T 1J4 - Canada)","institution_ids":["https://openalex.org/I45683168","https://openalex.org/I70931966"]},{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139970044","display_name":"Wuyang Dai","orcid":"https://orcid.org/0009-0002-0283-7288","observed_orcids":["https://orcid.org/0009-0002-0283-7288"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Dai, Wuyang","raw_affiliation_strings":["York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119688635","display_name":"Jiho Shin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Shin, Jiho","raw_affiliation_strings":["York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041089984","display_name":"Nikta Akbarpour","orcid":"https://orcid.org/0009-0009-9988-3379","observed_orcids":["https://orcid.org/0009-0009-9988-3379"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Akbarpour, Nikta","raw_affiliation_strings":["York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5151989274","display_name":"Claire Davies","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Davies, Claire","raw_affiliation_strings":["Queen's University [Kingston, Canada] (99 University Ave, Kingston, ON K7L 3N6 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Queen's University [Kingston, Canada] (99 University Ave, Kingston, ON K7L 3N6 - Canada)","institution_ids":["https://openalex.org/I204722609"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069795565","display_name":"Alvine Boaye Belle","orcid":"https://orcid.org/0000-0001-7533-7212","observed_orcids":["https://orcid.org/0000-0001-7533-7212","https://orcid.org/0000-0002-4772-0334"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Boaye Belle, Alvine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100495021","display_name":"Hung Viet Pham","orcid":"https://orcid.org/0000-0003-0861-8326","observed_orcids":["https://orcid.org/0000-0003-0861-8326"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Viet Pham, Hung","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067154597","display_name":"Gias Uddin","orcid":"https://orcid.org/0000-0003-1376-095X","observed_orcids":["https://orcid.org/0000-0003-1376-095X"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Uddin, Gias","raw_affiliation_strings":["York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035677127","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-3116-5038","observed_orcids":["https://orcid.org/0000-0003-3116-5038"]},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Wang, Song","raw_affiliation_strings":["York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"York University [Toronto] (4700 Keele Street, Toronto, ON, Canada M3J 1P3 - Canada)","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.9959999918937683},{"id":"https://openalex.org/keywords/software-testing","display_name":"software testing","score":0.7350000143051147},{"id":"https://openalex.org/keywords/model-quality-assurance","display_name":"model quality assurance","score":0.6110000014305115},{"id":"https://openalex.org/keywords/robustness-testing","display_name":"robustness testing","score":0.7799999713897705},{"id":"https://openalex.org/keywords/functional-reliability","display_name":"functional reliability","score":0.531000018119812},{"id":"https://openalex.org/keywords/evaluation-benchmark","display_name":"evaluation benchmark","score":0.5839999914169312},{"id":"https://openalex.org/keywords/automatic-test-generation","display_name":"automated test generation","score":0.6610000133514404},{"id":"https://openalex.org/keywords/regression-testing","display_name":"regression testing","score":0.8889999985694885},{"id":"https://openalex.org/keywords/uncertainty-calibration","display_name":"uncertainty calibration","score":0.5600000023841858}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.420199990272522},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.24809999763965607},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2460000067949295},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.24369999766349792},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.23960000276565552},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.23810000717639923},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.21879999339580536},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.21739999949932098},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.21209999918937683},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.20960000157356262}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:HAL:hal-05743141v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-05743141v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2026","raw_type":"info:eu-repo/semantics/preprint"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-05743141v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-05743141v1","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2026","raw_type":"info:eu-repo/semantics/preprint"},"sustainable_development_goals":[],"study_designs":[{"display_name":"Systematic Review","id":"https://openalex.org/study-designs/systematic-review"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7223294292","https://openalex.org/W4417149977","https://openalex.org/W4415800385","https://openalex.org/W7155558949","https://openalex.org/W4406420619","https://openalex.org/W4402670365","https://openalex.org/W4411104111","https://openalex.org/W4404340653","https://openalex.org/W7213951908","https://openalex.org/W4392120762"],"abstract_inverted_index":{"Vision-Language":[0],"Models":[1],"(VLMs)":[2],"have":[3],"rapidly":[4],"evolved":[5],"into":[6,235,264],"foundational":[7],"technologies":[8],"that":[9,221,251],"power":[10],"a":[11,90,100,117,144,167,278],"broad":[12],"spectrum":[13],"of":[14,69,93,147,241,288],"applications,":[15],"including":[16,34],"visual":[17],"question":[18],"answering,":[19],"multimodal":[20],"reasoning,":[21,173],"autonomous":[22],"driving,":[23],"robotics,":[24],"embodied":[25,189],"agents,":[26],"healthcare,":[27],"and":[28,41,55,77,97,111,115,131,134,161,175,188,199,210,229,274,284,292],"scientific":[29],"discovery.":[30],"Assuring":[31],"their":[32],"quality,":[33],"reliability,":[35,172,187],"robustness,":[36,174],"truthfulness,":[37],"safety,":[38],"fairness,":[39],"efficiency,":[40],"security-is":[42],"therefore":[43],"an":[44,265],"important":[45],"software-engineering":[46],"challenge.":[47],"Although":[48],"research":[49,96,222,282],"on":[50],"VLM":[51,94,152,252,294],"benchmarks,":[52,227],"evaluation":[53,186,230],"frameworks,":[54],"quality":[56,122],"assessment":[57],"has":[58,254,260],"grown":[59],"rapidly,":[60],"the":[61,121,128,135,140,286],"landscape":[62,168],"remains":[63,192],"fragmented,":[64],"with":[65,143,178],"limited":[66,180],"systematic":[67,91],"understanding":[68],"what":[70],"is":[71,75,245],"tested,":[72],"how":[73],"testing":[74,79,95,153,243,253,295],"performed,":[76],"whether":[78],"contributions":[80,223],"are":[81],"adopted":[82],"in":[83],"downstream":[84,148,217],"software":[85,236],"ecosystems.":[86],"This":[87],"paper":[88],"presents":[89],"mapping":[92],"practice":[98],"from":[99,158],"softwaretesting":[101],"perspective.":[102],"We":[103,138],"analyze":[104],"307":[105],"primary":[106],"studies":[107],"published":[108],"between":[109],"2020":[110],"June":[112],"26,":[113],"2026,":[114],"develop":[116],"unified":[118],"taxonomy":[119],"characterizing":[120],"properties":[123],"researchers":[124],"seek":[125],"to":[126,182],"assure,":[127],"system":[129],"components":[130],"representations":[132],"targeted,":[133],"methodologies":[136,244],"employed.":[137],"complement":[139],"literature":[141],"study":[142],"large-scale":[145],"analysis":[146,218],"GitHub":[149],"repositories":[150],"referencing":[151],"artifacts,":[154],"distinguishing":[155],"operational":[156],"reuse":[157],"evaluation/reporting":[159],"references":[160],"documentary":[162],"uptake.":[163],"Our":[164],"findings":[165,249],"reveal":[166],"dominated":[169],"by":[170],"functional":[171],"hallucination":[176],"assessment,":[177,201],"comparatively":[179,246],"attention":[181],"privacy,":[183],"uncertainty":[184],"calibration,":[185],"safety.":[190],"Testing":[191],"predominantly":[193],"evaluation-centered,":[194],"emphasizing":[195],"observable":[196],"model":[197],"behavior":[198],"benchmarkdriven":[200],"while":[202],"component-level":[203],"diagnosis,":[204],"automated":[205],"test":[206],"generation,":[207],"regression":[208],"testing,":[209],"deployment-oriented":[211],"assurance":[212],"receive":[213],"less":[214],"attention.":[215],"The":[216,271],"further":[219],"shows":[220],"diffuse":[224],"selectively:":[225],"reusable":[226],"datasets,":[228],"infrastructure":[231],"provide":[232,277],"prominent":[233],"pathways":[234],"ecosystems,":[237],"whereas":[238],"direct":[239],"transfer":[240],"complete":[242],"limited.":[247],"These":[248],"suggest":[250],"developed":[255],"substantial":[256],"capability-evaluation":[257],"resources":[258],"but":[259],"not":[261],"yet":[262],"matured":[263],"equally":[266],"reusable,":[267],"lifecycle-oriented":[268],"quality-assurance":[269],"ecosystem.":[270],"resulting":[272],"taxonomies":[273],"adoption":[275],"evidence":[276],"foundation":[279],"for":[280],"identifying":[281],"gaps":[283],"guiding":[285],"development":[287],"more":[289],"systematic,":[290],"integrable,":[291],"deployment-aware":[293],"practices.":[296]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-09-17T00:00:00"}