{"id":"https://openalex.org/W7160606966","doi":"https://doi.org/10.48550/arxiv.2605.06201","title":"Towards Annotation-Free Validation of MLLMs: A Vision-Language Logical Consistency Metric","display_name":"Towards Annotation-Free Validation of MLLMs: A Vision-Language Logical Consistency Metric","publication_year":2026,"publication_date":"2026-05-07","ids":{"openalex":"https://openalex.org/W7160606966","doi":"https://doi.org/10.48550/arxiv.2605.06201"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.06201","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06201","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.06201","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135716756","display_name":"Ying Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Ying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037514890","display_name":"Mei Chee Leong","orcid":"https://orcid.org/0000-0001-8123-8982"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leong, Mei Chee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026930610","display_name":"Hui Li Tan","orcid":"https://orcid.org/0000-0002-3012-9222"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Hui Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051758499","display_name":"Shangbo Mao","orcid":"https://orcid.org/0000-0002-9743-5451"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mao, Shangbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135669440","display_name":"Liyuan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Liyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135681198","display_name":"Nancy Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Nancy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.503000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.503000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1080000028014183,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07829999923706055,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.763700008392334},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6895999908447266},{"id":"https://openalex.org/keywords/consistency-model","display_name":"Consistency model","score":0.5256999731063843},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.5029000043869019},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.48330000042915344},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4772999882698059},{"id":"https://openalex.org/keywords/logical-conjunction","display_name":"Logical conjunction","score":0.4693000018596649},{"id":"https://openalex.org/keywords/sequential-consistency","display_name":"Sequential consistency","score":0.4287000000476837}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.763700008392334},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6895999908447266},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5861999988555908},{"id":"https://openalex.org/C37279795","wikidata":"https://www.wikidata.org/wiki/Q2492305","display_name":"Consistency model","level":3,"score":0.5256999731063843},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.5029000043869019},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.48330000042915344},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4772999882698059},{"id":"https://openalex.org/C21847791","wikidata":"https://www.wikidata.org/wiki/Q191081","display_name":"Logical conjunction","level":2,"score":0.4693000018596649},{"id":"https://openalex.org/C82029504","wikidata":"https://www.wikidata.org/wiki/Q4373882","display_name":"Sequential consistency","level":4,"score":0.4287000000476837},{"id":"https://openalex.org/C175652121","wikidata":"https://www.wikidata.org/wiki/Q4379351","display_name":"Causal consistency","level":5,"score":0.42179998755455017},{"id":"https://openalex.org/C134752490","wikidata":"https://www.wikidata.org/wiki/Q374182","display_name":"Logical consequence","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C122377713","wikidata":"https://www.wikidata.org/wiki/Q4422799","display_name":"Weak consistency","level":4,"score":0.37790000438690186},{"id":"https://openalex.org/C20693621","wikidata":"https://www.wikidata.org/wiki/Q6667502","display_name":"Logical framework","level":2,"score":0.36629998683929443},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3499999940395355},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34439998865127563},{"id":"https://openalex.org/C203702819","wikidata":"https://www.wikidata.org/wiki/Q17146953","display_name":"Logical data model","level":3,"score":0.3167000114917755},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31470000743865967},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.30790001153945923},{"id":"https://openalex.org/C56949724","wikidata":"https://www.wikidata.org/wiki/Q219079","display_name":"Truth table","level":2,"score":0.296999990940094},{"id":"https://openalex.org/C93361087","wikidata":"https://www.wikidata.org/wiki/Q4426698","display_name":"Data consistency","level":2,"score":0.2930000126361847},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2897999882698059},{"id":"https://openalex.org/C193429382","wikidata":"https://www.wikidata.org/wiki/Q232405","display_name":"Metric system","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C2983390893","wikidata":"https://www.wikidata.org/wiki/Q1092582","display_name":"Logical analysis","level":3,"score":0.26570001244544983},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C31388003","wikidata":"https://www.wikidata.org/wiki/Q7624548","display_name":"Strong consistency","level":3,"score":0.2572999894618988},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.06201","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06201","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.06201","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.06201","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Dominant":[0],"accuracy":[1,161],"evaluation":[2],"might":[3,13],"reward":[4],"unwarranted":[5],"guessing":[6],"of":[7,43,104,119,127,132,142],"Large":[8],"Language":[9],"Models,":[10],"and":[11,48,63,81,129,140,162,173],"it":[12],"not":[14],"be":[15,157,167],"applicable":[16],"to":[17,37],"novel":[18,35,178],"tasks":[19,179],"for":[20,70,159,169],"model":[21],"validation":[22],"without":[23,67,145,180],"ground-truth":[24],"(gt)":[25],"annotation.":[26,72,147,182],"Based":[27],"on":[28,45,59,76,107,116,123],"basic":[29],"logic":[30],"principle,":[31],"we":[32,87],"propose":[33],"a":[34],"framework":[36],"evaluate":[38],"the":[39,68,117,125,130,138],"vision-language":[40],"logical":[41,109,154],"consistency":[42,110,155],"MLLMs":[44,92,106],"both":[46,160],"sufficient":[47],"necessary":[49],"cause-effect":[50],"relations.":[51],"We":[52],"define":[53],"Vision-Language":[54],"Logical":[55],"Consistency":[56],"Metric":[57],"(VL-LCM)":[58],"traditional":[60],"MC-VQA":[61],"tests,":[62],"recent":[64,82,90,105],"NaturalBench":[65],"tests":[66],"need":[69],"gt":[71,146,181],"Through":[73],"systematic":[74],"experiments":[75],"representative":[77],"VL":[78,83],"benchmark":[79],"MMMU":[80],"challenges":[84],"like":[85],"NaturalBench,":[86],"evaluated":[88],"11":[89],"open-source":[91],"from":[93],"4":[94],"frontier":[95],"families.":[96],"Our":[97,148],"findings":[98,149],"reveal":[99],"that,":[100,151],"despite":[101],"significant":[102],"progress":[103],"accuracy,":[108,153],"lags":[111],"behind":[112],"significantly.":[113],"Extensive":[114],"evaluations":[115],"correlations":[118],"VL-LCM":[120,133,143,164],"with":[121,134],"metrics":[122],"gt,":[124],"reliability":[126],"LCM,":[128],"relation":[131],"response":[135],"distribution":[136],"justify":[137],"validity":[139],"applicability":[141],"even":[144],"suggest":[150],"beyond":[152],"could":[156],"employed":[158,168],"reliability.":[163],"can":[165],"also":[166],"MLLM":[170],"selection,":[171],"validation,":[172],"reliable":[174],"answer":[175],"justification":[176],"in":[177]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-09T00:00:00"}