{"id":"https://openalex.org/W7202324655","doi":null,"title":"Modality-Specific Confidence and Uncertainty Metrics for Vision-Language Models","display_name":"Modality-Specific Confidence and Uncertainty Metrics for Vision-Language Models","publication_year":2026,"publication_date":"2026-05-01","ids":{"openalex":"https://openalex.org/W7202324655"},"language":"en","primary_location":{"id":"pmh:oai:digitalcommons.usf.edu:bcreus-1026","doi":null,"is_oa":true,"landing_page_url":"https://digitalcommons.usf.edu/bcreus/2026/spring/14","pdf_url":"https://digitalcommons.usf.edu/context/bcreus/article/1026/viewcontent/group7_5360765_212978222_Modality_20Specific_20Confidence_20VLMs_20Abstract.pdf","source":{"id":"https://openalex.org/S4377196272","display_name":"Digital Commons - University of South Florida (University of South Florida)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2613432","host_organization_name":"University of South Florida","host_organization_lineage":["https://openalex.org/I2613432"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bellini College REU Symposium (BCREUS)","raw_type":"text"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://digitalcommons.usf.edu/context/bcreus/article/1026/viewcontent/group7_5360765_212978222_Modality_20Specific_20Confidence_20VLMs_20Abstract.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138878751","display_name":"Sahasra Kondapalli","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2613432","display_name":"University of South Florida","ror":"https://ror.org/032db5x82","country_code":"US","type":"education","lineage":["https://openalex.org/I2613432"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kondapalli, Sahasra","raw_affiliation_strings":["University of South Florida"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of South Florida","institution_ids":["https://openalex.org/I2613432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147052570","display_name":"Poyraz An","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2613432","display_name":"University of South Florida","ror":"https://ror.org/032db5x82","country_code":"US","type":"education","lineage":["https://openalex.org/I2613432"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"An, Poyraz","raw_affiliation_strings":["University of South Florida"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of South Florida","institution_ids":["https://openalex.org/I2613432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120536717","display_name":"Trung Dong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2613432","display_name":"University of South Florida","ror":"https://ror.org/032db5x82","country_code":"US","type":"education","lineage":["https://openalex.org/I2613432"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dong, Trung","raw_affiliation_strings":["University of South Florida"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of South Florida","institution_ids":["https://openalex.org/I2613432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138832270","display_name":"Lara Radovanovic","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2613432","display_name":"University of South Florida","ror":"https://ror.org/032db5x82","country_code":"US","type":"education","lineage":["https://openalex.org/I2613432"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Radovanovic, Lara","raw_affiliation_strings":["University of South Florida"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of South Florida","institution_ids":["https://openalex.org/I2613432"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5138802437","display_name":"Xiaomin Lin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2613432","display_name":"University of South Florida","ror":"https://ror.org/032db5x82","country_code":"US","type":"education","lineage":["https://openalex.org/I2613432"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lin, Xiaomin","raw_affiliation_strings":["University of South Florida"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of South Florida","institution_ids":["https://openalex.org/I2613432"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I2613432"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.55189514,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9682999849319458,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9682999849319458,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.014800000004470348,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.010300000198185444,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.972000002861023},{"id":"https://openalex.org/keywords/confidence-measures","display_name":"confidence measures","score":0.6159999966621399},{"id":"https://openalex.org/keywords/uncertainty-quantification","display_name":"uncertainty quantification","score":0.9869999885559082},{"id":"https://openalex.org/keywords/prompt-injection","display_name":"prompt injection","score":0.5070000290870667},{"id":"https://openalex.org/keywords/sycophancy","display_name":"sycophancy","score":0.8450000286102295},{"id":"https://openalex.org/keywords/autonomous-robots","display_name":"autonomous robots","score":0.5590000152587891},{"id":"https://openalex.org/keywords/human-robot-interaction","display_name":"human-robot interaction","score":0.9599999785423279},{"id":"https://openalex.org/keywords/medical-assistants","display_name":"medical assistants","score":0.7490000128746033},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"trustworthiness","score":0.6510000228881836},{"id":"https://openalex.org/keywords/coworking-spaces","display_name":"coworking spaces","score":0.5789999961853027}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6172999739646912},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.612500011920929},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6079999804496765},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49729999899864197},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.47850000858306885},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4677000045776367},{"id":"https://openalex.org/C34413123","wikidata":"https://www.wikidata.org/wiki/Q170978","display_name":"Robotics","level":3,"score":0.4462999999523163},{"id":"https://openalex.org/C2777026412","wikidata":"https://www.wikidata.org/wiki/Q2684591","display_name":"Statement (logic)","level":2,"score":0.43790000677108765},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.3790000081062317},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.3727000057697296},{"id":"https://openalex.org/C2909755999","wikidata":"https://www.wikidata.org/wiki/Q4751126","display_name":"Low Confidence","level":2,"score":0.34860000014305115},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3479999899864197},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3416999876499176},{"id":"https://openalex.org/C2779585090","wikidata":"https://www.wikidata.org/wiki/Q3457762","display_name":"Resilience (materials science)","level":2,"score":0.32030001282691956},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28940001130104065},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.28349998593330383},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.27639999985694885},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2653999924659729},{"id":"https://openalex.org/C32230216","wikidata":"https://www.wikidata.org/wiki/Q7882499","display_name":"Uncertainty quantification","level":2,"score":0.2535000145435333},{"id":"https://openalex.org/C2780154230","wikidata":"https://www.wikidata.org/wiki/Q513420","display_name":"Undo","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:digitalcommons.usf.edu:bcreus-1026","doi":null,"is_oa":true,"landing_page_url":"https://digitalcommons.usf.edu/bcreus/2026/spring/14","pdf_url":"https://digitalcommons.usf.edu/context/bcreus/article/1026/viewcontent/group7_5360765_212978222_Modality_20Specific_20Confidence_20VLMs_20Abstract.pdf","source":{"id":"https://openalex.org/S4377196272","display_name":"Digital Commons - University of South Florida (University of South Florida)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2613432","host_organization_name":"University of South Florida","host_organization_lineage":["https://openalex.org/I2613432"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bellini College REU Symposium (BCREUS)","raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:digitalcommons.usf.edu:bcreus-1026","doi":null,"is_oa":true,"landing_page_url":"https://digitalcommons.usf.edu/bcreus/2026/spring/14","pdf_url":"https://digitalcommons.usf.edu/context/bcreus/article/1026/viewcontent/group7_5360765_212978222_Modality_20Specific_20Confidence_20VLMs_20Abstract.pdf","source":{"id":"https://openalex.org/S4377196272","display_name":"Digital Commons - University of South Florida (University of South Florida)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2613432","host_organization_name":"University of South Florida","host_organization_lineage":["https://openalex.org/I2613432"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bellini College REU Symposium (BCREUS)","raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310847","display_name":"University of South Florida","ror":"https://ror.org/032db5x82"},{"id":"https://openalex.org/F4320318228","display_name":"Florida High Tech Corridor Council","ror":"https://ror.org/03q0tnp04"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7202324655.pdf","grobid_xml":"https://content.openalex.org/works/W7202324655.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4414691287"],"related_works":["https://openalex.org/W4416037108","https://openalex.org/W4415285811","https://openalex.org/W7128625439","https://openalex.org/W7140101510","https://openalex.org/W7079505790","https://openalex.org/W7171394009","https://openalex.org/W7212874325","https://openalex.org/W7164901229","https://openalex.org/W7213502555","https://openalex.org/W4411120259"],"abstract_inverted_index":{"Introduction":[0],"and":[1,22,110,130,163,174,202,269,273,283,332,365,371,379,387,393,434,449,471,475],"Problem":[2],"Statement":[3],"Vision-Language":[4,178],"Models":[5],"(VLMs)":[6],"are":[7,86,146],"increasingly":[8],"being":[9],"integrated":[10,151],"into":[11],"high-stakes":[12],"environments,":[13],"such":[14],"as":[15,23,132,148,164,236,445],"autonomous":[16,158,270],"robotics":[17],"used":[18,147],"in":[19,41,53,77,121,127,150,154,172,189,210,267,356,452,455,469],"shared":[20,57],"workspaces":[21],"multimodal":[24],"medical":[25,54,165],"assistant":[26],"systems.":[27],"However,":[28],"current":[29,170],"models":[30],"still":[31],"have":[32],"inherent":[33],"uncertainties,":[34],"often":[35],"producing":[36],"errors":[37],"or":[38,66,249],"false":[39,59],"confidence":[40,60,128,175,232,281,287,363],"responses":[42],"[1].":[43],"Such":[44,84],"errors,":[45],"when":[46],"passed":[47],"undetected,":[48],"can":[49,61,106,443],"cause":[50],"serious":[51],"issues":[52],"implementations.":[55],"In":[56],"workspaces,":[58],"lead":[62],"to":[63,206,225,238,245,261,292,301,308,340,348,391,413,421,446,461],"safety":[64,424],"hazards,":[65],"disruption.":[67],"Du":[68],"et":[69],"al.":[70],"present":[71],"an":[72,388,419],"evaluation":[73,389],"of":[74,80,92,112,124,213,229,265,313,329,352,411,465],"uncertainty,":[75],"specifically":[76,135,153],"the":[78,90,93,98,108,122,169,182,211,227,257,263,278,284,294,310,314,326,330,350,353,385,399,427,462],"field":[79],"human-robot":[81],"interaction":[82],"[2].":[83],"faults":[85],"only":[87],"magnified":[88],"with":[89,157],"addition":[91],"vision":[94],"modality,":[95],"thus":[96,241],"highlighting":[97],"need":[99],"for":[100,114,136,177,255,367,426],"a":[101,133,190,203,214,223,230,253,303,323,342,403,408,423],"multi-modal":[102],"uncertainty":[103,113,131,173],"quantification,":[104],"which":[105],"identify":[107],"source":[109],"quantities":[111],"each":[115],"mode.":[116],"Proposed":[117],"Solution":[118],"The":[119,296],"deficit":[120,171],"amount":[123],"research":[125,459],"done":[126],"metrics,":[129],"whole,":[134],"Vision":[137],"Language":[138],"Models,":[139,179],"risks":[140],"undeserving":[141],"supervised":[142],"systems,":[143],"where":[144],"VLMs":[145,266],"assistants":[149,468],"spaces,":[152],"collaborative":[155,268],"offices":[156],"robots":[159],"working":[160],"alongside":[161],"humans,":[162],"assistants.":[166],"To":[167,397],"address":[168],"quantification":[176],"we":[180,382,401],"propose":[181,383,418],"Modality-Specific":[183],"Confidence":[184],"(MSC)":[185],"framework.":[186],"It":[187],"works":[188],"dynamic,":[191],"three-stage":[192],"diagnostic":[193,472],"pipeline":[194],"involving":[195],"stochastic":[196],"semantic":[197],"pooling,":[198],"leading-prompt":[199],"counterfactual":[200,290,297],"probing,":[201],"mathematical":[204],"metric":[205,254,339,442],"measure":[207,293,349,361,394],"linguistic":[208],"reliance,":[209],"form":[212],"Modality":[215],"Reliance":[216],"Score":[217],"(MRS).":[218],"An":[219],"MRS":[220,428],"score":[221,282,288,429],"allows":[222,299,346],"user":[224],"judge":[226],"dependence":[228],"model's":[231],"on":[233,325,437],"visual":[234,315],"input":[235],"opposed":[237],"textual":[239,246],"input,":[240],"measuring":[242],"its":[243,395],"resilience":[244],"prompt":[247,331],"injection":[248],"sycophancy.":[250],"By":[251],"providing":[252],"reliability,":[256],"MSC":[258,354],"framework":[259,355,386],"aims":[260],"enhance":[262],"trustworthiness":[264],"workspaces.":[271],"Methods":[272],"Projected":[274],"Metrics":[275],"We":[276,336,359,416,439],"use":[277,302,337,402],"stage":[279,285],"one":[280],"two":[286],"post":[289],"probing":[291,298],"MRS.":[295],"us":[300,347],"distractor":[304],"label":[305],"that":[306,318,345,406],"tries":[307],"change":[309],"model\u2019s":[311],"understanding":[312],"input.":[316],"If":[317],"is":[319],"successful,":[320],"it":[321],"suggests":[322],"reliance":[324],"language":[327],"modality":[328],"lowers":[333],"true":[334],"confidence.":[335],"this":[338,441],"implement":[341,398],"feedback":[343,404],"loop":[344,405],"effectiveness":[351],"improving":[357],"accuracy.":[358,372,438],"also":[360],"overall":[362],"metrics":[364],"check":[366],"correlation":[368],"between":[369],"them":[370],"\\[MRS":[373],"=":[374],"\\frac{CS_{base}":[375],"-":[376],"CS_{counter}}{CS_{base}}\\]":[377],"Impact":[378],"Conclusion":[380],"Here,":[381],"both":[384],"methodology":[390],"ensure":[392],"effectiveness.":[396],"framework,":[400],"runs":[407],"set":[409],"number":[410],"times":[412],"conserve":[414],"resources.":[415],"further":[417],"experiment":[420],"calibrate":[422],"threshold":[425],"by":[430],"testing":[431],"various":[432],"thresholds":[433],"their":[435],"effect":[436],"hope":[440],"serve":[444],"improve":[447],"accuracy":[448],"highlight":[450],"unreliability":[451],"VLMs,":[453],"especially":[454],"high-risk":[456],"scenarios.":[457],"This":[458],"contributes":[460],"broader":[463],"goal":[464],"incorporating":[466],"VLM":[467],"repetitive":[470],"tasks":[473],"safely":[474],"incorporate":[476],"risk":[477],"management.":[478]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-13T00:00:00"}