{"id":"https://openalex.org/W4415249257","doi":"https://doi.org/10.48550/arxiv.2505.03788","title":"Calibrating Uncertainty Quantification of Multi-Modal LLMs using Grounding","display_name":"Calibrating Uncertainty Quantification of Multi-Modal LLMs using Grounding","publication_year":2025,"publication_date":"2025-04-30","ids":{"openalex":"https://openalex.org/W4415249257","doi":"https://doi.org/10.48550/arxiv.2505.03788"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.03788","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.03788","pdf_url":"https://arxiv.org/pdf/2505.03788","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.03788","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093887415","display_name":"Trilok Padhi","orcid":"https://orcid.org/0009-0003-2789-4825","observed_orcids":["https://orcid.org/0009-0003-2789-4825"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Padhi, Trilok","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108816777","display_name":"Ramneet Kaur","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kaur, Ramneet","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030699148","display_name":"Adam D. Cobb","orcid":"https://orcid.org/0000-0003-2868-6983","observed_orcids":["https://orcid.org/0000-0003-2868-6983","https://orcid.org/0000-0002-6638-1788"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cobb, Adam D.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019134968","display_name":"Manoj Acharya","orcid":"https://orcid.org/0009-0001-5239-5919","observed_orcids":["https://orcid.org/0009-0001-5239-5919","https://orcid.org/0000-0003-0223-3556"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Acharya, Manoj","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Roy, Anirban","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roy, Anirban","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011915615","display_name":"Colin Samplawski","orcid":"https://orcid.org/0000-0001-8486-1673","observed_orcids":["https://orcid.org/0000-0001-8486-1673"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Samplawski, Colin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084050272","display_name":"Brian Matejek","orcid":"https://orcid.org/0000-0002-3517-9229","observed_orcids":["https://orcid.org/0000-0002-3517-9229"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matejek, Brian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048112493","display_name":"Alexander Berenbeim","orcid":"https://orcid.org/0000-0001-8573-4270","observed_orcids":["https://orcid.org/0000-0001-8573-4270"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Berenbeim, Alexander M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032194186","display_name":"Nathaniel D. Bastian","orcid":"https://orcid.org/0000-0001-9957-2778","observed_orcids":["https://orcid.org/0000-0001-9957-2778"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bastian, Nathaniel D.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5035902535","display_name":"Susmit Jha","orcid":"https://orcid.org/0000-0001-5983-9095","observed_orcids":["https://orcid.org/0000-0001-5983-9095"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jha, Susmit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9398999810218811,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9398999810218811,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.03579999879002571,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/uncertainty-quantification","display_name":"uncertainty quantification","score":0.968999981880188},{"id":"https://openalex.org/keywords/calibration","display_name":"calibration","score":0.6389999985694885},{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9940000176429749},{"id":"https://openalex.org/keywords/cross-modal-consistency","display_name":"cross-modal consistency","score":0.7850000262260437},{"id":"https://openalex.org/keywords/self-consistency","display_name":"self-consistency","score":0.859000027179718},{"id":"https://openalex.org/keywords/grounding","display_name":"grounding","score":0.5199999809265137},{"id":"https://openalex.org/keywords/temperature-scaling","display_name":"temperature scaling","score":0.8769999742507935},{"id":"https://openalex.org/keywords/confidence-calibration","display_name":"confidence calibration","score":0.6620000004768372},{"id":"https://openalex.org/keywords/visual-question-answering","display_name":"visual question answering","score":0.8320000171661377},{"id":"https://openalex.org/keywords/medical-question-answering","display_name":"medical question answering","score":0.921999990940094},{"id":"https://openalex.org/keywords/llava","display_name":"LLaVA","score":0.9549999833106995}],"concepts":[{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.6729000210762024},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6402999758720398},{"id":"https://openalex.org/C32230216","wikidata":"https://www.wikidata.org/wiki/Q7882499","display_name":"Uncertainty quantification","level":2,"score":0.6225000023841858},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6225000023841858},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6202999949455261},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.5806999802589417},{"id":"https://openalex.org/C168993435","wikidata":"https://www.wikidata.org/wiki/Q6501125","display_name":"Ground","level":2,"score":0.5324000120162964},{"id":"https://openalex.org/C177803969","wikidata":"https://www.wikidata.org/wiki/Q29205","display_name":"Uncertainty analysis","level":2,"score":0.48989999294281006},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.483599990606308},{"id":"https://openalex.org/C137209882","wikidata":"https://www.wikidata.org/wiki/Q1403517","display_name":"Measurement uncertainty","level":2,"score":0.4074000120162964},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.3944000005722046},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.385699987411499},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3504999876022339},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3472999930381775},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.3303999900817871},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.32089999318122864},{"id":"https://openalex.org/C143380155","wikidata":"https://www.wikidata.org/wiki/Q5160088","display_name":"Confidence region","level":3,"score":0.30320000648498535},{"id":"https://openalex.org/C24574437","wikidata":"https://www.wikidata.org/wiki/Q7135228","display_name":"Parametric model","level":3,"score":0.29600000381469727},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C13662513","wikidata":"https://www.wikidata.org/wiki/Q5160087","display_name":"Confidence distribution","level":3,"score":0.2732999920845032},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.2685000002384186},{"id":"https://openalex.org/C2909755999","wikidata":"https://www.wikidata.org/wiki/Q4751126","display_name":"Low Confidence","level":2,"score":0.2533999979496002}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.03788","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.03788","pdf_url":"https://arxiv.org/pdf/2505.03788","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.03788","doi":"https://doi.org/10.48550/arxiv.2505.03788","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.03788","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.03788","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.03788","pdf_url":"https://arxiv.org/pdf/2505.03788","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,143],"introduce":[1],"a":[2,56,109,124],"novel":[3],"approach":[4,147],"for":[5,11],"calibrating":[6],"uncertainty":[7,115],"quantification":[8],"(UQ)":[9],"tailored":[10],"multi-modal":[12,81,150,164],"large":[13],"language":[14],"models":[15,165],"(LLMs).":[16],"Existing":[17],"state-of-the-art":[18],"UQ":[19],"methods":[20],"rely":[21],"on":[22,31,182],"consistency":[23,70],"among":[24],"multiple":[25,149],"responses":[26,88],"generated":[27,141],"by":[28],"the":[29,48,77,80,86,90,96,103,117,133,138,145,175],"LLM":[30,49],"an":[32],"input":[33],"query":[34],"under":[35],"diverse":[36],"settings.":[37],"However,":[38],"these":[39],"approaches":[40],"often":[41],"report":[42],"higher":[43],"confidence":[44,59,94,136],"in":[45,71,116,137],"scenarios":[46],"where":[47],"is":[50,99],"consistently":[51],"incorrect.":[52],"This":[53],"leads":[54],"to":[55,62,73,75,89,101,131],"poorly":[57],"calibrated":[58],"with":[60],"respect":[61],"accuracy.":[63],"To":[64],"address":[65],"this,":[66],"we":[67,84,119],"leverage":[68],"cross-modal":[69],"addition":[72],"self-consistency":[74],"improve":[76],"calibration":[78,128,181],"of":[79,140],"models.":[82],"Specifically,":[83],"ground":[85],"textual":[87],"visual":[91,159],"inputs.":[92],"The":[93,171],"from":[95],"grounding":[97,110,134],"model":[98,111],"used":[100],"calibrate":[102,132],"overall":[104],"confidence.":[105],"Given":[106],"that":[107,174],"using":[108],"adds":[112],"its":[113],"own":[114],"pipeline,":[118],"apply":[120],"temperature":[121],"scaling":[122],"-":[123,130],"widely":[125],"accepted":[126],"parametric":[127],"technique":[129],"model's":[135],"accuracy":[139],"responses.":[142],"evaluate":[144],"proposed":[146,176],"across":[148],"tasks,":[151],"such":[152,166],"as":[153,167],"medical":[154],"question":[155,160],"answering":[156,161],"(Slake)":[157],"and":[158,169],"(VQAv2),":[162],"considering":[163],"LLaVA-Med":[168],"LLaVA.":[170],"experiments":[172],"demonstrate":[173],"framework":[177],"achieves":[178],"significantly":[179],"improved":[180],"both":[183],"tasks.":[184]},"counts_by_year":[],"updated_date":"2026-10-12T09:40:04.860316","created_date":"2025-10-16T00:00:00"}