{"id":"https://openalex.org/W4409581368","doi":"https://doi.org/10.1109/tpami.2025.3562422","title":"Prophet: Prompting Large Language Models With Complementary Answer Heuristics for Knowledge-Based Visual Question Answering","display_name":"Prophet: Prompting Large Language Models With Complementary Answer Heuristics for Knowledge-Based Visual Question Answering","publication_year":2025,"publication_date":"2025-04-18","ids":{"openalex":"https://openalex.org/W4409581368","doi":"https://doi.org/10.1109/tpami.2025.3562422","pmid":"https://pubmed.ncbi.nlm.nih.gov/40249675"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3562422","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3562422","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061025828","display_name":"Yu Zhou","orcid":"https://orcid.org/0000-0001-8407-1137"},"institutions":[{"id":"https://openalex.org/I50760025","display_name":"Hangzhou Dianzi University","ror":"https://ror.org/0576gt767","country_code":"CN","type":"education","lineage":["https://openalex.org/I50760025"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhou Yu","raw_affiliation_strings":["Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science, Hangzhou Dianzi University, Zhejiang, China","Key Laboratory of Complex Systems Modeling and Simulation, the School of Computer Science, Hangzhou Dianzi University, China"],"raw_orcid":"https://orcid.org/0000-0001-8407-1137","affiliations":[{"raw_affiliation_string":"Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science, Hangzhou Dianzi University, Zhejiang, China","institution_ids":["https://openalex.org/I50760025"]},{"raw_affiliation_string":"Key Laboratory of Complex Systems Modeling and Simulation, the School of Computer Science, Hangzhou Dianzi University, China","institution_ids":["https://openalex.org/I50760025"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101363884","display_name":"Xuecheng Ouyang","orcid":null},"institutions":[{"id":"https://openalex.org/I50760025","display_name":"Hangzhou Dianzi University","ror":"https://ror.org/0576gt767","country_code":"CN","type":"education","lineage":["https://openalex.org/I50760025"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuecheng Ouyang","raw_affiliation_strings":["HDU-ITMO Joint Institute, Hangzhou Dianzi University, Zhejiang, China","HDU-ITMO Joint Institute, Hangzhou Dianzi University, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HDU-ITMO Joint Institute, Hangzhou Dianzi University, Zhejiang, China","institution_ids":["https://openalex.org/I50760025"]},{"raw_affiliation_string":"HDU-ITMO Joint Institute, Hangzhou Dianzi University, China","institution_ids":["https://openalex.org/I50760025"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098749084","display_name":"Zhenwei Shao","orcid":null},"institutions":[{"id":"https://openalex.org/I50760025","display_name":"Hangzhou Dianzi University","ror":"https://ror.org/0576gt767","country_code":"CN","type":"education","lineage":["https://openalex.org/I50760025"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenwei Shao","raw_affiliation_strings":["Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science, Hangzhou Dianzi University, Zhejiang, China","Key Laboratory of Complex Systems Modeling and Simulation, the School of Computer Science, Hangzhou Dianzi University, China"],"raw_orcid":"https://orcid.org/0009-0005-3069-9347","affiliations":[{"raw_affiliation_string":"Key Laboratory of Complex Systems Modeling and Simulation, School of Computer Science, Hangzhou Dianzi University, Zhejiang, China","institution_ids":["https://openalex.org/I50760025"]},{"raw_affiliation_string":"Key Laboratory of Complex Systems Modeling and Simulation, the School of Computer Science, Hangzhou Dianzi University, China","institution_ids":["https://openalex.org/I50760025"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Meng Wang","orcid":"https://orcid.org/0000-0002-3094-7735"},"institutions":[{"id":"https://openalex.org/I16365422","display_name":"Hefei University of Technology","ror":"https://ror.org/02czkny70","country_code":"CN","type":"education","lineage":["https://openalex.org/I16365422"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meng Wang","raw_affiliation_strings":["School of Computer Science and Information Engineering, Hefei University of Technology, Anhui, China","School of Computer Science and Information Engineering, Hefei University of Technology, China"],"raw_orcid":"https://orcid.org/0000-0002-3094-7735","affiliations":[{"raw_affiliation_string":"School of Computer Science and Information Engineering, Hefei University of Technology, Anhui, China","institution_ids":["https://openalex.org/I16365422"]},{"raw_affiliation_string":"School of Computer Science and Information Engineering, Hefei University of Technology, China","institution_ids":["https://openalex.org/I16365422"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050817770","display_name":"Jun Yu","orcid":"https://orcid.org/0000-0003-1922-7283"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Yu","raw_affiliation_strings":["School of Intelligence Science and Engineering, Harbin Institute of Technology, Shenzhen, China","Department of Computer Science and Technology Harbin Institute of Technology Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-1922-7283","affiliations":[{"raw_affiliation_string":"School of Intelligence Science and Engineering, Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]},{"raw_affiliation_string":"Department of Computer Science and Technology Harbin Institute of Technology Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":4.615,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.95331847,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"47","issue":"8","first_page":"6797","last_page":"6808"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9750999808311462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.7345876097679138},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.706705629825592},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7045179605484009},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6697218418121338},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.521471381187439},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4452647566795349},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4388563632965088}],"concepts":[{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.7345876097679138},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.706705629825592},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7045179605484009},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6697218418121338},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.521471381187439},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4452647566795349},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4388563632965088},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2025.3562422","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3562422","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40249675","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40249675","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5400000214576721}],"awards":[{"id":"https://openalex.org/G1690291968","display_name":null,"funder_award_id":"62422204","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6151690139","display_name":null,"funder_award_id":"62020106007","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6570941292","display_name":null,"funder_award_id":"62125201","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7336325476","display_name":"\u5916\u90e8\u201c\u6570\u636e-\u77e5\u8bc6\u201d\u8054\u5408\u589e\u5f3a\u7684\u89c6\u89c9\u95ee\u7b54\u65b9\u6cd5\u7814\u7a76","funder_award_id":"62072147","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8792811732","display_name":null,"funder_award_id":"U24B20174","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W2016089260","https://openalex.org/W2080133951","https://openalex.org/W2277195237","https://openalex.org/W2560730294","https://openalex.org/W2561715562","https://openalex.org/W2745461083","https://openalex.org/W2896457183","https://openalex.org/W2946086442","https://openalex.org/W2947312908","https://openalex.org/W2963477107","https://openalex.org/W2963518342","https://openalex.org/W2964303913","https://openalex.org/W2966683369","https://openalex.org/W2970231061","https://openalex.org/W2979382951","https://openalex.org/W2981165461","https://openalex.org/W2988326850","https://openalex.org/W3004349648","https://openalex.org/W3034336960","https://openalex.org/W3087365719","https://openalex.org/W3090449556","https://openalex.org/W3091588028","https://openalex.org/W3139224848","https://openalex.org/W3172845486","https://openalex.org/W3173220247","https://openalex.org/W3181159501","https://openalex.org/W3196798856","https://openalex.org/W3199693760","https://openalex.org/W3207798279","https://openalex.org/W4226321975","https://openalex.org/W4283777642","https://openalex.org/W4312263373","https://openalex.org/W4312846625","https://openalex.org/W4312971273","https://openalex.org/W4384820618","https://openalex.org/W4385245566","https://openalex.org/W4385574358","https://openalex.org/W4386065596","https://openalex.org/W4389520282","https://openalex.org/W4390873086","https://openalex.org/W6679436768","https://openalex.org/W6752083267","https://openalex.org/W6766904570","https://openalex.org/W6778883912","https://openalex.org/W6791353385","https://openalex.org/W6798350552","https://openalex.org/W6810334672","https://openalex.org/W6810738896","https://openalex.org/W6811013733","https://openalex.org/W6838989559","https://openalex.org/W6839015040","https://openalex.org/W6839091850","https://openalex.org/W6842585177","https://openalex.org/W6843405348","https://openalex.org/W6851592950","https://openalex.org/W6851948999","https://openalex.org/W6853116092","https://openalex.org/W6854866820"],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2280422768","https://openalex.org/W3143197806","https://openalex.org/W4252555497","https://openalex.org/W2387743295","https://openalex.org/W3121175838","https://openalex.org/W4288267738","https://openalex.org/W2964413124","https://openalex.org/W4388937922","https://openalex.org/W3113264705"],"abstract_inverted_index":{"Knowledge-based":[0],"visual":[1,96],"question":[2],"answering":[3],"(VQA)":[4],"requires":[5],"external":[6,139],"knowledge":[7,19,22,55,61],"beyond":[8],"the":[9,13,31,35,59,65,80,83,86,94,100,152,175,180,192,220,262],"image":[10,181],"to":[11,30,44,57,92,98,115,173,259],"answer":[12,99,119,149,155,164],"question.":[14,101],"Early":[15],"studies":[16],"retrieve":[17],"required":[18,95],"from":[20,151],"explicit":[21],"bases":[23],"(KBs),":[24],"which":[25,255],"often":[26],"introduces":[27],"irrelevant":[28],"information":[29,97],"question,":[32,183],"hence":[33],"restricting":[34],"performance":[36],"of":[37,82,147,163,178,222],"their":[38],"models.":[39],"Recent":[40],"works":[41],"have":[42,76],"resorted":[43],"using":[45],"a":[46,128,133,170,186],"powerful":[47],"large":[48,249],"language":[49],"model":[50,131],"(LLM)":[51],"as":[52,85],"an":[53],"implicit":[54],"engine":[56],"acquire":[58],"necessary":[60],"for":[62,121],"answering.":[63],"Despite":[64],"encouraging":[66],"results":[67],"achieved":[68],"by":[69],"these":[70],"methods,":[71],"we":[72,105,125,143],"argue":[73],"that":[74,215],"they":[75],"not":[77],"fully":[78],"activated":[79],"capacity":[81],"LLM":[84,117,194],"provided":[87],"textual":[88],"input":[89],"is":[90,213,256],"insufficient":[91],"depict":[93],"In":[102],"this":[103],"paper,":[104],"present":[106],"Prophet-a":[107],"conceptually":[108],"simple,":[109],"flexible,":[110],"and":[111,157,182,229,232,238],"general":[112,214],"framework":[113],"designed":[114],"prompt":[116,172],"with":[118,219,247],"heuristics":[120,150,165],"knowledge-based":[122,135,209,265],"VQA.":[123],"Specifically,":[124],"first":[126],"train":[127],"vanilla":[129],"VQA":[130,136,153,210,224,266],"on":[132,206,264],"specific":[134],"dataset":[137],"without":[138],"knowledge.":[140],"After":[141],"that,":[142],"extract":[144],"two":[145,161],"types":[146,162],"complementary":[148],"model:":[154],"candidates":[156],"answer-aware":[158],"examples.":[159],"The":[160],"are":[166],"jointly":[167],"encoded":[168],"into":[169],"formatted":[171],"facilitate":[174],"LLM's":[176],"understanding":[177],"both":[179,227,236],"thus":[184],"generating":[185],"more":[187],"accurate":[188],"answer.":[189],"By":[190],"incorporating":[191],"state-of-the-art":[193,204],"GPT-3":[195],"(Brown":[196],"et":[197],"al.":[198],"2020),":[199],"Prophet":[200,212,242],"significantly":[201],"outperforms":[202],"existing":[203],"methods":[205],"four":[207],"challenging":[208],"datasets.":[211],"can":[216,243],"be":[217,245],"instantiated":[218],"combinations":[221],"different":[223,233,253],"models":[225,251],"(i.e.,":[226,235],"discriminative":[228],"generative":[230],"ones)":[231],"LLMs":[234],"commercial":[237],"open-source":[239],"ones).":[240],"Moreover,":[241],"also":[244],"integrated":[246],"modern":[248],"multimodal":[250],"in":[252],"stages,":[254],"named":[257],"Prophet++,":[258],"further":[260],"improve":[261],"capabilities":[263],"tasks.":[267]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}