{"id":"https://openalex.org/W4407450063","doi":"https://doi.org/10.1109/jbhi.2025.3538324","title":"Enhancing Visual Reasoning With LLM-Powered Knowledge Graphs for Visual Question Localized-Answering in Robotic Surgery","display_name":"Enhancing Visual Reasoning With LLM-Powered Knowledge Graphs for Visual Question Localized-Answering in Robotic Surgery","publication_year":2025,"publication_date":"2025-02-13","ids":{"openalex":"https://openalex.org/W4407450063","doi":"https://doi.org/10.1109/jbhi.2025.3538324","pmid":"https://pubmed.ncbi.nlm.nih.gov/40031828"},"language":"en","primary_location":{"id":"doi:10.1109/jbhi.2025.3538324","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2025.3538324","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Pengfei Hao","orcid":"https://orcid.org/0009-0007-8523-818X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]},{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN","HK"],"is_corresponding":true,"raw_author_name":"Pengfei Hao","raw_affiliation_strings":["ROAS Thrust, Systems Hub, The Hong Kong University of Science and Technology, Guangzhou, China","ROAS Thrust, Systems Hub, Hong Kong University of Science and Technology (Guangzhou), China"],"raw_orcid":"https://orcid.org/0009-0007-8523-818X","affiliations":[{"raw_affiliation_string":"ROAS Thrust, Systems Hub, The Hong Kong University of Science and Technology, Guangzhou, China","institution_ids":["https://openalex.org/I90610280","https://openalex.org/I889458895"]},{"raw_affiliation_string":"ROAS Thrust, Systems Hub, Hong Kong University of Science and Technology (Guangzhou), China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074311923","display_name":"Hongqiu Wang","orcid":"https://orcid.org/0000-0001-9726-4253"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]},{"id":"https://openalex.org/I90610280","display_name":"South China University of Technology","ror":"https://ror.org/0530pts50","country_code":"CN","type":"education","lineage":["https://openalex.org/I90610280"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Hongqiu Wang","raw_affiliation_strings":["ROAS Thrust, Systems Hub, The Hong Kong University of Science and Technology, Guangzhou, China","ROAS Thrust, Systems Hub, Hong Kong University of Science and Technology (Guangzhou), China"],"raw_orcid":"https://orcid.org/0000-0001-9726-4253","affiliations":[{"raw_affiliation_string":"ROAS Thrust, Systems Hub, The Hong Kong University of Science and Technology, Guangzhou, China","institution_ids":["https://openalex.org/I90610280","https://openalex.org/I889458895"]},{"raw_affiliation_string":"ROAS Thrust, Systems Hub, Hong Kong University of Science and Technology (Guangzhou), China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100436460","display_name":"Guang Yang","orcid":"https://orcid.org/0000-0001-7344-7733"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Guang Yang","raw_affiliation_strings":["Bioengineering/Imperial-X, Imperial College London, London, U.K","Bioengineering/Imperial-X, Imperial College London, U.K"],"raw_orcid":"https://orcid.org/0000-0001-7344-7733","affiliations":[{"raw_affiliation_string":"Bioengineering/Imperial-X, Imperial College London, London, U.K","institution_ids":["https://openalex.org/I47508984"]},{"raw_affiliation_string":"Bioengineering/Imperial-X, Imperial College London, U.K","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016081925","display_name":"Lei Zhu","orcid":"https://orcid.org/0000-0003-3871-663X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Lei Zhu","raw_affiliation_strings":["Henan Key Laboratory of Imaging and Intelligent Processing, Department of Electronic and Computer Engineering, The Hong Kong University of Science and Technology, Hong Kong, SAR, China","Thrust of Robotics and Autonomous Systems (ROAS), The Hong Kong University of Science and Technology (Guangzhou), Guangzhou, Guangdong, China"],"raw_orcid":"https://orcid.org/0000-0003-3871-663X","affiliations":[{"raw_affiliation_string":"Henan Key Laboratory of Imaging and Intelligent Processing, Department of Electronic and Computer Engineering, The Hong Kong University of Science and Technology, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]},{"raw_affiliation_string":"Thrust of Robotics and Autonomous Systems (ROAS), The Hong Kong University of Science and Technology (Guangzhou), Guangzhou, Guangdong, China","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895","https://openalex.org/I90610280"],"apc_list":null,"apc_paid":null,"fwci":6.7995,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.96539064,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"29","issue":"12","first_page":"9027","last_page":"9040"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9233999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7068692445755005},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5838057398796082},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4938414692878723},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.4138854146003723},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.35398542881011963}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7068692445755005},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5838057398796082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4938414692878723},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.4138854146003723},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.35398542881011963}],"mesh":[{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D065287","descriptor_name":"Robotic Surgical Procedures","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true},{"descriptor_ui":"D065287","descriptor_name":"Robotic Surgical Procedures","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1109/jbhi.2025.3538324","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2025.3538324","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},{"id":"pmid:40031828","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40031828","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE journal of biomedical and health informatics","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-146729","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-146729","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7605291260","display_name":null,"funder_award_id":"2024ZDZX2004","funder_id":"https://openalex.org/F4320324202","funder_display_name":"Guangdong Science and Technology Department"}],"funders":[{"id":"https://openalex.org/F4320324202","display_name":"Guangdong Science and Technology Department","ror":"https://ror.org/00tjzgn92"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2772917594","https://openalex.org/W2036807459","https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Expert":[0],"surgeons":[1],"often":[2],"have":[3],"heavy":[4],"workloads":[5],"and":[6,15,36,49,61,67,157,166,222,234],"cannot":[7],"promptly":[8],"respond":[9],"to":[10,32,45,59,76,148,183,206],"queries":[11],"from":[12,154],"medical":[13,34],"students":[14,35],"junior":[16,37],"doctors":[17,38],"about":[18],"surgical":[19,41,55,63,79,124,131,191,220],"procedures.":[20,68],"Thus,":[21],"research":[22],"on":[23,86,231],"Visual":[24,105],"Question":[25],"Localized-Answering":[26],"in":[27,39,53,193],"Surgery":[28],"(Surgical-VQLA)":[29],"is":[30,72],"essential":[31],"assist":[33],"understanding":[40,121,218],"scenarios.":[42],"Surgical-VQLA":[43,82,114],"aims":[44],"generate":[46],"accurate":[47],"answers":[48],"locate":[50],"relevant":[51,152],"areas":[52],"the":[54,73,113,119,123,135,150,162,189,194,198,201,217,232],"scene,":[56],"requiring":[57],"models":[58,83],"identify":[60],"understand":[62],"instruments,":[64],"operative":[65],"organs,":[66],"A":[69],"key":[70],"issue":[71],"model's":[74,120],"ability":[75],"accurately":[77],"distinguish":[78],"instruments.":[80],"Current":[81],"rely":[84],"primarily":[85],"sparse":[87],"textual":[88],"information,":[89],"limiting":[90],"their":[91],"visual":[92,208,225],"reasoning":[93,226],"capabilities.":[94,227],"To":[95],"address":[96],"this":[97],"issue,":[98],"we":[99,141,170],"propose":[100],"a":[101,143,172],"framework":[102,117],"called":[103],"Enhancing":[104],"Reasoning":[106],"with":[107,161,210],"LLM-Powered":[108],"Knowledge":[109,145],"Graphs":[110],"(EnVR-LPKG)":[111],"for":[112,251],"task.":[115],"This":[116],"enhances":[118],"of":[122,130,188,219],"scenario":[125],"by":[126,134],"utilizing":[127],"knowledge":[128,155,164,181,211],"graphs":[129,156,165,182],"instruments":[132,221],"constructed":[133],"Large":[136],"Language":[137],"Model":[138],"(LLM).":[139],"Specifically,":[140],"design":[142,171],"Fine-grained":[144],"Extractor":[146],"(FKE)":[147],"extract":[149],"most":[151],"information":[153],"perform":[158],"contrastive":[159],"learning":[160],"extracted":[163],"local":[167],"image.":[168],"Furthermore,":[169],"Multi-attention-based":[173],"Surgical":[174],"Instrument":[175],"Enhancer":[176],"(MSIE)":[177],"module,":[178,200],"which":[179],"employs":[180],"obtain":[184],"an":[185],"enhanced":[186],"representation":[187],"corresponding":[190],"instrument":[192],"global":[195],"scene.":[196],"Through":[197],"MSIE":[199],"model":[202],"can":[203],"learn":[204],"how":[205],"fuse":[207],"features":[209],"graph":[212],"text":[213],"features,":[214],"thereby":[215],"strengthening":[216],"further":[223],"improving":[224],"Extensive":[228],"experimental":[229],"results":[230],"EndoVis-17-VQLA":[233],"EndoVis-18-VQLA":[235],"datasets":[236],"demonstrate":[237],"that":[238],"our":[239,249],"proposed":[240],"method":[241],"outperforms":[242],"other":[243],"state-of-the-art":[244],"methods.":[245],"We":[246],"will":[247],"release":[248],"code":[250],"future":[252],"research.":[253]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}