{"id":"https://openalex.org/W7125595817","doi":"https://doi.org/10.1109/miot.2026.3650753","title":"Privacy-Preserving Multimodal Reasoning for Internet of Things: A Retrieval-Augmented Large Language and Vision Assistant Framework","display_name":"Privacy-Preserving Multimodal Reasoning for Internet of Things: A Retrieval-Augmented Large Language and Vision Assistant Framework","publication_year":2026,"publication_date":"2026-01-23","ids":{"openalex":"https://openalex.org/W7125595817","doi":"https://doi.org/10.1109/miot.2026.3650753"},"language":null,"primary_location":{"id":"doi:10.1109/miot.2026.3650753","is_oa":false,"landing_page_url":"https://doi.org/10.1109/miot.2026.3650753","pdf_url":null,"source":{"id":"https://openalex.org/S4210201254","display_name":"IEEE Internet of Things Magazine","issn_l":"2576-3180","issn":["2576-3180","2576-3199"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Internet of Things Magazine","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114065101","display_name":"Tengjun Ni","orcid":null},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Tengjun Ni","raw_affiliation_strings":["Global Big Data Technologies Centre and the School of Electrical and Data Engineering, University of Technology Sydney, Sydney, NSW, Australia"],"raw_orcid":"https://orcid.org/0009-0005-7001-3039","affiliations":[{"raw_affiliation_string":"Global Big Data Technologies Centre and the School of Electrical and Data Engineering, University of Technology Sydney, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049765501","display_name":"Xin Yuan","orcid":"https://orcid.org/0000-0002-9167-1613"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I4210101388","display_name":"Health Sciences and Nutrition","ror":"https://ror.org/0152bt112","country_code":"AU","type":"facility","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4210101388","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Xin Yuan","raw_affiliation_strings":["Commonwealth Scientific and Industrial Research Organization (CSIRO), Marsfield, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0002-9167-1613","affiliations":[{"raw_affiliation_string":"Commonwealth Scientific and Industrial Research Organization (CSIRO), Marsfield, NSW, Australia","institution_ids":["https://openalex.org/I4210101388","https://openalex.org/I1292875679"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053636716","display_name":"S. Li","orcid":null},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I4210101388","display_name":"Health Sciences and Nutrition","ror":"https://ror.org/0152bt112","country_code":"AU","type":"facility","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4210101388","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Shenghong Li","raw_affiliation_strings":["Commonwealth Scientific and Industrial Research Organization (CSIRO), Marsfield, NSW, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5516-9394","affiliations":[{"raw_affiliation_string":"Commonwealth Scientific and Industrial Research Organization (CSIRO), Marsfield, NSW, Australia","institution_ids":["https://openalex.org/I4210101388","https://openalex.org/I1292875679"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101544825","display_name":"Wei Ni","orcid":"https://orcid.org/0000-0002-4933-594X"},"institutions":[{"id":"https://openalex.org/I12079687","display_name":"Edith Cowan University","ror":"https://ror.org/05jhnwe22","country_code":"AU","type":"education","lineage":["https://openalex.org/I12079687"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Wei Ni","raw_affiliation_strings":["School of Engineering, Edith Cowan University, Perth, WA, Australia"],"raw_orcid":"https://orcid.org/0000-0002-4933-594X","affiliations":[{"raw_affiliation_string":"School of Engineering, Edith Cowan University, Perth, WA, Australia","institution_ids":["https://openalex.org/I12079687"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.09482226,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"9","issue":"3","first_page":"113","last_page":"122"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7767999768257141,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7767999768257141,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.04280000180006027,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.024700000882148743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.4611000120639801},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4406999945640564},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.40130001306533813},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.3804999887943268},{"id":"https://openalex.org/keywords/information-privacy","display_name":"Information privacy","score":0.37130001187324524},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.3619999885559082},{"id":"https://openalex.org/keywords/context-model","display_name":"Context model","score":0.35370001196861267},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.353300005197525},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.3402999937534332}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8848999738693237},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49309998750686646},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.4611000120639801},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4406999945640564},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.40130001306533813},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C123201435","wikidata":"https://www.wikidata.org/wiki/Q456632","display_name":"Information privacy","level":2,"score":0.37130001187324524},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.3619999885559082},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.35749998688697815},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.35370001196861267},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.353300005197525},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3465999960899353},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.3402999937534332},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3271999955177307},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C40305131","wikidata":"https://www.wikidata.org/wiki/Q2616305","display_name":"Obfuscation","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.301800012588501},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C2779599972","wikidata":"https://www.wikidata.org/wiki/Q82231","display_name":"NoSQL","level":3,"score":0.2964000105857849},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.29600000381469727},{"id":"https://openalex.org/C25810664","wikidata":"https://www.wikidata.org/wiki/Q44325","display_name":"Ontology","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2937000095844269},{"id":"https://openalex.org/C2129575","wikidata":"https://www.wikidata.org/wiki/Q54837","display_name":"Semantic Web","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2808000147342682},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2791999876499176},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.27219998836517334},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26260000467300415},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.25279998779296875},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/miot.2026.3650753","is_oa":false,"landing_page_url":"https://doi.org/10.1109/miot.2026.3650753","pdf_url":null,"source":{"id":"https://openalex.org/S4210201254","display_name":"IEEE Internet of Things Magazine","issn_l":"2576-3180","issn":["2576-3180","2576-3199"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Internet of Things Magazine","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"Internet":[1],"of":[2,198,256],"Things":[3],"(IoT)":[4],"generates":[5],"massive":[6],"multimodal":[7],"data":[8,165,239],"that":[9,91,169,220,250],"can":[10],"potentially":[11],"power":[12],"sophisticated":[13],"analytics":[14],"using":[15],"Multimodal":[16],"Large":[17],"Language":[18],"Models":[19],"(MLLMs)":[20],"and":[21,37,107,129,158,188,213,241,267,278],"give":[22],"rise":[23],"to":[24,109,131,153,162,182,208,287],"new":[25],"paradigms":[26],"where":[27],"users":[28],"interact":[29],"with":[30],"IoT":[31,39,85,175,210,258],"systems":[32],"through":[33],"natural":[34],"language":[35],"prompts":[36],"query":[38],"applications":[40],"for":[41,63,84,280],"complex":[42],"insights.":[43,190],"However,":[44],"these":[45],"models":[46],"pose":[47],"significant":[48],"privacy":[49,94,224,235,254,276,282],"risks":[50],"when":[51],"sensitive":[52,111,133],"data,":[53],"such":[54],"as":[55],"personal":[56],"identifiers":[57],"or":[58,66,96,116,138],"proprietary":[59],"information,":[60],"is":[61,114,136],"shared":[62],"model":[64,122,144],"training":[65],"finetuning":[67,242],"at":[68,236],"scale.":[69],"In":[70],"this":[71,244],"paper,":[72],"we":[73,177],"propose":[74],"a":[75,155,194,204,214,247,281,291],"novel":[76],"privacy-enhanced":[77],"Visual":[78],"Question":[79],"Answering":[80],"(VQA)":[81],"framework":[82,103,125,222,271],"tailored":[83],"scenarios.":[86],"Unlike":[87],"prior":[88],"VQA":[89],"pipelines":[90],"omit":[92],"explicit":[93],"controls":[95],"rely":[97],"on":[98,193],"coarse":[99],"box":[100],"masking,":[101],"our":[102,221,270],"applies":[104,126],"semantic":[105,127],"segmentation":[106,128],"labeling":[108,130],"isolate":[110,132],"content,":[112,134],"which":[113,135,263],"removed":[115,137],"obfuscated":[117,139],"via":[118,140,203],"diffusion":[119,141],"inpainting":[120,142],"before":[121,143],"processing.":[123,145],"Our":[124],"We":[146],"then":[147],"leverage":[148],"parameter-efficient":[149],"Low-Rank":[150],"Adaptation":[151],"(LoRA)":[152],"finetune":[154],"language\u2013vision":[156],"assistant":[157],"incorporate":[159],"Ghost":[160],"Clipping":[161],"reduce":[163],"gradient-based":[164],"leakage.":[166],"For":[167],"questions":[168],"require":[170],"context":[171],"beyond":[172],"the":[173,199,238,252,261],"processed":[174],"images,":[176],"introduce":[178],"Retrieval-Augmented":[179],"Generation":[180],"(RAG)":[181],"draw":[183],"upon":[184],"external":[185],"knowledge":[186],"sources":[187],"deliver":[189],"Experimental":[191],"results":[192],"customized":[195],"IoT-oriented":[196],"subset":[197],"OK-VQA":[200],"dataset":[201],"constructed":[202],"CLIP":[205],"similarity":[206],"score":[207],"an":[209],"phrase":[211],"bank":[212],"light":[215],"COCO":[216],"super-category":[217],"filter":[218],"illustrate":[219],"preserves":[223],"while":[225],"achieving":[226],"competitive":[227],"accuracy,":[228],"even":[229,289],"outperforming":[230],"standard":[231],"non-private":[232,292],"baselines.":[233],"Protecting":[234],"both":[237],"preprocessing":[240],"stages,":[243],"work":[245],"offers":[246],"scalable":[248],"solution":[249],"meets":[251],"stringent":[253],"demands":[255],"real-world":[257],"environments.":[259],"On":[260],"OK-VQA,":[262],"contains":[264],"14,055":[265],"images":[266],"QA":[268],"pairs,":[269],"attains":[272],"76.1%":[273],"accuracy":[274],"without":[275],"consideration,":[277],"49.3%\u201375.2%":[279],"budget":[283],"ranging":[284],"from":[285],"0.5":[286],"5,":[288],"surpassing":[290],"LLaVA-7B":[293],"baseline":[294],"(73.7%).":[295]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-01-25T00:00:00"}