{"id":"https://openalex.org/W7166859504","doi":"https://doi.org/10.1016/j.jvcir.2026.104887","title":"OCC-MLLM-V2: Joint understanding and generation for occluded objects via multi-modal token learning","display_name":"OCC-MLLM-V2: Joint understanding and generation for occluded objects via multi-modal token learning","publication_year":2026,"publication_date":"2026-07-01","ids":{"openalex":"https://openalex.org/W7166859504","doi":"https://doi.org/10.1016/j.jvcir.2026.104887"},"language":"en","primary_location":{"id":"doi:10.1016/j.jvcir.2026.104887","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jvcir.2026.104887","pdf_url":null,"source":{"id":"https://openalex.org/S105424869","display_name":"Journal of Visual Communication and Image Representation","issn_l":"1047-3203","issn":["1047-3203","1095-9076"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Visual Communication and Image Representation","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.jvcir.2026.104887","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045229314","display_name":"Chaoyi Wang","orcid":"https://orcid.org/0000-0003-0164-1953"},"institutions":[{"id":"https://openalex.org/I4210147322","display_name":"Shanghai Institute of Microsystem and Information Technology","ror":"https://ror.org/04nytyj38","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210147322"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chaoyi Wang","raw_affiliation_strings":["School of Electronic, Electrical, and Communication, University of Chinese Academy of Sciences, Beijing, 100049, China","Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China"],"raw_orcid":"https://orcid.org/0000-0003-0164-1953","affiliations":[{"raw_affiliation_string":"School of Electronic, Electrical, and Communication, University of Chinese Academy of Sciences, Beijing, 100049, China","institution_ids":["https://openalex.org/I4210165038"]},{"raw_affiliation_string":"Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China","institution_ids":["https://openalex.org/I4210147322"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139798937","display_name":"Qingdong He","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingdong He","raw_affiliation_strings":["Youtu Lab, Tencent, Longteng Avenue, Shanghai, 200032, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Youtu Lab, Tencent, Longteng Avenue, Shanghai, 200032, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139849923","display_name":"Yifan Yang","orcid":"https://orcid.org/0009-0006-9752-1524"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifan Yang","raw_affiliation_strings":["Shanghai Jiao Tong University, Dongchuan Road 800, Shanghai, 200240, China"],"raw_orcid":"https://orcid.org/0009-0006-9752-1524","affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University, Dongchuan Road 800, Shanghai, 200240, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139831003","display_name":"Jun Pei","orcid":null},"institutions":[{"id":"https://openalex.org/I4210147322","display_name":"Shanghai Institute of Microsystem and Information Technology","ror":"https://ror.org/04nytyj38","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210147322"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Pei","raw_affiliation_strings":["Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China","institution_ids":["https://openalex.org/I4210147322"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137933292","display_name":"Lijie Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I4210147322","display_name":"Shanghai Institute of Microsystem and Information Technology","ror":"https://ror.org/04nytyj38","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210147322"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lijie Xia","raw_affiliation_strings":["Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China","institution_ids":["https://openalex.org/I4210147322"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121174994","display_name":"Jianpo Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210147322","display_name":"Shanghai Institute of Microsystem and Information Technology","ror":"https://ror.org/04nytyj38","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210147322"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianpo Liu","raw_affiliation_strings":["Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China","institution_ids":["https://openalex.org/I4210147322"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139796316","display_name":"Baoqing Li","orcid":null},"institutions":[{"id":"https://openalex.org/I4210147322","display_name":"Shanghai Institute of Microsystem and Information Technology","ror":"https://ror.org/04nytyj38","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210147322"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Baoqing Li","raw_affiliation_strings":["Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences, Pingcheng Road 1455, Shanghai, 201899, China","institution_ids":["https://openalex.org/I4210147322"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021368755","display_name":"Xinhan Di","orcid":"https://orcid.org/0009-0001-8855-8628"},"institutions":[{"id":"https://openalex.org/I5343935","display_name":"Guilin University of Electronic Technology","ror":"https://ror.org/05arjae42","country_code":"CN","type":"education","lineage":["https://openalex.org/I5343935"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xinhan Di","raw_affiliation_strings":["Giant Network, Guilin Road 396, Shanghai, 200233, China"],"raw_orcid":"https://orcid.org/0009-0001-8855-8628","affiliations":[{"raw_affiliation_string":"Giant Network, Guilin Road 396, Shanghai, 200233, China","institution_ids":["https://openalex.org/I5343935"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5021368755","https://openalex.org/A5139796316"],"corresponding_institution_ids":["https://openalex.org/I4210147322","https://openalex.org/I5343935"],"apc_list":{"value":2150,"currency":"USD","value_usd":2150},"apc_paid":{"value":2150,"currency":"USD","value_usd":2150},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.87192447,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"119","issue":null,"first_page":"104887","last_page":"104887"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5616999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5616999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.10819999873638153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.05920000001788139,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.7748000025749207},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4447000026702881},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3066999912261963},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.30169999599456787},{"id":"https://openalex.org/keywords/view-synthesis","display_name":"View synthesis","score":0.27320000529289246}],"concepts":[{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.7748000025749207},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7063999772071838},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5835000276565552},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5760999917984009},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4447000026702881},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.30169999599456787},{"id":"https://openalex.org/C2776449333","wikidata":"https://www.wikidata.org/wiki/Q7928781","display_name":"View synthesis","level":3,"score":0.27320000529289246},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2669999897480011},{"id":"https://openalex.org/C137878579","wikidata":"https://www.wikidata.org/wiki/Q9636076","display_name":"Joint attention","level":3,"score":0.23469999432563782}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.jvcir.2026.104887","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jvcir.2026.104887","pdf_url":null,"source":{"id":"https://openalex.org/S105424869","display_name":"Journal of Visual Communication and Image Representation","issn_l":"1047-3203","issn":["1047-3203","1095-9076"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Visual Communication and Image Representation","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.jvcir.2026.104887","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jvcir.2026.104887","pdf_url":null,"source":{"id":"https://openalex.org/S105424869","display_name":"Journal of Visual Communication and Image Representation","issn_l":"1047-3203","issn":["1047-3203","1095-9076"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Visual Communication and Image Representation","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320337104","display_name":"Shanghai Institute of Microsystem and Information Technology, Chinese Academy of Sciences","ror":"https://ror.org/04nytyj38"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W2227547437","https://openalex.org/W2516753435","https://openalex.org/W2963660453","https://openalex.org/W2968722025","https://openalex.org/W3006871679","https://openalex.org/W3034479523","https://openalex.org/W3111278072","https://openalex.org/W3136792391","https://openalex.org/W3180172155","https://openalex.org/W4206982465","https://openalex.org/W4214630662","https://openalex.org/W4312331262","https://openalex.org/W4386066654","https://openalex.org/W4390873312","https://openalex.org/W4398138340","https://openalex.org/W4399492477","https://openalex.org/W4402727536","https://openalex.org/W4402727574","https://openalex.org/W4403791995","https://openalex.org/W4405426927","https://openalex.org/W4409975538","https://openalex.org/W4413145497","https://openalex.org/W4413147805","https://openalex.org/W4413156242","https://openalex.org/W4415796345"],"related_works":[],"abstract_inverted_index":{"Comprehending":[0],"occluded":[1],"objects":[2],"remains":[3],"a":[4],"critical":[5],"challenge":[6],"for":[7,78,83],"multi-modal":[8],"large":[9],"language":[10],"models":[11],"due":[12],"to":[13,33],"missing":[14],"visual":[15],"representations.":[16],"Current":[17],"approaches":[18],"either":[19],"rely":[20],"on":[21,111,115,137],"multi-stage":[22],"pipelines":[23],"with":[24,45,89],"error":[25],"propagation,":[26],"or":[27],"use":[28],"unified":[29,96],"encoders":[30],"that":[31],"fail":[32],"balance":[34],"understanding":[35,79],"and":[36,59,67,80,101,113,125,134,142],"generation.":[37],"We":[38],"propose":[39],"OCC-MLLM-V2,":[40],"an":[41],"end-to-end":[42,103],"autoregressive":[43],"framework":[44,97,131],"three":[46],"key":[47],"innovations:":[48],"(1)":[49],"Hierarchical":[50],"Trinity":[51],"Fusion":[52,66],"Architecture":[53],"integrating":[54],"multi-view":[55],"RGB,":[56],"hand":[57],"masks,":[58],"3D":[60],"reconstructions":[61],"via":[62],"Adaptive":[63],"Weight":[64],"Image":[65],"Spatial":[68],"Attention":[69],"Affine":[70],"Fusion;":[71],"(2)":[72],"Visual":[73,86],"Dual":[74,87],"Encoder":[75],"employing":[76],"SigLIP":[77],"VQ":[81],"tokenizers":[82],"generation;":[84],"(3)":[85],"Decoder":[88],"joint":[90],"optimization.":[91],"Unlike":[92],"pipeline":[93],"methods,":[94],"our":[95],"eliminates":[98],"sequential":[99],"dependencies":[100],"enables":[102],"gradient":[104],"flow.":[105],"Experiments":[106],"demonstrate":[107],"improvements:":[108],"6.27%":[109],"gains":[110],"ObMan":[112],"6.35%":[114],"DexYCB":[116],"across":[117],"1B-8B":[118],"models,":[119],"while":[120],"reducing":[121],"FLOPs":[122],"by":[123,128],"32%\u201359%":[124],"inference":[126],"time":[127],"9.5%\u201337%.":[129],"Our":[130],"surpasses":[132],"GPT-4o":[133],"Gemini":[135],"2.5-Pro":[136],"multiple":[138],"occlusion":[139],"benchmarks.":[140],"Code":[141],"data":[143],"are":[144],"available":[145],"at":[146],"https://github.com/chaoyiwang09/OCC-MLLM":[147],".":[148]},"counts_by_year":[],"updated_date":"2026-07-04T06:09:54.619538","created_date":"2026-07-02T00:00:00"}