{"id":"https://openalex.org/W4415180041","doi":"https://doi.org/10.1109/tpami.2025.3621246","title":"A Unified Masked Jigsaw Puzzle Framework for Vision and Language Models","display_name":"A Unified Masked Jigsaw Puzzle Framework for Vision and Language Models","publication_year":2025,"publication_date":"2025-10-14","ids":{"openalex":"https://openalex.org/W4415180041","doi":"https://doi.org/10.1109/tpami.2025.3621246","pmid":"https://pubmed.ncbi.nlm.nih.gov/41086081"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2025.3621246","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3621246","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://ieeexplore.ieee.org/document/11202736/","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043804740","display_name":"Weixin Ye","orcid":null},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Weixin Ye","raw_affiliation_strings":["Beijing Jiaotong University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wei Wang","orcid":"https://orcid.org/0000-0002-9837-9615"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Wang","raw_affiliation_strings":["Beijing Jiaotong University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9837-9615","affiliations":[{"raw_affiliation_string":"Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100375513","display_name":"Yahui Liu","orcid":"https://orcid.org/0000-0003-4398-8867"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yahui Liu","raw_affiliation_strings":["Kuaishou, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4398-8867","affiliations":[{"raw_affiliation_string":"Kuaishou, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089428065","display_name":"Yue Song","orcid":"https://orcid.org/0000-0003-1573-5643"},"institutions":[{"id":"https://openalex.org/I122411786","display_name":"California Institute of Technology","ror":"https://ror.org/05dxps055","country_code":"US","type":"education","lineage":["https://openalex.org/I122411786"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yue Song","raw_affiliation_strings":["Caltech, Pasadena, CA, USA","Caltech, USA"],"raw_orcid":"https://orcid.org/0000-0003-1573-5643","affiliations":[{"raw_affiliation_string":"Caltech, Pasadena, CA, USA","institution_ids":["https://openalex.org/I122411786"]},{"raw_affiliation_string":"Caltech, USA","institution_ids":["https://openalex.org/I122411786"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101795339","display_name":"Bin Ren","orcid":"https://orcid.org/0000-0002-9790-1504"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Bin Ren","raw_affiliation_strings":["University of Trento, Trento, Italy","University of Trento, Italy"],"raw_orcid":"https://orcid.org/0000-0002-9790-1504","affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy","institution_ids":["https://openalex.org/I193223587"]},{"raw_affiliation_string":"University of Trento, Italy","institution_ids":["https://openalex.org/I193223587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101671512","display_name":"Wei Bi","orcid":"https://orcid.org/0000-0001-8457-0630"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wei Bi","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong, SAR, China","Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, SAR, China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030948871","display_name":"Rita Cucchiara","orcid":"https://orcid.org/0000-0002-2239-283X"},"institutions":[{"id":"https://openalex.org/I122346577","display_name":"University of Modena and Reggio Emilia","ror":"https://ror.org/02d4c4y02","country_code":"IT","type":"education","lineage":["https://openalex.org/I122346577"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Rita Cucchiara","raw_affiliation_strings":["University of Modena and Reggio Emilia, Modena, Italy"],"raw_orcid":"https://orcid.org/0000-0002-2239-283X","affiliations":[{"raw_affiliation_string":"University of Modena and Reggio Emilia, Modena, Italy","institution_ids":["https://openalex.org/I122346577"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027171279","display_name":"Nicu Sebe","orcid":"https://orcid.org/0000-0002-6597-7248"},"institutions":[{"id":"https://openalex.org/I193223587","display_name":"University of Trento","ror":"https://ror.org/05trd4x28","country_code":"IT","type":"education","lineage":["https://openalex.org/I193223587"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Nicu Sebe","raw_affiliation_strings":["University of Trento, Trento, Italy","University of Trento, Italy"],"raw_orcid":"https://orcid.org/0000-0002-6597-7248","affiliations":[{"raw_affiliation_string":"University of Trento, Trento, Italy","institution_ids":["https://openalex.org/I193223587"]},{"raw_affiliation_string":"University of Trento, Italy","institution_ids":["https://openalex.org/I193223587"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5043804740"],"corresponding_institution_ids":["https://openalex.org/I21193070"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.24621193,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"48","issue":"2","first_page":"1873","last_page":"1887"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14339","display_name":"Image Processing and 3D Reconstruction","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14339","display_name":"Image Processing and 3D Reconstruction","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11211","display_name":"3D Surveying and Cultural Heritage","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1907","display_name":"Geology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.8465999960899353},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6769000291824341},{"id":"https://openalex.org/keywords/jigsaw","display_name":"Jigsaw","score":0.6696000099182129},{"id":"https://openalex.org/keywords/shuffling","display_name":"Shuffling","score":0.588699996471405},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.5687000155448914},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4555000066757202},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.4431999921798706},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.3928999900817871}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.8465999960899353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.781499981880188},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6769000291824341},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6729000210762024},{"id":"https://openalex.org/C2779405079","wikidata":"https://www.wikidata.org/wiki/Q356040","display_name":"Jigsaw","level":2,"score":0.6696000099182129},{"id":"https://openalex.org/C167927819","wikidata":"https://www.wikidata.org/wiki/Q1930567","display_name":"Shuffling","level":2,"score":0.588699996471405},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5687000155448914},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4555000066757202},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.4431999921798706},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4156000018119812},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3928999900817871},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3801000118255615},{"id":"https://openalex.org/C182548165","wikidata":"https://www.wikidata.org/wiki/Q2261483","display_name":"Scrambling","level":2,"score":0.3555000126361847},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3544999957084656},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.33219999074935913},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31619998812675476},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.30160000920295715},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.29829999804496765},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.2833999991416931},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.28290000557899475},{"id":"https://openalex.org/C9417928","wikidata":"https://www.wikidata.org/wiki/Q1070689","display_name":"Image processing","level":3,"score":0.27970001101493835},{"id":"https://openalex.org/C176809094","wikidata":"https://www.wikidata.org/wiki/Q15401496","display_name":"Traverse","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C160633673","wikidata":"https://www.wikidata.org/wiki/Q355198","display_name":"Pixel","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.2685999870300293},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tpami.2025.3621246","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2025.3621246","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:41086081","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41086081","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null},{"id":"pmh:oai:iris.unitn.it:11572/474154","is_oa":true,"landing_page_url":"https://ieeexplore.ieee.org/document/11202736/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401913","display_name":"Institutional Research Information System (Universit\u00e0 degli Studi di Trento)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193223587","host_organization_name":"University of Trento","host_organization_lineage":["https://openalex.org/I193223587"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:iris.unitn.it:11572/474154","is_oa":true,"landing_page_url":"https://ieeexplore.ieee.org/document/11202736/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401913","display_name":"Institutional Research Information System (Universit\u00e0 degli Studi di Trento)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193223587","host_organization_name":"University of Trento","host_organization_lineage":["https://openalex.org/I193223587"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7784432583","display_name":null,"funder_award_id":"2022XKRC015","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2117539524","https://openalex.org/W2134149606","https://openalex.org/W2194775991","https://openalex.org/W2251939518","https://openalex.org/W2296283641","https://openalex.org/W2507296351","https://openalex.org/W2737258237","https://openalex.org/W2783522756","https://openalex.org/W2884822772","https://openalex.org/W2896457183","https://openalex.org/W2962785568","https://openalex.org/W2963159690","https://openalex.org/W2963563276","https://openalex.org/W2963748441","https://openalex.org/W2963925437","https://openalex.org/W2970408908","https://openalex.org/W2978670439","https://openalex.org/W2980088508","https://openalex.org/W2992308087","https://openalex.org/W3016632787","https://openalex.org/W3035452548","https://openalex.org/W3096609285","https://openalex.org/W3102094970","https://openalex.org/W3105816068","https://openalex.org/W3108154605","https://openalex.org/W3131500599","https://openalex.org/W3132607382","https://openalex.org/W3138516171","https://openalex.org/W3170841864","https://openalex.org/W3171125843","https://openalex.org/W3171516518","https://openalex.org/W3172312230","https://openalex.org/W3175192640","https://openalex.org/W3175515048","https://openalex.org/W3188511781","https://openalex.org/W3200520312","https://openalex.org/W3207918547","https://openalex.org/W4206199121","https://openalex.org/W4206706211","https://openalex.org/W4214736485","https://openalex.org/W4221161778","https://openalex.org/W4225793049","https://openalex.org/W4312604272","https://openalex.org/W4312705808","https://openalex.org/W4313156423","https://openalex.org/W4384080537","https://openalex.org/W4385245566","https://openalex.org/W4386083168","https://openalex.org/W4388979610"],"related_works":[],"abstract_inverted_index":{"In":[0,98],"federated":[1],"learning,":[2],"Transformer,":[3],"as":[4,162],"a":[5,62,81,184],"popular":[6],"architecture,":[7],"faces":[8],"critical":[9],"challenges":[10],"in":[11,20,42,108,154,191],"defending":[12],"against":[13,146],"gradient":[14,37,147],"attacks":[15],"and":[16,25,79,114,157,169,176,194],"improving":[17],"model":[18],"performance":[19,153],"both":[21,155,192],"Computer":[22],"Vision":[23],"(CV)":[24],"Natural":[26],"Language":[27],"Processing":[28],"(NLP)":[29],"tasks.":[30,196],"It":[31],"has":[32],"been":[33],"revealed":[34],"that":[35,123,181],"the":[36,53,76,92,95,101,109,115,128,134],"of":[38,94,137],"Position":[39],"Embeddings":[40],"(PEs)":[41],"Transformer":[43],"contains":[44],"sufficient":[45],"information,":[46],"which":[47,105],"can":[48,140],"be":[49],"used":[50,88],"to":[51,74,89,119],"reconstruct":[52],"input":[54],"data.":[55],"To":[56],"mitigate":[57],"this":[58,99],"issue,":[59],"we":[60,139],"introduce":[61],"Masked":[63],"Jigsaw":[64],"Puzzle":[65],"(MJP)":[66],"framework.":[67],"MJP":[68,182],"starts":[69],"with":[70,133],"random":[71],"token":[72,77],"shuffling":[73],"break":[75],"order,":[78],"then":[80],"learnable":[82],"unknown":[83],"(unk)":[84],"position":[85,110],"embedding":[86],"is":[87,106,112,183],"mask":[90],"out":[91],"PEs":[93],"shuffled":[96],"tokens.":[97],"manner,":[100],"local":[102,129],"spatial":[103,130],"information":[104],"encoded":[107],"embeddings":[111],"disrupted,":[113],"models":[116,190],"are":[117,124],"forced":[118],"learn":[120],"feature":[121],"representations":[122],"less":[125],"reliant":[126],"on":[127],"information.":[131],"Notably,":[132],"careful":[135],"use":[136],"MJP,":[138],"not":[141],"only":[142],"improve":[143],"models'":[144],"robustness":[145],"attacks,":[148],"but":[149],"also":[150],"boost":[151],"their":[152],"vision":[156,193],"text":[158,173],"application":[159],"scenarios,":[160],"such":[161],"classification":[163],"for":[164,172,187],"images":[165],"(e.g.,":[166,174],"ImageNet-1":[167],"K)":[168],"sentiment":[170],"analysis":[171],"Yelp":[175],"Amazon).":[177],"Experimental":[178],"results":[179],"suggest":[180],"unified":[185],"framework":[186],"different":[188],"Transformer-based":[189],"language":[195]},"counts_by_year":[],"updated_date":"2026-06-04T09:04:59.091469","created_date":"2025-10-15T00:00:00"}