{"id":"https://openalex.org/W4390660317","doi":"https://doi.org/10.1109/tip.2023.3348656","title":"Mind the Gap: Learning Modality-Agnostic Representations With a Cross-Modality UNet","display_name":"Mind the Gap: Learning Modality-Agnostic Representations With a Cross-Modality UNet","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4390660317","doi":"https://doi.org/10.1109/tip.2023.3348656","pmid":"https://pubmed.ncbi.nlm.nih.gov/38190674"},"language":"en","primary_location":{"id":"doi:10.1109/tip.2023.3348656","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tip.2023.3348656","pdf_url":null,"source":{"id":"https://openalex.org/S4210173141","display_name":"IEEE Transactions on Image Processing","issn_l":"1057-7149","issn":["1057-7149","1941-0042"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Image Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101665349","display_name":"Xin Niu","orcid":"https://orcid.org/0009-0007-7260-6617"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xin Niu","raw_affiliation_strings":["Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104225787","display_name":"Enyi Li","orcid":null},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Enyi Li","raw_affiliation_strings":["Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101599789","display_name":"Jinchao Liu","orcid":"https://orcid.org/0000-0003-2559-7512"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinchao Liu","raw_affiliation_strings":["Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100322975","display_name":"Yan Wang","orcid":"https://orcid.org/0000-0003-4346-692X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yan Wang","raw_affiliation_strings":["VisionMetric Ltd., Canterbury, Kent, U.K"],"affiliations":[{"raw_affiliation_string":"VisionMetric Ltd., Canterbury, Kent, U.K","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068276075","display_name":"Margarita Osadchy","orcid":"https://orcid.org/0000-0001-5480-5099"},"institutions":[{"id":"https://openalex.org/I91203450","display_name":"University of Haifa","ror":"https://ror.org/02f009v59","country_code":"IL","type":"education","lineage":["https://openalex.org/I91203450"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Margarita Osadchy","raw_affiliation_strings":["Department of Computer Science, Haifa University, Haifa, Israel"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Haifa University, Haifa, Israel","institution_ids":["https://openalex.org/I91203450"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058644546","display_name":"Yongchun Fang","orcid":"https://orcid.org/0000-0002-3061-2708"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongchun Fang","raw_affiliation_strings":["Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory of Intelligent Robotics, the Engineering Research Center of Trusted Behavior Intelligence, Ministry of Education, and the College of Artificial Intelligence, Nankai University, Tianjin, China","institution_ids":["https://openalex.org/I205237279"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5101665349"],"corresponding_institution_ids":["https://openalex.org/I205237279"],"apc_list":null,"apc_paid":null,"fwci":2.7801,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.90886649,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"33","issue":null,"first_page":"655","last_page":"670"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.7404000759124756},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5863283276557922},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5059086680412292}],"concepts":[{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.7404000759124756},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5863283276557922},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5059086680412292}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tip.2023.3348656","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tip.2023.3348656","pdf_url":null,"source":{"id":"https://openalex.org/S4210173141","display_name":"IEEE Transactions on Image Processing","issn_l":"1057-7149","issn":["1057-7149","1941-0042"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Image Processing","raw_type":"journal-article"},{"id":"pmid:38190674","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38190674","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on image processing : a publication of the IEEE Signal Processing Society","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4699999988079071,"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities"},{"score":0.46000000834465027,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G2303075371","display_name":null,"funder_award_id":"62233011","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7645675549","display_name":null,"funder_award_id":"62076140","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":82,"referenced_works":["https://openalex.org/W38226588","https://openalex.org/W1484953274","https://openalex.org/W1529583686","https://openalex.org/W1882958252","https://openalex.org/W1980093854","https://openalex.org/W2039108558","https://openalex.org/W2076631638","https://openalex.org/W2085810284","https://openalex.org/W2121604810","https://openalex.org/W2141345255","https://openalex.org/W2153288431","https://openalex.org/W2158096215","https://openalex.org/W2211092169","https://openalex.org/W2266728343","https://openalex.org/W2395611524","https://openalex.org/W2485822847","https://openalex.org/W2494566811","https://openalex.org/W2510111515","https://openalex.org/W2523746035","https://openalex.org/W2551624499","https://openalex.org/W2593768305","https://openalex.org/W2626578807","https://openalex.org/W2743674619","https://openalex.org/W2752532133","https://openalex.org/W2777534232","https://openalex.org/W2803714405","https://openalex.org/W2808260522","https://openalex.org/W2886192117","https://openalex.org/W2891747104","https://openalex.org/W2903091095","https://openalex.org/W2904791642","https://openalex.org/W2911514609","https://openalex.org/W2913468054","https://openalex.org/W2914001277","https://openalex.org/W2914664131","https://openalex.org/W2921310231","https://openalex.org/W2962750014","https://openalex.org/W2962793481","https://openalex.org/W2963073614","https://openalex.org/W2963364701","https://openalex.org/W2963809521","https://openalex.org/W2963953476","https://openalex.org/W2963968629","https://openalex.org/W2964057616","https://openalex.org/W2964323748","https://openalex.org/W2982482221","https://openalex.org/W2997877744","https://openalex.org/W3001208314","https://openalex.org/W3034956129","https://openalex.org/W3039060838","https://openalex.org/W3047180765","https://openalex.org/W3087056823","https://openalex.org/W3092709185","https://openalex.org/W3100410998","https://openalex.org/W3119971933","https://openalex.org/W3125695697","https://openalex.org/W3128412859","https://openalex.org/W3134567495","https://openalex.org/W3139438386","https://openalex.org/W3174441232","https://openalex.org/W3174572181","https://openalex.org/W3175888430","https://openalex.org/W3176028309","https://openalex.org/W3176633985","https://openalex.org/W3177151075","https://openalex.org/W3180849682","https://openalex.org/W3199096350","https://openalex.org/W3202750592","https://openalex.org/W3203255241","https://openalex.org/W3204075450","https://openalex.org/W3207905476","https://openalex.org/W3210946531","https://openalex.org/W4205635927","https://openalex.org/W4225809603","https://openalex.org/W4226422926","https://openalex.org/W4312700302","https://openalex.org/W4312802153","https://openalex.org/W4312936309","https://openalex.org/W4382467684","https://openalex.org/W4386065398","https://openalex.org/W4390872471","https://openalex.org/W6745620410"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2385859805","https://openalex.org/W2530972254","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857"],"abstract_inverted_index":{"Cross-modality":[0],"recognition":[1],"has":[2,242],"many":[3],"important":[4],"applications":[5],"in":[6,115,246],"science,":[7],"law":[8],"enforcement":[9],"and":[10,102,143,164,169,260,262,277],"entertainment.":[11],"Popular":[12],"methods":[13,265],"to":[14,88,130,179,194,205,215,222,239],"bridge":[15,234],"the":[16,21,42,49,58,116,140,235,247],"modality":[17,34,60,69,210,236],"gap":[18],"include":[19],"reducing":[20],"distributional":[22],"differences":[23],"of":[24,26,44,113,207,228],"representations":[25,31,91,114,142],"different":[27],"modalities,":[28],"learning":[29],"indistinguishable":[30],"or":[32,74,200],"explicit":[33,68],"transfer.":[35],"The":[36,52,271],"first":[37],"two":[38],"approaches":[39],"suffer":[40],"from":[41,198],"loss":[43,109],"discriminant":[45,196],"information":[46,197],"while":[47,92],"removing":[48],"modality-specific":[50],"variations.":[51],"third":[53],"one":[54],"heavily":[55],"relies":[56],"on":[57,153],"successful":[59],"transfer,":[61],"could":[62,191],"face":[63,171],"catastrophic":[64],"performance":[65,177],"drop":[66],"when":[67],"transfers":[70],"are":[71],"not":[72],"possible":[73],"difficult.":[75],"To":[76],"tackle":[77],"this":[78],"problem,":[79],"we":[80,123],"proposed":[81,272],"a":[82,131,187,230,267,275,282],"compact":[83],"encoder-decoder":[84],"neural":[85],"module":[86],"(cmUNet)":[87],"learn":[89],"modality-agnostic":[90,141],"retaining":[93],"identity-related":[94],"information.":[95],"This":[96],"is":[97,128,184,274],"achieved":[98],"through":[99],"cross-modality":[100,121,161,188],"transformation":[101],"in-modality":[103],"reconstruction,":[104],"enhanced":[105],"by":[106],"an":[107,226],"adversarial/perceptual":[108],"which":[110,136,212],"encourages":[111],"indistinguishability":[112],"original":[117],"sample":[118],"space.":[119],"For":[120],"matching,":[122,160],"propose":[124],"MarrNet":[125,174,254],"where":[126,173],"cmUNet":[127,273],"connected":[129],"standard":[132],"feature":[133],"extraction":[134],"network":[135],"takes":[137],"as":[138,281],"inputs":[139],"outputs":[144],"similarity":[145],"scores":[146],"for":[147,285],"matching.":[148],"We":[149,218],"validated":[150],"our":[151,240],"method":[152,190,231],"five":[154],"challenging":[155],"tasks,":[156],"namely":[157],"Raman-infrared":[158],"spectrum":[159],"person":[162],"re-identification":[163],"heterogeneous":[165],"(photo-sketch,":[166],"visible-near":[167],"infrared":[168],"visible-thermal)":[170],"recognition,":[172],"showed":[175],"superior":[176],"compared":[178],"state-of-the-art":[180],"methods.":[181],"Furthermore,":[182],"it":[183],"observed":[185],"that":[186,220,253],"matching":[189],"be":[192,225,279],"biased":[193],"extract":[195],"partial":[199],"even":[201],"wrong":[202],"regions,":[203],"due":[204],"incompetence":[206],"dealing":[208],"with":[209,266],"gaps,":[211],"subsequently":[213],"leads":[214],"poor":[216],"generalization.":[217],"show":[219],"robustness":[221,257],"occlusions":[223],"can":[224,232,278],"indicator":[227],"whether":[229],"well":[233],"gap.":[237],"This,":[238],"knowledge,":[241],"been":[243],"largely":[244],"neglected":[245],"previous":[248],"works.":[249],"Our":[250],"experiments":[251],"demonstrated":[252],"exhibited":[255],"excellent":[256],"against":[258],"disguises":[259],"occlusions,":[261],"outperformed":[263],"existing":[264],"large":[268],"margin":[269],"(>10%).":[270],"meta-approach":[276],"used":[280],"building":[283],"block":[284],"various":[286],"applications.":[287]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}