{"id":"https://openalex.org/W4383503740","doi":"https://doi.org/10.1109/tmm.2023.3293335","title":"Multimodal Reaction: Information Modulation for Cross-Modal Representation Learning","display_name":"Multimodal Reaction: Information Modulation for Cross-Modal Representation Learning","publication_year":2023,"publication_date":"2023-07-07","ids":{"openalex":"https://openalex.org/W4383503740","doi":"https://doi.org/10.1109/tmm.2023.3293335"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3293335","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3293335","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085190039","display_name":"Ying Zeng","orcid":"https://orcid.org/0000-0001-8842-2045"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ying Zeng","raw_affiliation_strings":["School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010270301","display_name":"Sijie Mai","orcid":"https://orcid.org/0000-0001-9763-375X"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sijie Mai","raw_affiliation_strings":["School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-9763-375X","affiliations":[{"raw_affiliation_string":"School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101481658","display_name":"Wenjun Yan","orcid":"https://orcid.org/0000-0003-3551-7408"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjun Yan","raw_affiliation_strings":["School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-3551-7408","affiliations":[{"raw_affiliation_string":"School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056953478","display_name":"Haifeng Hu","orcid":"https://orcid.org/0000-0002-4884-323X"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haifeng Hu","raw_affiliation_strings":["School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-4884-323X","affiliations":[{"raw_affiliation_string":"School of Electronics and Information Technology, Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157773358"],"apc_list":null,"apc_paid":null,"fwci":2.0412,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.86188981,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"26","issue":null,"first_page":"2178","last_page":"2191"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11667","display_name":"Advanced Chemical Sensor Technologies","score":0.9603999853134155,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11667","display_name":"Advanced Chemical Sensor Technologies","score":0.9603999853134155,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9549000263214111,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7625541687011719},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6146361827850342},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6019377112388611},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5118793249130249},{"id":"https://openalex.org/keywords/filter","display_name":"Filter (signal processing)","score":0.43876007199287415},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.43639567494392395},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4193114638328552},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4121396243572235},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.08931306004524231}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7625541687011719},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6146361827850342},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6019377112388611},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5118793249130249},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.43876007199287415},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.43639567494392395},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4193114638328552},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4121396243572235},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.08931306004524231},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2023.3293335","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3293335","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5799999833106995,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G5355820501","display_name":"\u4eba\u8138\u56fe\u50cf\u7279\u5f81\u62bd\u53d6\u53ca\u5206\u7c7b\u5668\u8bbe\u8ba1\u82e5\u5e72\u95ee\u9898\u7814\u7a76","funder_award_id":"60802069","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5678804896","display_name":null,"funder_award_id":"61673402","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G709867051","display_name":null,"funder_award_id":"61273270","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8307738215","display_name":null,"funder_award_id":"62076262","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":77,"referenced_works":["https://openalex.org/W753847829","https://openalex.org/W1679826675","https://openalex.org/W2079725295","https://openalex.org/W2146334809","https://openalex.org/W2168465881","https://openalex.org/W2250539671","https://openalex.org/W2546919788","https://openalex.org/W2556418146","https://openalex.org/W2583643061","https://openalex.org/W2740550900","https://openalex.org/W2752782242","https://openalex.org/W2767249564","https://openalex.org/W2772633765","https://openalex.org/W2787581402","https://openalex.org/W2789221157","https://openalex.org/W2832876791","https://openalex.org/W2883409523","https://openalex.org/W2886193235","https://openalex.org/W2896457183","https://openalex.org/W2946218857","https://openalex.org/W2946675767","https://openalex.org/W2949391930","https://openalex.org/W2962749469","https://openalex.org/W2962931510","https://openalex.org/W2963063161","https://openalex.org/W2963128932","https://openalex.org/W2963710346","https://openalex.org/W2963736842","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964130424","https://openalex.org/W2964216663","https://openalex.org/W2964346351","https://openalex.org/W2966864260","https://openalex.org/W2971047694","https://openalex.org/W2971050617","https://openalex.org/W2981206386","https://openalex.org/W2981448908","https://openalex.org/W2982104868","https://openalex.org/W2997573100","https://openalex.org/W3007282427","https://openalex.org/W3025665229","https://openalex.org/W3034266838","https://openalex.org/W3034368386","https://openalex.org/W3037572520","https://openalex.org/W3048195943","https://openalex.org/W3093051361","https://openalex.org/W3093400813","https://openalex.org/W3141688548","https://openalex.org/W3170224286","https://openalex.org/W3175196926","https://openalex.org/W3175706072","https://openalex.org/W3186953717","https://openalex.org/W3190537575","https://openalex.org/W3197875592","https://openalex.org/W3211692312","https://openalex.org/W3216298502","https://openalex.org/W4214737784","https://openalex.org/W4214825151","https://openalex.org/W4225650823","https://openalex.org/W4283328744","https://openalex.org/W4285149123","https://openalex.org/W4285269381","https://openalex.org/W4292794050","https://openalex.org/W4312287430","https://openalex.org/W4312527924","https://openalex.org/W6638523607","https://openalex.org/W6677929280","https://openalex.org/W6719057275","https://openalex.org/W6729014267","https://openalex.org/W6739901393","https://openalex.org/W6745499552","https://openalex.org/W6763701032","https://openalex.org/W6773204560","https://openalex.org/W6784106106","https://openalex.org/W6790156507","https://openalex.org/W6955071965"],"related_works":["https://openalex.org/W2081900870","https://openalex.org/W2183306018","https://openalex.org/W2549990292","https://openalex.org/W2345479200","https://openalex.org/W2951819827","https://openalex.org/W2849310602","https://openalex.org/W2419146053","https://openalex.org/W2088247287","https://openalex.org/W3006008237","https://openalex.org/W2963903416"],"abstract_inverted_index":{"In":[0,111,275],"multimodal":[1,49,82,155],"machine":[2],"learning,":[3,210],"proper":[4],"handling":[5],"of":[6,44,69,74,142,182,201,217,234,240,259,293],"cross-modal":[7,45,144,185],"information":[8,58,194,242,283],"is":[9,52,59,163,212],"essential":[10],"for":[11,78,273],"obtaining":[12],"an":[13],"ideal":[14],"joint":[15,80],"embedding.":[16],"Despite":[17],"the":[18,29,32,41,57,66,72,79,138,143,150,171,180,190,196,199,215,218,225,238,244,256,266,281,291,294],"progress":[19],"made":[20],"by":[21,214],"recent":[22],"fusion":[23,30,245],"strategies,":[24],"we":[25,114],"hold":[26],"that":[27,38,192,236],"before":[28,176],"stage,":[31,248],"unimodal":[33],"representation":[34,257],"inevitably":[35],"contains":[36],"noise":[37],"may":[39,97],"hinder":[40],"correct":[42],"learning":[43,83,134,183,258],"dynamics":[46],"and":[47,62,94,105,140,148,152,167,227,246,270],"affect":[48],"fusion.":[50],"It":[51],"worthwhile":[53],"to":[54,64,102,108,136,165,208,231,279,284],"investigate":[55],"how":[56,63],"being":[60],"utilized":[61],"make":[65],"full":[67],"use":[68],"it.":[70],"Rethinking":[71],"process":[73,93],"leveraging":[75],"multiple":[76],"modalities":[77],"embedding,":[81],"can":[84],"be":[85],"regarded":[86],"as":[87],"a":[88,116,158,206,232],"<italic":[89],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[90,118,122,126,131],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">chemical":[91],"reaction</i>":[92],"two":[95],"steps":[96],"benefit":[98],"learning:":[99],"1)":[100],"purification":[101],"filter":[103,169],"impurity,":[104],"2)":[106],"catalyst":[107],"facilitate":[109,209],"learning.":[110,156],"this":[112],"paper,":[113],"propose":[115],"<bold":[117,121,125,130],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">M</b>":[119,127],"ultimodal":[120],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">I</b>":[123],"nformation":[124],"odulation":[128],"(":[129],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">MIM</b>":[132],")":[133],"framework":[135],"modulate":[137,280],"contribution":[139],"utilization":[141],"information,":[145],"which":[146,178,211],"identifies":[147],"handles":[149],"\u2018impurity\u2019":[151],"\u2018catalyst\u2019":[153],"in":[154,198,243,254],"Specifically,":[157],"Unimodal":[159],"Purification":[160],"Network":[161],"(UPN)":[162],"proposed":[164,295],"identify":[166],"explicitly":[168],"out":[170],"impurity":[172],"within":[173],"each":[174],"modality":[175,267],"fusion,":[177],"reduces":[179],"possibility":[181],"incorrect":[184],"dynamics.":[186],"Besides,":[187,262],"based":[188],"on":[189],"intuition":[191],"useful":[193,241],"has":[195],"potential":[197,252],"guidance":[200],"model":[202],"updating,":[203],"it":[204,263],"plays":[205],"role":[207,239,253],"achieved":[213],"design":[216],"Knowledge":[219],"Guidance":[220],"Scheme":[221],"(KGS)":[222],"considering":[223],"both":[224],"intra-":[226],"inter-modal":[228],"scenarios.":[229],"Different":[230],"majority":[233],"works":[235],"emphasize":[237],"inference":[247],"KGS":[249],"considers":[250,265],"its":[251],"assisting":[255],"weaker":[260],"components.":[261],"fully":[264],"dominance":[268],"problem":[269],"sample":[271],"variations":[272],"optimization.":[274],"short,":[276],"MIM":[277],"manages":[278],"useless/useful":[282],"minimize/emphasize":[285],"their":[286],"contribution.":[287],"Experimental":[288],"results":[289],"verify":[290],"effectiveness":[292],"method.":[296]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}