{"id":"https://openalex.org/W4402832821","doi":"https://doi.org/10.1109/tpami.2024.3468315","title":"On-the-Fly Modulation for Balanced Multimodal Learning","display_name":"On-the-Fly Modulation for Balanced Multimodal Learning","publication_year":2024,"publication_date":"2024-09-25","ids":{"openalex":"https://openalex.org/W4402832821","doi":"https://doi.org/10.1109/tpami.2024.3468315","pmid":"https://pubmed.ncbi.nlm.nih.gov/39321012"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2024.3468315","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3468315","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006944766","display_name":"Yake Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yake Wei","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-8779-0637","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100670614","display_name":"Di Hu","orcid":"https://orcid.org/0000-0002-7118-6733"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Di Hu","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-7118-6733","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113315402","display_name":"Henghui Du","orcid":"https://orcid.org/0009-0003-9815-0575"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Henghui Du","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025631695","display_name":"Ji-Rong Wen","orcid":"https://orcid.org/0000-0002-9777-9676"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ji-Rong Wen","raw_affiliation_strings":["Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9777-9676","affiliations":[{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]},{"raw_affiliation_string":"Gaoling School of Artificial Intelligence, and Beijing Key Laboratory of Big Data Management and Analysis Methods, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I78988378"],"apc_list":null,"apc_paid":null,"fwci":3.6017,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.93601387,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"47","issue":"1","first_page":"469","last_page":"485"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10711","display_name":"Target Tracking and Data Fusion in Sensor Networks","score":0.7487000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10711","display_name":"Target Tracking and Data Fusion in Sensor Networks","score":0.7487000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.7164999842643738,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10326","display_name":"Indoor and Outdoor Localization Technologies","score":0.6761999726295471,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6138160824775696},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5366081595420837},{"id":"https://openalex.org/keywords/on-the-fly","display_name":"On the fly","score":0.5268900394439697},{"id":"https://openalex.org/keywords/modulation","display_name":"Modulation (music)","score":0.4421424865722656},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37793049216270447},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.333626389503479},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.10149657726287842}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6138160824775696},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5366081595420837},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.5268900394439697},{"id":"https://openalex.org/C123079801","wikidata":"https://www.wikidata.org/wiki/Q750240","display_name":"Modulation (music)","level":2,"score":0.4421424865722656},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37793049216270447},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.333626389503479},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.10149657726287842},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tpami.2024.3468315","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3468315","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:39321012","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39321012","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5586465760","display_name":null,"funder_award_id":"62106272","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":60,"referenced_works":["https://openalex.org/W1979482308","https://openalex.org/W2030931454","https://openalex.org/W2194775991","https://openalex.org/W2474638510","https://openalex.org/W2507009361","https://openalex.org/W2594097440","https://openalex.org/W2619697695","https://openalex.org/W2760103357","https://openalex.org/W2768267830","https://openalex.org/W2888471892","https://openalex.org/W2911840775","https://openalex.org/W2931433835","https://openalex.org/W2962865004","https://openalex.org/W2963177640","https://openalex.org/W2963192057","https://openalex.org/W2964109005","https://openalex.org/W2967308614","https://openalex.org/W2971680695","https://openalex.org/W2977259558","https://openalex.org/W2980339970","https://openalex.org/W2990152177","https://openalex.org/W3015371781","https://openalex.org/W3015707499","https://openalex.org/W3025520547","https://openalex.org/W3034658206","https://openalex.org/W3035333188","https://openalex.org/W3035570025","https://openalex.org/W3090227645","https://openalex.org/W3094612121","https://openalex.org/W3170349722","https://openalex.org/W3175514052","https://openalex.org/W3194841206","https://openalex.org/W3196591432","https://openalex.org/W4214604251","https://openalex.org/W4231090517","https://openalex.org/W4285602007","https://openalex.org/W4312380001","https://openalex.org/W4312639100","https://openalex.org/W6600983433","https://openalex.org/W6605737493","https://openalex.org/W6637500735","https://openalex.org/W6674330103","https://openalex.org/W6682864246","https://openalex.org/W6692201896","https://openalex.org/W6692956712","https://openalex.org/W6719667659","https://openalex.org/W6734280532","https://openalex.org/W6743842864","https://openalex.org/W6751983209","https://openalex.org/W6767185500","https://openalex.org/W6773226109","https://openalex.org/W6773837390","https://openalex.org/W6774337062","https://openalex.org/W6774451759","https://openalex.org/W6784187948","https://openalex.org/W6793736971","https://openalex.org/W6797528369","https://openalex.org/W6809497388","https://openalex.org/W6810226654","https://openalex.org/W6955071965"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W2033914206"],"abstract_inverted_index":{"Multimodal":[0],"learning":[1],"is":[2,17],"expected":[3],"to":[4,36,112],"boost":[5],"model":[6],"performance":[7,178],"by":[8,119,137],"integrating":[9],"information":[10],"from":[11,89],"different":[12],"modalities.":[13],"However,":[14],"its":[15,139,151],"potential":[16],"not":[18,175],"fully":[19],"exploited":[20],"because":[21],"the":[22,68,86,91,94,114,121,131,134,145,154],"widely-used":[23],"joint":[24,69],"training":[25,70],"strategy,":[26],"which":[27],"has":[28],"a":[29,165],"uniform":[30],"objective":[31],"for":[32],"all":[33],"modalities,":[34],"leads":[35],"imbalanced":[37],"and":[38,60,93,104,181,195],"under-optimized":[39,87],"uni-modal":[40],"representations.":[41],"Specifically,":[42],"we":[43,83],"point":[44],"out":[45],"that":[46],"there":[47],"often":[48],"exists":[49],"modality":[50,136],"with":[51,141],"more":[52,188],"discriminative":[53,122],"information,":[54],"e.g.,":[55],"vision":[56],"of":[57,62,116,133,167],"playing":[58],"football":[59],"sound":[61],"blowing":[63],"wind.":[64],"They":[65],"could":[66],"dominate":[67],"process,":[71],"resulting":[72],"in":[73,144,153,179,187],"other":[74],"modalities":[75,125],"being":[76],"significantly":[77],"under-optimized.":[78],"To":[79],"alleviate":[80],"this":[81],"problem,":[82],"first":[84],"analyze":[85],"phenomenon":[88],"both":[90],"feed-forward":[92,146],"back-propagation":[95,155],"stages":[96],"during":[97,126],"optimization.":[98],"Then,":[99],"On-the-fly":[100,105],"Prediction":[101],"Modulation":[102,107],"(OPM)":[103],"Gradient":[106],"(OGM)":[108],"strategies":[109,174],"are":[110],"proposed":[111],"modulate":[113],"optimization":[115],"each":[117],"modality,":[118],"monitoring":[120],"discrepancy":[123],"between":[124],"training.":[127],"Concretely,":[128],"OPM":[129],"weakens":[130],"influence":[132],"dominant":[135],"dropping":[138],"feature":[140],"dynamical":[142],"probability":[143],"stage,":[147],"while":[148],"OGM":[149],"mitigates":[150],"gradient":[152],"stage.":[156],"In":[157],"experiments,":[158],"our":[159],"methods":[160],"demonstrate":[161],"considerable":[162],"improvement":[163],"across":[164],"variety":[166],"multimodal":[168,183,190],"tasks.":[169],"These":[170],"simple":[171],"yet":[172],"effective":[173],"only":[176],"enhance":[177],"vanilla":[180],"task-oriented":[182],"models,":[184],"but":[185],"also":[186],"complex":[189],"tasks,":[191],"showcasing":[192],"their":[193],"effectiveness":[194],"flexibility.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":7}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}