{"id":"https://openalex.org/W3208239235","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","title":"MSD: Saliency-aware Knowledge Distillation for Multimodal Understanding","display_name":"MSD: Saliency-aware Knowledge Distillation for Multimodal Understanding","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3208239235","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","mag":"3208239235"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2021.findings-emnlp.302","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","pdf_url":"https://aclanthology.org/2021.findings-emnlp.302.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2021","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2021.findings-emnlp.302.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087823830","display_name":"Woojeong Jin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Woojeong Jin","raw_affiliation_strings":["University of Southern California"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046195860","display_name":"Maziar Sanjabi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Maziar Sanjabi","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018913504","display_name":"Shaoliang Nie","orcid":"https://orcid.org/0000-0002-5513-3439","observed_orcids":["https://orcid.org/0000-0002-5513-3439"]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Shaoliang Nie","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031865317","display_name":"Liang Tan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Liang Tan","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009408707","display_name":"Xiang Ren","orcid":"https://orcid.org/0000-0001-8655-663X","observed_orcids":["https://orcid.org/0000-0001-8655-663X"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiang Ren","raw_affiliation_strings":["University of Southern California"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043413179","display_name":"Hamed Firooz","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Hamed Firooz","raw_affiliation_strings":["Facebook AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook AI","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3851,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.65161114,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"3557","last_page":"3569"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8920999765396118,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.8920999765396118,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.07280000299215317,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.029500000178813934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.9850000143051147},{"id":"https://openalex.org/keywords/vision-language-models","display_name":"vision-language models","score":0.6510000228881836},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.7300000190734863},{"id":"https://openalex.org/keywords/multimodal-understanding","display_name":"multimodal understanding","score":0.5339999794960022}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6872164607048035},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6826214790344238},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4378805458545685},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3659588694572449},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.16671034693717957},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.1190960705280304}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2021.findings-emnlp.302","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","pdf_url":"https://aclanthology.org/2021.findings-emnlp.302.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2021","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2101.01881","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2101.01881","pdf_url":"https://arxiv.org/pdf/2101.01881","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2101.01881","doi":"https://doi.org/10.48550/arxiv.2101.01881","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2101.01881","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2021.findings-emnlp.302","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.findings-emnlp.302","pdf_url":"https://aclanthology.org/2021.findings-emnlp.302.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2021","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.4156000018119812,"display_name":"Quality education"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G1002423621","display_name":"CAREER: Teaching Machines through Human Explanation for Information Extraction","funder_award_id":"2048211","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1270017660","display_name":null,"funder_award_id":"2019-19051600007","funder_id":"https://openalex.org/F4320333051","funder_display_name":"Intelligence Advanced Research Projects Activity"},{"id":"https://openalex.org/G1272498319","display_name":"Collaborative Research: Modeling the Invention, Dissemination, and Translation of Scientific Concepts","funder_award_id":"1829268","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2261877036","display_name":null,"funder_award_id":"2019-19051600007","funder_id":"https://openalex.org/F4320312530","funder_display_name":"Office of the Director of National Intelligence"},{"id":"https://openalex.org/G390747679","display_name":null,"funder_award_id":"W911NF\u201019\u20102\u20100271","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G5865625667","display_name":null,"funder_award_id":"N660011924033","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G608057831","display_name":null,"funder_award_id":"W911NF-19","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320312530","display_name":"Office of the Director of National Intelligence","ror":"https://ror.org/01v3fsc55"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320333051","display_name":"Intelligence Advanced Research Projects Activity","ror":"https://ror.org/01v3fsc55"},{"id":"https://openalex.org/F4320337349","display_name":"NIH Office of the Director","ror":"https://ror.org/00fj8a872"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3208239235.pdf","grobid_xml":"https://content.openalex.org/works/W3208239235.grobid-xml"},"referenced_works_count":41,"referenced_works":["https://openalex.org/W753847829","https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W1861492603","https://openalex.org/W1933349210","https://openalex.org/W2134797427","https://openalex.org/W2185175083","https://openalex.org/W2560730294","https://openalex.org/W2562979205","https://openalex.org/W2587372848","https://openalex.org/W2745461083","https://openalex.org/W2771951981","https://openalex.org/W2788643321","https://openalex.org/W2795282075","https://openalex.org/W2885593519","https://openalex.org/W2912371042","https://openalex.org/W2936135081","https://openalex.org/W2955124656","https://openalex.org/W2963140444","https://openalex.org/W2963341956","https://openalex.org/W2963386594","https://openalex.org/W2963780286","https://openalex.org/W2964161024","https://openalex.org/W2964222566","https://openalex.org/W2968124245","https://openalex.org/W2970711760","https://openalex.org/W2981794819","https://openalex.org/W2982358316","https://openalex.org/W3023989664","https://openalex.org/W3030163527","https://openalex.org/W3034860466","https://openalex.org/W3035160371","https://openalex.org/W3035333188","https://openalex.org/W3036049548","https://openalex.org/W3041342983","https://openalex.org/W3098903812","https://openalex.org/W3103700490","https://openalex.org/W3105966348","https://openalex.org/W3176456321","https://openalex.org/W3204610735","https://openalex.org/W4394663826"],"related_works":["https://openalex.org/W3120825179","https://openalex.org/W4393187487","https://openalex.org/W4411266625","https://openalex.org/W7083186660","https://openalex.org/W4408378525","https://openalex.org/W4300973271","https://openalex.org/W7136652148","https://openalex.org/W4403747428","https://openalex.org/W4402727638","https://openalex.org/W4387211459"],"abstract_inverted_index":{"To":[0],"reduce":[1],"a":[2,19,24,57,75,87,104,142],"model":[3,22],"size":[4],"but":[5],"retain":[6],"performance,":[7],"we":[8,55,73,124,158],"often":[9],"rely":[10],"on":[11,29,89,151,185],"knowledge":[12,17,71,77,85],"distillation":[13,78,81],"(KD)":[14],"which":[15],"transfers":[16],"from":[18,86],"large":[20],"\"teacher\"":[21],"to":[23,61,83,146],"smaller":[25],"\"student\"":[26],"model.However,":[27],"KD":[28,184],"multimodal":[30,41,76,90,187],"datasets":[31],"such":[32],"as":[33],"vision-language":[34],"tasks":[35,91],"is":[36,43],"relatively":[37],"unexplored,":[38],"and":[39,65,131,176],"digesting":[40],"information":[42],"challenging":[44],"since":[45],"different":[46,49,120],"modalities":[47],"present":[48],"types":[50],"of":[51,67,162,170],"information.In":[52],"this":[53],"paper,":[54],"perform":[56],"large-scale":[58],"empirical":[59,156],"study":[60,141],"investigate":[62,132],"the":[63,94,137,148,160,168,171],"importance":[64],"effects":[66],"each":[68,98,114,117,129,163],"modality":[69,118,130,164],"in":[70,165,174],"distillation.Furthermore,":[72],"introduce":[74],"framework,":[79],"modality-specific":[80],"(MSD),":[82],"transfer":[84],"teacher":[88],"by":[92,108],"learning":[93,144],"teacher's":[95,105],"behavior":[96],"within":[97],"modality.The":[99],"idea":[100],"aims":[101],"at":[102],"mimicking":[103],"modalityspecific":[106],"predictions":[107],"introducing":[109],"auxiliary":[110,138],"loss":[111,153],"terms":[112],"for":[113,122,128,136],"modality.Furthermore,":[115],"because":[116],"has":[119],"saliency":[121,126,161],"predictions,":[123],"define":[125],"scores":[127],"saliency-based":[133],"weighting":[134,172],"schemes":[135],"losses.We":[139],"further":[140],"weight":[143],"approach":[145],"learn":[147],"optimal":[149],"weights":[150],"these":[152],"terms.In":[154],"our":[155],"analysis,":[157],"examine":[159],"KD,":[166],"demonstrate":[167],"effectiveness":[169],"scheme":[173],"MSD,":[175],"show":[177],"that":[178],"it":[179],"achieves":[180],"better":[181],"performance":[182],"than":[183],"four":[186],"datasets.":[188]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":3}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2021-11-08T00:00:00"}