{"id":"https://openalex.org/W3206298025","doi":"https://doi.org/10.1109/avss52988.2021.9663793","title":"From Multimodal to Unimodal Attention in Transformers using Knowledge Distillation","display_name":"From Multimodal to Unimodal Attention in Transformers using Knowledge Distillation","publication_year":2021,"publication_date":"2021-11-16","ids":{"openalex":"https://openalex.org/W3206298025","doi":"https://doi.org/10.1109/avss52988.2021.9663793","mag":"3206298025"},"language":"en","primary_location":{"id":"doi:10.1109/avss52988.2021.9663793","is_oa":false,"landing_page_url":"https://doi.org/10.1109/avss52988.2021.9663793","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 17th IEEE International Conference on Advanced Video and Signal Based Surveillance (AVSS)","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2110.08270","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014154126","display_name":"D. P. Agarwal","orcid":"https://orcid.org/0000-0002-1090-3583"},"institutions":[{"id":"https://openalex.org/I26072440","display_name":"Indian Institute of Information Technology Allahabad","ror":"https://ror.org/03rgjt374","country_code":"IN","type":"education","lineage":["https://openalex.org/I26072440"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en informatique et en automatique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4210106545","display_name":"Research Centre Inria Sophia Antipolis - M\u00e9diterran\u00e9e","ror":"https://ror.org/01nzkaw91","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283","https://openalex.org/I201841394","https://openalex.org/I4210106545"]}],"countries":["FR","IN"],"is_corresponding":true,"raw_author_name":"Dhruv Agarwal","raw_affiliation_strings":["INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","Indian Institute of Information Technology, Allahabad, India","Indian Institute of Information Technology Allahabad"],"affiliations":[{"raw_affiliation_string":"INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","institution_ids":["https://openalex.org/I4210106545","https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Indian Institute of Information Technology, Allahabad, India","institution_ids":["https://openalex.org/I26072440"]},{"raw_affiliation_string":"Indian Institute of Information Technology Allahabad","institution_ids":["https://openalex.org/I26072440"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057455578","display_name":"Tanay Agrawal","orcid":null},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en informatique et en automatique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4210106545","display_name":"Research Centre Inria Sophia Antipolis - M\u00e9diterran\u00e9e","ror":"https://ror.org/01nzkaw91","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283","https://openalex.org/I201841394","https://openalex.org/I4210106545"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Tanay Agrawal","raw_affiliation_strings":["INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","Spatio-Temporal Activity Recognition Systems"],"affiliations":[{"raw_affiliation_string":"INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","institution_ids":["https://openalex.org/I4210106545","https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Spatio-Temporal Activity Recognition Systems","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088264735","display_name":"Laura M. Ferrari","orcid":"https://orcid.org/0000-0001-8521-9666"},"institutions":[{"id":"https://openalex.org/I4210106545","display_name":"Research Centre Inria Sophia Antipolis - M\u00e9diterran\u00e9e","ror":"https://ror.org/01nzkaw91","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283","https://openalex.org/I201841394","https://openalex.org/I4210106545"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en informatique et en automatique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4210126779","display_name":"Observatoire de la C\u00f4te d\u2019Azur","ror":"https://ror.org/039fj2469","country_code":"FR","type":"facility","lineage":["https://openalex.org/I201841394","https://openalex.org/I4210126779"]},{"id":"https://openalex.org/I201841394","display_name":"Universit\u00e9 C\u00f4te d'Azur","ror":"https://ror.org/019tgvf94","country_code":"FR","type":"education","lineage":["https://openalex.org/I201841394"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Laura M. Ferrari","raw_affiliation_strings":["INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","Universit\u00e9 C\u00f4te d\u2019Azur, France","Spatio-Temporal Activity Recognition Systems"],"affiliations":[{"raw_affiliation_string":"INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","institution_ids":["https://openalex.org/I4210106545","https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Universit\u00e9 C\u00f4te d\u2019Azur, France","institution_ids":["https://openalex.org/I4210126779","https://openalex.org/I201841394"]},{"raw_affiliation_string":"Spatio-Temporal Activity Recognition Systems","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037298642","display_name":"Fran\u00e7ois Br\u00e9mond","orcid":"https://orcid.org/0000-0003-2988-2142"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en informatique et en automatique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4210126779","display_name":"Observatoire de la C\u00f4te d\u2019Azur","ror":"https://ror.org/039fj2469","country_code":"FR","type":"facility","lineage":["https://openalex.org/I201841394","https://openalex.org/I4210126779"]},{"id":"https://openalex.org/I4210106545","display_name":"Research Centre Inria Sophia Antipolis - M\u00e9diterran\u00e9e","ror":"https://ror.org/01nzkaw91","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283","https://openalex.org/I201841394","https://openalex.org/I4210106545"]},{"id":"https://openalex.org/I201841394","display_name":"Universit\u00e9 C\u00f4te d'Azur","ror":"https://ror.org/019tgvf94","country_code":"FR","type":"education","lineage":["https://openalex.org/I201841394"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Francois Bremond","raw_affiliation_strings":["INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","Universit\u00e9 C\u00f4te d\u2019Azur, France","Spatio-Temporal Activity Recognition Systems"],"affiliations":[{"raw_affiliation_string":"INRIA Sophia Antipolis - M\u00e9editerran\u00e9e, France","institution_ids":["https://openalex.org/I4210106545","https://openalex.org/I1326498283"]},{"raw_affiliation_string":"Universit\u00e9 C\u00f4te d\u2019Azur, France","institution_ids":["https://openalex.org/I4210126779","https://openalex.org/I201841394"]},{"raw_affiliation_string":"Spatio-Temporal Activity Recognition Systems","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5014154126"],"corresponding_institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I26072440","https://openalex.org/I4210106545"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.1512873,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"2","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.802237868309021},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7467465400695801},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7156369090080261},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.7007184028625488},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6859821081161499},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5987045168876648},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.557411253452301},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.5263206362724304},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.49931764602661133},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.46632295846939087},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.15247967839241028},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1337214708328247},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.11990785598754883}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.802237868309021},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7467465400695801},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7156369090080261},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.7007184028625488},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6859821081161499},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5987045168876648},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.557411253452301},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5263206362724304},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.49931764602661133},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.46632295846939087},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.15247967839241028},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1337214708328247},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.11990785598754883},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.1109/avss52988.2021.9663793","is_oa":false,"landing_page_url":"https://doi.org/10.1109/avss52988.2021.9663793","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 17th IEEE International Conference on Advanced Video and Signal Based Surveillance (AVSS)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2110.08270","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.08270","pdf_url":"https://arxiv.org/pdf/2110.08270","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3206298025","is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2110.08270.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:HAL:hal-03389126v1","is_oa":true,"landing_page_url":"https://hal.science/hal-03389126","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"AVSS 2021 - 17th IEEE International Conference on Advanced Video and Signal-based Surveillance, Nov 2021, Virtual, United States. ⟨10.1109/AVSS52988.2021.9663793⟩","raw_type":"Conference papers"},{"id":"doi:10.48550/arxiv.2110.08270","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2110.08270","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"},{"id":"mag:3210084807","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2110.08270","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2110.08270","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.08270","pdf_url":"https://arxiv.org/pdf/2110.08270","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W2095176743","https://openalex.org/W2096733369","https://openalex.org/W2098689807","https://openalex.org/W2119288237","https://openalex.org/W2122563357","https://openalex.org/W2146334809","https://openalex.org/W2151096985","https://openalex.org/W2250539671","https://openalex.org/W2294370754","https://openalex.org/W2546919788","https://openalex.org/W2619383789","https://openalex.org/W2842511635","https://openalex.org/W2883409523","https://openalex.org/W2947084868","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964216663","https://openalex.org/W2964260444","https://openalex.org/W3098042509","https://openalex.org/W3099206234","https://openalex.org/W3119997354","https://openalex.org/W3168391820","https://openalex.org/W3190537575","https://openalex.org/W6788556936","https://openalex.org/W6844194202"],"related_works":["https://openalex.org/W3210084807","https://openalex.org/W3159344844","https://openalex.org/W3094216089","https://openalex.org/W3162707322","https://openalex.org/W3103513401","https://openalex.org/W2902232562","https://openalex.org/W3184145304","https://openalex.org/W2999963857","https://openalex.org/W3203309275","https://openalex.org/W3112170415","https://openalex.org/W2940411505","https://openalex.org/W3196420731","https://openalex.org/W2766839578","https://openalex.org/W2968295315","https://openalex.org/W20159809","https://openalex.org/W3047814055","https://openalex.org/W2978302660","https://openalex.org/W2808568644","https://openalex.org/W2464862435","https://openalex.org/W3080768309"],"abstract_inverted_index":{"Multimodal":[0],"Deep":[1],"Learning":[2],"has":[3],"garnered":[4],"much":[5],"interest,":[6],"and":[7,35,77,100,115],"transformers":[8,53],"have":[9],"triggered":[10],"novel":[11],"approaches,":[12],"thanks":[13],"to":[14,23,54],"the":[15,30,36,44,47,81,86,93,101,125],"cross-attention":[16],"mechanism.":[17],"Here":[18],"we":[19,84,91,116],"propose":[20],"an":[21],"approach":[22],"deal":[24],"with":[25,130],"two":[26],"key":[27],"existing":[28],"challenges:":[29],"high":[31],"computational":[32],"resource":[33],"demanded":[34,134],"issue":[37],"of":[38,49,95,127],"missing":[39],"modalities.":[40],"We":[41,62],"introduce":[42],"for":[43],"first":[45],"time":[46,103],"concept":[48],"knowledge":[50],"distillation":[51,74],"in":[52,112],"use":[55],"only":[56],"one":[57],"modality":[58],"at":[59,72,118],"inference":[60,102],"time.":[61],"report":[63],"a":[64,120],"full":[65],"study":[66],"analyzing":[67],"multiple":[68],"student-teacher":[69],"configurations,":[70],"levels":[71],"which":[73],"is":[75,133],"applied,":[76],"different":[78],"methodologies.":[79],"With":[80],"best":[82],"configuration,":[83],"improved":[85],"state-of-the-art":[87],"accuracy":[88],"by":[89,97,104],"3%,":[90],"reduced":[92],"number":[94],"parameters":[96],"2.5":[98],"times":[99],"22%.":[105],"Such":[106],"performance-computation":[107],"tradeoff":[108],"can":[109],"be":[110],"exploited":[111],"many":[113],"applications":[114],"aim":[117],"opening":[119],"new":[121],"research":[122],"area":[123],"where":[124],"deployment":[126],"complex":[128],"models":[129],"limited":[131],"resources":[132]},"counts_by_year":[],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}