{"id":"https://openalex.org/W4403488566","doi":"https://doi.org/10.3233/faia240493","title":"MIM-HD: Making Smaller Masked Autoencoder Better with Efficient Distillation","display_name":"MIM-HD: Making Smaller Masked Autoencoder Better with Efficient Distillation","publication_year":2024,"publication_date":"2024-10-16","ids":{"openalex":"https://openalex.org/W4403488566","doi":"https://doi.org/10.3233/faia240493"},"language":"en","primary_location":{"id":"doi:10.3233/faia240493","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia240493","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA240493","source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA240493","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082391073","display_name":"Zherui Zhang","orcid":"https://orcid.org/0009-0000-3385-6161"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zherui Zhang","raw_affiliation_strings":["Artificial Intelligence, Beijing University of Posts and Telecommunications"],"affiliations":[{"raw_affiliation_string":"Artificial Intelligence, Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020628396","display_name":"Changwei Wang","orcid":"https://orcid.org/0000-0001-8259-7717"},"institutions":[{"id":"https://openalex.org/I152269853","display_name":"Qilu University of Technology","ror":"https://ror.org/04hyzq608","country_code":"CN","type":"education","lineage":["https://openalex.org/I152269853"]},{"id":"https://openalex.org/I4210142748","display_name":"Shandong Academy of Sciences","ror":"https://ror.org/04y8d6y55","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changwei Wang","raw_affiliation_strings":["Key Laboratory of Computing Power Network and Information Security, Ministry of Education, Shandong Computer Science Center, Qilu University of Technology (Shandong Academy of Sciences)","Shandong Provincial Key Laboratory of Computer Networks, Shandong Fundamental Research Center for Computer Science"],"affiliations":[{"raw_affiliation_string":"Key Laboratory of Computing Power Network and Information Security, Ministry of Education, Shandong Computer Science Center, Qilu University of Technology (Shandong Academy of Sciences)","institution_ids":["https://openalex.org/I152269853","https://openalex.org/I4210142748"]},{"raw_affiliation_string":"Shandong Provincial Key Laboratory of Computer Networks, Shandong Fundamental Research Center for Computer Science","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038918940","display_name":"Rongtao Xu","orcid":"https://orcid.org/0000-0003-4619-9679"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"funder","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rongtao Xu","raw_affiliation_strings":["Institute of Automation, Chinese Academy of Sciences"],"affiliations":[{"raw_affiliation_string":"Institute of Automation, Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113366662","display_name":"Wenhao Xu","orcid":"https://orcid.org/0009-0003-5789-1609"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenhao Xu","raw_affiliation_strings":["Artificial Intelligence, Beijing University of Posts and Telecommunications"],"affiliations":[{"raw_affiliation_string":"Artificial Intelligence, Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011919230","display_name":"Shibiao Xu","orcid":"https://orcid.org/0000-0003-4037-9900"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shibiao Xu","raw_affiliation_strings":["Artificial Intelligence, Beijing University of Posts and Telecommunications"],"affiliations":[{"raw_affiliation_string":"Artificial Intelligence, Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107958000","display_name":"Li Guo","orcid":null},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Guo","raw_affiliation_strings":["Artificial Intelligence, Beijing University of Posts and Telecommunications"],"affiliations":[{"raw_affiliation_string":"Artificial Intelligence, Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032583158","display_name":"Ji\u2010Guang Zhang","orcid":"https://orcid.org/0000-0001-7343-4609"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"funder","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiguang Zhang","raw_affiliation_strings":["Institute of Automation, Chinese Academy of Sciences"],"affiliations":[{"raw_affiliation_string":"Institute of Automation, Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047655109","display_name":"Xiaoqiang Teng","orcid":"https://orcid.org/0000-0002-5876-7605"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoqiang Teng","raw_affiliation_strings":["Artificial Intelligence, Beijing University of Posts and Telecommunications"],"affiliations":[{"raw_affiliation_string":"Artificial Intelligence, Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103234483","display_name":"Wenbo Xu","orcid":"https://orcid.org/0000-0002-3994-6077"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenbo Xu","raw_affiliation_strings":["Artificial Intelligence, Beijing University of Posts and Telecommunications"],"affiliations":[{"raw_affiliation_string":"Artificial Intelligence, Beijing University of Posts and Telecommunications","institution_ids":["https://openalex.org/I139759216"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5082391073"],"corresponding_institution_ids":["https://openalex.org/I139759216"],"apc_list":null,"apc_paid":null,"fwci":0.7962,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.75113879,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6787999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6787999868392944,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.786013126373291},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5622945427894592},{"id":"https://openalex.org/keywords/process-engineering","display_name":"Process engineering","score":0.3983083963394165},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3783193528652191},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.329402893781662},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.293997198343277},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.283853143453598},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.12477430701255798},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.049485623836517334}],"concepts":[{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.786013126373291},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5622945427894592},{"id":"https://openalex.org/C21880701","wikidata":"https://www.wikidata.org/wiki/Q2144042","display_name":"Process engineering","level":1,"score":0.3983083963394165},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3783193528652191},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.329402893781662},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.293997198343277},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.283853143453598},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.12477430701255798},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.049485623836517334}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.3233/faia240493","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia240493","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA240493","source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"}],"best_oa_location":{"id":"doi:10.3233/faia240493","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia240493","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA240493","source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403488566.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4391375266","https://openalex.org/W3085764877","https://openalex.org/W2514414740","https://openalex.org/W2377414158","https://openalex.org/W3199615306","https://openalex.org/W77207468","https://openalex.org/W3212781313","https://openalex.org/W4307725381"],"abstract_inverted_index":{"Self-supervised":[0],"learning":[1],"and":[2,79,123,128],"knowledge":[3,52],"distillation":[4,41],"intersect":[5],"to":[6,48,72,107,134,154],"achieve":[7],"exceptional":[8],"performance":[9],"on":[10,57,76,111,120,140],"downstream":[11],"tasks":[12],"across":[13],"diverse":[14],"network":[15],"capacities.":[16],"This":[17],"paper":[18],"introduces":[19],"MIM-HD,":[20],"which":[21],"implements":[22],"enhancements":[23],"for":[24,97],"masked":[25],"image":[26],"modeling":[27],"(MIM)":[28],"distillation,":[29],"in":[30,85,89],"two":[31],"key":[32],"aspects.":[33],"First,":[34],"a":[35,93],"vision":[36],"transformer":[37,66],"head-level":[38],"relation":[39],"adaptive":[40],"approach":[42],"is":[43,101,116],"proposed,":[44],"allowing":[45],"the":[46,54,74,77,82,104],"student":[47],"dynamically":[49],"draw":[50],"multi-source":[51],"from":[53,152],"teacher":[55],"based":[56],"its":[58],"evolving":[59],"state,":[60],"compatible":[61],"with":[62,137],"scenarios":[63],"where":[64],"teacher-student":[65],"block":[67],"head":[68],"count":[69],"differs.":[70],"Second,":[71],"address":[73],"overemphasis":[75],"encoder":[78],"neglect":[80],"of":[81],"decoder":[83,106],"role":[84],"maintaining":[86],"representation":[87],"consistency":[88],"previous":[90],"MIM":[91,109],"distillations,":[92],"dual-view":[94],"decoding":[95],"strategy":[96],"latent":[98],"visual":[99],"representations":[100],"introduced,":[102],"reusing":[103],"teacher\u2019s":[105],"alleviate":[108],"burdens":[110],"smaller":[112,141],"networks.":[113],"MIM-HD":[114,145],"effectiveness":[115],"demonstrated":[117],"through":[118],"evaluations":[119],"ADE20K":[121],"(mIoU)":[122],"ImageNet-1K":[124],"(Acc),":[125],"achieving":[126],"+1.4%":[127],"+0.5%":[129],"improved":[130],"performance,":[131],"respectively,":[132],"compared":[133],"state-of-the-art":[135],"methods,":[136],"substantial":[138],"advantages":[139],"pre-training":[142,150],"datasets.":[143],"Moreover,":[144],"achieves":[146],"superior":[147],"efficiency,":[148],"reducing":[149],"epochs":[151],"300":[153],"100.":[155]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}