{"id":"https://openalex.org/W4395447509","doi":"https://doi.org/10.1109/tcsvt.2024.3393474","title":"Generic-to-Specific Distillation of Masked Autoencoders","display_name":"Generic-to-Specific Distillation of Masked Autoencoders","publication_year":2024,"publication_date":"2024-04-25","ids":{"openalex":"https://openalex.org/W4395447509","doi":"https://doi.org/10.1109/tcsvt.2024.3393474"},"language":"en","primary_location":{"id":"doi:10.1109/tcsvt.2024.3393474","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3393474","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101651673","display_name":"Wei Huang","orcid":"https://orcid.org/0000-0001-8899-0069"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Huang","raw_affiliation_strings":["School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8899-0069","affiliations":[{"raw_affiliation_string":"School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038511307","display_name":"Zhiliang Peng","orcid":"https://orcid.org/0000-0002-6643-9329"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiliang Peng","raw_affiliation_strings":["School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6643-9329","affiliations":[{"raw_affiliation_string":"School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101751775","display_name":"Li Dong","orcid":"https://orcid.org/0000-0003-3083-7170"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Dong","raw_affiliation_strings":["Microsoft Research, BeiJing, China"],"raw_orcid":"https://orcid.org/0000-0003-3083-7170","affiliations":[{"raw_affiliation_string":"Microsoft Research, BeiJing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014662947","display_name":"Furu Wei","orcid":"https://orcid.org/0000-0002-7810-5852"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Furu Wei","raw_affiliation_strings":["Microsoft Research, BeiJing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, BeiJing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015317495","display_name":"Qixiang Ye","orcid":null},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qixiang Ye","raw_affiliation_strings":["School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1215-6259","affiliations":[{"raw_affiliation_string":"School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110088325","display_name":"Jianbin Jiao","orcid":"https://orcid.org/0000-0003-0454-3929"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianbin Jiao","raw_affiliation_strings":["School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0454-3929","affiliations":[{"raw_affiliation_string":"School of Electronic, Electrical and Communication Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.4087,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.56700758,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":"34","issue":"9","first_page":"8779","last_page":"8793"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9848999977111816,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5931911468505859},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5541048049926758},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5541003346443176},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.4157387912273407},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3313939869403839},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.12528041005134583},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.12035167217254639}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5931911468505859},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5541048049926758},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5541003346443176},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4157387912273407},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3313939869403839},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.12528041005134583},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.12035167217254639}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsvt.2024.3393474","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3393474","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Zero hunger","id":"https://metadata.un.org/sdg/2","score":0.5400000214576721}],"awards":[{"id":"https://openalex.org/G1295000622","display_name":null,"funder_award_id":"61836012","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G262448394","display_name":null,"funder_award_id":"XDA27000000","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6113573069","display_name":null,"funder_award_id":"62171431","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6735075065","display_name":null,"funder_award_id":"62225208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":71,"referenced_works":["https://openalex.org/W1690739335","https://openalex.org/W1821462560","https://openalex.org/W1861492603","https://openalex.org/W2117539524","https://openalex.org/W2194775991","https://openalex.org/W2507296351","https://openalex.org/W2765407302","https://openalex.org/W2884822772","https://openalex.org/W2896457183","https://openalex.org/W2936864631","https://openalex.org/W2942810103","https://openalex.org/W2963091558","https://openalex.org/W2963140444","https://openalex.org/W2963150697","https://openalex.org/W2963182550","https://openalex.org/W2982083293","https://openalex.org/W2982242214","https://openalex.org/W2984287396","https://openalex.org/W2986015886","https://openalex.org/W2990152177","https://openalex.org/W2990503944","https://openalex.org/W3005680577","https://openalex.org/W3009561768","https://openalex.org/W3034572008","https://openalex.org/W3035422918","https://openalex.org/W3037492894","https://openalex.org/W3105966348","https://openalex.org/W3138516171","https://openalex.org/W3159481202","https://openalex.org/W3170874841","https://openalex.org/W3171007011","https://openalex.org/W3172942063","https://openalex.org/W3173799170","https://openalex.org/W3174544005","https://openalex.org/W3175528717","https://openalex.org/W3177096435","https://openalex.org/W3177196641","https://openalex.org/W3213836217","https://openalex.org/W4214524539","https://openalex.org/W4214614183","https://openalex.org/W4214727094","https://openalex.org/W4221145109","https://openalex.org/W4221148504","https://openalex.org/W4226336732","https://openalex.org/W4280496682","https://openalex.org/W4281860797","https://openalex.org/W4286910290","https://openalex.org/W4286914341","https://openalex.org/W4287101401","https://openalex.org/W4292948576","https://openalex.org/W4297644256","https://openalex.org/W4301914798","https://openalex.org/W4312312750","https://openalex.org/W4312443924","https://openalex.org/W4312671791","https://openalex.org/W4312677967","https://openalex.org/W4312764321","https://openalex.org/W4312804044","https://openalex.org/W4313156423","https://openalex.org/W4313159787","https://openalex.org/W4320009770","https://openalex.org/W4322576919","https://openalex.org/W4378499239","https://openalex.org/W4385245566","https://openalex.org/W4386071576","https://openalex.org/W4386072286","https://openalex.org/W4386075861","https://openalex.org/W4386076509","https://openalex.org/W4386076625","https://openalex.org/W4386221015","https://openalex.org/W4390874177"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2033914206","https://openalex.org/W2042327336","https://openalex.org/W3204019825"],"abstract_inverted_index":{"To":[0],"transfer":[1,30],"the":[2,29,60,73,103,114,139],"representation":[3],"capacity":[4],"of":[5,31,62,75,87,102,113],"large":[6,63,89,115],"pre-trained":[7,65],"models":[8,58,64,170],"to":[9,25,37,55,81,108,117,135,173],"lightweight":[10,57,168],"models,":[11],"knowledge":[12,40,94],"distillation":[13,21,53],"has":[14],"been":[15],"widely":[16],"explored.":[17],"However,":[18],"conventional":[19],"single-stage":[20],"methods":[22],"are":[23,106,171],"prone":[24],"getting":[26],"stuck":[27],"in":[28],"task-specific":[32],"knowledge,":[33],"making":[34],"it":[35],"difficult":[36],"retain":[38],"task-agnostic":[39,93],"which":[41],"is":[42,79,122,183],"crucial":[43],"for":[44,125,156],"model":[45,78,105,141],"generalization.":[46],"In":[47,70,98],"this":[48],"study,":[49],"we":[50],"propose":[51],"generic-to-specific":[52],"(G2SD),":[54],"boost":[56],"under":[59],"assistance":[61],"by":[66],"masked":[67],"image":[68,157,178],"modeling.":[69],"generic":[71],"distillation,":[72,100],"decoder":[74],"a":[76,88,174],"small":[77,104],"encouraged":[80,107],"align":[82],"feature":[83],"predictions":[84,101],"with":[85,111,152],"that":[86,92],"model,":[90,116],"so":[91],"can":[95],"be":[96,109],"transferred.":[97],"specific":[99],"consistent":[110],"those":[112],"guarantee":[118],"task":[119],"performance.":[120],"G2SD":[121],"also":[123],"applicable":[124],"heterogeneous":[126],"settings(":[127],"<italic":[128],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[129],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">i.e</i>":[130],".,":[131],"distilling":[132],"from":[133],"ViT":[134],"CNN).":[136],"With":[137],"G2SD,":[138],"ViT-Small":[140],"respectively":[142],"achieves":[143],"98.9%,":[144],"98.4%,":[145],"99.3%":[146],"and":[147,163],"98.9%":[148],"accuracies":[149],"when":[150],"compared":[151],"its":[153],"teachers":[154],"(ViT-Base)":[155],"classification,":[158],"object":[159],"detection,":[160],"semantic":[161],"segmentation":[162],"video":[164],"recognition":[165],"tasks.":[166],"The":[167,181],"ResNet":[169],"improved":[172],"new":[175],"height":[176],"on":[177],"classification":[179],"task.":[180],"code":[182],"available":[184],"at":[185],"github.com/pengzhiliang/G2SD.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}