{"id":"https://openalex.org/W4406728024","doi":"https://doi.org/10.1109/tnnls.2025.3525737","title":"DenseKD: Dense Knowledge Distillation by Exploiting Region and Sample Importance","display_name":"DenseKD: Dense Knowledge Distillation by Exploiting Region and Sample Importance","publication_year":2025,"publication_date":"2025-01-22","ids":{"openalex":"https://openalex.org/W4406728024","doi":"https://doi.org/10.1109/tnnls.2025.3525737","pmid":"https://pubmed.ncbi.nlm.nih.gov/40031279"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2025.3525737","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3525737","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100705310","display_name":"Haonan Zhang","orcid":"https://orcid.org/0000-0002-4239-6141"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Haonan Zhang","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China"],"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080046377","display_name":"Longjun Liu","orcid":"https://orcid.org/0000-0002-7467-4994"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Longjun Liu","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China"],"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104126346","display_name":"Yi Zhang","orcid":"https://orcid.org/0009-0000-6788-0696"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yi Zhang","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China"],"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024647438","display_name":"Xinyu Lei","orcid":"https://orcid.org/0000-0002-8464-1344"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xinyu Lei","raw_affiliation_strings":["National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China"],"affiliations":[{"raw_affiliation_string":"National Key Laboratory of Human-Machine Hybrid Augmented Intelligence, the National Engineering Research Center for Visual Information and Applications, Institute of Artificial Intelligence and Robotics, Xi’an Jiaotong University, Xi’an, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031341608","display_name":"Fei Hui","orcid":"https://orcid.org/0000-0001-8981-4745"},"institutions":[{"id":"https://openalex.org/I25355098","display_name":"Chang'an University","ror":"https://ror.org/05mxya461","country_code":"CN","type":"education","lineage":["https://openalex.org/I25355098"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fei Hui","raw_affiliation_strings":["School of Electronics and Control Engineering, Chang’an University, Xi’an, China"],"affiliations":[{"raw_affiliation_string":"School of Electronics and Control Engineering, Chang’an University, Xi’an, China","institution_ids":["https://openalex.org/I25355098"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024709593","display_name":"Bihan Wen","orcid":"https://orcid.org/0000-0002-6874-6453"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Bihan Wen","raw_affiliation_strings":["School of Electrical and Electronic Engineering, Nanyang Technological University, Jurong West, Singapore"],"affiliations":[{"raw_affiliation_string":"School of Electrical and Electronic Engineering, Nanyang Technological University, Jurong West, Singapore","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100705310"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.6209,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.87348657,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"36","issue":"6","first_page":"11243","last_page":"11257"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6805276870727539},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6588550806045532},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.6329333782196045},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5982191562652588},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5695697665214539},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.568733274936676},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.5570851564407349},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.544163703918457},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.5139591097831726},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4899381995201111},{"id":"https://openalex.org/keywords/residual-neural-network","display_name":"Residual neural network","score":0.47961342334747314},{"id":"https://openalex.org/keywords/channel","display_name":"Channel (broadcasting)","score":0.46712878346443176},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4657391309738159},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.4589702785015106},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.45102518796920776},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4232885539531708},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.09423515200614929},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.09112757444381714},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07113698124885559}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6805276870727539},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6588550806045532},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.6329333782196045},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5982191562652588},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5695697665214539},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.568733274936676},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.5570851564407349},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.544163703918457},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.5139591097831726},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4899381995201111},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.47961342334747314},{"id":"https://openalex.org/C127162648","wikidata":"https://www.wikidata.org/wiki/Q16858953","display_name":"Channel (broadcasting)","level":2,"score":0.46712878346443176},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4657391309738159},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4589702785015106},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45102518796920776},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4232885539531708},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.09423515200614929},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.09112757444381714},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07113698124885559},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2025.3525737","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3525737","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:40031279","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40031279","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Zero hunger","score":0.4099999964237213,"id":"https://metadata.un.org/sdg/2"}],"awards":[{"id":"https://openalex.org/G3833898155","display_name":null,"funder_award_id":"62088102","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4867311355","display_name":null,"funder_award_id":"52172380","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7754692170","display_name":null,"funder_award_id":"xzy012024066","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W1861492603","https://openalex.org/W1901129140","https://openalex.org/W2102605133","https://openalex.org/W2117539524","https://openalex.org/W2194775991","https://openalex.org/W2294370754","https://openalex.org/W2295107390","https://openalex.org/W2565639579","https://openalex.org/W2750432752","https://openalex.org/W2791026767","https://openalex.org/W2887783173","https://openalex.org/W2921396149","https://openalex.org/W2936864631","https://openalex.org/W2955192706","https://openalex.org/W2959289524","https://openalex.org/W2962858109","https://openalex.org/W2963125010","https://openalex.org/W2963140444","https://openalex.org/W2963150697","https://openalex.org/W2963216524","https://openalex.org/W2963351448","https://openalex.org/W2964137095","https://openalex.org/W2982157312","https://openalex.org/W2982242214","https://openalex.org/W2986015886","https://openalex.org/W3030901202","https://openalex.org/W3034223443","https://openalex.org/W3034674511","https://openalex.org/W3034756453","https://openalex.org/W3034818206","https://openalex.org/W3035414587","https://openalex.org/W3091981646","https://openalex.org/W3118608800","https://openalex.org/W3127215335","https://openalex.org/W3173270634","https://openalex.org/W3174102142","https://openalex.org/W3177196641","https://openalex.org/W3205325461","https://openalex.org/W3212385273","https://openalex.org/W4205707194","https://openalex.org/W4224075564","https://openalex.org/W4226003804","https://openalex.org/W4226191776","https://openalex.org/W4226426325","https://openalex.org/W4246193833","https://openalex.org/W4296437248","https://openalex.org/W4306914511","https://openalex.org/W4307092353","https://openalex.org/W4313052529","https://openalex.org/W4318586078","https://openalex.org/W4385245566","https://openalex.org/W4386072392","https://openalex.org/W4402667890","https://openalex.org/W4402727442","https://openalex.org/W4402727744","https://openalex.org/W6620707391","https://openalex.org/W6637373629","https://openalex.org/W6637551013","https://openalex.org/W6638783484","https://openalex.org/W6684191040","https://openalex.org/W6684563725","https://openalex.org/W6685405536","https://openalex.org/W6726275242","https://openalex.org/W6743188669","https://openalex.org/W6747340706","https://openalex.org/W6748082341","https://openalex.org/W6769906912","https://openalex.org/W6787972765","https://openalex.org/W6803837815","https://openalex.org/W6842607031","https://openalex.org/W6858439355","https://openalex.org/W6862698143"],"related_works":["https://openalex.org/W2366107444","https://openalex.org/W4388145910","https://openalex.org/W2381570729","https://openalex.org/W1976205134","https://openalex.org/W4248336175","https://openalex.org/W2031260042","https://openalex.org/W2391445434","https://openalex.org/W3009369890","https://openalex.org/W4312490297","https://openalex.org/W2599472179"],"abstract_inverted_index":{"Knowledge":[0],"distillation":[1,66],"(KD)":[2],"can":[3],"compress":[4],"deep":[5],"neural":[6],"networks":[7],"(DNNs)":[8],"by":[9,142,167],"transferring":[10],"the":[11,14,19,35,45,69,106,131,137,143,168,189,204,213],"knowledge":[12],"of":[13,32,38,49,61,114,139,145,147,170,196,206,225],"redundant":[15],"teacher":[16,148,171],"model":[17],"to":[18,51,55,73,96,110,129,152,158],"resource-friendly":[20],"student":[21,115],"model,":[22],"where":[23],"cross-layer":[24],"KD":[25,28,87],"(CKD)":[26],"conducts":[27],"between":[29],"each":[30,112],"stage":[31],"students":[33,72,154],"and":[34],"multiple":[36,183],"stages":[37],"teachers.":[39],"However,":[40],"previous":[41,207],"CKD":[42,208],"schemes":[43],"select":[44],"coarse-grained":[46],"stagewise":[47],"features":[48,121],"teachers":[50],"teach":[52],"students,":[53],"leading":[54],"improper":[56],"channel":[57,113],"alignment.":[58],"Also,":[59],"most":[60],"these":[62,81],"methods":[63],"conduct":[64],"uniform":[65],"for":[67],"all":[68],"knowledge,":[70],"limiting":[71],"focus":[74],"more":[75,98,118,156],"on":[76,182,198],"important":[77],"knowledge.":[78],"To":[79],"address":[80],"problems,":[82],"we":[83,104,125,163],"propose":[84],"a":[85],"dense":[86,108],"(DenseKD)":[88],"in":[89,102,161,180,188,212],"this":[90],"article,":[91],"dubbed":[92],"as":[93],"DenseKD.":[94],"First,":[95],"achieve":[97],"accurate":[99],"feature":[100],"alignment":[101],"CKD,":[103],"construct":[105],"learnable":[107],"architecture":[109],"make":[111,153],"flexibly":[116],"capture":[117],"diverse":[119],"channelwise":[120],"from":[122],"teacher.":[123],"Moreover,":[124],"introduce":[126],"region":[127],"importance":[128,166],"investigate":[130],"region's":[132],"guiding":[133],"potential,":[134],"it":[135],"distinguishes":[136],"influence":[138],"different":[140],"regions":[141],"variation":[144],"representations":[146],"models.":[149,172],"In":[150,210],"addition,":[151,211],"pay":[155],"attention":[157],"useful":[159],"samples":[160],"KD,":[162],"calculate":[164],"sample":[165],"loss":[169],"Consistent":[173],"improvements":[174,224],"over":[175],"state-of-the-art":[176],"approaches":[177],"are":[178],"observed":[179],"experiments":[181],"vision":[184],"tasks.":[185],"For":[186],"example,":[187],"classification":[190],"task,":[191,216],"DenseKD":[192,217],"achieves":[193],"72.30%":[194],"accuracy":[195],"ResNet-20":[197],"CIFAR-100,":[199],"which":[200],"is":[201],"higher":[202],"than":[203],"results":[205],"methods.":[209],"object":[214],"detection":[215],"gains":[218],"2.84%":[219],"mean":[220],"average":[221],"precision":[222],"(mAP)":[223],"Faster":[226],"R-CNN":[227],"with":[228],"ResNet-18":[229],"against":[230],"vanilla":[231],"KD.":[232]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2025-12-22T23:10:17.713674","created_date":"2025-10-10T00:00:00"}