{"id":"https://openalex.org/W4402742685","doi":"https://doi.org/10.1109/tits.2024.3455416","title":"TransKD: Transformer Knowledge Distillation for Efficient Semantic Segmentation","display_name":"TransKD: Transformer Knowledge Distillation for Efficient Semantic Segmentation","publication_year":2024,"publication_date":"2024-09-23","ids":{"openalex":"https://openalex.org/W4402742685","doi":"https://doi.org/10.1109/tits.2024.3455416"},"language":"en","primary_location":{"id":"doi:10.1109/tits.2024.3455416","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2024.3455416","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Ruiping Liu","orcid":"https://orcid.org/0000-0001-5245-2277"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ruiping Liu","raw_affiliation_strings":["Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0001-5245-2277","affiliations":[{"raw_affiliation_string":"Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027010844","display_name":"Kailun Yang","orcid":"https://orcid.org/0000-0002-1090-667X"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kailun Yang","raw_affiliation_strings":["School of Robotics and the National Engineering Laboratory of Robot Visual Perception and Control Technology, Hunan University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-1090-667X","affiliations":[{"raw_affiliation_string":"School of Robotics and the National Engineering Laboratory of Robot Visual Perception and Control Technology, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076927137","display_name":"Alina Roitberg","orcid":"https://orcid.org/0000-0003-4724-9164"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Alina Roitberg","raw_affiliation_strings":["Institute for Artificial Intelligence, University of Stuttgart, Stuttgart, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Artificial Intelligence, University of Stuttgart, Stuttgart, Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jiaming Zhang","orcid":"https://orcid.org/0000-0003-3471-328X"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jiaming Zhang","raw_affiliation_strings":["Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0003-3471-328X","affiliations":[{"raw_affiliation_string":"Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017521271","display_name":"Kunyu Peng","orcid":"https://orcid.org/0000-0002-5419-9292"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Kunyu Peng","raw_affiliation_strings":["Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0002-5419-9292","affiliations":[{"raw_affiliation_string":"Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022997463","display_name":"Huayao Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huayao Liu","raw_affiliation_strings":["NIO, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NIO, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025640070","display_name":"Yaonan Wang","orcid":"https://orcid.org/0000-0002-0519-6458"},"institutions":[{"id":"https://openalex.org/I16609230","display_name":"Hunan University","ror":"https://ror.org/05htk5m33","country_code":"CN","type":"education","lineage":["https://openalex.org/I16609230"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaonan Wang","raw_affiliation_strings":["School of Robotics and the National Engineering Laboratory of Robot Visual Perception and Control Technology, Hunan University, Changsha, China"],"raw_orcid":"https://orcid.org/0000-0002-0519-6458","affiliations":[{"raw_affiliation_string":"School of Robotics and the National Engineering Laboratory of Robot Visual Perception and Control Technology, Hunan University, Changsha, China","institution_ids":["https://openalex.org/I16609230"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087051920","display_name":"Rainer Stiefelhagen","orcid":"https://orcid.org/0000-0001-8046-4945"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rainer Stiefelhagen","raw_affiliation_strings":["Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0001-8046-4945","affiliations":[{"raw_affiliation_string":"Institute for Anthropomatics and Robotics, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.1601,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.95699058,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"25","issue":"12","first_page":"20933","last_page":"20949"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7264999747276306,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.7264999747276306,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.6632000207901001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13382","display_name":"Robotics and Automated Systems","score":0.6026999950408936,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6403881311416626},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.6038476228713989},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5864536762237549},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5361943244934082},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.510480523109436},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4177861511707306},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.2149362564086914},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.13442721962928772},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.11457791924476624},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.06574144959449768},{"id":"https://openalex.org/keywords/chromatography","display_name":"Chromatography","score":0.06115725636482239}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6403881311416626},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.6038476228713989},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5864536762237549},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5361943244934082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.510480523109436},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4177861511707306},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2149362564086914},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.13442721962928772},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.11457791924476624},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.06574144959449768},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.06115725636482239}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tits.2024.3455416","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tits.2024.3455416","pdf_url":null,"source":{"id":"https://openalex.org/S144771191","display_name":"IEEE Transactions on Intelligent Transportation Systems","issn_l":"1524-9050","issn":["1524-9050","1558-0016"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Intelligent Transportation Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1548865700","display_name":null,"funder_award_id":"62473139","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":86,"referenced_works":["https://openalex.org/W125693051","https://openalex.org/W1821462560","https://openalex.org/W1903029394","https://openalex.org/W2031489346","https://openalex.org/W2108598243","https://openalex.org/W2144794286","https://openalex.org/W2340897893","https://openalex.org/W2560023338","https://openalex.org/W2752782242","https://openalex.org/W2762439315","https://openalex.org/W2787091153","https://openalex.org/W2883780447","https://openalex.org/W2886934227","https://openalex.org/W2922509574","https://openalex.org/W2952787292","https://openalex.org/W2954054736","https://openalex.org/W2955058313","https://openalex.org/W2963091558","https://openalex.org/W2963163009","https://openalex.org/W2963418739","https://openalex.org/W2963419596","https://openalex.org/W2963727650","https://openalex.org/W2963881378","https://openalex.org/W2963890956","https://openalex.org/W2964217532","https://openalex.org/W2981689412","https://openalex.org/W2995059005","https://openalex.org/W2997006708","https://openalex.org/W3015735225","https://openalex.org/W3034368386","https://openalex.org/W3034958977","https://openalex.org/W3105676814","https://openalex.org/W3105966348","https://openalex.org/W3112503277","https://openalex.org/W3131500599","https://openalex.org/W3138516171","https://openalex.org/W3159637683","https://openalex.org/W3164443777","https://openalex.org/W3165745140","https://openalex.org/W3170841864","https://openalex.org/W3175515048","https://openalex.org/W3177196641","https://openalex.org/W3190492058","https://openalex.org/W3194816901","https://openalex.org/W3198917403","https://openalex.org/W3202329792","https://openalex.org/W3207649350","https://openalex.org/W4200630704","https://openalex.org/W4206210197","https://openalex.org/W4214524539","https://openalex.org/W4214588794","https://openalex.org/W4214634256","https://openalex.org/W4214893857","https://openalex.org/W4221061906","https://openalex.org/W4225292349","https://openalex.org/W4281489207","https://openalex.org/W4311166797","https://openalex.org/W4312272720","https://openalex.org/W4312309807","https://openalex.org/W4312671791","https://openalex.org/W4312820606","https://openalex.org/W4312977443","https://openalex.org/W4312986923","https://openalex.org/W4313007769","https://openalex.org/W4379382445","https://openalex.org/W4385245566","https://openalex.org/W4385696135","https://openalex.org/W4386179772","https://openalex.org/W4389977645","https://openalex.org/W6717372056","https://openalex.org/W6751146195","https://openalex.org/W6755536945","https://openalex.org/W6757817989","https://openalex.org/W6760613829","https://openalex.org/W6762718338","https://openalex.org/W6773815586","https://openalex.org/W6788135285","https://openalex.org/W6791705549","https://openalex.org/W6794345597","https://openalex.org/W6795140394","https://openalex.org/W6797399245","https://openalex.org/W6798837711","https://openalex.org/W6802648153","https://openalex.org/W6811002592","https://openalex.org/W6838393215","https://openalex.org/W6839053282"],"related_works":["https://openalex.org/W3026162553","https://openalex.org/W2768175398","https://openalex.org/W2344382886","https://openalex.org/W19111321","https://openalex.org/W4379231730","https://openalex.org/W2412887479","https://openalex.org/W32245304","https://openalex.org/W2953684491","https://openalex.org/W4285338581","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Semantic":[0],"segmentation":[1,38],"benchmarks":[2],"in":[3],"the":[4,50,63,87,93,143,148,160,199,222],"realm":[5],"of":[6,42,82,178,202],"autonomous":[7],"driving":[8],"are":[9],"dominated":[10],"by":[11,21,74,95],"large":[12,83],"pre-trained":[13],"transformers,":[14,85],"yet":[15],"their":[16],"widespread":[17],"adoption":[18],"is":[19,229],"impeded":[20],"substantial":[22],"computational":[23],"costs":[24],"and":[25,46,56,79,91,108,127,175,195,209,220],"prolonged":[26],"training":[27],"durations.":[28],"To":[29],"lift":[30],"this":[31],"constraint,":[32],"we":[33,98,153],"look":[34],"at":[35,232],"efficient":[36],"semantic":[37],"from":[39,164],"a":[40,179],"perspective":[41],"comprehensive":[43],"knowledge":[44,54,119],"distillation":[45,107,130,163,218],"aim":[47],"to":[48,103,146,158,191],"bridge":[49,193],"gap":[51],"between":[52,121],"multi-source":[53],"extractions":[55],"transformer-specific":[57],"patch":[58,80,109,149,161],"embeddings.":[59],"We":[60],"put":[61],"forward":[62],"Transformer-based":[64],"Knowledge":[65],"Distillation":[66],"(TransKD)":[67],"framework":[68],"which":[69],"learns":[70],"compact":[71],"student":[72,196],"transformers":[73],"distilling":[75],"both":[76,173],"feature":[77,105,128],"maps":[78],"embeddings":[81],"teacher":[84,194],"bypassing":[86],"long":[88],"pre-training":[89,224],"process":[90,145],"reducing":[92],"FLOPs":[94],">85.0%.":[96],"Specifically,":[97],"propose":[99],"two":[100,155],"fundamental":[101],"modules":[102,157],"realize":[104],"map":[106,129],"embedding":[110,150,162,189],"distillation,":[111],"respectively:":[112],"1)":[113,167],"Cross":[114],"Selective":[115],"Fusion":[116],"(CSF)":[117],"enables":[118],"transfer":[120],"cross-stage":[122],"features":[123],"via":[124],"channel":[125],"attention":[126],"within":[131,142],"hierarchical":[132],"transformers;":[133],"2)":[134,182],"Patch":[135],"Embedding":[136,183],"Alignment":[137],"(PEA)":[138],"performs":[139],"dimensional":[140],"transformation":[141],"patchifying":[144],"facilitate":[147],"distillation.":[151],"Furthermore,":[152],"introduce":[154],"optimization":[156],"enhance":[159],"different":[165],"perspectives:":[166],"Global-Local":[168],"Context":[169],"Mixer":[170],"(GL-Mixer)":[171],"extracts":[172],"global":[174],"local":[176],"information":[177],"representative":[180],"embedding;":[181],"Assistant":[184],"(EA)":[185],"acts":[186],"as":[187],"an":[188],"method":[190],"seamlessly":[192],"models":[197],"with":[198],"teacher\u2019s":[200],"number":[201],"channels.":[203],"Experiments":[204],"on":[205],"Cityscapes,":[206],"ACDC,":[207],"NYUv2,":[208],"Pascal":[210],"VOC2012":[211],"datasets":[212],"show":[213],"that":[214],"TransKD":[215],"outperforms":[216],"state-of-the-art":[217],"frameworks":[219],"rivals":[221],"time-consuming":[223],"method.":[225],"The":[226],"source":[227],"code":[228],"publicly":[230],"available":[231],"<uri":[233],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[234],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">https://github.com/RuipingL/TransKD</uri>.":[235]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}