{"id":"https://openalex.org/W4387010609","doi":"https://doi.org/10.1109/les.2023.3299638","title":"Differentiable Slimming for Memory-Efficient Transformers","display_name":"Differentiable Slimming for Memory-Efficient Transformers","publication_year":2023,"publication_date":"2023-09-25","ids":{"openalex":"https://openalex.org/W4387010609","doi":"https://doi.org/10.1109/les.2023.3299638"},"language":"en","primary_location":{"id":"doi:10.1109/les.2023.3299638","is_oa":false,"landing_page_url":"https://doi.org/10.1109/les.2023.3299638","pdf_url":null,"source":{"id":"https://openalex.org/S22443479","display_name":"IEEE Embedded Systems Letters","issn_l":"1943-0663","issn":["1943-0663","1943-0671"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Embedded Systems Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002909243","display_name":"N Penkov","orcid":"https://orcid.org/0000-0002-5421-4253"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Nikolay Penkov","raw_affiliation_strings":["Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0002-5421-4253","affiliations":[{"raw_affiliation_string":"Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006315507","display_name":"Konstantinos Balaskas","orcid":"https://orcid.org/0000-0003-2886-6896"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Konstantinos Balaskas","raw_affiliation_strings":["Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0003-2886-6896","affiliations":[{"raw_affiliation_string":"Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017470419","display_name":"Martin Rapp","orcid":"https://orcid.org/0000-0002-5989-2950"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Martin Rapp","raw_affiliation_strings":["Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0002-5989-2950","affiliations":[{"raw_affiliation_string":"Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5109181333","display_name":"Joerg Henkel","orcid":null},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Joerg Henkel","raw_affiliation_strings":["Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany"],"raw_orcid":"https://orcid.org/0000-0001-9602-2922","affiliations":[{"raw_affiliation_string":"Chair for Embedded Systems, Karlsruhe Institute of Technology, Karlsruhe, Germany","institution_ids":["https://openalex.org/I102335020"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I102335020"],"apc_list":null,"apc_paid":null,"fwci":0.1064,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.38739115,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"15","issue":"4","first_page":"186","last_page":"189"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8396909832954407},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7516525983810425},{"id":"https://openalex.org/keywords/differentiable-function","display_name":"Differentiable function","score":0.6502822637557983},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5664485692977905},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.4874267876148224},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.44684073328971863},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.4445365071296692},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.3577588200569153},{"id":"https://openalex.org/keywords/computer-hardware","display_name":"Computer hardware","score":0.3374086022377014},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33106672763824463},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.13837933540344238},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.12441730499267578}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8396909832954407},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7516525983810425},{"id":"https://openalex.org/C202615002","wikidata":"https://www.wikidata.org/wiki/Q783507","display_name":"Differentiable function","level":2,"score":0.6502822637557983},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5664485692977905},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.4874267876148224},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.44684073328971863},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.4445365071296692},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.3577588200569153},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.3374086022377014},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33106672763824463},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.13837933540344238},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.12441730499267578},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/les.2023.3299638","is_oa":false,"landing_page_url":"https://doi.org/10.1109/les.2023.3299638","pdf_url":null,"source":{"id":"https://openalex.org/S22443479","display_name":"IEEE Embedded Systems Letters","issn_l":"1943-0663","issn":["1943-0663","1943-0671"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Embedded Systems Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W2946794439","https://openalex.org/W3035251378","https://openalex.org/W4226512186","https://openalex.org/W4385245566","https://openalex.org/W6752515464","https://openalex.org/W6762945437","https://openalex.org/W6768080748","https://openalex.org/W6769227307","https://openalex.org/W6775706467","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W2770234245","https://openalex.org/W96612179","https://openalex.org/W4285277090","https://openalex.org/W4229499248","https://openalex.org/W2566006169","https://openalex.org/W4327738859","https://openalex.org/W4206178588","https://openalex.org/W4287635093","https://openalex.org/W3094491777","https://openalex.org/W3214715529"],"abstract_inverted_index":{"Transformer":[0,85,93],"models":[1],"are":[2,46,95],"continuously":[3,24],"achieving":[4],"state-of-the-art":[5,29],"performance":[6,16,127],"on":[7,38],"a":[8,27,72,79,84,111],"wide":[9],"range":[10],"of":[11,20,54,119,143],"benchmarks.":[12],"To":[13],"meet":[14],"demanding":[15],"targets,":[17],"the":[18,42,52,60,64,117,133,140],"number":[19],"model":[21],"parameters":[22],"is":[23],"increased.":[25],"As":[26],"result,":[28],"Transformers":[30,45],"require":[31],"substantial":[32],"computational":[33],"resources":[34],"prohibiting":[35],"their":[36],"deployment":[37],"consumer-grade":[39],"hardware.":[40],"In":[41],"literature,":[43],"overparameterized":[44],"successfully":[47],"reduced":[48],"in":[49,71],"size":[50],"with":[51,97,125],"help":[53],"pruning":[55,130],"strategies.":[56],"Existing":[57],"works":[58],"lack":[59],"ability":[61],"to":[62],"optimize":[63],"full":[65],"architecture,":[66],"without":[67],"incurring":[68],"significant":[69],"overheads,":[70],"fully":[73],"differentiable":[74,112],"manner.":[75],"Our":[76],"work":[77],"proposes":[78],"single-stage":[80],"approach":[81],"for":[82,86],"training":[83,114],"memory-efficient":[87],"inference":[88],"and":[89,104,136],"various":[90,148],"resource-constrained":[91],"scenarios.":[92],"blocks":[94],"extended":[96],"trainable":[98],"gate":[99],"parameters,":[100],"which":[101],"attribute":[102],"importance":[103],"control":[105],"information":[106],"flow.":[107],"Their":[108],"integration":[109],"into":[110],"pruning-aware":[113],"scheme":[115],"allows":[116],"extraction":[118],"extremely":[120],"sparse":[121],"subnetworks":[122,146],"at":[123,132],"runtime,":[124],"minimal":[126],"degradation.":[128],"Evaluative":[129],"results,":[131],"attention":[134],"head":[135],"layer":[137],"levels,":[138],"illustrate":[139],"memory":[141,149],"efficiency":[142],"our":[144],"trained":[145],"under":[147],"budgets.":[150]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}