{"id":"https://openalex.org/W4385488605","doi":"https://doi.org/10.1109/ijcnn54540.2023.10191515","title":"Flexible BERT with Width- and Depth-dynamic Inference","display_name":"Flexible BERT with Width- and Depth-dynamic Inference","publication_year":2023,"publication_date":"2023-06-18","ids":{"openalex":"https://openalex.org/W4385488605","doi":"https://doi.org/10.1109/ijcnn54540.2023.10191515"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn54540.2023.10191515","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/ijcnn54540.2023.10191515","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101579165","display_name":"Ting Hu","orcid":"https://orcid.org/0000-0001-6266-2622"},"institutions":[{"id":"https://openalex.org/I143288331","display_name":"Hasso Plattner Institute","ror":"https://ror.org/058rn5r42","country_code":"DE","type":"facility","lineage":["https://openalex.org/I143288331","https://openalex.org/I176453806"]},{"id":"https://openalex.org/I176453806","display_name":"University of Potsdam","ror":"https://ror.org/03bnmw459","country_code":"DE","type":"education","lineage":["https://openalex.org/I176453806"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ting Hu","raw_affiliation_strings":["Hasso Plattner Institute University of Potsdam,Potsdam,Germany","Hasso Plattner Institute University of Potsdam, Potsdam, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hasso Plattner Institute University of Potsdam,Potsdam,Germany","institution_ids":["https://openalex.org/I143288331","https://openalex.org/I176453806"]},{"raw_affiliation_string":"Hasso Plattner Institute University of Potsdam, Potsdam, Germany","institution_ids":["https://openalex.org/I143288331","https://openalex.org/I176453806"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102919398","display_name":"Christoph Meinel","orcid":"https://orcid.org/0000-0002-3410-3193"},"institutions":[{"id":"https://openalex.org/I143288331","display_name":"Hasso Plattner Institute","ror":"https://ror.org/058rn5r42","country_code":"DE","type":"facility","lineage":["https://openalex.org/I143288331","https://openalex.org/I176453806"]},{"id":"https://openalex.org/I176453806","display_name":"University of Potsdam","ror":"https://ror.org/03bnmw459","country_code":"DE","type":"education","lineage":["https://openalex.org/I176453806"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Christoph Meinel","raw_affiliation_strings":["Hasso Plattner Institute University of Potsdam,Potsdam,Germany","Hasso Plattner Institute University of Potsdam, Potsdam, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hasso Plattner Institute University of Potsdam,Potsdam,Germany","institution_ids":["https://openalex.org/I143288331","https://openalex.org/I176453806"]},{"raw_affiliation_string":"Hasso Plattner Institute University of Potsdam, Potsdam, Germany","institution_ids":["https://openalex.org/I143288331","https://openalex.org/I176453806"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101039243","display_name":"Haojin Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I143288331","display_name":"Hasso Plattner Institute","ror":"https://ror.org/058rn5r42","country_code":"DE","type":"facility","lineage":["https://openalex.org/I143288331","https://openalex.org/I176453806"]},{"id":"https://openalex.org/I176453806","display_name":"University of Potsdam","ror":"https://ror.org/03bnmw459","country_code":"DE","type":"education","lineage":["https://openalex.org/I176453806"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Haojin Yang","raw_affiliation_strings":["Hasso Plattner Institute University of Potsdam,Potsdam,Germany","Hasso Plattner Institute University of Potsdam, Potsdam, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hasso Plattner Institute University of Potsdam,Potsdam,Germany","institution_ids":["https://openalex.org/I143288331","https://openalex.org/I176453806"]},{"raw_affiliation_string":"Hasso Plattner Institute University of Potsdam, Potsdam, Germany","institution_ids":["https://openalex.org/I143288331","https://openalex.org/I176453806"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.08465524,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8300901651382446},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.773339569568634},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.661817193031311},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5576509833335876},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.5319182276725769},{"id":"https://openalex.org/keywords/dimension","display_name":"Dimension (graph theory)","score":0.5203371047973633},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.5178309082984924},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.48843327164649963},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3186209797859192},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09207013249397278},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08612903952598572}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8300901651382446},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.773339569568634},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.661817193031311},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5576509833335876},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.5319182276725769},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.5203371047973633},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5178309082984924},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.48843327164649963},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3186209797859192},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09207013249397278},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08612903952598572},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn54540.2023.10191515","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/ijcnn54540.2023.10191515","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W2510886230","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2963748441","https://openalex.org/W2970454332","https://openalex.org/W2975381464","https://openalex.org/W2978017171","https://openalex.org/W2994749257","https://openalex.org/W2997006708","https://openalex.org/W3015609966","https://openalex.org/W3034292689","https://openalex.org/W3035030897","https://openalex.org/W3035038672","https://openalex.org/W3101163004","https://openalex.org/W3105966348","https://openalex.org/W3112673818","https://openalex.org/W3160843329","https://openalex.org/W3167266074","https://openalex.org/W3167851353","https://openalex.org/W3173563887","https://openalex.org/W3174708387","https://openalex.org/W3174730533","https://openalex.org/W3198917403","https://openalex.org/W4226126941","https://openalex.org/W4287021475","https://openalex.org/W4287391717","https://openalex.org/W4288089799","https://openalex.org/W4385245566","https://openalex.org/W6739901393","https://openalex.org/W6767064347","https://openalex.org/W6768080748","https://openalex.org/W6768851824","https://openalex.org/W6769627184","https://openalex.org/W6775706467","https://openalex.org/W6779313456","https://openalex.org/W6788811087","https://openalex.org/W6798211005"],"related_works":["https://openalex.org/W2055243143","https://openalex.org/W4231775656","https://openalex.org/W2046435967","https://openalex.org/W2383646825","https://openalex.org/W2371018915","https://openalex.org/W2354191502","https://openalex.org/W1972225038","https://openalex.org/W4321636575","https://openalex.org/W3134658850","https://openalex.org/W1986418932"],"abstract_inverted_index":{"Pre-trained":[0],"Language":[1],"Models":[2],"bring":[3],"about":[4],"an":[5],"in-creasing":[6],"computational":[7,22,156,179],"and":[8,62,115,126,155,162,194,201,206],"memory":[9],"cost.":[10],"The":[11,86],"recently":[12],"proposed":[13,87,139],"computation-flexible":[14],"BERT":[15,27,33,80,94,109],"models":[16,28,176],"facilitate":[17],"their":[18],"deployment":[19],"in":[20,130,150,171],"varied":[21],"environments.":[23],"Training":[24],"such":[25],"flexible":[26,108],"involves":[29],"jointly":[30],"optimizing":[31],"multiple":[32],"subnets":[34],"that":[35,111,137],"inevitably":[36],"interfere":[37],"with":[38,177],"one":[39],"another.":[40],"Besides,":[41],"the":[42,58,63,70,83,90,131,138,151,163,183,190],"performance":[43,55,92,154],"of":[44,93,100],"large":[45],"sub":[46,60,95],"nets":[47,96],"is":[48,52],"limited":[49],"when":[50],"there":[51],"a":[53,107,142],"significant":[54],"gap":[56],"between":[57,153],"smallest":[59],"net":[61],"supernet,":[64,184],"despite":[65],"methods":[66,149],"managing":[67],"to":[68,78,182],"enhance":[69],"smaller":[71],"subnets.":[72],"We":[73],"propose":[74],"layer-wise":[75],"Neural":[76],"grafting":[77],"boost":[79],"subnets,":[81],"particularly":[82],"larger":[84],"ones.":[85],"method":[88],"improves":[89,189],"average":[91,191],"on":[97,158,197],"six":[98],"out":[99],"eight":[101],"GLUE":[102,160,192],"tasks.":[103],"Furthermore,":[104],"we":[105],"build":[106],"model":[110],"enables":[112],"practical":[113],"width-":[114],"depth-dynamic":[116],"inference":[117,145,169,187],"regarding":[118],"different":[119],"inputs":[120],"by":[121,199,209],"combining":[122],"width-dynamic":[123],"gating":[124],"modules":[125],"early":[127],"exit":[128],"off-ramps":[129],"depth":[132],"dimension.":[133],"Experimental":[134],"results":[135],"demonstrate":[136],"framework":[140],"achieves":[141],"better":[143],"dynamic":[144],"range":[146],"than":[147],"other":[148],"trade-off":[152],"complexity":[157],"four":[159],"tasks":[161],"SQuAD":[164,198],"data":[165],"set.":[166],"Our":[167],"optimal-tradeoff":[168],"result,":[170],"particular,":[172],"outperforms":[173],"related":[174],"fixed-size":[175],"comparable":[178],"complexity.":[180],"Compared":[181],"BERT-Base,":[185],"this":[186],"result":[188],"score":[193,196],"Fl":[195],"1.3":[200],"2.2":[202],"absolute":[203],"points,":[204],"respectively,":[205],"decreases":[207],"computations":[208],"around":[210],"45%.":[211]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}