{"id":"https://openalex.org/W3208396733","doi":"https://doi.org/10.1109/tnnls.2021.3114188","title":"An Empirical Study on Adaptive Inference for Pretrained Language Model","display_name":"An Empirical Study on Adaptive Inference for Pretrained Language Model","publication_year":2021,"publication_date":"2021-10-25","ids":{"openalex":"https://openalex.org/W3208396733","doi":"https://doi.org/10.1109/tnnls.2021.3114188","mag":"3208396733","pmid":"https://pubmed.ncbi.nlm.nih.gov/34695001"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2021.3114188","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3114188","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100668786","display_name":"Weijie Liu","orcid":"https://orcid.org/0000-0002-8023-9913"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijie Liu","raw_affiliation_strings":["Technology and Engineering Group (TEG), Tencent, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-8023-9913","affiliations":[{"raw_affiliation_string":"Technology and Engineering Group (TEG), Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057368785","display_name":"Xin Zhao","orcid":"https://orcid.org/0000-0002-1621-2337"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Zhao","raw_affiliation_strings":["School of Information, Renmin University of China, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Information, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100631152","display_name":"Zhe Zhao","orcid":"https://orcid.org/0000-0003-4189-3258"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhe Zhao","raw_affiliation_strings":["Technology and Engineering Group (TEG), Tencent, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology and Engineering Group (TEG), Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071290194","display_name":"Qi Ju","orcid":null},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Ju","raw_affiliation_strings":["Technology and Engineering Group (TEG), Tencent, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology and Engineering Group (TEG), Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100715830","display_name":"Xuefeng Yang","orcid":"https://orcid.org/0000-0002-3832-2422"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuefeng Yang","raw_affiliation_strings":["Technology and Engineering Group (TEG), Tencent, Shenzhen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technology and Engineering Group (TEG), Tencent, Shenzhen, China","institution_ids":["https://openalex.org/I2250653659"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005529387","display_name":"Wei L\u00fc","orcid":"https://orcid.org/0000-0001-6769-2695"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei Lu","raw_affiliation_strings":["School of Information, Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6769-2695","affiliations":[{"raw_affiliation_string":"School of Information, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6824,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.75885272,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"34","issue":"8","first_page":"4321","last_page":"4331"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.8722237348556519},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8276784420013428},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5874691605567932},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5400868058204651},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.527763843536377},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5232937932014465},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4949638247489929},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4916229844093323}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.8722237348556519},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8276784420013428},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5874691605567932},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5400868058204651},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.527763843536377},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5232937932014465},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4949638247489929},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4916229844093323},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2021.3114188","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2021.3114188","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:34695001","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/34695001","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1690739335","https://openalex.org/W2567070169","https://openalex.org/W2739542029","https://openalex.org/W2752813595","https://openalex.org/W2759474451","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2945542139","https://openalex.org/W2962677625","https://openalex.org/W2964303773","https://openalex.org/W2965373594","https://openalex.org/W2970120757","https://openalex.org/W2970390493","https://openalex.org/W2970454332","https://openalex.org/W2970557265","https://openalex.org/W2978017171","https://openalex.org/W2981757109","https://openalex.org/W2996428491","https://openalex.org/W2997200074","https://openalex.org/W3006635614","https://openalex.org/W3011574394","https://openalex.org/W3034457371","https://openalex.org/W3035030897","https://openalex.org/W3035038672","https://openalex.org/W3039458202","https://openalex.org/W3088409176","https://openalex.org/W3098363382","https://openalex.org/W3101163004","https://openalex.org/W3101904655","https://openalex.org/W3105966348","https://openalex.org/W3114651185","https://openalex.org/W3120358876","https://openalex.org/W3163958022","https://openalex.org/W3177365697","https://openalex.org/W6631190155","https://openalex.org/W6637551013","https://openalex.org/W6731370813","https://openalex.org/W6741753902","https://openalex.org/W6744262642","https://openalex.org/W6755207826","https://openalex.org/W6759579507","https://openalex.org/W6766673545","https://openalex.org/W6768021236","https://openalex.org/W6768851824","https://openalex.org/W6769227307","https://openalex.org/W6773538725","https://openalex.org/W6779313456","https://openalex.org/W6781996221","https://openalex.org/W6788246489","https://openalex.org/W6795517269"],"related_works":["https://openalex.org/W4206178588","https://openalex.org/W4287635093","https://openalex.org/W3094491777","https://openalex.org/W3214715529","https://openalex.org/W3049463507","https://openalex.org/W2936497627","https://openalex.org/W4288365749","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W4287598411"],"abstract_inverted_index":{"Adaptive":[0],"inference":[1,13,50,107,144,172],"has":[2],"been":[3],"proven":[4],"to":[5,81,90,164],"improve":[6],"bidirectional":[7],"encoder":[8],"representations":[9],"from":[10,88],"transformers":[11],"(BERT)'s":[12],"speed":[14,82,96],"with":[15,110,170],"minimal":[16],"loss":[17],"of":[18,32,48],"accuracy.":[19],"However,":[20],"current":[21],"work":[22,109],"only":[23],"focuses":[24],"on":[25,45,67,102,122],"the":[26,46,120,130,136],"BERT":[27],"model":[28,114],"and":[29,61,70,73,116],"lacks":[30],"exploration":[31],"other":[33],"pretrained":[34,168],"language":[35],"models":[36,169],"(PLMs).":[37],"Therefore,":[38],"this":[39,146,156],"article":[40,147,157],"conducts":[41],"an":[42,159],"empirical":[43],"study":[44],"application":[47,101,121],"adaptive":[49,106,143,171],"mechanism":[51,64],"in":[52,174],"various":[53],"PLMs,":[54],"including":[55],"generative":[56],"pretraining":[57],"(GPT),":[58],"GCNN,":[59],"ALBERT,":[60],"TinyBERT.":[62],"This":[63],"is":[65,79],"verified":[66],"both":[68],"English":[69],"Chinese":[71],"benchmarks,":[72],"experimental":[74],"results":[75],"demonstrated":[76],"that":[77,105,125,138],"it":[78,126],"able":[80],"up":[83],"by":[84],"a":[85,150],"wide":[86],"range":[87],"1":[89],"10":[91],"times":[92],"if":[93],"given":[94],"different":[95],"thresholds.":[97],"In":[98],"addition,":[99],"its":[100],"ALBERT":[103],"shows":[104],"can":[108,127],"parameter":[111],"sharing,":[112],"achieving":[113],"compression":[115],"acceleration":[117],"simultaneously,":[118],"while":[119],"TinyBERT":[123],"proves":[124],"further":[128],"accelerate":[129],"distilled":[131],"small":[132],"model.":[133],"As":[134],"for":[135],"problem":[137],"too":[139],"many":[140],"labels":[141],"make":[142],"invalid,":[145],"also":[148],"proposes":[149],"solution,":[151],"namely":[152],"label":[153],"reduction.":[154],"Finally,":[155],"open-sources":[158],"easy-to-use":[160],"toolkit":[161],"called":[162],"FastPLM":[163],"help":[165],"developers":[166],"adopt":[167],"capabilities":[173],"their":[175],"applications.":[176]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}