{"id":"https://openalex.org/W3216434712","doi":null,"title":"Finding the Winning Ticket of BERT for Binary Text Classification via Adaptive Layer Truncation before Fine-tuning","display_name":"Finding the Winning Ticket of BERT for Binary Text Classification via Adaptive Layer Truncation before Fine-tuning","publication_year":2021,"publication_date":"2021-11-22","ids":{"openalex":"https://openalex.org/W3216434712","mag":"3216434712"},"language":"en","primary_location":{"id":"mag:3216434712","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2111.10951.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2111.10951.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101608873","display_name":"Jing Fan","orcid":"https://orcid.org/0000-0003-4604-401X","observed_orcids":["https://orcid.org/0000-0003-4604-401X"]},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jing Fan","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100327490","display_name":"Xin Zhang","orcid":"https://orcid.org/0000-0002-6070-1592","observed_orcids":["https://orcid.org/0000-0002-6070-1592"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xin Zhang","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-6070-1592","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100394046","display_name":"Sheng Zhang","orcid":"https://orcid.org/0000-0003-1732-0011","observed_orcids":["https://orcid.org/0000-0003-1732-0011"]},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sheng Zhang","raw_affiliation_strings":["National University of Defense Technology"],"raw_orcid":"https://orcid.org/0000-0003-1732-0011","affiliations":[{"raw_affiliation_string":"National University of Defense Technology","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7803000211715698,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7803000211715698,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.17679999768733978,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.021800000220537186,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.7919999957084656},{"id":"https://openalex.org/keywords/binary-text-classification","display_name":"binary text classification","score":0.7900000214576721},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.718999981880188},{"id":"https://openalex.org/keywords/pruning","display_name":"pruning","score":0.3610000014305115},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.6850000023841858},{"id":"https://openalex.org/keywords/subnetworks","display_name":"subnetworks","score":0.39899998903274536}],"concepts":[{"id":"https://openalex.org/C2776540713","wikidata":"https://www.wikidata.org/wiki/Q7800647","display_name":"Ticket","level":2,"score":0.8142823576927185},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7875145077705383},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.7051699161529541},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.6827138066291809},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.6814177632331848},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.598469614982605},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.5780867338180542},{"id":"https://openalex.org/C106195933","wikidata":"https://www.wikidata.org/wiki/Q7847935","display_name":"Truncation (statistics)","level":2,"score":0.5201176404953003},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4935518503189087},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4306495487689972},{"id":"https://openalex.org/C157524613","wikidata":"https://www.wikidata.org/wiki/Q2828883","display_name":"Fine-tuning","level":2,"score":0.42544135451316833},{"id":"https://openalex.org/C143724316","wikidata":"https://www.wikidata.org/wiki/Q312468","display_name":"Series (stratigraphy)","level":2,"score":0.41319340467453003},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.377052903175354},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3491976261138916},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10279417037963867},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0944828987121582},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.09252810478210449},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08642116189002991},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3216434712","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2111.10951.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3216434712","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2111.10951.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1520335995","https://openalex.org/W1573514622","https://openalex.org/W1599016936","https://openalex.org/W1821462560","https://openalex.org/W1832693441","https://openalex.org/W2015932775","https://openalex.org/W2096707493","https://openalex.org/W2130158090","https://openalex.org/W2251939518","https://openalex.org/W2294370754","https://openalex.org/W2396767181","https://openalex.org/W2525127255","https://openalex.org/W2607892599","https://openalex.org/W2806120502","https://openalex.org/W2886424491","https://openalex.org/W2896409484","https://openalex.org/W2907947679","https://openalex.org/W2946794439","https://openalex.org/W2948223045","https://openalex.org/W2950813464","https://openalex.org/W2951561177","https://openalex.org/W2963310665","https://openalex.org/W2963341956","https://openalex.org/W2963748441","https://openalex.org/W2963813662","https://openalex.org/W2965373594","https://openalex.org/W2970565456","https://openalex.org/W2975381464","https://openalex.org/W2978017171","https://openalex.org/W2979314664","https://openalex.org/W2979361799","https://openalex.org/W2979691890","https://openalex.org/W2982567551","https://openalex.org/W2996854111","https://openalex.org/W2996970889","https://openalex.org/W2997006708","https://openalex.org/W2997710335","https://openalex.org/W2998183051","https://openalex.org/W3008219293","https://openalex.org/W3015982254","https://openalex.org/W3034906024","https://openalex.org/W3034949522","https://openalex.org/W3035317912","https://openalex.org/W3044749682","https://openalex.org/W3103754749","https://openalex.org/W3105183573","https://openalex.org/W3105966348","https://openalex.org/W3114651185","https://openalex.org/W3151929433","https://openalex.org/W3174510164"],"related_works":["https://openalex.org/W4226330564","https://openalex.org/W4310043880","https://openalex.org/W3015233032","https://openalex.org/W4226073878","https://openalex.org/W4308847508","https://openalex.org/W3044749682","https://openalex.org/W3120519792","https://openalex.org/W3114304470","https://openalex.org/W3163479207","https://openalex.org/W3103754749"],"abstract_inverted_index":{"In":[0],"light":[1],"of":[2,5,49],"the":[3,15,21,34,75],"success":[4],"transferring":[6],"language":[7],"models":[8,51],"into":[9],"NLP":[10],"tasks,":[11],"we":[12,79],"ask":[13],"whether":[14],"full":[16,76],"BERT":[17,91],"model":[18],"is":[19],"always":[20],"best":[22],"and":[23,55,84,105],"does":[24],"it":[25],"exist":[26,69],"a":[27,47,81,86],"simple":[28,87],"but":[29],"effective":[30],"method":[31,88,101],"to":[32,89],"find":[33],"winning":[35],"ticket":[36],"in":[37],"state-of-the-art":[38],"deep":[39],"neural":[40],"networks":[41],"without":[42],"complex":[43],"calculations.":[44],"We":[45],"construct":[46],"series":[48],"BERT-based":[50],"with":[52,109],"different":[53],"size":[54],"compare":[56],"their":[57],"predictions":[58],"on":[59],"8":[60],"binary":[61],"classification":[62],"tasks.":[63],"The":[64],"results":[65],"show":[66],"there":[67],"truly":[68],"smaller":[70],"sub-networks":[71],"performing":[72],"better":[73],"than":[74],"model.":[77],"Then":[78],"present":[80],"further":[82],"study":[83],"propose":[85],"shrink":[90],"appropriately":[92],"before":[93],"fine-tuning.":[94],"Some":[95],"extended":[96],"experiments":[97],"indicate":[98],"that":[99],"our":[100],"could":[102],"save":[103],"time":[104],"storage":[106],"overhead":[107],"extraordinarily":[108],"little":[110],"even":[111],"no":[112],"accuracy":[113],"loss.":[114]},"counts_by_year":[],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2021-12-06T00:00:00"}