{"id":"https://openalex.org/W3006683367","doi":"https://doi.org/10.48550/arxiv.2004.11886","title":"Lite Transformer with Long-Short Range Attention","display_name":"Lite Transformer with Long-Short Range Attention","alternate_titles":[],"publication_year":2020,"publication_date":"2020-04-24","ids":{"openalex":"https://openalex.org/W3006683367","doi":"https://doi.org/10.48550/arxiv.2004.11886","mag":"3006683367"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2004.11886","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.11886","pdf_url":"https://arxiv.org/pdf/2004.11886","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W3006683367"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2004.11886","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082308834","display_name":"Zhanghao Wu","orcid":"https://orcid.org/0000-0003-2352-4002","observed_orcids":["https://orcid.org/0000-0003-2352-4002"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["CN","US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Wu, Zhanghao","raw_affiliation_strings":["Massachusetts Institute of Technology","Shanghai Jiao Tong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100750264","display_name":"Zhijian Liu","orcid":"https://orcid.org/0000-0003-3632-9986","observed_orcids":["https://orcid.org/0000-0003-3632-9986"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Liu, Zhijian","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065001783","display_name":"Ji Lin","orcid":"https://orcid.org/0000-0001-6053-4344","observed_orcids":["https://orcid.org/0000-0001-6053-4344"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lin, Ji","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021398655","display_name":"Yujun Lin","orcid":"https://orcid.org/0000-0001-6314-1722","observed_orcids":["https://orcid.org/0000-0001-6314-1722"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lin, Yujun","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070926896","display_name":"Song Han","orcid":"https://orcid.org/0000-0002-4186-7618","observed_orcids":["https://orcid.org/0000-0002-4186-7618"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Han, Song","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":434,"reviewed_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8335000276565552,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.8335000276565552,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08129999786615372,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0738999992609024,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/edge-devices","display_name":"edge devices","score":0.7099999785423279},{"id":"https://openalex.org/keywords/convolutional-attention","display_name":"convolutional attention","score":0.5669999718666077},{"id":"https://openalex.org/keywords/model-compression","display_name":"model compression","score":0.6060000061988831},{"id":"https://openalex.org/keywords/pruning","display_name":"pruning","score":0.6150000095367432},{"id":"https://openalex.org/keywords/quantization","display_name":"quantization","score":0.996999979019165},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.9480000138282776},{"id":"https://openalex.org/keywords/abstractive-summarization","display_name":"abstractive summarization","score":0.9959999918937683},{"id":"https://openalex.org/keywords/language-modeling","display_name":"language modeling","score":0.8980000019073486}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7679481506347656},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7600481510162354},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.7008618116378784},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.685149073600769},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.5618274211883545},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.4922126829624176},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.4920000731945038},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4801216125488281},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4719870686531067},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43328818678855896},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.3281414210796356},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3250502347946167},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.14284589886665344},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.1265004575252533},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10248687863349915},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2004.11886","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.11886","pdf_url":"https://arxiv.org/pdf/2004.11886","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3019527251","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2004.11886.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2004.11886","doi":"https://doi.org/10.48550/arxiv.2004.11886","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2004.11886","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2004.11886","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.11886","pdf_url":"https://arxiv.org/pdf/2004.11886","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3006683367.pdf","grobid_xml":"https://content.openalex.org/works/W3006683367.grobid-xml"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W1544827683","https://openalex.org/W1845051632","https://openalex.org/W1902237438","https://openalex.org/W2119144962","https://openalex.org/W2130942839","https://openalex.org/W2133564696","https://openalex.org/W2154652894","https://openalex.org/W2183341477","https://openalex.org/W2319920447","https://openalex.org/W2518108298","https://openalex.org/W2553303224","https://openalex.org/W2560017826","https://openalex.org/W2581377246","https://openalex.org/W2613904329","https://openalex.org/W2625477277","https://openalex.org/W2626778328","https://openalex.org/W2736941579","https://openalex.org/W2767989436","https://openalex.org/W2789541106","https://openalex.org/W2806311723","https://openalex.org/W2809624076","https://openalex.org/W2886851211","https://openalex.org/W2886953980","https://openalex.org/W2894175714","https://openalex.org/W2896060389","https://openalex.org/W2902251695","https://openalex.org/W2908336025","https://openalex.org/W2912521296","https://openalex.org/W2933138175","https://openalex.org/W2940744433","https://openalex.org/W2946567085","https://openalex.org/W2948223045","https://openalex.org/W2953337107","https://openalex.org/W2962746461","https://openalex.org/W2962784628","https://openalex.org/W2962851801","https://openalex.org/W2962861284","https://openalex.org/W2963034893","https://openalex.org/W2963091558","https://openalex.org/W2963341956","https://openalex.org/W2963363373","https://openalex.org/W2963768805","https://openalex.org/W2963821229","https://openalex.org/W2964121744","https://openalex.org/W2967733054","https://openalex.org/W2969624041","https://openalex.org/W2982399380","https://openalex.org/W2982479999","https://openalex.org/W2985462664","https://openalex.org/W2996167479","https://openalex.org/W3010650062","https://openalex.org/W7097023696"],"related_works":["https://openalex.org/W4385570909","https://openalex.org/W7163533105","https://openalex.org/W7164011506","https://openalex.org/W7162335977","https://openalex.org/W7161059102","https://openalex.org/W3086977883","https://openalex.org/W4392943591","https://openalex.org/W3007007518","https://openalex.org/W4407679557","https://openalex.org/W7213667577"],"abstract_inverted_index":{"Transformer":[0,53,123,134,162,169,183,188],"has":[1,211],"become":[2],"ubiquitous":[3],"in":[4,78,89],"natural":[5],"language":[6,108,115,166],"processing":[7],"(e.g.,":[8],"machine":[9,110],"translation,":[10,111],"question":[11],"answering);":[12],"however,":[13],"it":[14,26],"requires":[15,204],"enormous":[16],"amount":[17],"of":[18,75,138,160],"computations":[19],"to":[20,54],"achieve":[21],"high":[22],"performance,":[23],"which":[24],"makes":[25],"not":[27],"suitable":[28],"for":[29,193],"mobile":[30,49,57,195],"applications":[31,59],"that":[32,203],"are":[33],"tightly":[34],"constrained":[35,118],"by":[36,129,142,163,189],"the":[37,67,79,90,102,136,157,175,185,194,199],"hardware":[38],"resources":[39,119],"and":[40,114,152],"battery.":[41],"In":[42],"this":[43],"paper,":[44],"we":[45,154],"present":[46],"an":[47],"efficient":[48],"NLP":[50,58,196],"architecture,":[51],"Lite":[52,122,133,161,168,182],"facilitate":[55],"deploying":[56],"on":[60,105,126],"edge":[61],"devices.":[62],"The":[63],"key":[64],"primitive":[65],"is":[66],"Long-Short":[68],"Range":[69],"Attention":[70],"(LSRA),":[71],"where":[72],"one":[73],"group":[74,87],"heads":[76],"specializes":[77,88],"local":[80],"context":[81],"modeling":[82,93],"(by":[83,94],"convolution)":[84],"while":[85],"another":[86],"long-distance":[91],"relationship":[92],"attention).":[95],"Such":[96],"specialization":[97],"brings":[98],"consistent":[99],"improvement":[100],"over":[101],"vanilla":[103],"transformer":[104,125,139,176],"three":[106],"well-established":[107],"tasks:":[109],"abstractive":[112],"summarization,":[113],"modeling.":[116],"Under":[117],"(500M/100M":[120],"MACs),":[121],"outperforms":[124,184],"WMT'14":[127],"English-French":[128],"1.2/1.7":[130],"BLEU,":[131],"respectively.":[132],"reduces":[135],"computation":[137],"base":[140],"model":[141,158],"2.5x":[143],"with":[144,150],"0.3":[145],"BLEU":[146,192],"score":[147],"degradation.":[148],"Combining":[149],"pruning":[151],"quantization,":[153],"further":[155],"compressed":[156],"size":[159],"18.2x.":[164],"For":[165],"modeling,":[167],"achieves":[170],"1.8":[171],"lower":[172],"perplexity":[173],"than":[174,206],"at":[177,215],"around":[178],"500M":[179],"MACs.":[180],"Notably,":[181],"AutoML-based":[186],"Evolved":[187],"0.5":[190],"higher":[191],"setting":[197],"without":[198],"costly":[200],"architecture":[201],"search":[202],"more":[205],"250":[207],"GPU":[208],"years.":[209],"Code":[210],"been":[212],"made":[213],"available":[214],"https://github.com/mit-han-lab/lite-transformer.":[216]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3006683367","counts_by_year":[{"year":2026,"cited_by_count":55},{"year":2025,"cited_by_count":85},{"year":2024,"cited_by_count":99},{"year":2023,"cited_by_count":67},{"year":2022,"cited_by_count":57},{"year":2021,"cited_by_count":52},{"year":2020,"cited_by_count":19}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2020-03-06T00:00:00"}
