{"id":"https://openalex.org/W7220766868","doi":null,"title":"Dynamic sparsity: enabling efficient, interpretable and generalizable sequence models","display_name":"Dynamic sparsity: enabling efficient, interpretable and generalizable sequence models","publication_year":2024,"publication_date":"2024-07-12","ids":{"openalex":"https://openalex.org/W7220766868"},"language":"en","primary_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/125587","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/125587","pdf_url":"https://www.ideals.illinois.edu/items/132666/bitstreams/440044/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/132666/bitstreams/440044/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055976235","display_name":"Liliang Ren","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ren, Liliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5055976235"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.006099999882280827,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dynamic-sparsity","display_name":"dynamic sparsity","score":0.996999979019165},{"id":"https://openalex.org/keywords/sparse-neural-networks","display_name":"sparse neural networks","score":0.6779999732971191},{"id":"https://openalex.org/keywords/knowledge-graph-extraction","display_name":"knowledge graph extraction","score":0.8190000057220459},{"id":"https://openalex.org/keywords/pre-training-objectives","display_name":"pre-training objectives","score":0.48399999737739563},{"id":"https://openalex.org/keywords/few-shot-domain-adaptation","display_name":"few-shot domain adaptation","score":0.7419999837875366},{"id":"https://openalex.org/keywords/long-sequence-modeling","display_name":"long sequence modeling","score":0.8090000152587891},{"id":"https://openalex.org/keywords/sparse-attention","display_name":"sparse attention","score":0.5740000009536743},{"id":"https://openalex.org/keywords/linear-state-space-models","display_name":"linear state-space models","score":0.8790000081062317},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.7649999856948853},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.640999972820282},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.4860000014305115},{"id":"https://openalex.org/keywords/biological-neural-networks","display_name":"biological neural networks","score":0.40799999237060547}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6220999956130981},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6014999747276306},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.539900004863739},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2946999967098236},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.29120001196861267},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.27549999952316284},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2750999927520752},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2565000057220459},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.25609999895095825}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:www.ideals.illinois.edu:2142/125587","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/125587","pdf_url":"https://www.ideals.illinois.edu/items/132666/bitstreams/440044/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/125587","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/125587","pdf_url":"https://www.ideals.illinois.edu/items/132666/bitstreams/440044/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Thesis"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220766868.pdf","grobid_xml":"https://content.openalex.org/works/W7220766868.grobid-xml"},"referenced_works_count":95,"referenced_works":["https://openalex.org/W1940872118","https://openalex.org/W1959608418","https://openalex.org/W2104246439","https://openalex.org/W2133564696","https://openalex.org/W2325237720","https://openalex.org/W2548228487","https://openalex.org/W2567070169","https://openalex.org/W2775521672","https://openalex.org/W2792643794","https://openalex.org/W2794325560","https://openalex.org/W2796705611","https://openalex.org/W2857028992","https://openalex.org/W2890894339","https://openalex.org/W2896457183","https://openalex.org/W2936262349","https://openalex.org/W2946659172","https://openalex.org/W2949650786","https://openalex.org/W2949680561","https://openalex.org/W2950700477","https://openalex.org/W2952179106","https://openalex.org/W2952677853","https://openalex.org/W2962818478","https://openalex.org/W2962902328","https://openalex.org/W2963641152","https://openalex.org/W2963748441","https://openalex.org/W2964167098","https://openalex.org/W2965373594","https://openalex.org/W2968917279","https://openalex.org/W2974004142","https://openalex.org/W2976023236","https://openalex.org/W2981142451","https://openalex.org/W2988394319","https://openalex.org/W2994673210","https://openalex.org/W2996908057","https://openalex.org/W2998617917","https://openalex.org/W3006439205","https://openalex.org/W3015468748","https://openalex.org/W3024835274","https://openalex.org/W3030163527","https://openalex.org/W3033529678","https://openalex.org/W3034617555","https://openalex.org/W3035053871","https://openalex.org/W3035839559","https://openalex.org/W3086626290","https://openalex.org/W3091156754","https://openalex.org/W3103836967","https://openalex.org/W3105063288","https://openalex.org/W3134144764","https://openalex.org/W3174146318","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W3196318247","https://openalex.org/W3200543719","https://openalex.org/W3210904070","https://openalex.org/W4297941019","https://openalex.org/W4301194718","https://openalex.org/W4310510250","https://openalex.org/W4311991547","https://openalex.org/W4321593177","https://openalex.org/W4324312887","https://openalex.org/W4327810158","https://openalex.org/W4360836968","https://openalex.org/W4367060867","https://openalex.org/W4378505330","https://openalex.org/W4382603041","https://openalex.org/W4384648484","https://openalex.org/W4384648639","https://openalex.org/W4384918448","https://openalex.org/W4385573254","https://openalex.org/W4386435778","https://openalex.org/W4386655575","https://openalex.org/W4387294617","https://openalex.org/W4387561528","https://openalex.org/W4388444732","https://openalex.org/W4388979610","https://openalex.org/W4389326242","https://openalex.org/W4389518760","https://openalex.org/W4389599535","https://openalex.org/W4389713725","https://openalex.org/W4392090070","https://openalex.org/W4392822465","https://openalex.org/W4393399080","https://openalex.org/W4394782111","https://openalex.org/W4394782164","https://openalex.org/W4394867439","https://openalex.org/W4396881410","https://openalex.org/W4399655853","https://openalex.org/W4401042914","https://openalex.org/W4401157551","https://openalex.org/W6903985660","https://openalex.org/W7133192266","https://openalex.org/W7133201027","https://openalex.org/W7133224126","https://openalex.org/W7133231983","https://openalex.org/W7133241209"],"related_works":["https://openalex.org/W2554701276","https://openalex.org/W7020184947","https://openalex.org/W7133214195","https://openalex.org/W7163635551","https://openalex.org/W7220762720","https://openalex.org/W4385570354","https://openalex.org/W7152587185","https://openalex.org/W4416036880","https://openalex.org/W2266238587","https://openalex.org/W7100363414"],"abstract_inverted_index":{"As":[0],"humans,":[1],"we":[2,24,125],"consume":[3],"natural":[4,119],"signals":[5],"and":[6,14,19,33,100,159,172,187,229,262,304,361,386,456,466,494],"generate":[7],"structured":[8,116],"information":[9,117],"to":[10,39,79,108,200,227,245,270,279,283,300,320,389,412,427,473,477,510],"understand":[11],"our":[12,103,241,354],"world":[13,60,486],"propagate":[15],"knowledge.Equipped":[16],"with":[17,96,161,169,222,235,273,309,418],"dynamically":[18,230,305,346],"sparsely":[20,228,303,321,338],"activated":[21],"neural":[22,54,71,111,150,224,298,315],"networks,":[23,72],"conduct":[25],"the":[26,35,58,65,73,81,85,94,138,141,201,207,250,254,328,349,375,379,391,401,419,440,464,488,515],"learning":[27,175],"process":[28],"efficiently,":[29],"make":[30],"interpretable":[31,247],"decisions,":[32],"generalize":[34],"learned":[36,264,330],"knowledge":[37,130,177,265],"quickly":[38,268],"unseen":[40],"scenarios.Recent":[41],"advances":[42],"in":[43,56,69,144,190,463],"large":[44],"language":[45],"models":[46,286,367],"(LLMs)":[47],"have":[48,406,413],"shown":[49],"impressive":[50],"results":[51],"of":[52,75,84,137,157,176,206,209,249,381,491,502],"artificial":[53,86],"networks":[55,299],"understanding":[57],"real":[59],"through":[61,439],"sequential":[62],"data.Inspired":[63],"by":[64,105],"dynamic":[66,281],"sparsity":[67,282],"presented":[68],"biological":[70],"aim":[74],"this":[76,318],"thesis":[77],"is":[78,243,337,345,383,399],"improve":[80],"fundamental":[82],"architecture":[83,316],"sequence":[87,156,225,285,307,370],"models,":[88],"so":[89],"that":[90,113,132,148,152,240,296,344,353,394],"they":[91],"can":[92,114,165,266,356,461],"complete":[93,511],"tasks":[95],"improved":[97],"efficiency,":[98],"interpretability,":[99],"generalizability.We":[101],"start":[102],"research":[104,392,416],"investigating":[106],"how":[107,278],"develop":[109],"efficient":[110,288],"architectures":[112],"extract":[115,231],"from":[118,331,517],"text":[120,191],"sequences":[121],"under":[122],"human":[123,182,212,260],"supervision.Particularly,":[124],"focus":[126],"on":[127,327,368],"extracting":[128],"a":[129,134,145,149,218,223,293,314,332,341,414,453,457,497],"graph":[131,178],"provides":[133],"sparse":[135],"representation":[136],"relationships":[139],"between":[140],"important":[142],"entities":[143],"sentence.We":[146],"show":[147,239,352],"network":[151],"generates":[153],"an":[154,323],"alternating":[155],"nodes":[158],"edges":[160],"hybrid":[162],"span":[163],"decoding":[164],"achieve":[166],"state-of-the-art":[167,366],"performance":[168],"linear":[170,333],"time":[171,377,442,503],"space":[173],"complexities.Supervised":[174],"extraction":[179],"requires":[180],"expensive":[181],"efforts":[183],"for":[184,214,287,306,482,495],"tagging":[185,248],"relations":[186],"entity":[188],"types":[189],"sequences.This":[192],"problem":[193],"becomes":[194],"more":[195],"severe":[196],"when":[197],"it":[198],"comes":[199],"science":[202],"discovery":[203],"domain":[204],"because":[205],"cost":[208],"finding":[210],"capable":[211],"experts":[213],"data":[215],"annotation.We":[216],"propose":[217,292],"pre-training":[219],"objective":[220],"together":[221],"model":[226,242,355],"sentence-level":[232],"keyword":[233],"representations":[234,329],"diverse":[236],"latent":[237,255],"types.We":[238],"able":[244,509],"learn":[246],"sentence":[251],"elements":[252,308],"at":[253,374,431],"level":[256],"without":[257,514],"using":[258],"any":[259],"annotations,":[261],"its":[263],"be":[267,508],"adapted":[269],"new":[271],"domains":[272],"few":[274],"labels.We":[275],"further":[276],"explore":[277,390],"introduce":[280],"general":[284,294],"long-sequence":[289],"modeling.We":[290],"first":[291],"mechanism":[295,319],"enables":[297],"activate":[301,322],"submodules":[302],"endto-end":[310],"differentiability.We":[311],"then":[312],"design":[313],"employing":[317],"attention":[324],"module":[325],"based":[326],"state-space":[334],"model.The":[335],"input":[336],"selected":[339],"into":[340,484],"First-In-First-Out":[342],"memory":[343],"updated":[347],"throughout":[348],"generation":[350],"process.We":[351],"bring":[357],"significantly":[358],"better":[359],"training":[360],"inference":[362],"quality-efficiency":[363],"trade-offs":[364],"than":[365],"various":[369],"modeling":[371],"tasks,":[372],"while":[373],"same":[376],"revealing":[378],"amount":[380],"who":[382,436,460],"really":[384],"supportive":[385],"allows":[387],"me":[388,438,449],"topics":[393],"I":[395,405,471,505],"am":[396,423],"passionate":[397],"about.He":[398],"probably":[400],"most":[402,420],"ideal":[403],"advisor":[404],"ever":[407],"met":[408],"if":[409],"one":[410],"wants":[411],"curiositydriven":[415],"journey":[417],"possible":[421],"freedom.I":[422],"also":[424],"very":[425],"grateful":[426],"my":[428,444,478,485,492,512,518],"internship":[429],"mentor":[430],"Mircosoft,":[432],"Dr.":[433],"Yang":[434],"Liu,":[435],"helped":[437],"hardest":[441],"during":[443,487],"PhD":[445,513],"journey.The":[446],"experience":[447],"helps":[448],"grow":[450],"as":[451],"both":[452],"mature":[454],"researcher":[455],"tough":[458],"person":[459],"stay":[462],"clam":[465],"overcome":[467],"seemingly":[468],"impossible":[469],"challenges.Finally,":[470],"want":[472],"give":[474],"special":[475],"thanks":[476],"girlfriend,":[479],"Songlin":[480],"Yang,":[481],"coming":[483],"final":[489],"year":[490],"PhD,":[493],"navigating":[496],"busy":[498],"but":[499],"fruitful":[500],"period":[501],"together.":[504],"will":[506],"not":[507],"company":[516],"friends:":[519],"Shizhuo":[520]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-07T00:00:00"}