{"id":"https://openalex.org/W4416615434","doi":"https://doi.org/10.52202/085713-0349","title":"FlyLoRA: Boosting Task Decoupling and Parameter Efficiency via Implicit Rank-Wise Mixture-of-Experts","display_name":"FlyLoRA: Boosting Task Decoupling and Parameter Efficiency via Implicit Rank-Wise Mixture-of-Experts","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416615434","doi":"https://doi.org/10.52202/085713-0349"},"language":"en","primary_location":{"id":"doi:10.52202/085713-0349","doi":"https://doi.org/10.52202/085713-0349","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0349","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.08396","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112472161","display_name":"Heming Zou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Heming Zou","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042965930","display_name":"Yunliang Zang","orcid":"https://orcid.org/0000-0001-8999-1936","observed_orcids":["https://orcid.org/0000-0001-8999-1936"]},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunliang Zang","raw_affiliation_strings":["Tianjin University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tianjin University","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007288064","display_name":"Wutong Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wutong Xu","raw_affiliation_strings":["University of Maryland, College Park"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland, College Park","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101567070","display_name":"Yao Zhu","orcid":"https://orcid.org/0009-0000-6731-4475","observed_orcids":["https://orcid.org/0009-0000-6731-4475"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yao Zhu","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102859000","display_name":"Xiangyang Ji","orcid":"https://orcid.org/0000-0001-9542-5260","observed_orcids":["https://orcid.org/0000-0001-9542-5260","https://orcid.org/0000-0002-7333-9975"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangyang Ji","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9483,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.78184347,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"11540","last_page":"11573"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0017000000225380063,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/low-rank-adaptation","display_name":"low-rank adaptation","score":0.9620000123977661},{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.8450000286102295},{"id":"https://openalex.org/keywords/parameter-efficient-fine-tuning","display_name":"parameter-efficient fine-tuning","score":0.8920000195503235},{"id":"https://openalex.org/keywords/foundation-models","display_name":"foundation models","score":0.4580000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7002000212669373},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.5702000260353088},{"id":"https://openalex.org/C17137986","wikidata":"https://www.wikidata.org/wiki/Q215067","display_name":"Orthogonality","level":2,"score":0.48159998655319214},{"id":"https://openalex.org/C2775896111","wikidata":"https://www.wikidata.org/wiki/Q642560","display_name":"Router","level":2,"score":0.4447000026702881},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4415000081062317},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.39089998602867126},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3831000030040741},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.3792000114917755},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3492000102996826},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.33410000801086426},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3276999890804291},{"id":"https://openalex.org/C177860922","wikidata":"https://www.wikidata.org/wiki/Q788608","display_name":"Decorrelation","level":2,"score":0.32510000467300415},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.32199999690055847},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3174000084400177},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.3140999972820282},{"id":"https://openalex.org/C162932704","wikidata":"https://www.wikidata.org/wiki/Q1058791","display_name":"Synchronizing","level":3,"score":0.3138999938964844},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.31369999051094055},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.31279999017715454},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C171018156","wikidata":"https://www.wikidata.org/wiki/Q7370306","display_name":"Rotation formalisms in three dimensions","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.29170000553131104},{"id":"https://openalex.org/C43711488","wikidata":"https://www.wikidata.org/wiki/Q7534783","display_name":"Skew","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.28630000352859497},{"id":"https://openalex.org/C176809094","wikidata":"https://www.wikidata.org/wiki/Q15401496","display_name":"Traverse","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.25459998846054077}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-0349","doi":"https://doi.org/10.52202/085713-0349","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0349","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2510.08396","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.08396","pdf_url":"https://arxiv.org/pdf/2510.08396","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.08396","doi":"https://doi.org/10.48550/arxiv.2510.08396","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.08396","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.08396","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.08396","pdf_url":"https://arxiv.org/pdf/2510.08396","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416615434.pdf","grobid_xml":"https://content.openalex.org/works/W4416615434.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4417151025","https://openalex.org/W4411120616","https://openalex.org/W4414834769","https://openalex.org/W7136845088","https://openalex.org/W4416844909","https://openalex.org/W4412888959","https://openalex.org/W4406880133","https://openalex.org/W4411120229","https://openalex.org/W7163942534","https://openalex.org/W4404781224"],"abstract_inverted_index":{"Low-Rank":[0],"Adaptation":[1],"(LoRA)":[2],"is":[3,174],"a":[4,90],"widely":[5],"used":[6],"parameter-efficient":[7],"fine-tuning":[8],"method":[9],"for":[10,117],"foundation":[11],"models,":[12],"but":[13],"it":[14],"suffers":[15],"from":[16],"parameter":[17],"interference,":[18],"resulting":[19],"in":[20,30,34,46,74,170],"suboptimal":[21],"performance.":[22],"Although":[23],"Mixture-of-Experts":[24],"(MoE)-based":[25],"LoRA":[26,66],"variants":[27],"show":[28],"promise":[29],"mitigating":[31,123],"intra-task":[32,108],"correlations":[33],"single-task":[35],"instruction":[36],"tuning,":[37],"they":[38],"introduce":[39],"additional":[40],"router":[41,82],"parameters":[42],"and":[43,78,87,110,148],"remain":[44],"ineffective":[45],"multi-task":[47],"model":[48],"merging":[49],"where":[50,89],"inter-task":[51,124],"interference":[52,125],"arises.":[53],"Inspired":[54],"by":[55,113],"the":[56,75,97,105,115,128],"fly":[57],"olfactory":[58],"circuit,":[59],"we":[60],"propose":[61],"FlyLoRA,":[62],"an":[63,80,118],"implicit":[64,81],"MoE-based":[65],"variant":[67],"that":[68,83],"introduces:":[69],"(1)":[70],"rank-wise":[71],"expert":[72,85],"activation":[73],"up-projection":[76],"matrix,":[77],"(2)":[79],"unifies":[84],"routing":[86],"down-projection,":[88],"frozen":[91],"sparse":[92],"random":[93,132],"projection":[94],"matrix":[95],"replaces":[96],"traditional":[98],"dense":[99],"trainable":[100],"version.":[101],"This":[102],"design":[103],"resolves":[104],"trade-off":[106],"between":[107],"decorrelation":[109],"computational":[111],"efficiency":[112],"eliminating":[114],"need":[116],"explicit":[119],"router,":[120],"while":[121],"inherently":[122],"due":[126],"to":[127],"orthogonality":[129],"property":[130],"of":[131],"matrices.":[133],"Extensive":[134],"experiments":[135],"across":[136],"four":[137],"domains":[138],"--":[139,151],"general":[140],"knowledge":[141],"understanding,":[142],"scientific":[143],"question":[144],"answering,":[145],"mathematical":[146],"reasoning,":[147],"code":[149],"generation":[150],"demonstrate":[152],"consistent":[153],"performance":[154],"improvements":[155],"over":[156],"existing":[157],"methods.":[158],"Beyond":[159],"empirical":[160],"gains,":[161],"FlyLoRA":[162],"highlights":[163],"how":[164],"biological":[165],"structures":[166],"can":[167],"inspire":[168],"innovations":[169],"AI":[171],"technologies.":[172],"Code":[173],"available":[175],"at":[176],"https://github.com/gfyddha/FlyLoRA.":[177]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-11T00:00:00"}