{"id":"https://openalex.org/W4404569111","doi":"https://doi.org/10.48550/arxiv.2411.09816","title":"Learning Fine-grained Parameter Sharing via Sparse Tensor Decomposition","display_name":"Learning Fine-grained Parameter Sharing via Sparse Tensor Decomposition","publication_year":2024,"publication_date":"2024-11-14","ids":{"openalex":"https://openalex.org/W4404569111","doi":"https://doi.org/10.48550/arxiv.2411.09816"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.09816","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.09816","pdf_url":"https://arxiv.org/pdf/2411.09816","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.09816","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114729831","display_name":"Cem \u00dcy\u00fck","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u00dcy\u00fck, Cem","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019355289","display_name":"Mike Lasby","orcid":"https://orcid.org/0000-0002-7090-8773"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lasby, Mike","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013748571","display_name":"Mohamed Yassin","orcid":"https://orcid.org/0000-0002-6489-3499"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yassin, Mohamed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069716783","display_name":"Utku Evci","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Evci, Utku","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5063025899","display_name":"Yani Ioannou","orcid":"https://orcid.org/0000-0002-9797-5888"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ioannou, Yani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12303","display_name":"Tensor decomposition and applications","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/2605","display_name":"Computational Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12303","display_name":"Tensor decomposition and applications","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/2605","display_name":"Computational Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13650","display_name":"Computational Physics and Python Applications","score":0.9516000151634216,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.9085999727249146,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/tensor","display_name":"Tensor (intrinsic definition)","score":0.6312403082847595},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4718468189239502},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.38406693935394287},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35919272899627686},{"id":"https://openalex.org/keywords/pure-mathematics","display_name":"Pure mathematics","score":0.2941961884498596}],"concepts":[{"id":"https://openalex.org/C155281189","wikidata":"https://www.wikidata.org/wiki/Q3518150","display_name":"Tensor (intrinsic definition)","level":2,"score":0.6312403082847595},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4718468189239502},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.38406693935394287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35919272899627686},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.2941961884498596}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2411.09816","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.09816","pdf_url":"https://arxiv.org/pdf/2411.09816","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:arXiv.org:2411.09816","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2411.09816","pdf_url":"https://arxiv.org/pdf/2411.09816","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2411.09816","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.09816","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.09816","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.09816","pdf_url":"https://arxiv.org/pdf/2411.09816","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320325651","display_name":"Alberta Innovates","ror":null},{"id":"https://openalex.org/F4320331257","display_name":"Alliance de recherche num\u00e9rique du Canada","ror":"https://ror.org/010r6td27"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404569111.pdf","grobid_xml":"https://content.openalex.org/works/W4404569111.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4391375266","https://openalex.org/W1979597421","https://openalex.org/W2007980826","https://openalex.org/W2061531152","https://openalex.org/W3002753104","https://openalex.org/W2077600819","https://openalex.org/W2142036596","https://openalex.org/W2072657027"],"abstract_inverted_index":{"Large":[0,132],"neural":[1],"networks":[2],"achieve":[3],"state-of-the-art":[4],"performance":[5],"on":[6,15,118,160],"many":[7],"tasks,":[8],"yet":[9],"their":[10],"sheer":[11],"size":[12],"hinders":[13],"deployment":[14],"resource-constrained":[16],"devices.":[17],"Among":[18],"existing":[19,142],"compression":[20],"approaches,":[21],"cross-layer":[22],"parameter":[23,52,179],"sharing":[24,180],"remains":[25],"relatively":[26],"unexplored":[27],"for":[28,43,187],"transformer":[29,45,71,188],"models.":[30],"In":[31],"this":[32],"paper,":[33],"we":[34],"introduce":[35],"Fine-grained":[36],"Parameter":[37],"Sharing":[38],"(FiPS),":[39],"a":[40,59,68,77,182],"unified":[41],"framework":[42],"compressing":[44],"Multi-Layer":[46],"Perceptrons":[47],"(MLPs)":[48],"that":[49],"combines":[50],"cross-block":[51],"sharing,":[53],"low-rank":[54],"factorization,":[55],"and":[56,73,80,94,120,147,184],"sparsity":[57],"in":[58,145],"single":[60],"optimization.":[61],"FiPS":[62,102,159],"concatenates":[63],"MLP":[64,189],"weight":[65],"matrices":[66],"across":[67],"group":[69],"of":[70],"blocks":[72],"factorizes":[74],"them":[75],"into":[76],"shared":[78],"basis":[79],"sparse,":[81],"layer-specific":[82],"projection":[83],"matrices.":[84],"Both":[85],"factors":[86],"are":[87],"initialized":[88],"via":[89],"singular":[90],"value":[91],"decomposition":[92],"(SVD)":[93],"jointly":[95],"optimized":[96],"by":[97,107,121,136],"block-wise":[98],"reconstruction":[99],"error":[100],"minimization.":[101],"compresses":[103,131],"Vision":[104],"Transformers":[105],"(ViTs)":[106],"up":[108,122,137],"to":[109,123,138],"33%":[110],"with":[111,127,154],"less":[112],"than":[113,165],"1%":[114],"top-1":[115],"accuracy":[116],"loss":[117],"ImageNet-1k,":[119],"57%":[124],"when":[125],"combined":[126],"fine-tuning.":[128],"It":[129],"also":[130],"Language":[133],"Models":[134],"(LLMs)":[135],"20%":[139],"while":[140,169],"outperforming":[141],"SVD-based":[143],"methods":[144],"perplexity":[146,164],"downstream":[148],"benchmarks":[149],"at":[150],"matched":[151],"compression.":[152,174,190],"Combined":[153],"Quantization-Aware":[155],"Training":[156],"(QAT),":[157],"3-bit":[158],"Gemma-2-2B":[161],"achieves":[162],"lower":[163],"2-bit":[166],"QAT":[167],"alone":[168],"matching":[170],"the":[171],"same":[172],"8x":[173],"These":[175],"results":[176],"establish":[177],"fine-grained":[178],"as":[181],"practical":[183],"effective":[185],"approach":[186]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-11T18:08:03.149640","created_date":"2025-10-10T00:00:00"}