{"id":"https://openalex.org/W4404335040","doi":"https://doi.org/10.48550/arxiv.2410.21271","title":"EoRA: Fine-tuning-free Compensation for Compressed LLM with Eigenspace Low-Rank Approximation","display_name":"EoRA: Fine-tuning-free Compensation for Compressed LLM with Eigenspace Low-Rank Approximation","publication_year":2024,"publication_date":"2024-10-28","ids":{"openalex":"https://openalex.org/W4404335040","doi":"https://doi.org/10.48550/arxiv.2410.21271"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.21271","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.21271","pdf_url":"https://arxiv.org/pdf/2410.21271","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.21271","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016722419","display_name":"Shih-Yang Liu","orcid":"https://orcid.org/0000-0003-1997-0843"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Liu, Shih-Yang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Khadkevich, Maksim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khadkevich, Maksim","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114630280","display_name":"Nai Chit Fung","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fung, Nai Chit","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047303294","display_name":"Charbel Sakr","orcid":"https://orcid.org/0000-0001-5641-0541"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sakr, Charbel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yang, Chao-Han Huck","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Chao-Han Huck","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Chien-Yi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chien-Yi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099241641","display_name":"Saurav Muralidharan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Muralidharan, Saurav","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002444694","display_name":"Hongxu Yin","orcid":"https://orcid.org/0000-0002-6481-6389"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Hongxu","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077687075","display_name":"Kwang\u2010Ting Cheng","orcid":"https://orcid.org/0000-0002-3885-4912"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Kwang-Ting","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056503617","display_name":"Jan Kautz","orcid":"https://orcid.org/0000-0002-8830-429X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kautz, Jan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090045508","display_name":"Yu-Chiang Frank Wang","orcid":"https://orcid.org/0000-0002-2333-157X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yu-Chiang Frank","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066945976","display_name":"Pavlo Molchanov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Molchanov, Pavlo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5041446638","display_name":"Min-Hung Chen","orcid":"https://orcid.org/0000-0003-4981-215X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Min-Hung","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":13,"corresponding_author_ids":["https://openalex.org/A5016722419"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11447","display_name":"Blind Source Separation Techniques","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11447","display_name":"Blind Source Separation Techniques","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10500","display_name":"Sparse and Compressive Sensing Techniques","score":0.9855999946594238,"subfield":{"id":"https://openalex.org/subfields/2206","display_name":"Computational Mechanics"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11105","display_name":"Advanced Image Processing Techniques","score":0.9836000204086304,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.7410261034965515},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6424254775047302},{"id":"https://openalex.org/keywords/compensation","display_name":"Compensation (psychology)","score":0.5532214641571045},{"id":"https://openalex.org/keywords/eigenvalues-and-eigenvectors","display_name":"Eigenvalues and eigenvectors","score":0.538232147693634},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.428924024105072},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3995892405509949},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.38283926248550415},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3479864299297333},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.34160375595092773},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.33203473687171936},{"id":"https://openalex.org/keywords/combinatorics","display_name":"Combinatorics","score":0.19425299763679504},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.18769985437393188},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.1796705424785614},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.12040668725967407},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.057377010583877563}],"concepts":[{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.7410261034965515},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6424254775047302},{"id":"https://openalex.org/C2780023022","wikidata":"https://www.wikidata.org/wiki/Q1338171","display_name":"Compensation (psychology)","level":2,"score":0.5532214641571045},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.538232147693634},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.428924024105072},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3995892405509949},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.38283926248550415},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3479864299297333},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.34160375595092773},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.33203473687171936},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.19425299763679504},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.18769985437393188},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.1796705424785614},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.12040668725967407},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.057377010583877563},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.21271","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.21271","pdf_url":"https://arxiv.org/pdf/2410.21271","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.21271","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.21271","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.21271","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.21271","pdf_url":"https://arxiv.org/pdf/2410.21271","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404335040.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W100620283","https://openalex.org/W2495260952","https://openalex.org/W4366179611","https://openalex.org/W2996078371"],"abstract_inverted_index":{"While":[0],"post-training":[1],"compression":[2,36,88],"techniques":[3],"effectively":[4],"reduce":[5],"the":[6,77,85,100,153],"memory":[7,140],"footprint,":[8],"latency,":[9],"and":[10,25,30,74,81,116,138,165],"power":[11],"consumption":[12],"of":[13,46,87,102,155,168],"Large":[14],"Language":[15],"Models":[16],"(LLMs),":[17],"they":[18],"often":[19],"result":[20],"in":[21,98],"noticeable":[22],"accuracy":[23,80,101,107,154],"degradation":[24],"remain":[26],"limited":[27],"by":[28,134],"hardware":[29],"kernel":[31],"constraints":[32,86],"that":[33,60],"restrict":[34],"supported":[35],"formats":[37],"-":[38,55],"ultimately":[39],"reducing":[40,139],"flexibility":[41],"across":[42],"a":[43,56,148],"wide":[44],"range":[45],"deployment":[47,167],"scenarios.":[48],"In":[49],"this":[50],"work,":[51],"we":[52],"propose":[53],"EoRA":[54,90,146],"novel":[57],"$\\textbf{fine-tuning-free}$":[58],"method":[59],"augments":[61],"compressed":[62,103,122,156],"LLMs":[63],"with":[64],"low-rank":[65],"matrices,":[66],"allowing":[67],"users":[68],"to":[69,123,136],"rapidly":[70],"enhance":[71],"task-specific":[72],"performance":[73],"freely":[75],"balance":[76],"trade-off":[78],"between":[79],"computational":[82],"overhead":[83,141],"beyond":[84],"formats.":[89],"consistently":[91],"outperforms":[92],"prior":[93],"fine-tuning-free":[94],"low":[95],"rank":[96],"methods":[97],"recovering":[99],"LLMs,":[104],"achieving":[105],"notable":[106],"improvements":[108],"(e.g.,":[109],"$\\mathbf{10.84\\%}$":[110],"on":[111,114,118],"ARC-Challenge,":[112],"$\\mathbf{6.74\\%}$":[113],"MathQA,":[115],"$\\mathbf{11.45\\%}$":[117],"GSM8K":[119],"for":[120,151],"LLaMA3-8B":[121],"3-bit).":[124],"We":[125],"also":[126],"introduce":[127],"an":[128],"optimized":[129],"CUDA":[130],"kernel,":[131],"accelerating":[132],"inference":[133],"up":[135],"1.4x":[137],"through":[142],"quantizing":[143],"EoRA.":[144],"Overall,":[145],"offers":[147],"prompt":[149],"solution":[150],"improving":[152],"models":[157],"under":[158],"varying":[159],"user":[160],"requirements,":[161],"enabling":[162],"more":[163],"efficient":[164],"flexible":[166],"LLMs.":[169],"Code":[170],"is":[171],"available":[172],"at":[173],"https://github.com/NVlabs/EoRA.":[174]},"counts_by_year":[],"updated_date":"2026-03-25T23:56:10.502304","created_date":"2024-11-14T00:00:00"}