{"id":"https://openalex.org/W7166658627","doi":"https://doi.org/10.48550/arxiv.2606.28932","title":"DLR: Zero-Inference-Cost Latent Residuals for Low-Rank Pre-Training","display_name":"DLR: Zero-Inference-Cost Latent Residuals for Low-Rank Pre-Training","publication_year":2026,"publication_date":"2026-06-27","ids":{"openalex":"https://openalex.org/W7166658627","doi":"https://doi.org/10.48550/arxiv.2606.28932"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.28932","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.28932","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.28932","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139653286","display_name":"Dong Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Dong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060609967","display_name":"Wenwu Tang","orcid":"https://orcid.org/0000-0001-7870-177X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Wenwu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139637847","display_name":"Yun Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Yun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5074037460","display_name":"Olga Saukh","orcid":"https://orcid.org/0000-0001-7849-3368"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saukh, Olga","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.23849999904632568,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.14489999413490295,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1386999934911728,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.6988999843597412},{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.5526999831199646},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.47119998931884766},{"id":"https://openalex.org/keywords/latent-variable","display_name":"Latent variable","score":0.4253000020980835},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.4189999997615814},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.35199999809265137},{"id":"https://openalex.org/keywords/zero","display_name":"Zero (linguistics)","score":0.32330000400543213},{"id":"https://openalex.org/keywords/variable","display_name":"Variable (mathematics)","score":0.31779998540878296}],"concepts":[{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6988999843597412},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.5526999831199646},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5029000043869019},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.47119998931884766},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.4253000020980835},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.4189999997615814},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4120999872684479},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.41110000014305115},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.35199999809265137},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3416999876499176},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33079999685287476},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.32330000400543213},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.3093000054359436},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3068000078201294},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C3826847","wikidata":"https://www.wikidata.org/wiki/Q188768","display_name":"FLOPS","level":2,"score":0.2858000099658966},{"id":"https://openalex.org/C65965080","wikidata":"https://www.wikidata.org/wiki/Q1806885","display_name":"Latent variable model","level":3,"score":0.2849000096321106},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.27799999713897705},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.27070000767707825},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.26600000262260437},{"id":"https://openalex.org/C2780551164","wikidata":"https://www.wikidata.org/wiki/Q2306599","display_name":"Column (typography)","level":3,"score":0.25110000371932983}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.28932","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.28932","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.28932","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.28932","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1,8],"models":[2,131],"have":[3],"driven":[4],"recent":[5],"progress":[6],"in":[7,46,107,145],"and":[9,35,121,154],"multimodal":[10],"AI,":[11],"yet":[12],"pre-training":[13,140],"them":[14],"at":[15,152],"scale":[16],"is":[17,37,102],"prohibitively":[18],"expensive.":[19],"Low-rank":[20],"pre-training,":[21],"which":[22],"factorizes":[23],"each":[24,79],"weight":[25],"matrix":[26],"into":[27,104],"a":[28,38,54,70],"rank-r":[29],"product":[30],"to":[31,134,160],"reduce":[32],"both":[33],"parameters":[34,96],"FLOPs,":[36],"promising":[39],"response":[40],"but":[41],"typically":[42],"lags":[43],"full-rank":[44],"training":[45],"quality.":[47],"We":[48],"propose":[49],"Duplicated":[50],"Latent":[51],"Residual":[52],"(DLR),":[53],"training-only,":[55],"parameter-free,":[56],"foldable":[57],"plug-in":[58],"for":[59],"low-rank":[60,66,126,139],"pre-training.":[61],"DLR":[62,92,137],"augments":[63],"the":[64,87,105,124,149],"standard":[65,164],"output":[67],"Bz":[68],"with":[69,148],"fixed":[71],"structured":[72],"residual":[73],"alpha/sqrt(K)":[74,114],"*":[75],"Expand_K(z)":[76],"that":[77],"replicates":[78],"latent":[80],"coordinate":[81],"K":[82],"=":[83,111],"ceil(d_out/r)":[84],"times":[85],"across":[86],"output.":[88],"With":[89],"alpha":[90],"fixed,":[91],"adds":[93],"zero":[94],"learnable":[95],"per":[97],"layer;":[98],"after":[99],"training,":[100],"it":[101],"absorbed":[103],"up-projection":[106],"closed":[108],"form,":[109],"B*":[110],"B":[112],"+":[113],"R^T,":[115],"so":[116],"deployment":[117],"parameter":[118],"count,":[119],"FLOPs":[120],"memory":[122],"match":[123],"underlying":[125],"backbone":[127],"exactly.":[128],"Across":[129],"LLaMA":[130],"from":[132],"60M":[133],"7B":[135],"parameters,":[136],"strengthens":[138],"on":[141,163],"C4":[142],"validation":[143],"perplexity":[144],"most":[146],"settings,":[147],"clearest":[150],"gains":[151],"130M":[153],"above;":[155],"folded":[156],"checkpoints":[157],"transfer":[158],"cleanly":[159],"supervised":[161],"fine-tuning":[162],"benchmarks.":[165]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}