{"id":"https://openalex.org/W4401201832","doi":"https://doi.org/10.48550/arxiv.2407.19580","title":"Mini-batch Coresets for Memory-efficient Language Model Training on Data Mixtures","display_name":"Mini-batch Coresets for Memory-efficient Language Model Training on Data Mixtures","publication_year":2024,"publication_date":"2024-07-28","ids":{"openalex":"https://openalex.org/W4401201832","doi":"https://doi.org/10.48550/arxiv.2407.19580"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.19580","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19580","pdf_url":"https://arxiv.org/pdf/2407.19580","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.19580","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101861939","display_name":"Dang Khoa Nguyen","orcid":"https://orcid.org/0000-0002-4659-2465","observed_orcids":["https://orcid.org/0000-0002-4659-2465"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nguyen, Dang","raw_affiliation_strings":["Computer Science Department , UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department , UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070884682","display_name":"Wenhan Yang","orcid":"https://orcid.org/0000-0002-1692-0069","observed_orcids":["https://orcid.org/0000-0002-1692-0069","https://orcid.org/0009-0000-9961-8852"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Wenhan","raw_affiliation_strings":["Computer Science Department , UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department , UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106945781","display_name":"Rathul Anand","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anand, Rathul","raw_affiliation_strings":["Computer Science Department , UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department , UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100342259","display_name":"Yu Yang","orcid":"https://orcid.org/0000-0001-9113-6023","observed_orcids":["https://orcid.org/0000-0001-9113-6023","https://orcid.org/0000-0002-8624-1467","https://orcid.org/0009-0003-4543-8815","https://orcid.org/0009-0004-7495-8534"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Yu","raw_affiliation_strings":["Computer Science Department , UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department , UCLA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009166220","display_name":"Baharan Mirzasoleiman","orcid":"https://orcid.org/0000-0002-9374-9496","observed_orcids":["https://orcid.org/0000-0002-9374-9496"]},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mirzasoleiman, Baharan","raw_affiliation_strings":["Computer Science Department , UCLA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department , UCLA","institution_ids":["https://openalex.org/I161318765"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I161318765"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.5990999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.5990999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.26660001277923584,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0502999983727932,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-training","display_name":"large language model training","score":0.8889999985694885},{"id":"https://openalex.org/keywords/memory-efficient-training","display_name":"memory-efficient training","score":0.9070000052452087},{"id":"https://openalex.org/keywords/gradient-matching","display_name":"gradient matching","score":0.8930000066757202},{"id":"https://openalex.org/keywords/adam-optimizer","display_name":"Adam optimizer","score":0.8140000104904175},{"id":"https://openalex.org/keywords/derivative-free-optimization","display_name":"derivative-free optimization","score":0.6840000152587891},{"id":"https://openalex.org/keywords/gradient-sparsification","display_name":"gradient sparsification","score":0.7310000061988831},{"id":"https://openalex.org/keywords/low-rank-adaptation","display_name":"low-rank adaptation","score":0.8040000200271606},{"id":"https://openalex.org/keywords/gpu-memory-reduction","display_name":"GPU memory reduction","score":0.7350000143051147}],"concepts":[{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6747899055480957},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35715776681900024},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33248370885849},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.1417376697063446},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.19580","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19580","pdf_url":"https://arxiv.org/pdf/2407.19580","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.19580","doi":"https://doi.org/10.48550/arxiv.2407.19580","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.19580","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.19580","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.19580","pdf_url":"https://arxiv.org/pdf/2407.19580","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401201832.pdf","grobid_xml":"https://content.openalex.org/works/W4401201832.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W230091440","https://openalex.org/W2390279801","https://openalex.org/W2233261550","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2376932109"],"abstract_inverted_index":{"Training":[0,97],"with":[1,15,153,176,193,214,222],"larger":[2,50,216],"mini-batches":[3,17],"improves":[4],"the":[5,27,47,63,74,78,90,116,126,130,141,165,174,177,203,231],"convergence":[6],"rate":[7],"and":[8,57,77,120,169,190,197,210],"can":[9],"yield":[10],"superior":[11],"performance.":[12,137],"However,":[13,52],"training":[14,152,213,225,235],"large":[16,28,80],"becomes":[18,55],"prohibitive":[19],"for":[20,59,96,135,151],"Large":[21],"Language":[22],"Models":[23],"(LLMs),":[24],"due":[25,61],"to":[26,62,147,160,172,185],"GPU":[29],"memory":[30,204,232],"requirement.":[31],"To":[32],"address":[33,89],"this":[34,53,86],"problem,":[35],"an":[36],"effective":[37],"approach":[38,54],"is":[39,133,239],"finding":[40],"small":[41,117,127],"mini-batch":[42,104,131,149],"coresets":[43,105,132,150],"that":[44,103],"closely":[45],"match":[46],"gradient":[48,81,108,163,180],"of":[49,67,73,83,115,125,164,206,234],"mini-batches.":[51,217],"infeasible":[56],"ineffective":[58],"LLMs,":[60],"highly":[64],"imbalanced":[65],"mixture":[66],"sources":[68,118,128],"in":[69,129],"language":[70],"data,":[71],"use":[72],"Adam":[75],"optimizer,":[76],"very":[79],"dimensionality":[82],"LLMs.":[84,236],"In":[85],"work,":[87],"we":[88,101,139,156],"above":[91],"challenges":[92],"by":[93,107,143,208],"proposing":[94],"Coresets":[95],"LLMs":[98],"(CoLM).":[99],"First,":[100],"show":[102],"found":[106],"matching":[109],"do":[110],"not":[111],"contain":[112],"representative":[113],"examples":[114,124],"w.h.p.,":[119],"thus":[121],"including":[122],"all":[123],"crucial":[134],"optimal":[136],"Second,":[138],"normalize":[140],"gradients":[142],"their":[144],"historical":[145],"exponential":[146],"find":[148,161],"Adam.":[154],"Finally,":[155],"leverage":[157],"zeroth-order":[158],"methods":[159,226],"smooth":[162],"last":[166],"V-projection":[167],"matrix":[168],"sparsify":[170],"it":[171],"keep":[173],"dimensions":[175],"largest":[178],"normalized":[179],"magnitude.":[181],"We":[182],"apply":[183],"CoLM":[184,201,219],"fine-tuning":[186,207],"Phi-2,":[187],"Phi-3,":[188],"Zephyr,":[189],"Llama-3":[191],"models":[192],"LoRA":[194],"on":[195],"MathInstruct":[196],"SuperGLUE":[198],"benchmark.":[199],"Remarkably,":[200],"reduces":[202],"requirement":[205],"2x":[209],"even":[211],"outperforms":[212],"4x":[215],"Moreover,":[218],"seamlessly":[220],"integrates":[221],"existing":[223],"memory-efficient":[224],"like":[227],"LoRA,":[228],"further":[229],"reducing":[230],"requirements":[233],"Our":[237],"code":[238],"available":[240],"at":[241],"https://github.com/BigML-CS-UCLA/CoLM.":[242]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-10-05T09:50:58.205359","created_date":"2024-08-01T00:00:00"}