{"id":"https://openalex.org/W4417448830","doi":"https://doi.org/10.48550/arxiv.2512.12476","title":"HetRL: Efficient Reinforcement Learning for LLMs in Heterogeneous Environments","display_name":"HetRL: Efficient Reinforcement Learning for LLMs in Heterogeneous Environments","publication_year":2025,"publication_date":"2025-12-13","ids":{"openalex":"https://openalex.org/W4417448830","doi":"https://doi.org/10.48550/arxiv.2512.12476"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2512.12476","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2512.12476","pdf_url":"https://arxiv.org/pdf/2512.12476","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2512.12476","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101590823","display_name":"Yongjun He","orcid":"https://orcid.org/0000-0002-2697-5012"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yongjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115978436","display_name":"Shuai Zhang","orcid":"https://orcid.org/0009-0002-4920-2669"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Gai, Jiading","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gai, Jiading","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100725661","display_name":"Xiyuan Zhang","orcid":"https://orcid.org/0000-0002-3503-8856"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100781695","display_name":"Boran Han","orcid":"https://orcid.org/0000-0002-3621-3134"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Boran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055825320","display_name":"Bernie Wang","orcid":"https://orcid.org/0000-0002-0291-7184"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Bernie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006581225","display_name":"Huzefa Rangwala","orcid":"https://orcid.org/0000-0003-0435-0035"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rangwala, Huzefa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5082384108","display_name":"George Karypis","orcid":"https://orcid.org/0000-0003-2753-1437"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karypis, George","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.23720000684261322,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.23720000684261322,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11779999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.09889999777078629,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.680899977684021},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.6743000149726868},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6725999712944031},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6473000049591064},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.4966999888420105},{"id":"https://openalex.org/keywords/homogeneous","display_name":"Homogeneous","score":0.45739999413490295},{"id":"https://openalex.org/keywords/symmetric-multiprocessor-system","display_name":"Symmetric multiprocessor system","score":0.426800012588501},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.3750999867916107}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7738000154495239},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.680899977684021},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6743000149726868},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6725999712944031},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6473000049591064},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5317000150680542},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4966999888420105},{"id":"https://openalex.org/C66882249","wikidata":"https://www.wikidata.org/wiki/Q169336","display_name":"Homogeneous","level":2,"score":0.45739999413490295},{"id":"https://openalex.org/C172430144","wikidata":"https://www.wikidata.org/wiki/Q17111997","display_name":"Symmetric multiprocessor system","level":2,"score":0.426800012588501},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.3750999867916107},{"id":"https://openalex.org/C194051981","wikidata":"https://www.wikidata.org/wiki/Q1337691","display_name":"Economic shortage","level":3,"score":0.365200012922287},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3553999960422516},{"id":"https://openalex.org/C3017813396","wikidata":"https://www.wikidata.org/wiki/Q17078173","display_name":"Resource constraints","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2921999990940094},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.2903999984264374},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.2825999855995178},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.2615000009536743},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C3018263672","wikidata":"https://www.wikidata.org/wiki/Q1296251","display_name":"Efficient algorithm","level":2,"score":0.25780001282691956}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2512.12476","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2512.12476","pdf_url":"https://arxiv.org/pdf/2512.12476","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2512.12476","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.12476","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2512.12476","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2512.12476","pdf_url":"https://arxiv.org/pdf/2512.12476","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4417448830.pdf","grobid_xml":"https://content.openalex.org/works/W4417448830.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"large":[1],"language":[2],"models":[3,68],"(LLMs)":[4],"continue":[5],"to":[6,27,164],"scale":[7],"and":[8,35,69,74,95,113,132,151,171,181],"new":[9],"GPUs":[10,34,42,94],"are":[11],"released":[12],"even":[13],"more":[14],"frequently,":[15],"there":[16],"is":[17],"an":[18,134],"increasing":[19],"demand":[20],"for":[21,55,86,118],"LLM":[22],"post-training":[23],"in":[24,90,104],"heterogeneous":[25,93,105],"environments":[26,106],"fully":[28],"leverage":[29],"underutilized":[30],"mid-range":[31],"or":[32],"previous-generation":[33],"alleviate":[36],"the":[37,64,99,166],"shortage":[38],"of":[39,101,168,179],"homogeneous":[40],"high-end":[41],"within":[43],"a":[44,83,108,123,176],"single":[45],"availability":[46],"zone.":[47],"However,":[48],"achieving":[49],"high-performance":[50],"reinforcement":[51],"learning":[52],"(RL)":[53],"training":[54,89,103],"LLMs":[56],"on":[57,173],"such":[58],"computing":[59],"resources":[60],"remains":[61],"challenging":[62],"because":[63],"workflow":[65],"involves":[66],"multiple":[67],"tasks":[70],"with":[71,92],"complex":[72],"computation":[73],"data":[75],"dependencies.":[76],"In":[77],"this":[78,120],"paper,":[79],"we":[80],"present":[81],"HetRL,":[82],"distributed":[84],"system":[85],"efficient":[87],"RL":[88,102],"infrastructures":[91],"networks.":[96],"HetRL":[97,161],"formulates":[98],"scheduling":[100,125,139],"as":[107],"constrained":[109],"joint":[110],"optimization":[111],"problem":[112],"provides":[114],"two":[115],"complementary":[116],"approaches":[117],"addressing":[119],"problem:":[121],"(1)":[122],"hybrid":[124],"algorithm":[126,140],"that":[127,141,160],"efficiently":[128],"identifies":[129],"near-optimal":[130],"solutions,":[131,144],"(2)":[133],"integer":[135],"linear":[136],"programming":[137],"(ILP)-based":[138],"obtains":[142],"optimal":[143],"enabling":[145],"flexible":[146],"trade-offs":[147],"between":[148],"solution":[149],"optimality":[150],"efficiency.":[152],"Our":[153],"extensive":[154],"evaluation,":[155],"consuming":[156],"20,000":[157],"GPU-hours,":[158],"shows":[159],"achieves":[162],"up":[163],"9.17x":[165],"throughput":[167],"state-of-the-art":[169],"systems,":[170],"3.17x":[172],"average,":[174],"across":[175],"wide":[177],"range":[178],"workloads":[180],"settings.":[182]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-12-17T00:00:00"}