{"id":"https://openalex.org/W4403883066","doi":"https://doi.org/10.1145/3689031.3696075","title":"HybridFlow: A Flexible and Efficient RLHF Framework","display_name":"HybridFlow: A Flexible and Efficient RLHF Framework","publication_year":2025,"publication_date":"2025-03-26","ids":{"openalex":"https://openalex.org/W4403883066","doi":"https://doi.org/10.1145/3689031.3696075"},"language":"en","primary_location":{"id":"doi:10.1145/3689031.3696075","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689031.3696075","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth European Conference on Computer Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.19256","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020661868","display_name":"Guangming Sheng","orcid":"https://orcid.org/0000-0003-3395-3994"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Guangming Sheng","raw_affiliation_strings":["The University of Hong Kong"],"raw_orcid":"https://orcid.org/0000-0003-3395-3994","affiliations":[{"raw_affiliation_string":"The University of Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100458134","display_name":"Chi Zhang","orcid":"https://orcid.org/0000-0001-7374-1940"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chi Zhang","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0000-0001-7374-1940","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039789909","display_name":"Zilingfeng Ye","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zilingfeng Ye","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0009-0003-5437-0445","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063756360","display_name":"Xibin Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xibin Wu","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0009-0007-9702-5288","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wang Zhang","orcid":"https://orcid.org/0009-0001-2436-8761"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang Zhang","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0009-0001-2436-8761","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ru Zhang","orcid":"https://orcid.org/0009-0005-8543-0097"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ru Zhang","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0009-0005-8543-0097","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013856812","display_name":"Yanghua Peng","orcid":"https://orcid.org/0000-0003-3989-4358"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yanghua Peng","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0000-0003-3989-4358","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084047386","display_name":"Haibin Lin","orcid":"https://orcid.org/0000-0003-4879-5335"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Haibin Lin","raw_affiliation_strings":["ByteDance"],"raw_orcid":"https://orcid.org/0000-0003-4879-5335","affiliations":[{"raw_affiliation_string":"ByteDance","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5012597518","display_name":"Chuan Wu","orcid":"https://orcid.org/0000-0002-3144-4398"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chuan Wu","raw_affiliation_strings":["The University of Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-3144-4398","affiliations":[{"raw_affiliation_string":"The University of Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":33.2901,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":{"value":0.99665924,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1279","last_page":"1297"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/dataflow","display_name":"Dataflow","score":0.9147015810012817},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8736863136291504},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.5705822706222534},{"id":"https://openalex.org/keywords/node","display_name":"Node (physics)","score":0.5550866723060608},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5245773196220398},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.4696739614009857},{"id":"https://openalex.org/keywords/multicast","display_name":"Multicast","score":0.4315565824508667},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.42363011837005615},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.4134443402290344},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.127544105052948},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.12188884615898132}],"concepts":[{"id":"https://openalex.org/C96324660","wikidata":"https://www.wikidata.org/wiki/Q205446","display_name":"Dataflow","level":2,"score":0.9147015810012817},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8736863136291504},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5705822706222534},{"id":"https://openalex.org/C62611344","wikidata":"https://www.wikidata.org/wiki/Q1062658","display_name":"Node (physics)","level":2,"score":0.5550866723060608},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5245773196220398},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.4696739614009857},{"id":"https://openalex.org/C32295351","wikidata":"https://www.wikidata.org/wiki/Q899288","display_name":"Multicast","level":2,"score":0.4315565824508667},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.42363011837005615},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.4134443402290344},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.127544105052948},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12188884615898132},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3689031.3696075","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689031.3696075","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth European Conference on Computer Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2409.19256","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.19256","pdf_url":"https://arxiv.org/pdf/2409.19256","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.19256","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.19256","pdf_url":"https://arxiv.org/pdf/2409.19256","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403883066.pdf"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W122021961","https://openalex.org/W1606512765","https://openalex.org/W2078762661","https://openalex.org/W2082171780","https://openalex.org/W2100830825","https://openalex.org/W2150871235","https://openalex.org/W2152517358","https://openalex.org/W2170796499","https://openalex.org/W2173213060","https://openalex.org/W2323693848","https://openalex.org/W2517617279","https://openalex.org/W2542459869","https://openalex.org/W2605178034","https://openalex.org/W2606555609","https://openalex.org/W2612076670","https://openalex.org/W2734941459","https://openalex.org/W2883830791","https://openalex.org/W2969388332","https://openalex.org/W2975712713","https://openalex.org/W3081168214","https://openalex.org/W3101104221","https://openalex.org/W3129831491","https://openalex.org/W3132107458","https://openalex.org/W3135499934","https://openalex.org/W3204998121","https://openalex.org/W3214226605","https://openalex.org/W4221158240","https://openalex.org/W4226278401","https://openalex.org/W4308426193","https://openalex.org/W4321446170","https://openalex.org/W4386840193","https://openalex.org/W4387302750","https://openalex.org/W4387321091","https://openalex.org/W4389520781","https://openalex.org/W4404401018","https://openalex.org/W6810475161"],"related_works":["https://openalex.org/W2293118914","https://openalex.org/W2998381397","https://openalex.org/W4236419692","https://openalex.org/W2171015181","https://openalex.org/W3167919718","https://openalex.org/W4251718783","https://openalex.org/W4239447582","https://openalex.org/W4233095166","https://openalex.org/W2074451312","https://openalex.org/W2351519177"],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"from":[2],"Human":[3],"Feedback":[4],"(RLHF)":[5],"is":[6,227],"widely":[7],"used":[8],"in":[9,87,127,159],"Large":[10],"Language":[11],"Model":[12],"(LLM)":[13],"alignment.":[14],"Traditional":[15,65],"RL":[16,66],"can":[17,84,107],"be":[18,85,108],"modeled":[19],"as":[20,219],"a":[21,29,51,62,72,103,128,146,184],"dataflow,":[22,163],"where":[23],"each":[24,34,48,59],"node":[25,49],"represents":[26],"computation":[27,79,114,155,177],"of":[28,138,148,175],"neural":[30],"network":[31],"(NN)":[32],"and":[33,58,80,115,124,135,153,156,172,193,200],"edge":[35,60],"denotes":[36],"data":[37,116,141,157],"dependencies":[38,158],"between":[39,191],"the":[40,44,69,139,160,176],"NNs.":[41],"RLHF":[42,88,100,140,162,170,215],"complicates":[43],"dataflow":[45,70],"by":[46],"expanding":[47],"into":[50,61],"distributed":[52,96,113],"LLM":[53],"training":[54,192],"or":[55],"generation":[56,194],"program,":[57],"many-to-many":[63],"multicast.":[64],"frameworks":[67],"execute":[68],"using":[71,217],"single":[73],"controller":[74],"to":[75,90,111,131,168],"instruct":[76],"both":[77],"intra-node":[78,97],"inter-node":[81],"communication,":[82],"which":[83,106,121],"inefficient":[86],"due":[89,110],"large":[91],"control":[92],"dispatch":[93],"overhead":[94],"for":[95,186],"computation.":[98],"Existing":[99],"systems":[101],"adopt":[102],"multi-controller":[104,125],"paradigm,":[105],"inflexible":[109],"nesting":[112],"communication.":[117],"We":[118,143,181],"propose":[119],"HybridFlow,":[120,218],"combines":[122],"single-controller":[123],"paradigms":[126],"hybrid":[129],"manner":[130],"enable":[132],"flexible":[133,173],"representation":[134],"efficient":[136,165,187],"execution":[137],"flow.":[142],"carefully":[144],"design":[145,183],"set":[147],"hierarchical":[149],"APIs":[150],"that":[151],"decouple":[152],"encapsulate":[154],"complex":[161],"allowing":[164],"operation":[166],"orchestration":[167],"implement":[169],"algorithms":[171,216],"mapping":[174],"onto":[178],"various":[179,214],"devices.":[180],"further":[182],"3D-HybridEngine":[185],"actor":[188],"model":[189],"resharding":[190],"phases,":[195],"with":[196,221],"zero":[197],"memory":[198],"redundancy":[199],"significantly":[201],"reduced":[202],"communication":[203],"overhead.":[204],"Our":[205],"experimental":[206],"results":[207],"demonstrate":[208],"1.53x~20.57\u00d7":[209],"throughput":[210],"improvement":[211],"when":[212],"running":[213],"compared":[220],"state-of-the-art":[222],"baselines.":[223],"HybridFlow":[224],"source":[225],"code":[226],"available":[228],"at":[229],"https://github.com/volcengine/verl":[230]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":9}],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-10-10T00:00:00"}