{"id":"https://openalex.org/W4392406041","doi":"https://doi.org/10.1109/tase.2024.3371250","title":"Hybrid Task Scheduling in Cloud Manufacturing With Sparse-Reward Deep Reinforcement Learning","display_name":"Hybrid Task Scheduling in Cloud Manufacturing With Sparse-Reward Deep Reinforcement Learning","publication_year":2024,"publication_date":"2024-03-04","ids":{"openalex":"https://openalex.org/W4392406041","doi":"https://doi.org/10.1109/tase.2024.3371250"},"language":"en","primary_location":{"id":"doi:10.1109/tase.2024.3371250","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tase.2024.3371250","pdf_url":null,"source":{"id":"https://openalex.org/S34881539","display_name":"IEEE Transactions on Automation Science and Engineering","issn_l":"1545-5955","issn":["1545-5955","1558-3783"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automation Science and Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100354385","display_name":"Xiaohan Wang","orcid":"https://orcid.org/0000-0002-8688-047X"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaohan Wang","raw_affiliation_strings":["School of Automation Science and Electrical Engineering, Beihang University, Beijing, China","Key Laboratory of Intelligent Manufacturing Systems Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8688-047X","affiliations":[{"raw_affiliation_string":"School of Automation Science and Electrical Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"Key Laboratory of Intelligent Manufacturing Systems Technology, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017308450","display_name":"Yuanjun Laili","orcid":"https://orcid.org/0000-0003-3834-4602"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanjun Laili","raw_affiliation_strings":["School of Automation Science and Electrical Engineering, Beihang University, Beijing, China","Zhongguancun Laboratory, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3834-4602","affiliations":[{"raw_affiliation_string":"School of Automation Science and Electrical Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"Zhongguancun Laboratory, Beijing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100418969","display_name":"Zhang Li","orcid":"https://orcid.org/0000-0003-1989-6102"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Zhang","raw_affiliation_strings":["School of Automation Science and Electrical Engineering, Beihang University, Beijing, China","Key Laboratory of Intelligent Manufacturing Systems Technology, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1989-6102","affiliations":[{"raw_affiliation_string":"School of Automation Science and Electrical Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]},{"raw_affiliation_string":"Key Laboratory of Intelligent Manufacturing Systems Technology, Beijing, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082897139","display_name":"Yongkui Liu","orcid":"https://orcid.org/0000-0003-2165-775X"},"institutions":[{"id":"https://openalex.org/I149594827","display_name":"Xidian University","ror":"https://ror.org/05s92vm98","country_code":"CN","type":"education","lineage":["https://openalex.org/I149594827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongkui Liu","raw_affiliation_strings":["School of Mechano-Electronic Engineering, Xidian University, Xi&#x2019;an, China"],"raw_orcid":"https://orcid.org/0000-0003-2165-775X","affiliations":[{"raw_affiliation_string":"School of Mechano-Electronic Engineering, Xidian University, Xi&#x2019;an, China","institution_ids":["https://openalex.org/I149594827"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.7545,"has_fulltext":false,"cited_by_count":24,"citation_normalized_percentile":{"value":0.96967065,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"22","issue":null,"first_page":"1878","last_page":"1892"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10763","display_name":"Digital Transformation in Industry","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10763","display_name":"Digital Transformation in Industry","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10551","display_name":"Scheduling and Optimization Algorithms","score":0.9034000039100647,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7527287602424622},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.6460778117179871},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6403077840805054},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.5914040803909302},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5664923787117004},{"id":"https://openalex.org/keywords/partially-observable-markov-decision-process","display_name":"Partially observable Markov decision process","score":0.4734439551830292},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3789949417114258},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.341368705034256},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.16817423701286316},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.1670943796634674},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16344216465950012},{"id":"https://openalex.org/keywords/markov-model","display_name":"Markov model","score":0.11529672145843506}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7527287602424622},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.6460778117179871},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6403077840805054},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.5914040803909302},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5664923787117004},{"id":"https://openalex.org/C17098449","wikidata":"https://www.wikidata.org/wiki/Q176814","display_name":"Partially observable Markov decision process","level":4,"score":0.4734439551830292},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3789949417114258},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.341368705034256},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.16817423701286316},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.1670943796634674},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16344216465950012},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.11529672145843506},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tase.2024.3371250","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tase.2024.3371250","pdf_url":null,"source":{"id":"https://openalex.org/S34881539","display_name":"IEEE Transactions on Automation Science and Engineering","issn_l":"1545-5955","issn":["1545-5955","1558-3783"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automation Science and Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1316869953","display_name":null,"funder_award_id":"61973243","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2792437373","display_name":null,"funder_award_id":"62173017","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7358085591","display_name":null,"funder_award_id":"2023YFB3308201","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W1533220406","https://openalex.org/W1985556890","https://openalex.org/W2022796131","https://openalex.org/W2067311046","https://openalex.org/W2115694415","https://openalex.org/W2178807383","https://openalex.org/W2195423816","https://openalex.org/W2362247909","https://openalex.org/W2524282687","https://openalex.org/W2531359190","https://openalex.org/W2588845958","https://openalex.org/W2735842644","https://openalex.org/W2761097593","https://openalex.org/W2792859614","https://openalex.org/W2898279621","https://openalex.org/W2900225706","https://openalex.org/W2904246096","https://openalex.org/W2910683613","https://openalex.org/W2966352676","https://openalex.org/W2968986602","https://openalex.org/W2997337786","https://openalex.org/W3000708199","https://openalex.org/W3022972893","https://openalex.org/W3045843229","https://openalex.org/W3049282540","https://openalex.org/W3096926878","https://openalex.org/W3118561874","https://openalex.org/W3151076185","https://openalex.org/W3206520044","https://openalex.org/W3216218228","https://openalex.org/W4206587050","https://openalex.org/W4211242905","https://openalex.org/W4214806295","https://openalex.org/W4221066462","https://openalex.org/W4285064947","https://openalex.org/W4285115899","https://openalex.org/W4285262488","https://openalex.org/W4288773268","https://openalex.org/W4293213710","https://openalex.org/W4295347215","https://openalex.org/W4295736419","https://openalex.org/W4295836056","https://openalex.org/W4306752035","https://openalex.org/W4312958300","https://openalex.org/W4321180396","https://openalex.org/W4324116470","https://openalex.org/W4378804868","https://openalex.org/W6740801417","https://openalex.org/W6748839928"],"related_works":["https://openalex.org/W2123651102","https://openalex.org/W2953027552","https://openalex.org/W2287992105","https://openalex.org/W2570299561","https://openalex.org/W2392831491","https://openalex.org/W2117555338","https://openalex.org/W2130339357","https://openalex.org/W1512585804","https://openalex.org/W1526144783","https://openalex.org/W3204654320"],"abstract_inverted_index":{"Cloud":[0],"manufacturing":[1,6,12,16,237],"(CMfg)":[2],"converts":[3],"the":[4,66,75,83,94,98,110,122,128,151,159,165,189,193,228,231,267,299,303,315,372],"traditional":[5],"system":[7],"into":[8],"an":[9,360],"Internet-of-things-enabled":[10],"(IoT-enabled)":[11],"system,":[13],"where":[14],"both":[15],"and":[17,25,53,150,167,181,207,238,314],"computational":[18,239,328],"tasks":[19],"must":[20],"be":[21,132,212,243,319,350],"scheduled":[22,244],"among":[23],"distributed":[24],"heterogeneous":[26],"resources.":[27],"Deep":[28],"reinforcement":[29,69],"learning":[30,70],"(DRL)":[31],"has":[32],"recently":[33],"become":[34],"a":[35,102,208,338],"promising":[36],"idea":[37],"for":[38,308],"task":[39,77,85,249,329],"scheduling":[40,78,86,129,176,187,209,250,264,280,312,330],"in":[41,80,88,214,262],"CMfg.":[42,81],"However,":[43,246],"existing":[44,256],"DRL-based":[45,279],"methods":[46],"depend":[47],"heavily":[48],"on":[49,270],"problem-specific":[50,136,271],"reward":[51,124,137,272],"engineering":[52,273],"struggle":[54],"to":[55,73,92,120,145,157,173,184,225,242,288,321,341,352,367,370,374],"represent":[56,146,290],"hybrid":[57,76,84,147,234,248,291],"decision":[58,106,148,292],"variables.":[59,293],"To":[60],"this":[61,63,247],"end,":[62],"paper":[64],"proposed":[65,119,190,304,333],"sparse-reward":[67],"deep":[68],"(SDRL)":[71],"method":[72,171,305,373],"solve":[74],"problem":[79,100,251,296],"First,":[82],"model":[87],"CMfg":[89,232,300],"is":[90,118,143,155,252],"constructed":[91],"minimize":[93],"makespan.":[95],"We":[96,362],"reformulate":[97],"studied":[99,295],"as":[101,337],"partially":[103],"observable":[104],"Markov":[105],"process":[107],"(POMDP).":[108],"Then,":[109],"objective":[111,334],"hindsight":[112],"experience":[113],"replay":[114],"(objective":[115],"HER)":[116],"mechanism":[117],"alleviate":[121,158],"sparse":[123,346],"issue,":[125],"through":[126],"which":[127,348],"policy":[130],"can":[131,211,318,349],"effectively":[133,289],"trained":[134],"without":[135],"engineering.":[138],"The":[139,294,311,332],"continuous":[140],"action":[141],"space":[142],"defined":[144],"variables,":[149],"implicit":[152],"action-selection":[153],"mapping":[154],"utilized":[156],"boundary":[160],"effect.":[161],"Numerical":[162],"experiments":[163],"validated":[164],"effectiveness":[166],"superiority":[168],"of":[169,230],"our":[170,365],"compared":[172],"eleven":[174],"popular":[175],"algorithms":[177,180,281],"including":[178,236,324],"evolutionary":[179],"DRL.":[182],"Compared":[183],"mainstream":[185],"DRL":[186,258],"methods,":[188],"SDRL":[191],"outperforms":[192],"second-best":[194],"one":[195],"at":[196,358],"most":[197,278],"by":[198,255],"<inline-formula":[199,215],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[200,216,223],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">":[201,217],"<tex-math":[202,218],"notation=\"LaTeX\">$23.6\\%$</tex-math>":[203],"</inline-formula>":[204,220],"regarding":[205],"generalization,":[206],"solution":[210],"generated":[213],"notation=\"LaTeX\">$0.5$</tex-math>":[219],"seconds.":[221],"<italic":[222],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Note":[224],"Practitioners</i>":[226],"\u2014With":[227],"intelligentization":[229],"platform,":[233,301],"tasks,":[235,240],"need":[241],"simultaneously.":[245],"rarely":[253],"considered":[254],"works.":[257],"exhibits":[259],"many":[260],"benefits":[261],"addressing":[263,342],"problems,":[265,323],"but":[266,302],"strong":[268],"dependency":[269],"limits":[274],"its":[275],"application.":[276,310],"Additionally,":[277],"are":[282],"discrete-action":[283],"DRL,":[284],"restricting":[285],"their":[286],"capacity":[287],"originates":[297],"from":[298],"holds":[306],"potential":[307],"broader":[309],"framework":[313],"POMDP":[316],"modeling":[317],"applied":[320],"similar":[322],"hybrid,":[325],"manufacturing,":[326],"or":[327],"problems.":[331],"HER":[335],"serves":[336],"general":[339],"approach":[340],"challenges":[343],"associated":[344],"with":[345],"rewards,":[347],"extended":[351],"diverse":[353],"combinatorial":[354],"optimization":[355],"problems":[356],"aimed":[357],"optimizing":[359],"objective.":[361],"will":[363],"open-source":[364],"codes":[366],"help":[368],"others":[369],"apply":[371],"other":[375],"fields.":[376]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-13T07:31:44.756512","created_date":"2025-10-10T00:00:00"}