{"id":"https://openalex.org/W4379233584","doi":"https://doi.org/10.1109/access.2023.3277953","title":"PHH: Policy-Based Hyper-Heuristic With Reinforcement Learning","display_name":"PHH: Policy-Based Hyper-Heuristic With Reinforcement Learning","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4379233584","doi":"https://doi.org/10.1109/access.2023.3277953"},"language":"en","primary_location":{"id":"doi:10.1109/access.2023.3277953","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3277953","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10129938.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10129938.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028400275","display_name":"Orachun Udomkasemsub","orcid":"https://orcid.org/0000-0001-6452-9438"},"institutions":[{"id":"https://openalex.org/I60837268","display_name":"King Mongkut's University of Technology Thonburi","ror":"https://ror.org/0057ax056","country_code":"TH","type":"education","lineage":["https://openalex.org/I60837268"]}],"countries":["TH"],"is_corresponding":false,"raw_author_name":"Orachun Udomkasemsub","raw_affiliation_strings":["Department of Computer Engineering, King Mongkut’s University of Technology Thonburi, Bangkok, Thailand"],"raw_orcid":"https://orcid.org/0000-0001-6452-9438","affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, King Mongkut’s University of Technology Thonburi, Bangkok, Thailand","institution_ids":["https://openalex.org/I60837268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083855056","display_name":"Booncharoen Sirinaovakul","orcid":"https://orcid.org/0000-0001-8541-1922"},"institutions":[{"id":"https://openalex.org/I60837268","display_name":"King Mongkut's University of Technology Thonburi","ror":"https://ror.org/0057ax056","country_code":"TH","type":"education","lineage":["https://openalex.org/I60837268"]}],"countries":["TH"],"is_corresponding":false,"raw_author_name":"Booncharoen Sirinaovakul","raw_affiliation_strings":["Department of Computer Engineering, King Mongkut’s University of Technology Thonburi, Bangkok, Thailand"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, King Mongkut’s University of Technology Thonburi, Bangkok, Thailand","institution_ids":["https://openalex.org/I60837268"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014448123","display_name":"Tiranee Achalakul","orcid":null},"institutions":[{"id":"https://openalex.org/I2801080794","display_name":"Government of Thailand","ror":"https://ror.org/05fp08r12","country_code":"TH","type":"government","lineage":["https://openalex.org/I2801080794"]},{"id":"https://openalex.org/I60837268","display_name":"King Mongkut's University of Technology Thonburi","ror":"https://ror.org/0057ax056","country_code":"TH","type":"education","lineage":["https://openalex.org/I60837268"]}],"countries":["TH"],"is_corresponding":false,"raw_author_name":"Tiranee Achalakul","raw_affiliation_strings":["Department of Computer Engineering, King Mongkut’s University of Technology Thonburi, Bangkok, Thailand","Government Big Data Institute, Bangkok, Thailand"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, King Mongkut’s University of Technology Thonburi, Bangkok, Thailand","institution_ids":["https://openalex.org/I60837268"]},{"raw_affiliation_string":"Government Big Data Institute, Bangkok, Thailand","institution_ids":["https://openalex.org/I2801080794"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":1.7646,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.86130708,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"11","issue":null,"first_page":"52026","last_page":"52049"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10551","display_name":"Scheduling and Optimization Algorithms","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10551","display_name":"Scheduling and Optimization Algorithms","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10567","display_name":"Vehicle Routing Optimization Methods","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10100","display_name":"Metaheuristic Optimization Algorithms Research","score":0.9902999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7771495580673218},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6808458566665649},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.6380698680877686},{"id":"https://openalex.org/keywords/generality","display_name":"Generality","score":0.5240266919136047},{"id":"https://openalex.org/keywords/hyper-heuristic","display_name":"Hyper-heuristic","score":0.5018973350524902},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.4863350987434387},{"id":"https://openalex.org/keywords/job-shop-scheduling","display_name":"Job shop scheduling","score":0.4774366319179535},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.47403669357299805},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.44958072900772095},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.437216579914093},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34851717948913574},{"id":"https://openalex.org/keywords/routing","display_name":"Routing (electronic design automation)","score":0.2090664505958557},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1250356137752533}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7771495580673218},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6808458566665649},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.6380698680877686},{"id":"https://openalex.org/C2780767217","wikidata":"https://www.wikidata.org/wiki/Q5532421","display_name":"Generality","level":2,"score":0.5240266919136047},{"id":"https://openalex.org/C117270229","wikidata":"https://www.wikidata.org/wiki/Q5957539","display_name":"Hyper-heuristic","level":5,"score":0.5018973350524902},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4863350987434387},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.4774366319179535},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.47403669357299805},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.44958072900772095},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.437216579914093},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34851717948913574},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.2090664505958557},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1250356137752533},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.0},{"id":"https://openalex.org/C19966478","wikidata":"https://www.wikidata.org/wiki/Q4810574","display_name":"Mobile robot","level":3,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C188888258","wikidata":"https://www.wikidata.org/wiki/Q7353390","display_name":"Robot learning","level":4,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2023.3277953","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3277953","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10129938.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:63d7d98c07a94f949f77c4cb039b8bbc","is_oa":true,"landing_page_url":"https://doaj.org/article/63d7d98c07a94f949f77c4cb039b8bbc","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 11, Pp 52026-52049 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2023.3277953","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2023.3277953","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/10005208/10129938.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4737341966","display_name":null,"funder_award_id":"PHD/0065/2554","funder_id":"https://openalex.org/F4320327548","funder_display_name":"Royal Golden Jubilee (RGJ) Ph.D. Programme"}],"funders":[{"id":"https://openalex.org/F4320322614","display_name":"Thailand Research Fund","ror":"https://ror.org/03gd1bh37"},{"id":"https://openalex.org/F4320327548","display_name":"Royal Golden Jubilee (RGJ) Ph.D. Programme","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4379233584.pdf","grobid_xml":"https://content.openalex.org/works/W4379233584.grobid-xml"},"referenced_works_count":57,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1481976097","https://openalex.org/W1536647479","https://openalex.org/W1829921621","https://openalex.org/W1936679285","https://openalex.org/W1970275772","https://openalex.org/W1974758710","https://openalex.org/W2020320008","https://openalex.org/W2028351695","https://openalex.org/W2032639018","https://openalex.org/W2072441740","https://openalex.org/W2093284687","https://openalex.org/W2114224867","https://openalex.org/W2119717200","https://openalex.org/W2140759234","https://openalex.org/W2143560894","https://openalex.org/W2145339207","https://openalex.org/W2149294210","https://openalex.org/W2151975141","https://openalex.org/W2163428398","https://openalex.org/W2170749044","https://openalex.org/W2186998007","https://openalex.org/W2544812900","https://openalex.org/W2546571074","https://openalex.org/W2736601468","https://openalex.org/W2753452838","https://openalex.org/W2766552321","https://openalex.org/W2796528486","https://openalex.org/W2858821137","https://openalex.org/W2885965377","https://openalex.org/W2891971415","https://openalex.org/W2937450972","https://openalex.org/W2953722837","https://openalex.org/W2956161617","https://openalex.org/W2967726092","https://openalex.org/W2967783393","https://openalex.org/W2971617596","https://openalex.org/W2975266424","https://openalex.org/W3003257820","https://openalex.org/W3099098707","https://openalex.org/W3128766876","https://openalex.org/W3135613628","https://openalex.org/W3136908367","https://openalex.org/W3162719517","https://openalex.org/W3170112077","https://openalex.org/W3179983824","https://openalex.org/W3203365843","https://openalex.org/W3216772467","https://openalex.org/W4206941916","https://openalex.org/W4214717370","https://openalex.org/W4246565613","https://openalex.org/W4251343851","https://openalex.org/W4297964528","https://openalex.org/W4380768317","https://openalex.org/W6685444567","https://openalex.org/W6741002519","https://openalex.org/W6804601995"],"related_works":["https://openalex.org/W1628567638","https://openalex.org/W2109775141","https://openalex.org/W3186374002","https://openalex.org/W2526745971","https://openalex.org/W2521454154","https://openalex.org/W2107678394","https://openalex.org/W2521049309","https://openalex.org/W2020754965","https://openalex.org/W3083497736","https://openalex.org/W4297052805"],"abstract_inverted_index":{"Hyper-heuristics":[0],"have":[1],"a":[2,14,46,128,136,140],"high":[3],"level":[4],"of":[5,17,57,195,244],"generality":[6,241],"and":[7,28,39,91,106,153,158,164,176,202,242],"adaptability,":[8],"allowing":[9,35],"them":[10],"to":[11,31,37,53,64,72,127,161,197,204,237],"effectively":[12],"solve":[13,73,162],"wide":[15],"range":[16],"complex":[18],"optimization":[19,75,130,208,248],"problems.":[20,76,94,249],"With":[21],"reinforcement":[22,51],"learning,":[23],"hyper-heuristics":[24,36],"can":[25,102],"use":[26],"experience":[27],"knowledge":[29],"gained":[30],"tackle":[32],"unforeseen":[33,168,223],"problems,":[34,201],"adapt":[38],"improve":[40,54,238],"over":[41],"time.":[42],"Our":[43],"paper":[44],"proposes":[45],"framework":[47,60,78,101,120,190,219],"for":[48,109,132,199,206,222],"using":[49,82],"policy-based":[50],"learning":[52],"the":[55,66,99,182,218,235,240],"performance":[56,243],"hyper-heuristics.":[58],"The":[59,77,95,118,184,230],"trains":[61],"hyper-heuristic":[62,159],"agents":[63,144],"select":[65],"best":[67],"generalized":[68],"constructive":[69],"low-level":[70],"heuristics":[71],"combinatorial":[74,247],"evaluation":[79],"was":[80,121],"performed":[81,220],"three":[83,173,177],"benchmarking":[84,200],"problems:":[85],"traveling":[86],"salesman,":[87],"capacitated":[88],"vehicle":[89],"routing,":[90],"bin":[92],"packing":[93],"results":[96,185],"showed":[97,186],"that":[98,187],"proposed":[100,119,189,231],"outperform":[103],"existing":[104],"meta-heuristic":[105,157],"hyper-heuristic-based":[107],"algorithms":[108],"all":[110,115],"large":[111,246],"problem":[112,116,131,216,225],"instances":[113,226],"in":[114,181,209],"domains.":[117],"also":[122],"evaluated":[123],"by":[124],"applying":[125],"it":[126],"cost":[129,207],"workflow":[133,171,174,210],"scheduling":[134],"on":[135,147],"hybrid":[137],"cloud":[138],"with":[139,150,167,214],"deadline":[141],"constraint.":[142],"Eight":[143],"were":[145,179],"trained":[146,213],"medium-sized":[148],"workflows":[149,166],"two":[151],"deadlines":[152,178],"tested":[154],"against":[155],"traditional":[156],"methods":[160],"smaller":[163],"larger":[165,224],"deadlines.":[169],"Four":[170],"applications,":[172],"sizes,":[175],"used":[180],"evaluation.":[183],"our":[188],"provided":[191],"significantly":[192],"better":[193],"solutions":[194],"up":[196,203],"98%":[198],"22%":[205],"scheduling.":[211],"Moreover,":[212],"small":[215],"instances,":[217],"well":[221],"implying":[227],"its":[228],"generalization.":[229],"framework,":[232],"thus,":[233],"has":[234],"potential":[236],"both":[239],"solving":[245]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}