{"id":"https://openalex.org/W3002440596","doi":"https://doi.org/10.1109/tse.2020.2969178","title":"Falsification of Cyber-Physical Systems Using Deep Reinforcement Learning","display_name":"Falsification of Cyber-Physical Systems Using Deep Reinforcement Learning","publication_year":2020,"publication_date":"2020-01-23","ids":{"openalex":"https://openalex.org/W3002440596","doi":"https://doi.org/10.1109/tse.2020.2969178","mag":"3002440596"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2020.2969178","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2020.2969178","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028729272","display_name":"Yoriyuki Yamagata","orcid":"https://orcid.org/0000-0003-2096-677X"},"institutions":[{"id":"https://openalex.org/I73613424","display_name":"National Institute of Advanced Industrial Science and Technology","ror":"https://ror.org/01703db54","country_code":"JP","type":"government","lineage":["https://openalex.org/I73613424"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Yoriyuki Yamagata","raw_affiliation_strings":["National Institute of Advanced Industrial Science and Technology (AIST), Ikeda, Osaka, Japan"],"affiliations":[{"raw_affiliation_string":"National Institute of Advanced Industrial Science and Technology (AIST), Ikeda, Osaka, Japan","institution_ids":["https://openalex.org/I73613424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100319073","display_name":"Shuang Liu","orcid":"https://orcid.org/0000-0001-8766-7235"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuang Liu","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Nankai, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Nankai, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081191192","display_name":"Takumi Akazaki","orcid":"https://orcid.org/0000-0002-0782-1915"},"institutions":[{"id":"https://openalex.org/I2252096349","display_name":"Fujitsu (Japan)","ror":"https://ror.org/038e2g226","country_code":"JP","type":"company","lineage":["https://openalex.org/I2252096349"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Takumi Akazaki","raw_affiliation_strings":["Fujitsu Laboratories, Kawasaki, Kanagawa, Japan"],"affiliations":[{"raw_affiliation_string":"Fujitsu Laboratories, Kawasaki, Kanagawa, Japan","institution_ids":["https://openalex.org/I2252096349"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019579988","display_name":"Yihai Duan","orcid":null},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yihai Duan","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Nankai, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Nankai, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047509839","display_name":"Jianye Hao","orcid":"https://orcid.org/0000-0002-0422-8235"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]},{"id":"https://openalex.org/I4210153212","display_name":"Noah's Path","ror":"https://ror.org/05wv5ps25","country_code":"ES","type":"healthcare","lineage":["https://openalex.org/I4210153212"]},{"id":"https://openalex.org/I4210159102","display_name":"Huawei Technologies (Sweden)","ror":"https://ror.org/0500fyd17","country_code":"SE","type":"company","lineage":["https://openalex.org/I2250955327","https://openalex.org/I4210159102"]}],"countries":["CN","ES","SE"],"is_corresponding":false,"raw_author_name":"Jianye Hao","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Nankai, Tianjin, China","Noah&#x0027;s Ark Lab, Huawei","Noah&#x0027","s Ark Lab, Huawei"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Nankai, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]},{"raw_affiliation_string":"Noah&#x0027;s Ark Lab, Huawei","institution_ids":["https://openalex.org/I4210159102"]},{"raw_affiliation_string":"Noah&#x0027","institution_ids":["https://openalex.org/I4210153212"]},{"raw_affiliation_string":"s Ark Lab, Huawei","institution_ids":["https://openalex.org/I4210159102"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5028729272"],"corresponding_institution_ids":["https://openalex.org/I73613424"],"apc_list":null,"apc_paid":null,"fwci":5.058,"has_fulltext":false,"cited_by_count":53,"citation_normalized_percentile":{"value":0.9609414,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"47","issue":"12","first_page":"2823","last_page":"2840"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13295","display_name":"Safety Systems Engineering in Autonomy","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/2213","display_name":"Safety, Risk, Reliability and Quality"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7977390289306641},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7557600736618042},{"id":"https://openalex.org/keywords/cyber-physical-system","display_name":"Cyber-physical system","score":0.6214942336082458},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5959312319755554},{"id":"https://openalex.org/keywords/counterexample","display_name":"Counterexample","score":0.5683535933494568},{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.5147585868835449},{"id":"https://openalex.org/keywords/physical-system","display_name":"Physical system","score":0.49610480666160583},{"id":"https://openalex.org/keywords/model-checking","display_name":"Model checking","score":0.4518156349658966},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4027641713619232},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.3302809000015259},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3293980360031128},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.2991567850112915},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12829264998435974}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7977390289306641},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7557600736618042},{"id":"https://openalex.org/C179768478","wikidata":"https://www.wikidata.org/wiki/Q1120057","display_name":"Cyber-physical system","level":2,"score":0.6214942336082458},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5959312319755554},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.5683535933494568},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.5147585868835449},{"id":"https://openalex.org/C116672817","wikidata":"https://www.wikidata.org/wiki/Q1454986","display_name":"Physical system","level":2,"score":0.49610480666160583},{"id":"https://openalex.org/C110251889","wikidata":"https://www.wikidata.org/wiki/Q1569697","display_name":"Model checking","level":2,"score":0.4518156349658966},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4027641713619232},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3302809000015259},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3293980360031128},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2991567850112915},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12829264998435974},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2020.2969178","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2020.2969178","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.800000011920929,"display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G8462351438","display_name":null,"funder_award_id":"61702362","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8895623615","display_name":null,"funder_award_id":"61802275","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8921145797","display_name":null,"funder_award_id":"U1836214","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W175120007","https://openalex.org/W189973795","https://openalex.org/W618136976","https://openalex.org/W1506101803","https://openalex.org/W1508592466","https://openalex.org/W1519302343","https://openalex.org/W1533557569","https://openalex.org/W1547304883","https://openalex.org/W1548600288","https://openalex.org/W1783413852","https://openalex.org/W1884784351","https://openalex.org/W1902482618","https://openalex.org/W1966224165","https://openalex.org/W1992777946","https://openalex.org/W2026629052","https://openalex.org/W2039287452","https://openalex.org/W2067971887","https://openalex.org/W2068619069","https://openalex.org/W2112656403","https://openalex.org/W2121863487","https://openalex.org/W2131399618","https://openalex.org/W2145339207","https://openalex.org/W2147833661","https://openalex.org/W2153803621","https://openalex.org/W2172184261","https://openalex.org/W2189285614","https://openalex.org/W2236968622","https://openalex.org/W2257979135","https://openalex.org/W2295828805","https://openalex.org/W2522650173","https://openalex.org/W2524638160","https://openalex.org/W2554968732","https://openalex.org/W2567705466","https://openalex.org/W2571592482","https://openalex.org/W2734058336","https://openalex.org/W2759950027","https://openalex.org/W2798302610","https://openalex.org/W2883605651","https://openalex.org/W2888839644","https://openalex.org/W2912640545","https://openalex.org/W2950471160","https://openalex.org/W2963100367","https://openalex.org/W2964043796","https://openalex.org/W2994240296","https://openalex.org/W3105473318","https://openalex.org/W3125246424","https://openalex.org/W4211221179","https://openalex.org/W4244457856","https://openalex.org/W4250589301","https://openalex.org/W4298857966","https://openalex.org/W6637967152","https://openalex.org/W6689927692","https://openalex.org/W6692846177","https://openalex.org/W6696324988","https://openalex.org/W6732118020","https://openalex.org/W6740881549","https://openalex.org/W6771522568"],"related_works":["https://openalex.org/W2953113086","https://openalex.org/W1552191998","https://openalex.org/W2056825861","https://openalex.org/W2765989158","https://openalex.org/W2952737788","https://openalex.org/W4226455345","https://openalex.org/W2756692340","https://openalex.org/W2994120531","https://openalex.org/W2108326965","https://openalex.org/W2086962923"],"abstract_inverted_index":{"A":[0],"<i>Cyber-Physical":[1],"System</i>":[2],"(CPS)":[3],"is":[4,50,59,84,171,230,246],"a":[5,80,85,110,116,168,174],"system":[6,16,111,235],"which":[7,217],"consists":[8],"of":[9,39,48,74,119,152,167,184,215,233,249],"software":[10],"components":[11,35],"and":[12,72,113,142,196,211],"physical":[13,34],"components.":[14],"Traditional":[15],"verification":[17],"techniques":[18,97],"such":[19,158],"as":[20,173],"model":[21],"checking":[22],"or":[23],"theorem":[24],"proving":[25],"are":[26],"difficult":[27],"to":[28,30,65,88,98,122,148,156,238],"apply":[29],"CPS":[31,49,203,216],"because":[32],"the":[33,56,67,75,90,124,150,182,208,231,234,239,247,253],"have":[36],"infinite":[37],"number":[38,118,151],"states.":[40],"To":[41],"solve":[42],"this":[43,127],"problem,":[44],"robustness":[45,68,82,164],"guided":[46,62,165],"falsification":[47,63,166],"introduced.":[51],"Robustness":[52,61],"measures":[53],"how":[54,163],"robustly":[55],"given":[57],"specification":[58],"satisfied.":[60],"tries":[64],"minimize":[66,99,123],"by":[69],"changing":[70],"inputs":[71],"parameters":[73],"system.":[76],"The":[77,226,243],"input":[78,112],"with":[79,187],"minimal":[81],"(counterexample)":[83],"good":[86],"candidate":[87],"violate":[89],"specification.":[91],"Existing":[92],"methods":[93,103,186,221],"use":[94,106],"several":[95],"optimization":[96],"robustness.":[100,125],"However,":[101],"those":[102],"do":[104],"not":[105],"temporal":[107],"structures":[108],"in":[109,252],"often":[114],"require":[115],"large":[117],"simulation":[120,153],"runs":[121,154],"In":[126],"paper,":[128],"we":[129,179],"explore":[130],"state-of-the-art":[131],"<i>Deep":[132],"Reinforcement":[133],"Learning</i>":[134],"(DRL)":[135],"techniques,":[136],"i.e.,":[137,191],"<i>Asynchronous":[138],"Advantage":[139],"Actor-Critic</i>":[140],"(A3C)":[141],"<i>Double":[143],"Deep":[144],"Q":[145],"Network</i>":[146],"(DDQN),":[147],"reduce":[149],"required":[155],"find":[157],"counterexamples.":[159],"We":[160,205],"theoretically":[161],"show":[162],"safety":[169],"property":[170],"formatted":[172],"reinforcement":[175,240],"learning":[176,241],"problem.":[177],"Then,":[178],"experimentally":[180],"compare":[181],"effectiveness":[183],"our":[185],"three":[188,200],"baseline":[189],"methods,":[190],"random":[192],"sampling,":[193],"cross":[194],"entropy":[195],"simulated":[197],"annealing,":[198],"on":[199],"well":[201],"known":[202],"systems.":[204],"thoroughly":[206],"analyse":[207],"experiment":[209],"results":[210],"identify":[212],"two":[213],"factors":[214],"make":[218],"DRL":[219],"based":[220],"better":[222],"than":[223],"existing":[224],"methods.":[225],"most":[227],"important":[228],"factor":[229,245],"availability":[232],"internal":[236],"dynamics":[237],"algorithm.":[242],"other":[244],"existence":[248],"learnable":[250],"structure":[251],"counterexample.":[254]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":14},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":5}],"updated_date":"2026-04-18T07:56:08.524223","created_date":"2025-10-10T00:00:00"}