{"id":"https://openalex.org/W3118732669","doi":"https://doi.org/10.1109/tse.2021.3114353","title":"RNN-Test: Towards Adversarial Testing for Recurrent Neural Network Systems","display_name":"RNN-Test: Towards Adversarial Testing for Recurrent Neural Network Systems","publication_year":2021,"publication_date":"2021-09-21","ids":{"openalex":"https://openalex.org/W3118732669","doi":"https://doi.org/10.1109/tse.2021.3114353","mag":"3118732669"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2021.3114353","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2021.3114353","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1911.06155","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100705670","display_name":"Jianmin Guo","orcid":"https://orcid.org/0000-0001-6872-6383"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianmin Guo","raw_affiliation_strings":["Key Laboratory for Information System Security, Ministry of Education, School of Software, and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6872-6383","affiliations":[{"raw_affiliation_string":"Key Laboratory for Information System Security, Ministry of Education, School of Software, and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100406666","display_name":"Quan Zhang","orcid":"https://orcid.org/0000-0001-7778-4243"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Quan Zhang","raw_affiliation_strings":["Key Laboratory for Information System Security, Ministry of Education, School of Software, and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Key Laboratory for Information System Security, Ministry of Education, School of Software, and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101995035","display_name":"Yue Zhao","orcid":"https://orcid.org/0000-0002-1705-0536"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yue Zhao","raw_affiliation_strings":["Huawei Technologies Company, Ltd., Shenzhen, Guangdong, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei Technologies Company, Ltd., Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Heyuan Shi","orcid":"https://orcid.org/0000-0002-9040-7247"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Heyuan Shi","raw_affiliation_strings":["School of Software, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9040-7247","affiliations":[{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060117799","display_name":"Yu Jiang","orcid":"https://orcid.org/0000-0003-0955-503X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Jiang","raw_affiliation_strings":["Key Laboratory for Information System Security, Ministry of Education, School of Software, and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0955-503X","affiliations":[{"raw_affiliation_string":"Key Laboratory for Information System Security, Ministry of Education, School of Software, and the Beijing National Research Center for Information Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":null,"display_name":"Jiaguang Sun","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaguang Sun","raw_affiliation_strings":["School of Software, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Software, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.26,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.6230317,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"48","issue":"10","first_page":"4167","last_page":"4180"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9751999974250793,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9621999859809875,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recurrent-neural-network","display_name":"Recurrent neural network","score":0.9082790613174438},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7988630533218384},{"id":"https://openalex.org/keywords/mnist-database","display_name":"MNIST database","score":0.7344423532485962},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.7053093314170837},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6061043739318848},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5634483098983765},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4845206141471863},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.4160999655723572},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.3389548063278198},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.30671197175979614},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.21791884303092957}],"concepts":[{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.9082790613174438},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7988630533218384},{"id":"https://openalex.org/C190502265","wikidata":"https://www.wikidata.org/wiki/Q17069496","display_name":"MNIST database","level":3,"score":0.7344423532485962},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.7053093314170837},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6061043739318848},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5634483098983765},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4845206141471863},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4160999655723572},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3389548063278198},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.30671197175979614},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.21791884303092957}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tse.2021.3114353","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2021.3114353","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1911.06155","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1911.06155","pdf_url":"https://arxiv.org/pdf/1911.06155","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3118732669","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1911.06155v2","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1911.06155","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1911.06155","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1911.06155","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1911.06155","pdf_url":"https://arxiv.org/pdf/1911.06155","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3118732669.pdf","grobid_xml":"https://content.openalex.org/works/W3118732669.grobid-xml"},"referenced_works_count":80,"referenced_works":["https://openalex.org/W1538131130","https://openalex.org/W1591801644","https://openalex.org/W1686810756","https://openalex.org/W1821462560","https://openalex.org/W1922655562","https://openalex.org/W1972085588","https://openalex.org/W2060300932","https://openalex.org/W2064675550","https://openalex.org/W2117130368","https://openalex.org/W2143612262","https://openalex.org/W2157331557","https://openalex.org/W2163605009","https://openalex.org/W2171928131","https://openalex.org/W2180612164","https://openalex.org/W2183341477","https://openalex.org/W2184045248","https://openalex.org/W2187089797","https://openalex.org/W2193413348","https://openalex.org/W2194775991","https://openalex.org/W2229162816","https://openalex.org/W2243397390","https://openalex.org/W2342840547","https://openalex.org/W2460937040","https://openalex.org/W2503523779","https://openalex.org/W2533800772","https://openalex.org/W2609368435","https://openalex.org/W2616028256","https://openalex.org/W2618530766","https://openalex.org/W2735135478","https://openalex.org/W2765384636","https://openalex.org/W2767951891","https://openalex.org/W2774644650","https://openalex.org/W2793633339","https://openalex.org/W2799420921","https://openalex.org/W2799640043","https://openalex.org/W2804337238","https://openalex.org/W2809182766","https://openalex.org/W2888824816","https://openalex.org/W2905526464","https://openalex.org/W2905569079","https://openalex.org/W2949346385","https://openalex.org/W2954678575","https://openalex.org/W2962761044","https://openalex.org/W2963083752","https://openalex.org/W2963207607","https://openalex.org/W2963327228","https://openalex.org/W2963823140","https://openalex.org/W2963834268","https://openalex.org/W2963857521","https://openalex.org/W2963913218","https://openalex.org/W2963969878","https://openalex.org/W2964082701","https://openalex.org/W2964153729","https://openalex.org/W2964301649","https://openalex.org/W2964308564","https://openalex.org/W2968940383","https://openalex.org/W2988723074","https://openalex.org/W2994987245","https://openalex.org/W3013948624","https://openalex.org/W3035422387","https://openalex.org/W3099444373","https://openalex.org/W3101118235","https://openalex.org/W3103557498","https://openalex.org/W3103934057","https://openalex.org/W3104423855","https://openalex.org/W3105249702","https://openalex.org/W3127569499","https://openalex.org/W3172091474","https://openalex.org/W6632100814","https://openalex.org/W6635446068","https://openalex.org/W6637162671","https://openalex.org/W6637373629","https://openalex.org/W6640090968","https://openalex.org/W6640425456","https://openalex.org/W6686210252","https://openalex.org/W6704559304","https://openalex.org/W6741419198","https://openalex.org/W6745795967","https://openalex.org/W6763887731","https://openalex.org/W6770449242"],"related_works":["https://openalex.org/W3200167504","https://openalex.org/W2229372569","https://openalex.org/W2768320620","https://openalex.org/W2749590927","https://openalex.org/W2951539240","https://openalex.org/W3130033958","https://openalex.org/W3045726014","https://openalex.org/W3214321642","https://openalex.org/W2894378449","https://openalex.org/W2793805095","https://openalex.org/W2968262827","https://openalex.org/W2761286645","https://openalex.org/W2747545581","https://openalex.org/W3004084553","https://openalex.org/W1445143197","https://openalex.org/W2763213899","https://openalex.org/W3004223849","https://openalex.org/W3128207656","https://openalex.org/W3177296467","https://openalex.org/W3166767227"],"abstract_inverted_index":{"While":[0],"massive":[1],"efforts":[2],"have":[3],"been":[4],"investigated":[5],"in":[6,151],"adversarial":[7,37,80,117,153],"testing":[8,14,38],"of":[9,49,71,94,122,125,136],"convolutional":[10],"neural":[11,17],"networks":[12,18],"(CNN),":[13],"for":[15,26,41,64,119],"recurrent":[16],"(RNN)":[19],"is":[20,146],"still":[21],"limited":[22],"and":[23,112,115,157],"leaves":[24],"threats":[25],"vast":[27],"sequential":[28],"application":[29],"domains.":[30,55],"In":[31],"this":[32],"paper,":[33],"we":[34,57,83,129],"propose":[35],"an":[36],"framework":[39],"RNN-Test":[40,102,131,144,176],"RNN":[42,65,72,134],"systems,":[43],"focusing":[44],"on":[45,132,188,207],"sequence-to-sequence":[46],"(seq2seq)":[47],"tasks":[48,121],"widespread":[50],"deployments,":[51],"not":[52],"only":[53],"classification":[54],"First,":[56],"design":[58],"a":[59],"novel":[60],"search":[61],"methodology":[62],"customized":[63],"models":[66,135],"by":[67],"maximizing":[68],"the":[69,91,104,140,143,162,212,216],"inconsistency":[70,111],"states":[73],"against":[74],"their":[75],"inner":[76],"dependencies":[77],"to":[78,90,96,108,148,160,169,181,196,226],"produce":[79],"inputs.":[81,126],"Next,":[82],"introduce":[84],"two":[85],"state-based":[86],"coverage":[87,219],"metrics":[88,220],"according":[89],"distinctive":[92],"structure":[93],"RNNs":[95],"exercise":[97],"more":[98,165,198],"system":[99],"behaviors.":[100],"Finally,":[101],"solves":[103],"joint":[105],"optimization":[106],"problem":[107],"maximize":[109],"state":[110,113,218],"coverage,":[114,215],"crafts":[116],"inputs":[118],"various":[120],"different":[123],"kinds":[124],"For":[127],"evaluations,":[128],"apply":[130],"four":[133],"common":[137],"structures.":[138],"On":[139],"tested":[141],"models,":[142],"approach":[145],"demonstrated":[147],"be":[149],"competitive":[150],"generating":[152],"inputs,":[154],"outperforming":[155],"FGSM-based":[156],"DLFuzz-based":[158],"methods":[159],"reduce":[161],"model":[163],"performance":[164],"sharply":[166],"with":[167,200,211,224],"2.78%":[168],"37.94%":[170],"higher":[171,183,202,228],"success":[172,229],"(or":[173,230],"generation)":[174,231],"rate.":[175,232],"could":[177],"also":[178],"achieve":[179],"52.65%":[180],"66.45%":[182],"adversary":[184],"rate":[185,204],"than":[186,205],"testRNN":[187],"MNIST":[189],"LSTM":[190],"model,":[191],"as":[192,194,221],"well":[193],"53.76%":[195],"58.02%":[197],"perplexity":[199],"16%":[201],"generation":[203],"DeepStellar":[206],"PTB":[208],"language":[209],"model.Compared":[210],"traditional":[213],"neuron":[214],"proposed":[217],"guidance":[222],"excel":[223],"4.17%":[225],"97.22%":[227]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}