{"id":"https://openalex.org/W7118193374","doi":"https://doi.org/10.1109/tse.2026.3651022","title":"Deep Learning Framework Testing via Model Mutation: How Far Are We?","display_name":"Deep Learning Framework Testing via Model Mutation: How Far Are We?","publication_year":2026,"publication_date":"2026-01-05","ids":{"openalex":"https://openalex.org/W7118193374","doi":"https://doi.org/10.1109/tse.2026.3651022"},"language":null,"primary_location":{"id":"doi:10.1109/tse.2026.3651022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2026.3651022","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075895554","display_name":"Yanzhou Mu","orcid":"https://orcid.org/0000-0003-1816-2246"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanzhou Mu","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-1816-2246","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014248149","display_name":"R J Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rong Wang","raw_affiliation_strings":["School of Information Science and Technology, Nantong University, Nantong, China"],"raw_orcid":"https://orcid.org/0009-0004-0543-1894","affiliations":[{"raw_affiliation_string":"School of Information Science and Technology, Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121947575","display_name":"Juan Zhai","orcid":null},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan Zhai","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5017-8016","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075174750","display_name":"Chunrong Fang","orcid":"https://orcid.org/0000-0002-9930-7111"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunrong Fang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-9930-7111","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121949574","display_name":"Xiang Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang Chen","raw_affiliation_strings":["School of Artificial Intelligence and Computer Science, Nantong University, Nantong, China"],"raw_orcid":"https://orcid.org/0000-0002-1180-3891","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence and Computer Science, Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102657040","display_name":"Peiran Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peiran Yang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0008-8242-9543","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhixiang Cao","orcid":"https://orcid.org/0009-0002-6810-7608"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhixiang Cao","raw_affiliation_strings":["School of Electronic Science and Engineering, Xi’an Jiaotong University, Xi’an, China"],"raw_orcid":"https://orcid.org/0009-0002-6810-7608","affiliations":[{"raw_affiliation_string":"School of Electronic Science and Engineering, Xi’an Jiaotong University, Xi’an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121938088","display_name":"Ruixiang Qian","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruixiang Qian","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0003-5040-3123","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102816452","display_name":"Shaoyu Yang","orcid":"https://orcid.org/0009-0008-7651-8582"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaoyu Yang","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0008-7651-8582","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":null,"display_name":"Zhenyu Chen","orcid":"https://orcid.org/0000-0002-9592-7022"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenyu Chen","raw_affiliation_strings":["State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-9592-7022","affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.03975723,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"52","issue":"3","first_page":"835","last_page":"854"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5454999804496765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5454999804496765,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.21580000221729279,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.03480000048875809,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.8418999910354614},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.651199996471405},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.6409000158309937},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5616000294685364},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4490000009536743},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.446399986743927}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.8418999910354614},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8112999796867371},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.651199996471405},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.6409000158309937},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5813999772071838},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5616000294685364},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47940000891685486},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4490000009536743},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.446399986743927},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.42399999499320984},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3882000148296356},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3612000048160553},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33000001311302185},{"id":"https://openalex.org/C110251889","wikidata":"https://www.wikidata.org/wiki/Q1569697","display_name":"Model checking","level":2,"score":0.2994999885559082},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.29440000653266907},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2904999852180481},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.25870001316070557},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2556000053882599}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2026.3651022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2026.3651022","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3894403112","display_name":null,"funder_award_id":"14380029","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G8662163125","display_name":null,"funder_award_id":"62372228","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1975879668","https://openalex.org/W2119112357","https://openalex.org/W2135194391","https://openalex.org/W2145339207","https://openalex.org/W2194775991","https://openalex.org/W2525984666","https://openalex.org/W2919115771","https://openalex.org/W2947815220","https://openalex.org/W2954903132","https://openalex.org/W2961870034","https://openalex.org/W3104663419","https://openalex.org/W3160029089","https://openalex.org/W3177562315","https://openalex.org/W3178469298","https://openalex.org/W3199980055","https://openalex.org/W3212565000","https://openalex.org/W4249495939","https://openalex.org/W4251827675","https://openalex.org/W4252684946","https://openalex.org/W4284707748","https://openalex.org/W4284708930","https://openalex.org/W4308643012","https://openalex.org/W4319594569","https://openalex.org/W4360604658","https://openalex.org/W4385245566","https://openalex.org/W4387735187","https://openalex.org/W4389158362","https://openalex.org/W4400582513","https://openalex.org/W4404914810"],"related_works":[],"abstract_inverted_index":{"Deep":[0],"Learning":[1],"(DL)":[2],"frameworks":[3,20,30,78],"are":[4,187,365],"fundamental":[5],"components":[6],"of":[7,28,129,206,230,236,293,304,324,362,378],"DL":[8,19,29,60],"systems":[9],"in":[10,18,148,154,165,177,191],"their":[11,103,254,321],"development,":[12],"deployment,":[13],"and":[14,68,120,140,158,214,220,233,272,320,327,340,369],"execution,":[15],"while":[16,151],"defects":[17,153,160,184,243,345],"can":[21],"cause":[22],"severe":[23],"consequences.":[24],"Ensuring":[25],"the":[26,38,88,125,136,182,204,242,276,281,291,300,312,375],"quality":[27],"has":[31,44],"therefore":[32],"become":[33],"a":[34,47,178,200,228],"pressing":[35],"challenge.":[36],"Among":[37],"various":[39],"testing":[40,209],"techniques,":[41],"model":[42,98,149,170],"mutation":[43,57,113,117,268,270,273,309],"emerged":[45],"as":[46,116,169],"widely":[48],"adopted":[49],"approach.":[50],"Such":[51],"methods":[52,85,248,326],"generate":[53],"mutants":[54],"by":[55,194,245,367],"applying":[56],"operators":[58,95],"to":[59,105,223,252,307,338],"models":[61],"(e.g.,":[62],"structural":[63],"changes":[64],"or":[65,73,79,172],"parameter":[66],"edits)":[67],"then":[69,240],"analyzing":[70],"inconsistencies,":[71],"crashes,":[72],"abnormal":[74],"behaviors":[75],"across":[76],"different":[77],"hardware.":[80],"Despite":[81],"its":[82],"effectiveness,":[83],"existing":[84,207,305,325],"suffer":[86],"from":[87,218,289],"following":[89],"limitations.":[90],"First,":[91],"they":[92,110,133],"mainly":[93],"reuse":[94],"designed":[96],"for":[97],"testing,":[99],"raising":[100],"doubts":[101],"about":[102,164,190],"ability":[104,128,303],"expose":[106],"framework-level":[107],"defects.":[108,238],"Besides,":[109],"insufficiently":[111],"consider":[112],"constraints,":[114],"such":[115,168],"type,":[118,269],"position,":[119,271],"order,":[121,274],"which":[122,363],"directly":[123],"affect":[124,275,311],"defect":[126,216,301],"detection":[127,138,255,302],"generated":[130,277,313],"mutants.":[131,278,314],"Finally,":[132],"rely":[134],"on":[135,145,280,295],"limited":[137],"range":[139],"narrow":[141],"test":[142],"oracles,":[143],"focusing":[144],"functional":[146],"correctness":[147],"inference":[150],"overlooking":[152],"efficiency,":[155],"resource":[156],"usage,":[157],"other":[159,166],"that":[161,185],"developers":[162,186,294,368],"care":[163],"stages,":[167],"training":[171],"deployment.":[173],"These":[174],"limitations":[175,319],"result":[176],"weak":[179],"alignment":[180],"with":[181],"critical":[183],"most":[188],"concerned":[189],"practice.":[192],"Motivated":[193],"these":[195,259,336],"observations,":[196],"this":[197,250],"study":[198,356],"conducts":[199],"comprehensive":[201],"investigation":[202],"into":[203,249],"effectiveness":[205],"mutation-based":[208],"methods.":[210],"We":[211,239,333],"first":[212],"collect":[213],"classify":[215],"reports":[217],"PyTorch":[219],"MindSpore":[221],"according":[222],"developers\u2019":[224],"priority":[225,292],"tags,":[226],"building":[227],"taxonomy":[229,251],"seven":[231],"categories":[232],"19":[234],"sub-categories":[235],"HP":[237],"map":[241],"reported":[244],"five":[246],"state-of-the-art":[247],"evaluate":[253],"abilities.":[256],"To":[257],"explain":[258],"limitations,":[260],"we":[261,284,316],"further":[262,334],"analyze":[263],"how":[264,308],"three":[265],"key":[266],"factors,":[267],"Based":[279],"experiment":[282],"results,":[283],"summarize":[285],"ten":[286],"findings":[287],"ranging":[288],"revealing":[290],"fixing":[296],"framework":[297,360],"defects,":[298,361],"evaluating":[299],"methods,":[306],"factors":[310],"Furthermore,":[315],"reveal":[317],"four":[318,329,347],"root":[322],"causes":[323],"propose":[328],"targeted":[330],"optimization":[331],"strategies.":[332],"apply":[335],"strategies":[337],"COMET":[339],"successfully":[341],"uncover":[342],"six":[343],"new":[344],"spanning":[346],"types,":[348],"including":[349],"two":[350],"previously":[351],"unreported":[352],"categories.":[353],"Overall,":[354],"our":[355,379],"identifies":[357],"38":[358],"unique":[359],"30":[364],"confirmed":[366],"12":[370],"have":[371],"been":[372],"fixed,":[373],"demonstrating":[374],"practical":[376],"value":[377],"findings.":[380]},"counts_by_year":[],"updated_date":"2026-03-20T20:47:17.329874","created_date":"2026-01-05T00:00:00"}