{"id":"https://openalex.org/W4411522836","doi":"https://doi.org/10.1145/3728972","title":"Improving Deep Learning Framework Testing with Model-Level Metamorphic Testing","display_name":"Improving Deep Learning Framework Testing with Model-Level Metamorphic Testing","publication_year":2025,"publication_date":"2025-06-22","ids":{"openalex":"https://openalex.org/W4411522836","doi":"https://doi.org/10.1145/3728972"},"language":"en","primary_location":{"id":"doi:10.1145/3728972","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728972","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3728972","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075895554","display_name":"Yanzhou Mu","orcid":"https://orcid.org/0000-0003-1816-2246"},"institutions":[{"id":"https://openalex.org/I4210105229","display_name":"City University of Hong Kong, Shenzhen Research Institute","ror":"https://ror.org/00xc0ma20","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210105229"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yanzhou Mu","raw_affiliation_strings":["Nanjing University, Nanjing, China","Shenzhen Research Institute of Nanjing University, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"Shenzhen Research Institute of Nanjing University, Shenzhen, China","institution_ids":["https://openalex.org/I4210105229","https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071575216","display_name":"Juan Zhai","orcid":"https://orcid.org/0000-0001-5017-8016"},"institutions":[{"id":"https://openalex.org/I177605424","display_name":"Amherst College","ror":"https://ror.org/028vqfs63","country_code":"US","type":"education","lineage":["https://openalex.org/I177605424"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan Zhai","raw_affiliation_strings":["University of Massachusetts at Amherst, Amherst, USA"],"affiliations":[{"raw_affiliation_string":"University of Massachusetts at Amherst, Amherst, USA","institution_ids":["https://openalex.org/I177605424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075174750","display_name":"Chunrong Fang","orcid":"https://orcid.org/0000-0002-9930-7111"},"institutions":[{"id":"https://openalex.org/I4210105229","display_name":"City University of Hong Kong, Shenzhen Research Institute","ror":"https://ror.org/00xc0ma20","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210105229"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunrong Fang","raw_affiliation_strings":["Nanjing University, Nanjing, China","Shenzhen Research Institute of Nanjing University, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"Shenzhen Research Institute of Nanjing University, Shenzhen, China","institution_ids":["https://openalex.org/I4210105229","https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100441911","display_name":"Xiang Chen","orcid":"https://orcid.org/0000-0002-1180-3891"},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang Chen","raw_affiliation_strings":["Nantong University, Nantong, China"],"affiliations":[{"raw_affiliation_string":"Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081742959","display_name":"Zhixiang Cao","orcid":"https://orcid.org/0009-0002-6810-7608"},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhixiang Cao","raw_affiliation_strings":["Nantong University, Nantong, China"],"affiliations":[{"raw_affiliation_string":"Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102657040","display_name":"Peiran Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peiran Yang","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kexin Zhao","orcid":"https://orcid.org/0009-0008-7793-2507"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kexin Zhao","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107127726","display_name":"An Guo","orcid":"https://orcid.org/0009-0005-8661-6133"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"An Guo","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100422935","display_name":"Zhenyu Chen","orcid":"https://orcid.org/0000-0002-9592-7022"},"institutions":[{"id":"https://openalex.org/I4210105229","display_name":"City University of Hong Kong, Shenzhen Research Institute","ror":"https://ror.org/00xc0ma20","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210105229"]},{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenyu Chen","raw_affiliation_strings":["Nanjing University, Nanjing, China","Shenzhen Research Institute of Nanjing University, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"Shenzhen Research Institute of Nanjing University, Shenzhen, China","institution_ids":["https://openalex.org/I4210105229","https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5075895554"],"corresponding_institution_ids":["https://openalex.org/I4210105229","https://openalex.org/I881766915"],"apc_list":null,"apc_paid":null,"fwci":2.2387,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.87255302,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"2","issue":"ISSTA","first_page":"2158","last_page":"2180"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.84123694896698},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6681894063949585},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.6407245397567749},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.5757706165313721},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5425726175308228},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4783720076084137},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.45629146695137024},{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.4288705289363861},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.13921138644218445}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.84123694896698},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6681894063949585},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.6407245397567749},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5757706165313721},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5425726175308228},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4783720076084137},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.45629146695137024},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.4288705289363861},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.13921138644218445},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C129307140","wikidata":"https://www.wikidata.org/wiki/Q6795880","display_name":"Maximum bubble pressure method","level":3,"score":0.0},{"id":"https://openalex.org/C157915830","wikidata":"https://www.wikidata.org/wiki/Q2928001","display_name":"Bubble","level":2,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3728972","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728972","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3728972","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3728972","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3894403112","display_name":null,"funder_award_id":"14380029","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G6646539178","display_name":null,"funder_award_id":"U24A20337, 62372228","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W40327287","https://openalex.org/W1991567646","https://openalex.org/W2014729633","https://openalex.org/W2039522160","https://openalex.org/W2041713059","https://openalex.org/W2119112357","https://openalex.org/W2169015875","https://openalex.org/W2194775991","https://openalex.org/W2257979135","https://openalex.org/W2547954490","https://openalex.org/W2596585349","https://openalex.org/W2599674900","https://openalex.org/W2608702473","https://openalex.org/W2794497357","https://openalex.org/W2954903132","https://openalex.org/W2963470893","https://openalex.org/W2973084513","https://openalex.org/W3104663419","https://openalex.org/W3123045479","https://openalex.org/W3177562315","https://openalex.org/W3197929891","https://openalex.org/W3199980055","https://openalex.org/W4214589259","https://openalex.org/W4231739827","https://openalex.org/W4246907949","https://openalex.org/W4251827675","https://openalex.org/W4253813365","https://openalex.org/W4284686707","https://openalex.org/W4284707748","https://openalex.org/W4284708930","https://openalex.org/W4285097780","https://openalex.org/W4285796263","https://openalex.org/W4300958603","https://openalex.org/W4308643012","https://openalex.org/W4327594616","https://openalex.org/W4377238789","https://openalex.org/W4387735187","https://openalex.org/W4411552405"],"related_works":["https://openalex.org/W3034924094","https://openalex.org/W3094954546","https://openalex.org/W1488708774","https://openalex.org/W1982811510","https://openalex.org/W4391100477","https://openalex.org/W2402189625","https://openalex.org/W4327779705","https://openalex.org/W4310560702","https://openalex.org/W1513698804","https://openalex.org/W2029712093"],"abstract_inverted_index":{"Deep":[0],"learning":[1],"(DL)":[2],"frameworks":[3,186,243],"are":[4,149],"essential":[5],"to":[6,15,37,54,60,76,97,151,206,261],"DL-based":[7],"software":[8],"systems,":[9],"and":[10,32,45,89,108,141,158,166,230,247,277,298,317],"framework":[11,95],"bugs":[12,148,221,306,316],"may":[13],"lead":[14],"substantial":[16],"disasters,":[17],"thus":[18],"requiring":[19],"effective":[20],"testing.":[21],"Researchers":[22],"adopt":[23],"DL":[24,185,196,242,251],"models":[25,201,252],"or":[26],"single":[27,94,156],"interfaces":[28,157],"as":[29],"test":[30,49,69,100,110,128,208,275,281],"inputs":[31,50,209],"analyze":[33,55],"their":[34,113],"execution":[35,56,104,231],"results":[36,57,105,322],"detect":[38],"bugs.":[39,320],"However,":[40],"floating-point":[41],"errors,":[42],"inherent":[43],"randomness,":[44],"the":[46,152,192,215,235,272,324],"complexity":[47],"of":[48,67,92,155,195,225,237,274,279,292,326],"make":[51],"it":[52],"challenging":[53],"effectively,":[58],"leading":[59],"existing":[61,307],"methods":[62,308],"suffering":[63],"from":[64,160,253,258,271],"a":[65,93,179],"lack":[66],"suitable":[68],"oracles.":[70],"Some":[71],"researchers":[72],"utilize":[73],"metamorphic":[74,181],"testing":[75,182],"tackle":[77],"this":[78],"challenge.":[79],"They":[80],"design":[81],"Metamorphic":[82],"Relations":[83],"(MRs)":[84],"based":[85],"on":[86,135,191,239],"input":[87,129],"data":[88],"parameter":[90],"settings":[91],"interface":[96,204],"generate":[98,207],"equivalent":[99],"inputs,":[101],"ensuring":[102],"consistent":[103,211],"between":[106],"original":[107],"generated":[109,280],"inputs.":[111,282],"Despite":[112],"promising":[114],"effectiveness,":[115],"they":[116],"still":[117],"face":[118],"certain":[119],"limitations.":[120],"(1)":[121],"Existing":[122,132],"MRs":[123,133,189],"overlook":[124],"structural":[125],"complexity,":[126],"limiting":[127],"diversity.":[130],"(2)":[131],"focus":[134],"limited":[136],"interfaces,":[137],"which":[138,293],"limits":[139],"generalization":[140],"necessitates":[142],"additional":[143],"adaptations.":[144],"(3)":[145],"Their":[146],"detected":[147],"related":[150],"result":[153],"consistency":[154],"far":[159],"those":[161],"exposed":[162],"in":[163],"multi-interface":[164],"combinations":[165,205],"runtime":[167],"metrics":[168],"(e.g.,":[169],"resource":[170,314],"usage).":[171],"To":[172],"address":[173],"these":[174],"limitations,":[175],"we":[176],"propose":[177],"ModelMeta,":[178],"model-level":[180],"method":[183],"for":[184],"with":[187,202,210,249],"four":[188],"focused":[190],"structure":[193],"characteristics":[194],"models.":[197],"ModelMeta":[198,238,267,286],"augments":[199],"seed":[200],"diverse":[203],"outputs,":[212],"guided":[213],"by":[214],"QR-DQN":[216],"strategy.":[217],"It":[218],"then":[219],"detects":[220],"through":[222],"fine-grained":[223],"analysis":[224],"training":[226],"loss/gradients,":[227],"memory/GPU":[228],"usage,":[229],"time.":[232],"We":[233],"evaluate":[234],"effectiveness":[236],"three":[240],"popular":[241],"(i.e.,":[244],"MindSpore,":[245],"PyTorch,":[246],"ONNX)":[248],"17":[250],"ten":[254],"real-world":[255],"tasks":[256],"ranging":[257],"image":[259],"classification":[260],"object":[262],"detection.":[263],"Results":[264],"demonstrate":[265,323],"that":[266],"outperforms":[268],"state-of-the-art":[269],"baselines":[270],"perspective":[273],"coverage":[276],"diversity":[278],"Regarding":[283],"bug":[284],"detection,":[285],"has":[287],"identified":[288],"31":[289],"new":[290],"bugs,":[291],"27":[294],"have":[295,300],"been":[296,301],"confirmed,":[297],"11":[299],"fixed.":[302],"Among":[303],"them,":[304],"seven":[305],"cannot":[309],"detect,":[310],"i.e.,":[311],"five":[312],"wrong":[313],"usage":[315],"two":[318],"low-efficiency":[319],"These":[321],"practicality":[325],"our":[327],"method.":[328]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}