{"id":"https://openalex.org/W4389161045","doi":"https://doi.org/10.1145/3611643.3616320","title":"Pitfalls in Experiments with DNN4SE: An Analysis of the State of the Practice","display_name":"Pitfalls in Experiments with DNN4SE: An Analysis of the State of the Practice","publication_year":2023,"publication_date":"2023-11-30","ids":{"openalex":"https://openalex.org/W4389161045","doi":"https://doi.org/10.1145/3611643.3616320"},"language":"en","primary_location":{"id":"doi:10.1145/3611643.3616320","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3611643.3616320","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076034572","display_name":"Sira Vegas","orcid":"https://orcid.org/0000-0001-8535-9386"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Sira Vegas","raw_affiliation_strings":["Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"affiliations":[{"raw_affiliation_string":"Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025318877","display_name":"Sebastian Elbaum","orcid":"https://orcid.org/0000-0001-9592-1352"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sebastian Elbaum","raw_affiliation_strings":["University of Virginia, Charlottesville, USA"],"affiliations":[{"raw_affiliation_string":"University of Virginia, Charlottesville, USA","institution_ids":["https://openalex.org/I51556381"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5076034572"],"corresponding_institution_ids":["https://openalex.org/I88060688"],"apc_list":null,"apc_paid":null,"fwci":1.359,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.86146057,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"528","last_page":"540"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7302753925323486},{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.6927980780601501},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5897223353385925},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5496439337730408},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5112264156341553},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4999215602874756},{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.49203774333000183},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4906665086746216},{"id":"https://openalex.org/keywords/variable","display_name":"Variable (mathematics)","score":0.4447070062160492},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4110743999481201},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3731898367404938},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10604360699653625},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0974884033203125},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.09457600116729736}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7302753925323486},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.6927980780601501},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5897223353385925},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5496439337730408},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5112264156341553},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4999215602874756},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.49203774333000183},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4906665086746216},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.4447070062160492},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4110743999481201},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3731898367404938},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10604360699653625},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0974884033203125},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09457600116729736},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3611643.3616320","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3611643.3616320","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1320060331","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/50110001103","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G1598597440","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/5011000110","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G1622782223","display_name":null,"funder_award_id":"MCIN/AEI/10","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G1643949827","display_name":null,"funder_award_id":"AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G1815864298","display_name":null,"funder_award_id":"CIN/AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G2262748287","display_name":null,"funder_award_id":"501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G2394899866","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320322138","funder_display_name":"Universidad Polit\u00e9cnica de Madrid"},{"id":"https://openalex.org/G2969783435","display_name":null,"funder_award_id":"CIN/AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G300979063","display_name":null,"funder_award_id":"10.13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G3219925899","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G3429648993","display_name":null,"funder_award_id":"PID202","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G3480869486","display_name":null,"funder_award_id":"13039","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G4045093968","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G410571670","display_name":null,"funder_award_id":"10.13039/501100011033","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G4126322094","display_name":null,"funder_award_id":"01100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G451917667","display_name":null,"funder_award_id":"13039/501100011033","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G5080475149","display_name":null,"funder_award_id":"10.13039","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G5539308778","display_name":null,"funder_award_id":"AEI/10.13039/501100011033","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G5967599077","display_name":null,"funder_award_id":"501100011033","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G661330594","display_name":null,"funder_award_id":"00110","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G6685425346","display_name":null,"funder_award_id":"0011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G6838289232","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/50110001103","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G7084143925","display_name":null,"funder_award_id":"AEI/10","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G7266728691","display_name":null,"funder_award_id":"13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G7377729401","display_name":"FMitF: Track I: Focusing Incremental Abstraction-based Verification on Neural Networks Input Distributions","funder_award_id":"2019239","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7535663061","display_name":null,"funder_award_id":"AEI/10.","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G8260616629","display_name":null,"funder_award_id":"011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8570012161","display_name":null,"funder_award_id":"unknown","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G8638682022","display_name":null,"funder_award_id":"10.13039/50110001103","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320322138","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02"},{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320335598","display_name":"Agencia Estatal de Investigaci\u00f3n","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1511267087","https://openalex.org/W1970196847","https://openalex.org/W2064161267","https://openalex.org/W2122626138","https://openalex.org/W2138428785","https://openalex.org/W2140984588","https://openalex.org/W2161222712","https://openalex.org/W2169090130","https://openalex.org/W2508327075","https://openalex.org/W2795252046","https://openalex.org/W2897889469","https://openalex.org/W2913668833","https://openalex.org/W2983928456","https://openalex.org/W3047010584","https://openalex.org/W3122184684","https://openalex.org/W3161632096","https://openalex.org/W4229772528","https://openalex.org/W4285247202","https://openalex.org/W4285248702","https://openalex.org/W4292872700","https://openalex.org/W4389161045","https://openalex.org/W6892374893"],"related_works":["https://openalex.org/W3034924094","https://openalex.org/W3094954546","https://openalex.org/W1488708774","https://openalex.org/W1982811510","https://openalex.org/W4391100477","https://openalex.org/W2402189625","https://openalex.org/W4327779705","https://openalex.org/W4310560702","https://openalex.org/W1513698804","https://openalex.org/W2029712093"],"abstract_inverted_index":{"Software":[0],"engineering":[1],"(SE)":[2],"techniques":[3,28,102,255],"are":[4,228,257,269],"increasingly":[5],"relying":[6],"on":[7,105],"deep":[8,106],"learning":[9,76],"approaches":[10],"to":[11,19,57,119,170,243,276],"support":[12],"many":[13],"SE":[14,117],"tasks,":[15],"from":[16,50],"bug":[17],"triaging":[18],"code":[20],"generation.":[21],"To":[22],"assess":[23],"the":[24,42,45,74,81,87,124,127,141,158,185,191,217,229,237,241,245,254,258,264],"efficacy":[25],"of":[26,44,47,61,66,84,123,126,140,160,184,193,216,222,226,240,266],"such":[27],"researchers":[29],"typically":[30],"perform":[31],"controlled":[32],"experiments.":[33],"Conducting":[34],"these":[35,278],"experiments,":[36,142],"however,":[37],"is":[38,78,174],"particularly":[39],"challenging":[40],"given":[41],"complexity":[43],"space":[46,192],"variables":[48,182,204,231],"involved,":[49],"specialized":[51],"and":[52,55,64,80,133,180,205,219,232],"intricate":[53],"architectures":[54],"algorithms":[56],"a":[58,95,121,175,200],"large":[59],"number":[60],"training":[62,88],"hyper-parameters":[63],"choices":[65],"evolving":[67],"datasets,":[68],"all":[69],"compounded":[70],"by":[71],"how":[72,263],"rapidly":[73],"machine":[75],"technology":[77],"advancing,":[79],"inherent":[82],"sources":[83,265],"randomness":[85],"in":[86,111,115,224,236],"process.":[89],"In":[90],"this":[91],"work":[92],"we":[93,165],"conduct":[94],"mapping":[96],"study,":[97],"examining":[98],"194":[99],"experiments":[100,238],"with":[101],"that":[103,138,145,154,172,207],"rely":[104],"neural":[107],"networks":[108],"(DNNs)":[109],"appearing":[110],"55":[112],"papers":[113],"published":[114],"premier":[116],"venues":[118],"provide":[120,272],"characterization":[122],"state":[125],"practice,":[128],"pinpointing":[129],"experiments\u2019":[130],"common":[131],"trends":[132],"pitfalls.":[134],"Our":[135],"study":[136],"reveals":[137],"most":[139],"including":[143],"those":[144],"have":[146,151],"received":[147],"ACM":[148],"artifact":[149],"badges,":[150],"fundamental":[152],"limitations":[153],"raise":[155],"doubts":[156],"about":[157],"reliability":[159],"their":[161,233],"findings.":[162],"More":[163],"specifically,":[164],"find:":[166],"1)":[167],"weak":[168],"analyses":[169],"determine":[171],"there":[173],"true":[176],"relationship":[177,201],"between":[178,202],"independent":[179],"dependent":[181,203],"(87%":[183],"experiments),":[186,218],"2)":[187],"limited":[188],"control":[189],"over":[190],"DNN":[194,230],"relevant":[195],"variables,":[196],"which":[197,249],"can":[198],"render":[199],"treatments":[206,246],"may":[208],"not":[209],"be":[210],"causal":[211],"but":[212],"rather":[213],"correlational":[214],"(100%":[215],"3)":[220],"lack":[221],"specificity":[223],"terms":[225],"what":[227],"values":[234],"utilized":[235],"(86%":[239],"experiments)":[242],"define":[244],"being":[247,260],"applied,":[248],"makes":[250],"it":[251],"unclear":[252],"whether":[253],"designed":[256],"ones":[259],"assessed,":[261],"or":[262],"extraneous":[267],"variation":[268],"controlled.":[270],"We":[271],"some":[273],"practical":[274],"recommendations":[275],"address":[277],"limitations.":[279]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}