{"id":"https://openalex.org/W4409478867","doi":"https://doi.org/10.1145/3729533","title":"Evaluating API-Level Deep Learning Fuzzers: A Comprehensive Benchmarking Study","display_name":"Evaluating API-Level Deep Learning Fuzzers: A Comprehensive Benchmarking Study","publication_year":2025,"publication_date":"2025-04-15","ids":{"openalex":"https://openalex.org/W4409478867","doi":"https://doi.org/10.1145/3729533"},"language":"en","primary_location":{"id":"doi:10.1145/3729533","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3729533","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054038436","display_name":"Nima Shiri Harzevili","orcid":"https://orcid.org/0000-0003-0484-3972"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Nima Shiri Harzevili","raw_affiliation_strings":["York University, Toronto, Ontario, Canada","York University, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0484-3972","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"York University, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045306475","display_name":"Moshi Wei","orcid":"https://orcid.org/0000-0003-1659-1960"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Moshi Wei","raw_affiliation_strings":["York University, Toronto, Ontario, Canada","York University, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1659-1960","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"York University, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025120794","display_name":"Mohammad Mahdi Mohajer","orcid":"https://orcid.org/0009-0000-8192-0164"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mohammad Mahdi Mohajer","raw_affiliation_strings":["York University, Toronto, Ontario, Canada","York University, Canada"],"raw_orcid":"https://orcid.org/0009-0000-8192-0164","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"York University, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hung Viet Pham","orcid":"https://orcid.org/0000-0003-0617-2877"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Hung Viet Pham","raw_affiliation_strings":["York University, Toronto, Ontario, Canada","York University, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0617-2877","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"York University, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":null,"display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-0861-8326"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["York University, Toronto, Ontario, Canada","York University, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0861-8326","affiliations":[{"raw_affiliation_string":"York University, Toronto, Ontario, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"York University, Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7588,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.86026879,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"35","issue":"2","first_page":"1","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9137108325958252},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8592822551727295},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5154212713241577},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4902098774909973},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.44538819789886475},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4209679961204529},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34555482864379883}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9137108325958252},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8592822551727295},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5154212713241577},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4902098774909973},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.44538819789886475},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4209679961204529},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34555482864379883},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3729533","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3729533","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":61,"referenced_works":["https://openalex.org/W90447038","https://openalex.org/W116656696","https://openalex.org/W1516061453","https://openalex.org/W1541063262","https://openalex.org/W2014577207","https://openalex.org/W2137726309","https://openalex.org/W2350778671","https://openalex.org/W2521352760","https://openalex.org/W2567575208","https://openalex.org/W2740329368","https://openalex.org/W2745044774","https://openalex.org/W2767072113","https://openalex.org/W2888223970","https://openalex.org/W2898686857","https://openalex.org/W2941957370","https://openalex.org/W2954903132","https://openalex.org/W2955972181","https://openalex.org/W2964241064","https://openalex.org/W2968594320","https://openalex.org/W2971815038","https://openalex.org/W2996060033","https://openalex.org/W3004949678","https://openalex.org/W3019428952","https://openalex.org/W3082737479","https://openalex.org/W3089794841","https://openalex.org/W3104663419","https://openalex.org/W3123045479","https://openalex.org/W3135004823","https://openalex.org/W3135325813","https://openalex.org/W3149582851","https://openalex.org/W3160029089","https://openalex.org/W3160700180","https://openalex.org/W3181528482","https://openalex.org/W3194771370","https://openalex.org/W3196239222","https://openalex.org/W3207811774","https://openalex.org/W3211521973","https://openalex.org/W4205513494","https://openalex.org/W4206639582","https://openalex.org/W4221144766","https://openalex.org/W4226075195","https://openalex.org/W4226255934","https://openalex.org/W4281763794","https://openalex.org/W4284705844","https://openalex.org/W4284706927","https://openalex.org/W4284707748","https://openalex.org/W4284708930","https://openalex.org/W4285490440","https://openalex.org/W4294959093","https://openalex.org/W4308643012","https://openalex.org/W4308643032","https://openalex.org/W4313442384","https://openalex.org/W4313563693","https://openalex.org/W4313563712","https://openalex.org/W4313563757","https://openalex.org/W4327594616","https://openalex.org/W4378591002","https://openalex.org/W4378770802","https://openalex.org/W4384026546","https://openalex.org/W4387735187","https://openalex.org/W4388212678"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W1993948687","https://openalex.org/W2000169967","https://openalex.org/W4380075502"],"abstract_inverted_index":{"In":[0,88],"recent":[1],"years,":[2],"the":[3,16,73,85,93,184,189,199,214],"practice":[4],"of":[5,84,97,181,183,201,216],"fuzzing":[6,239],"Deep":[7],"Learning":[8],"(DL)":[9],"APIs":[10,31],"has":[11],"received":[12],"significant":[13],"attention":[14],"in":[15,42,53,60,188,204,218],"software":[17],"engineering":[18],"community.":[19],"Many":[20],"API-level":[21,67,99],"DL":[22,30,61,68,100,107,126,146,228,240],"fuzzers":[23,38,69,101,175,203,217,229],"have":[24,39],"been":[25,40],"proposed":[26],"to":[27],"test":[28],"individual":[29],"by":[32,138,230],"generating":[33],"malformed":[34,139],"input.":[35],"Although":[36],"these":[37,66,174,202],"effective":[41],"detecting":[43,205],"bugs":[44,59,74,127,187],"and":[45,131,160,235],"outperforming":[46],"prior":[47],"work,":[48,90],"there":[49],"remains":[50],"a":[51,80],"gap":[52],"benchmarking":[54,236],"them":[55],"against":[56,113],"ground-truth,":[57],"real-world":[58,114,125,186,206],"libraries.":[62,241],"Existing":[63],"comparisons":[64],"among":[65],"primarily":[70],"focus":[71],"on":[72,110,227],"detected":[75],"but":[76],"do":[77],"not":[78],"offer":[79],"comprehensive,":[81],"in-depth":[82,95],"evaluation":[83,96,234],"fuzzers\u2019":[86],"effectiveness.":[87],"this":[89,222],"we":[91],"perform":[92],"first":[94],"state-of-the-art":[98,145],"that":[102,134,173,197],"generate":[103],"tests":[104],"for":[105,212,238],"single":[106],"APIs,":[108],"focusing":[109],"their":[111,167],"effectiveness":[112,200],"bugs.":[115,207],"We":[116,141],"manually":[117],"created":[118],"an":[119,232],"extensive":[120,233],"benchmark":[121,164],"dataset,":[122,165],"including":[123],"517":[124],"collected":[128],"from":[129],"PyTorch":[130],"TensorFlow":[132],"libraries":[133],"can":[135],"be":[136],"triggered":[137],"inputs.":[140],"then":[142],"apply":[143],"seven":[144],"fuzzers\u2014":[147],"FreeFuzz":[148],",":[149,151,153,155,157,159],"DeepRel":[150],"NablaFuzz":[152],"DocTer":[154],"ACETest":[156],"TitanFuzz":[158],"FuzzGPT":[161],"\u2014to":[162],"our":[163],"following":[166],"respective":[168],"instructions.":[169],"Our":[170,191],"results":[171],"show":[172],"detect":[176],"only":[177],"6.5%":[178],"(34":[179],"out":[180],"517)":[182],"unique":[185],"dataset.":[190],"analysis":[192],"identifies":[193],"two":[194],"dominant":[195],"factors":[196],"impact":[198],"These":[208],"findings":[209],"suggest":[210],"opportunities":[211],"improving":[213],"performance":[215],"future":[219],"work.":[220],"Overall,":[221],"study":[223],"extends":[224],"previous":[225],"work":[226],"providing":[231],"platform":[237]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}