{"id":"https://openalex.org/W4385768129","doi":"https://doi.org/10.24963/ijcai.2023/542","title":"Revisiting the Evaluation of Deep Learning-Based Compiler Testing","display_name":"Revisiting the Evaluation of Deep Learning-Based Compiler Testing","publication_year":2023,"publication_date":"2023-08-01","ids":{"openalex":"https://openalex.org/W4385768129","doi":"https://doi.org/10.24963/ijcai.2023/542"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2023/542","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/542","pdf_url":"https://www.ijcai.org/proceedings/2023/0542.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2023/0542.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052271946","display_name":"Yongqiang Tian","orcid":"https://orcid.org/0000-0003-1644-2965"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]},{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["CA","HK"],"is_corresponding":true,"raw_author_name":"Yongqiang Tian","raw_affiliation_strings":["The Hong Kong University of Science and Technology","University of Waterloo"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080550560","display_name":"Zhenyang Xu","orcid":"https://orcid.org/0000-0002-9451-4031"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zhenyang Xu","raw_affiliation_strings":["University of Waterloo"],"affiliations":[{"raw_affiliation_string":"University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053101204","display_name":"Yiwen Dong","orcid":"https://orcid.org/0000-0002-3205-9010"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Yiwen Dong","raw_affiliation_strings":["University of Waterloo"],"affiliations":[{"raw_affiliation_string":"University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101708632","display_name":"C. P. Sun","orcid":"https://orcid.org/0000-0002-0862-2491"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Chengnian Sun","raw_affiliation_strings":["University of Waterloo"],"affiliations":[{"raw_affiliation_string":"University of Waterloo","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034057959","display_name":"Shing-Chi Cheung","orcid":"https://orcid.org/0000-0002-3508-7172"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shing-Chi Cheung","raw_affiliation_strings":["The Hong Kong University of Science and Technology"],"affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology","institution_ids":["https://openalex.org/I200769079"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5052271946"],"corresponding_institution_ids":["https://openalex.org/I151746483","https://openalex.org/I200769079"],"apc_list":null,"apc_paid":null,"fwci":1.074,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.84335703,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"4873","last_page":"4882"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8932365775108337},{"id":"https://openalex.org/keywords/compiler","display_name":"Compiler","score":0.8610526323318481},{"id":"https://openalex.org/keywords/generator","display_name":"Generator (circuit theory)","score":0.6972969770431519},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.682587742805481},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6230521202087402},{"id":"https://openalex.org/keywords/optimizing-compiler","display_name":"Optimizing compiler","score":0.5991441011428833},{"id":"https://openalex.org/keywords/domain-specific-language","display_name":"Domain-specific language","score":0.46542972326278687},{"id":"https://openalex.org/keywords/compiler-construction","display_name":"Compiler construction","score":0.4141072928905487},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3779374361038208}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8932365775108337},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.8610526323318481},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.6972969770431519},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.682587742805481},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6230521202087402},{"id":"https://openalex.org/C190902152","wikidata":"https://www.wikidata.org/wiki/Q1325106","display_name":"Optimizing compiler","level":3,"score":0.5991441011428833},{"id":"https://openalex.org/C135257023","wikidata":"https://www.wikidata.org/wiki/Q691358","display_name":"Domain-specific language","level":2,"score":0.46542972326278687},{"id":"https://openalex.org/C9957790","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler construction","level":3,"score":0.4141072928905487},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3779374361038208},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.24963/ijcai.2023/542","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/542","pdf_url":"https://www.ijcai.org/proceedings/2023/0542.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-130024","is_oa":false,"landing_page_url":"https://repository.hkust.edu.hk/ir/Record/1783.1-130024","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference paper"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2023/542","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2023/542","pdf_url":"https://www.ijcai.org/proceedings/2023/0542.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-Second International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7900000214576721}],"awards":[{"id":"https://openalex.org/G2165548363","display_name":null,"funder_award_id":"Canada","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"},{"id":"https://openalex.org/G8284766523","display_name":null,"funder_award_id":"(NSERC)","funder_id":"https://openalex.org/F4320334593","funder_display_name":"Natural Sciences and Engineering Research Council of Canada"}],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4385768129.pdf"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W1491178396","https://openalex.org/W2461570336","https://openalex.org/W2532737545","https://openalex.org/W2808279976","https://openalex.org/W2899781671","https://openalex.org/W2904932877","https://openalex.org/W2963350015","https://openalex.org/W2964241064","https://openalex.org/W2997653900","https://openalex.org/W3008321987","https://openalex.org/W3087775079","https://openalex.org/W3089808874","https://openalex.org/W3090416608","https://openalex.org/W3107362213","https://openalex.org/W4244688039","https://openalex.org/W4245487037","https://openalex.org/W4293235851"],"related_works":["https://openalex.org/W2003848320","https://openalex.org/W4240253816","https://openalex.org/W2169584677","https://openalex.org/W2374859588","https://openalex.org/W4232954277","https://openalex.org/W2094199724","https://openalex.org/W2083681681","https://openalex.org/W2749133591","https://openalex.org/W4226439887","https://openalex.org/W2089705195"],"abstract_inverted_index":{"A":[0],"high-quality":[1],"program":[2,14,41,61,208],"generator":[3,15,62],"is":[4,16,87,153],"essential":[5],"to":[6,21,57,64,89,94,99,131,145,205,235],"effective":[7],"automated":[8],"compiler":[9,101,136,251],"testing.":[10],"Engineering":[11],"such":[12,238],"a":[13,49,60,180,193,207,239,259],"difficult,":[17],"time-consuming,":[18],"and":[19,138,178],"specific":[20],"the":[22,114,117,122,133,174,217,220,231],"language":[23,118,166],"under":[24],"testing,":[25],"thus":[26],"requiring":[27],"tremendous":[28],"efforts":[29],"from":[30,67],"human":[31],"experts":[32],"with":[33,224],"language-specific":[34],"domain":[35],"knowledge.":[36],"To":[37],"avoid":[38],"repeatedly":[39],"writing":[40],"generators":[42],"for":[43,189,262],"different":[44],"languages,":[45],"researchers":[46],"recently":[47],"proposed":[48],"language-agnostic":[50],"approach":[51],"based":[52],"on":[53],"deep":[54,202],"learning":[55,203],"techniques":[56,204],"automatically":[58],"learn":[59,206],"(referred":[63],"as":[65,92],"DLG)":[66],"existing":[68],"programs.":[69,124],"Evaluations":[70],"show":[71],"that":[72,85,156,230,255],"DLGs":[73,127,233,256],"outperform":[74],"Language-Specific":[75],"Program":[76],"Generators":[77],"(LSGs)":[78],"in":[79,121,162,219],"testing":[80],"compilers.":[81],"However,":[82],"we":[83],"argue":[84],"it":[86,152],"unfair":[88],"use":[90],"LSGs":[91,97],"baselines":[93],"evaluate":[95],"DLGs.":[96,191],"aim":[98,130],"validate":[100,132],"optimizations":[102],"by":[103,215],"only":[104],"generating":[105],"compilable,":[106],"well-defined":[107,147],"test":[108],"programs;":[109],"this":[110],"restriction":[111],"inevitably":[112],"impairs":[113],"diversity":[115],"of":[116,135,164,176,196,200],"features":[119,167],"used":[120,165],"generated":[123,140],"In":[125],"contrast,":[126],"do":[128],"not":[129,143,154],"correctness":[134],"optimizations,":[137],"its":[139],"programs":[141,158,214,218],"are":[142,159],"guaranteed":[144],"be":[146],"or":[148],"even":[149],"compilable.":[150],"Therefore,":[151],"surprising":[155],"DLG-generated":[157],"more":[160,225],"diverse":[161],"terms":[163],"than":[168,226],"LSG-generated":[169],"ones.":[170],"This":[171],"study":[172],"revisits":[173],"evaluation":[175],"DLGs,":[177],"proposes":[179],"new,":[181],"fair,":[182],"simple":[183,240],"yet":[184],"strong":[185],"baseline":[186],"named":[187],"Kitten":[188,210],"evaluating":[190],"Given":[192],"dataset":[194],"consisting":[195],"human-written":[197],"programs,":[198],"instead":[199],"using":[201],"generator,":[209],"directly":[211],"derives":[212],"new":[213],"mutating":[216],"dataset.":[221],"Extensive":[222],"experiments":[223],"1,500":[227],"CPU-hours":[228],"demonstrate":[229],"state-of-the-art":[232],"fail":[234],"compete":[236],"against":[237],"baseline:":[241],"3":[242],"v.s.":[243,248],"1,750":[244],"hang":[245],"bugs,":[246],"1":[247],"34":[249],"distinct":[250],"crashes.":[252],"We":[253],"believe":[254],"still":[257],"have":[258],"large":[260],"room":[261],"improvement.":[263]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}