{"id":"https://openalex.org/W4285605736","doi":"https://doi.org/10.24963/ijcai.2022/588","title":"Lyra: A Benchmark for Turducken-Style Code Generation","display_name":"Lyra: A Benchmark for Turducken-Style Code Generation","publication_year":2022,"publication_date":"2022-07-01","ids":{"openalex":"https://openalex.org/W4285605736","doi":"https://doi.org/10.24963/ijcai.2022/588"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2022/588","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2022/588","pdf_url":"https://www.ijcai.org/proceedings/2022/0588.pdf","source":{"id":"https://openalex.org/S4363608755","display_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://www.ijcai.org/proceedings/2022/0588.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034794084","display_name":"Qingyuan Liang","orcid":"https://orcid.org/0000-0002-1697-3451"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Qingyuan Liang","raw_affiliation_strings":["Peking University","School of Software & Microelectronics, Peking University. Beijing, PR China","School of Computer Science, Peking University. Beijing, PR China","Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Software & Microelectronics, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Computer Science, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047965752","display_name":"Zeyu Sun","orcid":"https://orcid.org/0000-0002-9990-9120"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyu Sun","raw_affiliation_strings":["Peking University","Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","School of Computer Science, Peking University. Beijing, PR China"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Computer Science, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047271595","display_name":"Qihao Zhu","orcid":"https://orcid.org/0000-0001-8036-2623"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qihao Zhu","raw_affiliation_strings":["Peking University","School of Computer Science, Peking University. Beijing, PR China","Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Computer Science, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385507","display_name":"Wenjie Zhang","orcid":"https://orcid.org/0000-0002-2669-1837"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Zhang","raw_affiliation_strings":["Peking University","School of Computer Science, Peking University. Beijing, PR China","Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Computer Science, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100758675","display_name":"Lian Yu","orcid":"https://orcid.org/0000-0002-7232-2549"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lian Yu","raw_affiliation_strings":["Peking University","School of Software & Microelectronics, Peking University. Beijing, PR China"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Software & Microelectronics, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100712724","display_name":"Yingfei Xiong","orcid":"https://orcid.org/0000-0001-8991-747X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yingfei Xiong","raw_affiliation_strings":["Peking University","School of Computer Science, Peking University. Beijing, PR China","Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Computer Science, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101261545","display_name":"Lu Zhang","orcid":"https://orcid.org/0000-0002-3985-3524"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lu Zhang","raw_affiliation_strings":["Peking University","Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","School of Computer Science, Peking University. Beijing, PR China"],"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"Key Laboratory of High Confidence Software Technologies, Ministry of Education (Peking University)","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"School of Computer Science, Peking University. Beijing, PR China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5034794084"],"corresponding_institution_ids":["https://openalex.org/I20231570"],"apc_list":null,"apc_paid":null,"fwci":0.7292,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.71565404,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"4238","last_page":"4244"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8706837892532349},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.7336138486862183},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6854245066642761},{"id":"https://openalex.org/keywords/sql","display_name":"SQL","score":0.5203768014907837},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4489266276359558},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4219781458377838},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4196537733078003},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4102940261363983},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3360848128795624}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8706837892532349},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.7336138486862183},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6854245066642761},{"id":"https://openalex.org/C510870499","wikidata":"https://www.wikidata.org/wiki/Q47607","display_name":"SQL","level":2,"score":0.5203768014907837},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4489266276359558},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4219781458377838},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4196537733078003},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4102940261363983},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3360848128795624},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.24963/ijcai.2022/588","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2022/588","pdf_url":"https://www.ijcai.org/proceedings/2022/0588.pdf","source":{"id":"https://openalex.org/S4363608755","display_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2108.12144","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.12144","pdf_url":"https://arxiv.org/pdf/2108.12144","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2022/588","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2022/588","pdf_url":"https://www.ijcai.org/proceedings/2022/0588.pdf","source":{"id":"https://openalex.org/S4363608755","display_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Thirty-First International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.6499999761581421,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G466649759","display_name":null,"funder_award_id":"2017Y","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G5817679519","display_name":null,"funder_award_id":"61872011","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7429600474","display_name":null,"funder_award_id":"2017YFB1","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G8340078520","display_name":null,"funder_award_id":"2017YF","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G8561260003","display_name":null,"funder_award_id":"2017YFB1001803","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G8823009050","display_name":null,"funder_award_id":"2017YFB","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4285605736.pdf","grobid_xml":"https://content.openalex.org/works/W4285605736.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2242083635","https://openalex.org/W2751448157","https://openalex.org/W2762513422","https://openalex.org/W2896457183","https://openalex.org/W2962784628","https://openalex.org/W2962936887","https://openalex.org/W2964315653","https://openalex.org/W2964325845","https://openalex.org/W2997847174","https://openalex.org/W3049551829","https://openalex.org/W3081454995","https://openalex.org/W3098605233","https://openalex.org/W3119507053","https://openalex.org/W3126675481","https://openalex.org/W3160974509","https://openalex.org/W3163659631","https://openalex.org/W3177813494","https://openalex.org/W4289494028","https://openalex.org/W4385245566","https://openalex.org/W4394638297"],"related_works":["https://openalex.org/W2341492732","https://openalex.org/W3187193180","https://openalex.org/W106542691","https://openalex.org/W1699080303","https://openalex.org/W4297799326","https://openalex.org/W3116064965","https://openalex.org/W4287027380","https://openalex.org/W3193760048","https://openalex.org/W2207495067","https://openalex.org/W1906486629"],"abstract_inverted_index":{"Recently,":[0],"neural":[1],"techniques":[2,202],"have":[3],"been":[4],"used":[5],"to":[6,55,81],"generate":[7,82],"source":[8],"code":[9,70,103,188,200],"automatically.":[10],"While":[11],"promising":[12,47],"for":[13,24,187,203],"declarative":[14,29,50,93],"languages,":[15],"these":[16],"approaches":[17],"achieve":[18,164],"much":[19],"poorer":[20],"performance":[21],"on":[22,49],"datasets":[23],"imperative":[25,36,88],"languages.":[26],"Since":[27],"a":[28,68,74,83,86,112,138,184],"language":[30,37,76,89],"is":[31,99,134],"typically":[32],"embedded":[33,92,117],"in":[34,42,85,114],"an":[35,91,142],"(i.e.,":[38],"the":[39,46,100,158,197],"turducken-style":[40,102],"programming)":[41],"real-world":[43,204],"software":[44,60,205],"development,":[45],"results":[48],"languages":[51],"can":[52,163],"hardly":[53],"lead":[54],"significant":[56],"reduction":[57],"of":[58,199],"manual":[59],"development":[61],"efforts.":[62],"In":[63,145,157],"this":[64,78,98,107,192],"paper,":[65],"we":[66,109,148,179],"define":[67],"new":[69,185],"generation":[71,104,201],"task:":[72],"given":[73],"natural":[75],"comment,":[77],"task":[79],"aims":[80],"program":[84,133],"base":[87],"with":[90,116,136],"language.":[94],"To":[95],"our":[96,146],"knowledge,":[97],"first":[101],"task.":[105],"For":[106],"task,":[108],"present":[110],"Lyra:":[111],"dataset":[113,120],"Python":[115],"SQL.":[118],"This":[119],"contains":[121],"2,000":[122],"carefully":[123],"annotated":[124],"database":[125],"manipulation":[126],"programs":[127],"from":[128],"real":[129],"usage":[130],"projects.":[131],"Each":[132],"paired":[135],"both":[137],"Chinese":[139,173],"comment":[140],"and":[141,152,166,174],"English":[143,175],"comment.":[144],"experiment,":[147],"adopted":[149],"Transformer,":[150],"BERT-style,":[151],"GPT-style":[153,161],"models":[154],"as":[155],"baselines.":[156],"best":[159],"setting,":[160],"model":[162],"24%":[165],"25.5%":[167],"AST":[168],"exact":[169],"matching":[170],"accuracy":[171],"using":[172],"comments,":[176],"respectively.":[177],"Therefore,":[178],"believe":[180],"that":[181],"Lyra":[182],"provides":[183],"challenge":[186,193],"generation.":[189],"Yet,":[190],"overcoming":[191],"may":[194],"significantly":[195],"boost":[196],"applicability":[198],"development.":[206]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}