{"id":"https://openalex.org/W4395029328","doi":"https://doi.org/10.1109/tse.2024.3392499","title":"No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT","display_name":"No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT","publication_year":2024,"publication_date":"2024-04-23","ids":{"openalex":"https://openalex.org/W4395029328","doi":"https://doi.org/10.1109/tse.2024.3392499"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3392499","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3392499","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://eprints.gla.ac.uk/view/author/72794.html>","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103105157","display_name":"Zhijie Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhijie Liu","raw_affiliation_strings":["ShanghaiTech University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"ShanghaiTech University, Shanghai, China","institution_ids":["https://openalex.org/I30809798"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000449889","display_name":"Yutian Tang","orcid":"https://orcid.org/0000-0001-5677-4564"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yutian Tang","raw_affiliation_strings":["University of Glasgow, Glasgow, U.K","University of Glasgow, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, U.K","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"University of Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100400376","display_name":"Xiapu Luo","orcid":"https://orcid.org/0000-0002-9082-3208"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiapu Luo","raw_affiliation_strings":["Department of Computing, Hong Kong Polytechnic University, Hong Kong SAR, China"],"affiliations":[{"raw_affiliation_string":"Department of Computing, Hong Kong Polytechnic University, Hong Kong SAR, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031391841","display_name":"Yuming Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuming Zhou","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032937625","display_name":"Liang Feng Zhang","orcid":"https://orcid.org/0000-0003-3543-1524"},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang Feng Zhang","raw_affiliation_strings":["ShanghaiTech University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"ShanghaiTech University, Shanghai, China","institution_ids":["https://openalex.org/I30809798"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5103105157"],"corresponding_institution_ids":["https://openalex.org/I30809798"],"apc_list":null,"apc_paid":null,"fwci":33.161,"has_fulltext":false,"cited_by_count":96,"citation_normalized_percentile":{"value":0.99817908,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"50","issue":"6","first_page":"1548","last_page":"1584"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8559305667877197},{"id":"https://openalex.org/keywords/lift","display_name":"Lift (data mining)","score":0.6643435955047607},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4859794080257416},{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.4650949239730835},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.459811806678772},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.4534570574760437},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4384014904499054},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.1576600968837738},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.144182026386261},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.11609774827957153}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8559305667877197},{"id":"https://openalex.org/C139002025","wikidata":"https://www.wikidata.org/wiki/Q3001212","display_name":"Lift (data mining)","level":2,"score":0.6643435955047607},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4859794080257416},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.4650949239730835},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.459811806678772},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.4534570574760437},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4384014904499054},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.1576600968837738},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.144182026386261},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.11609774827957153},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2024.3392499","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3392499","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:eprints.gla.ac.uk:324588","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/72794.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"}],"best_oa_location":{"id":"pmh:oai:eprints.gla.ac.uk:324588","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/72794.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4223574508","display_name":null,"funder_award_id":"62202306","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5997361112","display_name":null,"funder_award_id":"62172205","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":77,"referenced_works":["https://openalex.org/W1538836445","https://openalex.org/W1566773348","https://openalex.org/W1964962870","https://openalex.org/W1980224781","https://openalex.org/W2121044470","https://openalex.org/W2132525863","https://openalex.org/W2157331557","https://openalex.org/W2157976942","https://openalex.org/W2620986014","https://openalex.org/W2749460454","https://openalex.org/W2768572539","https://openalex.org/W2786248241","https://openalex.org/W2884958048","https://openalex.org/W2896457183","https://openalex.org/W2955426500","https://openalex.org/W2962728167","https://openalex.org/W2962739339","https://openalex.org/W2963935794","https://openalex.org/W2964150020","https://openalex.org/W2996428491","https://openalex.org/W3008733198","https://openalex.org/W3043638540","https://openalex.org/W3085192122","https://openalex.org/W3098136301","https://openalex.org/W3098605233","https://openalex.org/W3099302725","https://openalex.org/W3099636232","https://openalex.org/W3099655892","https://openalex.org/W3108032709","https://openalex.org/W3126095862","https://openalex.org/W3138815606","https://openalex.org/W3161457214","https://openalex.org/W3162962341","https://openalex.org/W3177813494","https://openalex.org/W3196706143","https://openalex.org/W3198685994","https://openalex.org/W4205403018","https://openalex.org/W4225108562","https://openalex.org/W4226278401","https://openalex.org/W4226485558","https://openalex.org/W4229675450","https://openalex.org/W4239900777","https://openalex.org/W4285734663","https://openalex.org/W4286750487","https://openalex.org/W4287668913","https://openalex.org/W4288057765","https://openalex.org/W4288089799","https://openalex.org/W4312438588","https://openalex.org/W4315815628","https://openalex.org/W4367860052","https://openalex.org/W4379471102","https://openalex.org/W4384345708","https://openalex.org/W4384345745","https://openalex.org/W4384499126","https://openalex.org/W4385187421","https://openalex.org/W4385245566","https://openalex.org/W4385572722","https://openalex.org/W4387356195","https://openalex.org/W4391282616","https://openalex.org/W4399557965","https://openalex.org/W6643979850","https://openalex.org/W6676373471","https://openalex.org/W6718692554","https://openalex.org/W6755207826","https://openalex.org/W6768021236","https://openalex.org/W6769627184","https://openalex.org/W6778883912","https://openalex.org/W6787335730","https://openalex.org/W6794686226","https://openalex.org/W6795855765","https://openalex.org/W6798182279","https://openalex.org/W6810738896","https://openalex.org/W6810874553","https://openalex.org/W6840793632","https://openalex.org/W6852887568","https://openalex.org/W6853685553","https://openalex.org/W6856985672"],"related_works":["https://openalex.org/W2019257047","https://openalex.org/W2129714608","https://openalex.org/W4226317003","https://openalex.org/W2149635184","https://openalex.org/W2795746234","https://openalex.org/W2729258247","https://openalex.org/W3161903544","https://openalex.org/W2785752813","https://openalex.org/W2097099770","https://openalex.org/W4391835699"],"abstract_inverted_index":{"Large":[0],"language":[1,11],"models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"capabilities":[7],"across":[8],"various":[9],"natural":[10],"processing":[12],"(NLP)":[13],"tasks,":[14,35],"such":[15],"as":[16],"machine":[17],"translation,":[18],"question":[19],"answering,":[20],"summarization,":[21],"and":[22,104,106,140,167,186,276,315,317,324,356,380,388,400,407],"so":[23],"on.":[24],"Additionally,":[25],"LLMs":[26],"are":[27],"also":[28,143],"highly":[29],"valuable":[30,194],"in":[31,37,96,153,203,233,243,283,307,371,376,392],"supporting":[32],"software":[33],"engineering":[34],"particularly":[36],"the":[38,76,114,183,188,197,209,272,288,300,326,337,393,402],"field":[39],"of":[40,49,78,124,176,199,274,303,312,322,352,373],"code":[41,44,53,56,79,111,115,125,178,185,205,228,259,281,304,327,343,358,374,398,409],"generation.":[42,179],"Automatic":[43],"generation":[45,80,116,206,359,399,410],"is":[46,222,268],"a":[47,71,86,121],"process":[48,156,263,291,340],"automatically":[50],"generating":[51,225],"source":[52],"or":[54,61,298],"executable":[55],"based":[57],"on":[58,134,246],"given":[59],"specifications":[60],"requirements,":[62],"improving":[63,405],"developer":[64],"productivity.":[65],"In":[66],"this":[67,191],"study,":[68],"we":[69],"perform":[70],"systematic":[72],"empirical":[73],"assessment":[74],"to":[75,151,255,264],"quality":[77],"using":[81],"<italic":[82,129,146,158,168,200,219,250,293,330,364,394],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[83,130,147,159,169,201,220,251,294,331,365,395],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">ChatGPT</i>":[84,131,148,160,170,202,221,252,295,332,366,396],",":[85,132],"recent":[87],"state-of-the-art":[88],"product":[89],"LLM.":[90],"We":[91,142],"leverage":[92],"728":[93],"algorithm":[94,308],"problems":[95,230,237],"five":[97],"languages":[98,235,285,311,321],"(i.e.,":[99,157],"C,":[100,313],"C++,":[101,314],"Java,":[102],"Python,":[103],"JavaScript)":[105],"18":[107],"CWEs":[108],"with":[109,240,260,292,310,320,348],"54":[110],"scenarios":[112,309,319],"for":[113,171,229,280,341,404],"task.":[117],"Our":[118],"evaluation":[119],"encompasses":[120],"comprehensive":[122],"analysis":[123],"snippets":[126,282,344,375],"generated":[127,173,184,328],"by":[128,329,363],"focusing":[133],"three":[135,210],"critical":[136,211],"aspects:":[137],"correctness,":[138,378],"complexity,":[139,379],"security.":[141,381],"specifically":[144],"investigate":[145],"\u2019s":[149,161,253,367],"ability":[150,254],"engage":[152],"multi-round":[154,261,289,338],"fixing":[155,172,262,290,339],"dialog":[162],"ability,":[163],"chatting":[164],"between":[165],"users":[166],"buggy":[174],"code)":[175],"facilitating":[177],"By":[180],"delving":[181],"into":[182,196],"examining":[187],"experimental":[189,214],"results,":[190,347],"work":[192],"provides":[193],"insights":[195],"performance":[198],"tackling":[204],"tasks":[207],"over":[208],"aspects.":[212],"The":[213],"results":[215],"demonstrate":[216],"that":[217,390],"(1)":[218],"better":[223],"at":[224],"functionally":[226],"correct":[227,266],"before":[231],"2021":[232,239],"different":[234,284],"than":[236,350],"after":[238],"48.14%":[241],"advantage":[242],"Accepted":[244],"rate":[245],"judgment":[247],"platform,":[248],"but":[249],"directly":[256],"fix":[257],"erroneous":[258],"achieve":[265],"functionality":[267],"relatively":[269],"weak;":[270],"(2)":[271],"distribution":[273],"cyclomatic":[275],"cognitive":[277],"complexity":[278,301],"levels":[279,302],"varies.":[286],"Furthermore,":[287],"generally":[296],"preserves":[297],"increases":[299],"snippets;":[305],"(3)":[306],"Jave,":[316],"CWE":[318],"C":[323],"Python3,":[325],"has":[333],"relevant":[334],"vulnerabilities.":[335],"However,":[336],"vulnerable":[342],"demonstrates":[345],"promising":[346],"more":[349],"89%":[351],"vulnerabilities":[353],"successfully":[354],"addressed;":[355],"(4)":[357],"may":[360],"be":[361],"affected":[362],"non-determinism":[368],"factor,":[369],"resulting":[370],"variations":[372],"functional":[377],"Overall,":[382],"our":[383],"findings":[384],"uncover":[385],"potential":[386],"issues":[387],"limitations":[389],"arise":[391],"-based":[397],"lay":[401],"groundwork":[403],"AI":[406],"LLM-based":[408],"techniques.":[411]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":63},{"year":2024,"cited_by_count":23}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}