{"id":"https://openalex.org/W4395029328","doi":"https://doi.org/10.1109/tse.2024.3392499","title":"No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT","display_name":"No Need to Lift a Finger Anymore? Assessing the Quality of Code Generation by ChatGPT","publication_year":2024,"publication_date":"2024-04-23","ids":{"openalex":"https://openalex.org/W4395029328","doi":"https://doi.org/10.1109/tse.2024.3392499"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3392499","doi":"https://doi.org/10.1109/tse.2024.3392499","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3392499","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","fnege-3","jufo-3","norway-2","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://eprints.gla.ac.uk/view/author/72794.html>","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103105157","display_name":"Zhijie Liu","orcid":"https://orcid.org/0000-0003-0972-8631","observed_orcids":["https://orcid.org/0000-0003-0972-8631"]},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhijie Liu","raw_affiliation_strings":["ShanghaiTech University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0972-8631","affiliations":[{"raw_affiliation_string":"ShanghaiTech University, Shanghai, China","institution_ids":["https://openalex.org/I30809798"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000449889","display_name":"Yutian Tang","orcid":"https://orcid.org/0000-0001-5677-4564","observed_orcids":["https://orcid.org/0000-0001-5677-4564"]},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yutian Tang","raw_affiliation_strings":["University of Glasgow, Glasgow, U.K","University of Glasgow, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0001-5677-4564","affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, U.K","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"University of Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100400376","display_name":"Xiapu Luo","orcid":"https://orcid.org/0000-0002-9082-3208","observed_orcids":["https://orcid.org/0000-0002-9082-3208"]},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xiapu Luo","raw_affiliation_strings":["Department of Computing, Hong Kong Polytechnic University, Hong Kong SAR, China"],"raw_orcid":"https://orcid.org/0000-0002-9082-3208","affiliations":[{"raw_affiliation_string":"Department of Computing, Hong Kong Polytechnic University, Hong Kong SAR, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031391841","display_name":"Yuming Zhou","orcid":"https://orcid.org/0000-0002-4645-2526","observed_orcids":["https://orcid.org/0000-0002-4645-2526"]},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuming Zhou","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-4645-2526","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032937625","display_name":"Liang Feng Zhang","orcid":"https://orcid.org/0000-0003-3543-1524","observed_orcids":["https://orcid.org/0000-0003-3543-1524"]},"institutions":[{"id":"https://openalex.org/I30809798","display_name":"ShanghaiTech University","ror":"https://ror.org/030bhh786","country_code":"CN","type":"education","lineage":["https://openalex.org/I30809798"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang Feng Zhang","raw_affiliation_strings":["ShanghaiTech University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-3543-1524","affiliations":[{"raw_affiliation_string":"ShanghaiTech University, Shanghai, China","institution_ids":["https://openalex.org/I30809798"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2495,"currency":"USD","value_usd":2495},"apc_paid":null,"fwci":48.17,"has_fulltext":true,"cited_by_count":126,"citation_normalized_percentile":{"value":0.99835118,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"50","issue":"6","first_page":"1548","last_page":"1584"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8190000057220459},{"id":"https://openalex.org/keywords/automatic-code-generation","display_name":"automatic code generation","score":0.8830000162124634},{"id":"https://openalex.org/keywords/code-quality-assessment","display_name":"code quality assessment","score":0.6710000038146973},{"id":"https://openalex.org/keywords/program-correctness","display_name":"program correctness","score":0.7710000276565552},{"id":"https://openalex.org/keywords/code-complexity","display_name":"code complexity","score":0.8859999775886536},{"id":"https://openalex.org/keywords/software-vulnerabilities","display_name":"software vulnerabilities","score":0.5360000133514404},{"id":"https://openalex.org/keywords/common-weakness-enumeration","display_name":"Common Weakness Enumeration","score":0.8199999928474426},{"id":"https://openalex.org/keywords/nondeterminism","display_name":"non-determinism","score":0.7720000147819519},{"id":"https://openalex.org/keywords/programming-languages","display_name":"programming languages","score":0.37299999594688416}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8559305667877197},{"id":"https://openalex.org/C139002025","wikidata":"https://www.wikidata.org/wiki/Q3001212","display_name":"Lift (data mining)","level":2,"score":0.6643435955047607},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4859794080257416},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.4650949239730835},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.459811806678772},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.4534570574760437},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4384014904499054},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.1576600968837738},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.144182026386261},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.11609774827957153},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tse.2024.3392499","doi":"https://doi.org/10.1109/tse.2024.3392499","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3392499","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","fnege-3","jufo-3","norway-2","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:eprints.gla.ac.uk:324588","doi":null,"is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/72794.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"listed_in":["doaj"],"host_organization":"https://openalex.org/I4405259220","host_organization_name":"Institut Agama Islam Negeri Langsa","host_organization_lineage":["https://openalex.org/I4405259220"],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Articles"},{"id":"pmh:oai:arXiv.org:2308.04838","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2308.04838","pdf_url":"https://arxiv.org/pdf/2308.04838","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2308.04838","doi":"https://doi.org/10.48550/arxiv.2308.04838","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2308.04838","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:eprints.gla.ac.uk:324588","doi":null,"is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/72794.html>","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"listed_in":["doaj"],"host_organization":"https://openalex.org/I4405259220","host_organization_name":"Institut Agama Islam Negeri Langsa","host_organization_lineage":["https://openalex.org/I4405259220"],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Articles"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4223574508","display_name":null,"funder_award_id":"62202306","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5997361112","display_name":null,"funder_award_id":"62172205","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":77,"referenced_works":["https://openalex.org/W1538836445","https://openalex.org/W1566773348","https://openalex.org/W1964962870","https://openalex.org/W1980224781","https://openalex.org/W2121044470","https://openalex.org/W2132525863","https://openalex.org/W2157331557","https://openalex.org/W2157976942","https://openalex.org/W2620986014","https://openalex.org/W2749460454","https://openalex.org/W2768572539","https://openalex.org/W2786248241","https://openalex.org/W2787560479","https://openalex.org/W2795143051","https://openalex.org/W2884958048","https://openalex.org/W2896457183","https://openalex.org/W2955426500","https://openalex.org/W2962728167","https://openalex.org/W2963935794","https://openalex.org/W2996428491","https://openalex.org/W3008733198","https://openalex.org/W3020908159","https://openalex.org/W3025993830","https://openalex.org/W3043638540","https://openalex.org/W3085192122","https://openalex.org/W3098136301","https://openalex.org/W3098605233","https://openalex.org/W3099302725","https://openalex.org/W3099636232","https://openalex.org/W3126095862","https://openalex.org/W3138815606","https://openalex.org/W3161457214","https://openalex.org/W3162962341","https://openalex.org/W3177813494","https://openalex.org/W3196706143","https://openalex.org/W3198685994","https://openalex.org/W4205403018","https://openalex.org/W4225108562","https://openalex.org/W4226278401","https://openalex.org/W4226485558","https://openalex.org/W4229675450","https://openalex.org/W4239900777","https://openalex.org/W4285734663","https://openalex.org/W4286750487","https://openalex.org/W4287668913","https://openalex.org/W4288057765","https://openalex.org/W4288089799","https://openalex.org/W4312438588","https://openalex.org/W4315815628","https://openalex.org/W4384345708","https://openalex.org/W4384345745","https://openalex.org/W4384499126","https://openalex.org/W4385187421","https://openalex.org/W4385245566","https://openalex.org/W4385572722","https://openalex.org/W4391282616","https://openalex.org/W4399557965","https://openalex.org/W4400582376","https://openalex.org/W4407392582","https://openalex.org/W6643979850","https://openalex.org/W6676373471","https://openalex.org/W6718692554","https://openalex.org/W6755207826","https://openalex.org/W6768021236","https://openalex.org/W6769627184","https://openalex.org/W6778883912","https://openalex.org/W6787335730","https://openalex.org/W6794686226","https://openalex.org/W6795855765","https://openalex.org/W6798182279","https://openalex.org/W6810738896","https://openalex.org/W6810874553","https://openalex.org/W6840793632","https://openalex.org/W6852887568","https://openalex.org/W6853685553","https://openalex.org/W6856985672","https://openalex.org/W7133198585"],"related_works":["https://openalex.org/W2019257047","https://openalex.org/W2129714608","https://openalex.org/W4226317003","https://openalex.org/W2149635184","https://openalex.org/W2795746234","https://openalex.org/W2729258247","https://openalex.org/W3161903544","https://openalex.org/W2785752813","https://openalex.org/W2097099770","https://openalex.org/W4391835699"],"abstract_inverted_index":{"Large":[0],"language":[1,11],"models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"capabilities":[7],"across":[8],"various":[9],"natural":[10],"processing":[12],"(NLP)":[13],"tasks,":[14,35],"such":[15],"as":[16],"machine":[17],"translation,":[18],"question":[19],"answering,":[20],"summarization,":[21],"and":[22,100,102,132,166,244,279,281,288,316,336,344,352,359],"so":[23],"on.":[24],"Additionally,":[25],"LLMs":[26],"are":[27],"also":[28,135],"highly":[29],"valuable":[30,174],"in":[31,37,92,141,205,215,251,271,327,332,348],"supporting":[32],"software":[33],"engineering":[34],"particularly":[36],"the":[38,76,110,163,168,177,185,240,256,264,290,297,354],"field":[39],"of":[40,49,78,120,156,242,267,276,286,312,329],"code":[41,44,53,56,79,107,111,121,158,165,181,200,227,249,268,291,303,318,330,350,361],"generation.":[42,159],"Automatic":[43],"generation":[45,80,112,182,319,351,362],"is":[46,236],"a":[47,71,82,117],"process":[48,144,231,259,300],"automatically":[50],"generating":[51,197],"source":[52],"or":[54,61,262],"executable":[55],"based":[57],"on":[58,126,218],"given":[59],"specifications":[60],"requirements,":[62],"improving":[63,357],"developer":[64],"productivity.":[65],"In":[66],"this":[67,171],"study,":[68],"we":[69],"perform":[70],"systematic":[72],"empirical":[73],"assessment":[74],"to":[75,139,223,232],"quality":[77],"usingChatGPT,":[81],"recent":[83],"state-of-the-art":[84],"product":[85],"LLM.":[86],"We":[87,134],"leverage":[88],"728":[89],"algorithm":[90,272],"problems":[91,202,209],"five":[93],"languages":[94,207,253,275,285],"(i.e.,":[95],"C,":[96,277],"C++,":[97,278],"Java,":[98],"Python,":[99],"JavaScript)":[101],"18":[103],"CWEs":[104],"with":[105,212,228,274,284,308],"54":[106],"scenarios":[108,273,283],"for":[109,201,248,301,356],"task.":[113],"Our":[114],"evaluation":[115],"encompasses":[116],"comprehensive":[118],"analysis":[119],"snippets":[122,250,304,331],"generated":[123,153,164,292],"byChatGPT,":[124],"focusing":[125],"three":[127,186],"critical":[128,187],"aspects:":[129],"correctness,":[130,334],"complexity,":[131,335],"security.":[133,337],"specifically":[136],"investigateChatGPT\u2019s":[137],"ability":[138,222],"engage":[140],"multi-round":[142,229,257,298],"fixing":[143,152,230,258,299],"(i.e.,ChatGPT\u2019s":[145],"dialog":[146],"ability,":[147],"chatting":[148],"between":[149],"users":[150],"andChatGPTfor":[151],"buggy":[154],"code)":[155],"facilitating":[157],"By":[160],"delving":[161],"into":[162,176],"examining":[167],"experimental":[169,190],"results,":[170,307],"work":[172],"provides":[173],"insights":[175],"performance":[178],"ofChatGPTin":[179],"tackling":[180],"tasks":[183],"over":[184],"aspects.":[188],"The":[189],"results":[191],"demonstrate":[192],"that":[193,346],"(1)ChatGPTis":[194],"better":[195],"at":[196],"functionally":[198],"correct":[199,234],"before":[203],"2021":[204,211],"different":[206,252],"than":[208,310],"after":[210],"48.14%":[213],"advantage":[214],"Accepted":[216],"rate":[217],"judgment":[219],"platform,":[220],"butChatGPT\u2019s":[221],"directly":[224],"fix":[225],"erroneous":[226],"achieve":[233],"functionality":[235],"relatively":[237],"weak;":[238],"(2)":[239],"distribution":[241],"cyclomatic":[243],"cognitive":[245],"complexity":[246,265],"levels":[247,266],"varies.":[254],"Furthermore,":[255],"withChatGPTgenerally":[260],"preserves":[261],"increases":[263],"snippets;":[269],"(3)":[270],"Jave,":[280],"CWE":[282],"C":[287],"Python3,":[289],"byChatGPThas":[293],"relevant":[294],"vulnerabilities.":[295],"However,":[296],"vulnerable":[302],"demonstrates":[305],"promising":[306],"more":[309],"89%":[311],"vulnerabilities":[313],"successfully":[314],"addressed;":[315],"(4)":[317],"may":[320],"be":[321],"affected":[322],"byChatGPT\u2019s":[323],"non-determinism":[324],"factor,":[325],"resulting":[326],"variations":[328],"functional":[333],"Overall,":[338],"our":[339],"findings":[340],"uncover":[341],"potential":[342],"issues":[343],"limitations":[345],"arise":[347],"theChatGPT-based":[349],"lay":[353],"groundwork":[355],"AI":[358],"LLM-based":[360],"techniques.":[363]},"counts_by_year":[{"year":2026,"cited_by_count":34},{"year":2025,"cited_by_count":68},{"year":2024,"cited_by_count":24}],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2025-10-10T00:00:00"}