{"id":"https://openalex.org/W4403210636","doi":"https://doi.org/10.1109/tse.2024.3475375","title":"Exploring the Effectiveness of LLMs in Automated Logging Statement Generation: An Empirical Study","display_name":"Exploring the Effectiveness of LLMs in Automated Logging Statement Generation: An Empirical Study","publication_year":2024,"publication_date":"2024-10-08","ids":{"openalex":"https://openalex.org/W4403210636","doi":"https://doi.org/10.1109/tse.2024.3475375"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3475375","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3475375","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://ink.library.smu.edu.sg/sis_research/11016","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100426730","display_name":"Yichen Li","orcid":"https://orcid.org/0009-0009-8370-644X"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Yichen Li","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0009-0009-8370-644X","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080873193","display_name":"Yintong Huo","orcid":"https://orcid.org/0009-0006-8798-5667"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yintong Huo","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0009-0006-8798-5667","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068670072","display_name":"Zhihan Jiang","orcid":"https://orcid.org/0009-0003-1988-6219"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhihan Jiang","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0009-0003-1988-6219","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098969559","display_name":"Renyi Zhong","orcid":"https://orcid.org/0000-0001-6626-4437"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Renyi Zhong","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-6626-4437","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101941950","display_name":"Pinjia He","orcid":"https://orcid.org/0000-0003-3377-8129"},"institutions":[{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pinjia He","raw_affiliation_strings":["The Chinese University of Hong Kong, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0003-3377-8129","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Shenzhen, China","institution_ids":["https://openalex.org/I4210116924"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yuxin Su","orcid":"https://orcid.org/0000-0002-3338-8561"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxin Su","raw_affiliation_strings":["School of Software Engineering, Sun Yat-sen University, Zhuhai, China"],"raw_orcid":"https://orcid.org/0000-0002-3338-8561","affiliations":[{"raw_affiliation_string":"School of Software Engineering, Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078533117","display_name":"Lionel Briand","orcid":"https://orcid.org/0000-0002-1393-1010"},"institutions":[{"id":"https://openalex.org/I153718931","display_name":"University of Ottawa","ror":"https://ror.org/03c4mmv16","country_code":"CA","type":"education","lineage":["https://openalex.org/I153718931"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Lionel C. Briand","raw_affiliation_strings":["University of Ottawa, Ottawa, ON, Canada","University of Ottawa, Canada"],"raw_orcid":"https://orcid.org/0000-0002-1393-1010","affiliations":[{"raw_affiliation_string":"University of Ottawa, Ottawa, ON, Canada","institution_ids":["https://openalex.org/I153718931"]},{"raw_affiliation_string":"University of Ottawa, Canada","institution_ids":["https://openalex.org/I153718931"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069596903","display_name":"Michael R. Lyu","orcid":"https://orcid.org/0000-0002-3666-5798"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Michael R. Lyu","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-3666-5798","affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5100426730"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":8.2784,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.97980705,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"50","issue":"12","first_page":"3188","last_page":"3207"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9431999921798706,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9431999921798706,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9075000286102295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7516451478004456},{"id":"https://openalex.org/keywords/logging","display_name":"Logging","score":0.6766219735145569},{"id":"https://openalex.org/keywords/statement","display_name":"Statement (logic)","score":0.6283969283103943},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.43064674735069275},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3941877484321594},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.38576260209083557},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.3411391079425812},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.3296492099761963},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.10497787594795227},{"id":"https://openalex.org/keywords/forestry","display_name":"Forestry","score":0.09931346774101257},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.08798936009407043},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.08769148588180542}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7516451478004456},{"id":"https://openalex.org/C125620115","wikidata":"https://www.wikidata.org/wiki/Q845249","display_name":"Logging","level":2,"score":0.6766219735145569},{"id":"https://openalex.org/C2777026412","wikidata":"https://www.wikidata.org/wiki/Q2684591","display_name":"Statement (logic)","level":2,"score":0.6283969283103943},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.43064674735069275},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3941877484321594},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.38576260209083557},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3411391079425812},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3296492099761963},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.10497787594795227},{"id":"https://openalex.org/C97137747","wikidata":"https://www.wikidata.org/wiki/Q38112","display_name":"Forestry","level":1,"score":0.09931346774101257},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.08798936009407043},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.08769148588180542},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2024.3475375","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3475375","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-12018","is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/11016","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-12018","is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/11016","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1482659561","display_name":null,"funder_award_id":"13/RC/2094-2","funder_id":"https://openalex.org/F4320320847","funder_display_name":"Science Foundation Ireland"},{"id":"https://openalex.org/G7609782405","display_name":null,"funder_award_id":"2024A1515010145","funder_id":"https://openalex.org/F4320337111","funder_display_name":"Basic and Applied Basic Research Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320320847","display_name":"Science Foundation Ireland","ror":"https://ror.org/0271asj38"},{"id":"https://openalex.org/F4320337111","display_name":"Basic and Applied Basic Research Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":91,"referenced_works":["https://openalex.org/W1886625064","https://openalex.org/W2039157918","https://openalex.org/W2096761130","https://openalex.org/W2170890990","https://openalex.org/W2508565087","https://openalex.org/W2531425405","https://openalex.org/W2618689158","https://openalex.org/W2626180518","https://openalex.org/W2759550170","https://openalex.org/W2763526563","https://openalex.org/W2786424616","https://openalex.org/W2888115557","https://openalex.org/W2888491619","https://openalex.org/W2888557792","https://openalex.org/W2899171197","https://openalex.org/W2954234581","https://openalex.org/W2973461969","https://openalex.org/W2991805784","https://openalex.org/W3011222537","https://openalex.org/W3089832981","https://openalex.org/W3095319910","https://openalex.org/W3098605233","https://openalex.org/W3105247453","https://openalex.org/W3147048975","https://openalex.org/W3159695152","https://openalex.org/W3162546773","https://openalex.org/W3183619936","https://openalex.org/W3196824794","https://openalex.org/W4205596491","https://openalex.org/W4221166942","https://openalex.org/W4243521886","https://openalex.org/W4246903272","https://openalex.org/W4252326246","https://openalex.org/W4281763794","https://openalex.org/W4284665614","https://openalex.org/W4284692184","https://openalex.org/W4285225959","https://openalex.org/W4285490493","https://openalex.org/W4286331401","https://openalex.org/W4288057765","https://openalex.org/W4295646147","https://openalex.org/W4312252175","https://openalex.org/W4366452022","https://openalex.org/W4378942602","https://openalex.org/W4384304728","https://openalex.org/W4384345619","https://openalex.org/W4384345639","https://openalex.org/W4384345684","https://openalex.org/W4384345708","https://openalex.org/W4384345785","https://openalex.org/W4385245566","https://openalex.org/W4385572142","https://openalex.org/W4386830495","https://openalex.org/W4388132131","https://openalex.org/W4388483105","https://openalex.org/W4388483492","https://openalex.org/W4389158953","https://openalex.org/W4391558438","https://openalex.org/W4391876619","https://openalex.org/W4398766423","https://openalex.org/W6637807111","https://openalex.org/W6679796720","https://openalex.org/W6682631176","https://openalex.org/W6739585900","https://openalex.org/W6763098168","https://openalex.org/W6763345169","https://openalex.org/W6766673545","https://openalex.org/W6782879696","https://openalex.org/W6783227185","https://openalex.org/W6788175385","https://openalex.org/W6798182279","https://openalex.org/W6799738915","https://openalex.org/W6809646742","https://openalex.org/W6810259726","https://openalex.org/W6810738896","https://openalex.org/W6810889274","https://openalex.org/W6838865847","https://openalex.org/W6839472138","https://openalex.org/W6843135089","https://openalex.org/W6849413121","https://openalex.org/W6850627172","https://openalex.org/W6852776802","https://openalex.org/W6854344711","https://openalex.org/W6854866820","https://openalex.org/W6855970221","https://openalex.org/W6856274983","https://openalex.org/W6857387928","https://openalex.org/W6857761454","https://openalex.org/W6863330218","https://openalex.org/W6870353149","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W2373969208","https://openalex.org/W2578023326","https://openalex.org/W167088980","https://openalex.org/W2475705533","https://openalex.org/W2800852182","https://openalex.org/W186129870","https://openalex.org/W3200522959","https://openalex.org/W4389944781","https://openalex.org/W2997993211","https://openalex.org/W120415280"],"abstract_inverted_index":{"Automated":[0],"logging":[1,30,56,79,86,108,168,179,189,204,223,226,267,288,310,320,328],"statement":[2,80,87],"generation":[3,43,88],"supports":[4],"developers":[5,54],"in":[6,32,40],"documenting":[7],"critical":[8],"software":[9],"runtime":[10],"behavior.":[11],"While":[12,216],"substantial":[13],"recent":[14],"research":[15],"has":[16,60],"focused":[17],"on":[18,75,222,297],"retrieval-based":[19,164],"and":[20,44,114,135,165,191,196,225,256,262,304,330],"learning-based":[21,167],"methods,":[22],"results":[23],"suggest":[24],"they":[25,232],"fail":[26],"to":[27,129,150,187],"provide":[28,305],"appropriate":[29],"statements":[31,109],"real-world":[33],"complex":[34],"software.":[35],"Given":[36],"the":[37,67,72,119,131,157,171,178,192,251,316,324],"great":[38],"success":[39],"natural":[41],"language":[42,46,49],"programming":[45,260],"comprehension,":[47],"large":[48],"models":[50],"(LLMs)":[51],"might":[52],"help":[53],"generate":[55],"statements,":[57],"but":[58],"this":[59,69],"not":[61],"yet":[62],"been":[63],"investigated.":[64],"To":[65],"fill":[66],"gap,":[68],"paper":[70,248],"performs":[71],"first":[73,83],"study":[74,229],"exploring":[76],"LLMs":[77,161,182,218,275],"for":[78,245,265,308,334],"generation.":[81],"We":[82,200],"build":[84],"a":[85,235,277],"dataset,":[89],"<italic":[90,97,100,105,116,132,136,141,240],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[91,98,101,106,117,133,137,142,241,282],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">LogBench</i>,":[92],"with":[93,104,287],"two":[94],"parts:":[95],"(1)":[96],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">LogBench-O</i>:":[99],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">3,870</i>":[102],"methods":[103,169],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">6,849</i>":[107],"collected":[110],"from":[111,123,148,170,212],"GitHub":[112],"repositories,":[113],"(2)":[115],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">LogBench-T</i>:":[118],"transformed":[120],"unseen":[121,208,289],"code":[122,213,263],"LogBench-O.":[124],"Then,":[125],"we":[126,155,176,271,300],"leverage":[127],"LogBench":[128],"evaluate":[130,177,202],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">effectiveness</i>":[134],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">generalization":[138],"capabilities</i>":[139],"(using":[140],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">LogBench-T</i>)":[143],"of":[144,159,181,194,239,253,318,326],"13":[145],"top-performing":[146],"LLMs,":[147],"60M":[149],"405B":[151],"parameters.":[152],"In":[153,269],"addition,":[154,270],"examine":[156],"performance":[158,279],"these":[160,298],"against":[162],"classical":[163],"machine":[166],"era":[172],"preceding":[173],"LLMs.":[174],"Specifically,":[175],"effectiveness":[180],"by":[183],"studying":[184],"their":[185,292],"ability":[186],"determine":[188],"ingredients":[190],"impact":[193],"prompts":[195],"external":[197,257],"program":[198],"information.":[199],"further":[201],"LLM's":[203],"generalization":[205,294],"capabilities":[206],"using":[207],"data":[209],"(LogBench-T)":[210],"derived":[211],"transformation":[214],"techniques.":[215],"existing":[217,274],"deliver":[219],"decent":[220],"predictions":[221],"levels":[224],"variables,":[227],"our":[228],"indicates":[230],"that":[231,273],"only":[233],"achieve":[234],"maximum":[236],"BLEU":[237],"score":[238],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">0.249</i>,":[242],"thus":[243],"calling":[244],"improvements.":[246],"The":[247],"also":[249],"highlights":[250],"importance":[252],"prompt":[254],"constructions":[255],"factors":[258],"(e.g.,":[259],"contexts":[261],"comments)":[264],"LLMs\u2019":[266],"performance.":[268],"observed":[272],"show":[276],"significant":[278],"drop":[280],"(<italic":[281],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">8.2%-16.2%</i>":[283],"decrease)":[284],"when":[285],"dealing":[286],"code,":[290],"revealing":[291],"unsatisfactory":[293],"capabilities.":[295],"Based":[296],"findings,":[299],"identify":[301],"five":[302],"implications":[303],"practical":[306,337],"advice":[307],"future":[309],"research.":[311],"Our":[312],"empirical":[313],"analysis":[314],"discloses":[315],"limitations":[317],"current":[319],"approaches":[321],"while":[322],"showcasing":[323],"potential":[325],"LLM-based":[327],"tools,":[329],"provides":[331],"actionable":[332],"guidance":[333],"building":[335],"more":[336],"models.":[338]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}