{"id":"https://openalex.org/W4416799243","doi":"https://doi.org/10.1109/snpd65828.2025.11253974","title":"Evaluating the Source Code Review Performance of LLM-based AI Chatbots","display_name":"Evaluating the Source Code Review Performance of LLM-based AI Chatbots","publication_year":2025,"publication_date":"2025-06-25","ids":{"openalex":"https://openalex.org/W4416799243","doi":"https://doi.org/10.1109/snpd65828.2025.11253974"},"language":null,"primary_location":{"id":"doi:10.1109/snpd65828.2025.11253974","is_oa":false,"landing_page_url":"https://doi.org/10.1109/snpd65828.2025.11253974","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACIS 29th International Conference on Software Engineering, Artificial Intelligence, Networking and Parallel/Distributed Computing (SNPD)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Toya Kakimoto","orcid":null},"institutions":[{"id":"https://openalex.org/I136446963","display_name":"Okayama University of Science","ror":"https://ror.org/05aevyc10","country_code":"JP","type":"education","lineage":["https://openalex.org/I136446963"]},{"id":"https://openalex.org/I163770644","display_name":"Okayama University","ror":"https://ror.org/02pc6pc55","country_code":"JP","type":"education","lineage":["https://openalex.org/I163770644"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Toya Kakimoto","raw_affiliation_strings":["Okayama University,Information Technology, Electrical Engineering, and Mathematical and Data Sciences Program, Faculty of Engineering,Okayama,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Okayama University,Information Technology, Electrical Engineering, and Mathematical and Data Sciences Program, Faculty of Engineering,Okayama,Japan","institution_ids":["https://openalex.org/I136446963","https://openalex.org/I163770644"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075037207","display_name":"Hiroki Inayoshi","orcid":"https://orcid.org/0000-0003-3355-8804"},"institutions":[{"id":"https://openalex.org/I163770644","display_name":"Okayama University","ror":"https://ror.org/02pc6pc55","country_code":"JP","type":"education","lineage":["https://openalex.org/I163770644"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hiroki Inayoshi","raw_affiliation_strings":["Okayama University,Faculty of Environmental, Life, Natural Science and Technology,Okayama,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Okayama University,Faculty of Environmental, Life, Natural Science and Technology,Okayama,Japan","institution_ids":["https://openalex.org/I163770644"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013633970","display_name":"Hidetake Uwano","orcid":null},"institutions":[{"id":"https://openalex.org/I4210140361","display_name":"National Institute of Technology, Nara College","ror":"https://ror.org/034jd0m14","country_code":"JP","type":"education","lineage":["https://openalex.org/I4210120810","https://openalex.org/I4210140361"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hidetake Uwano","raw_affiliation_strings":["Nara College,National Institute of Technology,Nara,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nara College,National Institute of Technology,Nara,Japan","institution_ids":["https://openalex.org/I4210140361"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078686662","display_name":"Akito Monden","orcid":"https://orcid.org/0000-0003-4295-207X"},"institutions":[{"id":"https://openalex.org/I163770644","display_name":"Okayama University","ror":"https://ror.org/02pc6pc55","country_code":"JP","type":"education","lineage":["https://openalex.org/I163770644"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Akito Monden","raw_affiliation_strings":["Okayama University,Faculty of Environmental, Life, Natural Science and Technology,Okayama,Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Okayama University,Faculty of Environmental, Life, Natural Science and Technology,Okayama,Japan","institution_ids":["https://openalex.org/I163770644"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.46074533,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"593","last_page":"600"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8273000121116638,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8273000121116638,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.06440000236034393,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.018400000408291817,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.6647999882698059},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.6345000267028809},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5928000211715698},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4715999960899353},{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.4081999957561493},{"id":"https://openalex.org/keywords/static-program-analysis","display_name":"Static program analysis","score":0.4059999883174896},{"id":"https://openalex.org/keywords/code-smell","display_name":"Code smell","score":0.4018999934196472},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.38679999113082886}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7663000226020813},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.6647999882698059},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.6345000267028809},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5928000211715698},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5220000147819519},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4715999960899353},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.46480000019073486},{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.4081999957561493},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.4059999883174896},{"id":"https://openalex.org/C133237599","wikidata":"https://www.wikidata.org/wiki/Q2295111","display_name":"Code smell","level":5,"score":0.4018999934196472},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.38679999113082886},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.374099999666214},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.37310001254081726},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3596000075340271},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.34769999980926514},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.34389999508857727},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.32280001044273376},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.2989000082015991},{"id":"https://openalex.org/C121957198","wikidata":"https://www.wikidata.org/wiki/Q14365593","display_name":"KPI-driven code analysis","level":5,"score":0.2671999931335449},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2551000118255615}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/snpd65828.2025.11253974","is_oa":false,"landing_page_url":"https://doi.org/10.1109/snpd65828.2025.11253974","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACIS 29th International Conference on Software Engineering, Artificial Intelligence, Networking and Parallel/Distributed Computing (SNPD)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W4391558404","https://openalex.org/W4393213239","https://openalex.org/W4396242417","https://openalex.org/W4400582613","https://openalex.org/W4400583006"],"related_works":[],"abstract_inverted_index":{"Source":[0],"code":[1,39,83,102,137,159],"review":[2,52,84,160],"plays":[3],"a":[4,70,80],"critical":[5],"role":[6],"in":[7,15,79],"ensuring":[8],"software":[9],"quality":[10],"by":[11],"identifying":[12],"bugs":[13],"early":[14],"the":[16,24,51,60,145,149],"development":[17],"process.":[18],"Although":[19],"recent":[20],"studies":[21,42],"have":[22,43],"explored":[23],"use":[25],"of":[26,64,124],"large":[27],"language":[28],"models":[29],"(LLMs)":[30],"for":[31,156],"tasks":[32],"such":[33,100],"as":[34,101],"vulnerability":[35],"detection":[36,62,119],"and":[37,93,107,141,147],"automated":[38],"refinement,":[40],"few":[41],"evaluated":[44],"their":[45],"performance":[46,120],"against":[47],"human":[48,125],"reviewers":[49],"during":[50],"phase.":[53],"In":[54],"this":[55],"study,":[56],"we":[57],"empirically":[58],"evaluate":[59],"bug":[61,118],"capabilities":[63],"several":[65],"LLM-based":[66],"AI":[67,88],"chatbots":[68,89,116],"using":[69],"benchmark":[71],"C":[72],"program":[73],"containing":[74],"27":[75],"bugs,":[76],"originally":[77],"used":[78],"prior":[81],"human-based":[82],"study.":[85],"We":[86,133],"tested":[87],"including":[90],"GPT-4o,":[91],"Grok-3,":[92],"Claude":[94],"3.7":[95],"Sonnet,":[96],"under":[97],"varying":[98],"conditions":[99],"granularity,":[103,138],"specification":[104,139],"document":[105],"availability,":[106,140],"Chain-of-Thought":[108],"(CoT)":[109],"reasoning.":[110],"Our":[111],"results":[112],"show":[113],"that":[114,123,151],"while":[115],"achieve":[117],"comparable":[121],"to":[122,153],"reviewers,":[126],"they":[127],"do":[128],"not":[129],"yet":[130],"surpass":[131],"them.":[132],"further":[134],"analyze":[135],"how":[136],"CoT":[142],"reasoning":[143],"affect":[144],"performance,":[146],"discuss":[148],"challenges":[150],"need":[152],"be":[154],"addressed":[155],"effective":[157],"LLM-assisted":[158],"automation.":[161]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-11-28T00:00:00"}