{"id":"https://openalex.org/W4415743892","doi":"https://doi.org/10.1109/qrs-c65679.2025.00063","title":"Empirical Evaluation of LLMs for Automated Program Fault Localisation","display_name":"Empirical Evaluation of LLMs for Automated Program Fault Localisation","publication_year":2025,"publication_date":"2025-07-16","ids":{"openalex":"https://openalex.org/W4415743892","doi":"https://doi.org/10.1109/qrs-c65679.2025.00063"},"language":null,"primary_location":{"id":"doi:10.1109/qrs-c65679.2025.00063","is_oa":false,"landing_page_url":"https://doi.org/10.1109/qrs-c65679.2025.00063","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 25th International Conference on Software Quality, Reliability, and Security Companion (QRS-C)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101616745","display_name":"Yong Liu","orcid":"https://orcid.org/0000-0002-3812-9622"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yong Liu","raw_affiliation_strings":["Beijing University of Chemical Technology,Beijing,China"],"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology,Beijing,China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100786298","display_name":"Xin Wang","orcid":"https://orcid.org/0000-0002-9679-7426"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Wang","raw_affiliation_strings":["Beijing University of Chemical Technology,Beijing,China"],"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology,Beijing,China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004951458","display_name":"Hengyuan Liu","orcid":"https://orcid.org/0000-0002-5884-2089"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hengyuan Liu","raw_affiliation_strings":["Beijing University of Chemical Technology,Beijing,China"],"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology,Beijing,China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090067435","display_name":"R. Stephanie Huang","orcid":"https://orcid.org/0000-0002-9862-0368"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruishi Huang","raw_affiliation_strings":["Beijing University of Chemical Technology,Beijing,China"],"affiliations":[{"raw_affiliation_string":"Beijing University of Chemical Technology,Beijing,China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055056809","display_name":"Yonghao Wu","orcid":"https://orcid.org/0000-0002-4665-2827"},"institutions":[{"id":"https://openalex.org/I130541836","display_name":"Beijing Institute of Petrochemical Technology","ror":"https://ror.org/025s55q11","country_code":"CN","type":"education","lineage":["https://openalex.org/I130541836"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yonghao Wu","raw_affiliation_strings":["Beijing Institute of Petrochemical Technology,Beijing,China"],"affiliations":[{"raw_affiliation_string":"Beijing Institute of Petrochemical Technology,Beijing,China","institution_ids":["https://openalex.org/I130541836"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101616745"],"corresponding_institution_ids":["https://openalex.org/I75390827"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.32279772,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"454","last_page":"463"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5691999793052673,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5691999793052673,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.27570000290870667,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.06809999793767929,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.8313000202178955},{"id":"https://openalex.org/keywords/fault","display_name":"Fault (geology)","score":0.5940999984741211},{"id":"https://openalex.org/keywords/software-fault-tolerance","display_name":"Software fault tolerance","score":0.5819000005722046},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5651999711990356},{"id":"https://openalex.org/keywords/fault-injection","display_name":"Fault injection","score":0.44020000100135803},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.424699991941452},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.41530001163482666},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.41200000047683716}],"concepts":[{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.8313000202178955},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6359999775886536},{"id":"https://openalex.org/C175551986","wikidata":"https://www.wikidata.org/wiki/Q47089","display_name":"Fault (geology)","level":2,"score":0.5940999984741211},{"id":"https://openalex.org/C50712370","wikidata":"https://www.wikidata.org/wiki/Q4269346","display_name":"Software fault tolerance","level":3,"score":0.5819000005722046},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5651999711990356},{"id":"https://openalex.org/C2775928411","wikidata":"https://www.wikidata.org/wiki/Q2041312","display_name":"Fault injection","level":3,"score":0.44020000100135803},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.424699991941452},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.41530001163482666},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.41200000047683716},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4027999937534332},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3944000005722046},{"id":"https://openalex.org/C126953365","wikidata":"https://www.wikidata.org/wiki/Q5438152","display_name":"Fault coverage","level":3,"score":0.3935999870300293},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.34630000591278076},{"id":"https://openalex.org/C167391956","wikidata":"https://www.wikidata.org/wiki/Q1401211","display_name":"Fault model","level":3,"score":0.3280999958515167},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.31630000472068787},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.2989000082015991},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2892000079154968},{"id":"https://openalex.org/C63540848","wikidata":"https://www.wikidata.org/wiki/Q3140932","display_name":"Fault tolerance","level":2,"score":0.2831000089645386},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2831000089645386},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.25850000977516174},{"id":"https://openalex.org/C152745839","wikidata":"https://www.wikidata.org/wiki/Q5438153","display_name":"Fault detection and isolation","level":3,"score":0.25440001487731934}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/qrs-c65679.2025.00063","is_oa":false,"landing_page_url":"https://doi.org/10.1109/qrs-c65679.2025.00063","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 25th International Conference on Software Quality, Reliability, and Security Companion (QRS-C)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320330914","display_name":"Beijing Institute of Petrochemical Technology","ror":"https://ror.org/025s55q11"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W100454585","https://openalex.org/W1950030762","https://openalex.org/W1990785546","https://openalex.org/W2013655083","https://openalex.org/W2128049346","https://openalex.org/W2156723666","https://openalex.org/W2343875716","https://openalex.org/W2467903332","https://openalex.org/W2620081107","https://openalex.org/W2745750801","https://openalex.org/W2762786131","https://openalex.org/W2958754741","https://openalex.org/W2962715466","https://openalex.org/W3095453482","https://openalex.org/W3195442242","https://openalex.org/W4243127898","https://openalex.org/W4284702776","https://openalex.org/W4293072403","https://openalex.org/W4308641610","https://openalex.org/W4386436496","https://openalex.org/W4391579639","https://openalex.org/W4400582623","https://openalex.org/W4407691659","https://openalex.org/W4411551762","https://openalex.org/W4411552726","https://openalex.org/W4414281281"],"related_works":[],"abstract_inverted_index":{"Many":[0],"recent":[1],"service":[2],"interruptions":[3],"caused":[4],"by":[5,187,213,229],"software":[6,42],"faults":[7],"have":[8,27],"shown":[9],"that":[10,33,166,201,234],"fault":[11,64,79,102,138,149,159,177,241,259],"localisation":[12,65,80,103,139,160,260],"is":[13],"crucial":[14],"for":[15,62,239],"automated":[16,240],"debugging":[17],"and":[18,52,116,205,215,248],"repair.":[19],"In":[20],"this":[21,91],"context,":[22,170],"Large":[23],"Language":[24],"Models":[25],"(LLMs)":[26],"emerged":[28],"as":[29,46,151,153],"a":[30,39,96,168],"promising":[31,237],"tool":[32],"has":[34,58],"demonstrated":[35],"strong":[36],"capabilities":[37,104,238,261],"in":[38,148,175,191],"variety":[40],"of":[41,76,100,146,182,193,262],"engineering":[43,156],"tasks":[44],"such":[45],"code":[47,72],"generation,":[48],"program":[49],"repair,":[50],"summarisation":[51],"test":[53,203],"generation.":[54],"While":[55],"existing":[56,136],"research":[57,92],"either":[59],"leveraged":[60],"LLMs":[61,147,235],"function-level":[63,169],"or":[66],"fine-tuned":[67],"them":[68],"using":[69],"exclusively":[70],"faulty":[71,225,253],"snippets,":[73],"the":[74,110,113,124,144,158,180,209,258],"efficacy":[75],"LLM-based":[77],"statement-level":[78,101,137,176],"with":[81,179],"dynamic":[82,245],"execution":[83,246],"information":[84,247],"remains":[85],"inadequately":[86],"investigated.":[87],"Therefore,":[88],"to":[89,223,251],"address":[90],"gap,":[93],"we":[94,130],"present":[95],"comprehensive":[97],"empirical":[98,163],"evaluation":[99],"across":[105],"multiple":[106],"LLM":[107],"architectures,":[108],"including":[109],"ChatGPT":[111],"series,":[112,115],"DeepSeek":[114],"small-scale,":[117],"locally":[118],"deployed":[119],"open-source":[120],"language":[121],"models.":[122],"Utilising":[123],"extensively":[125],"validated":[126],"Defects4J":[127],"v1.5.0":[128],"benchmark,":[129],"systematically":[131],"compare":[132],"these":[133],"models":[134],"against":[135],"techniques.":[140],"We":[141],"also":[142],"investigate":[143],"consistency":[145],"localisation,":[150,178],"well":[152],"how":[154],"prompt":[155],"affects":[157],"effectiveness.":[161],"Our":[162],"findings":[164],"reveal":[165],"within":[167],"DeepSeek-R1":[171],"surpasses":[172],"all":[173],"baselines":[174],"integration":[181],"error":[183,206],"logs":[184,207],"enhancing":[185],"accuracy":[186,212,228],"72.6%":[188],"over":[189],"SmartFL":[190],"terms":[192],"TOP-1":[194,211],"metric.":[195],"Moreover,":[196],"our":[197],"ablation":[198],"experiments":[199],"demonstrate":[200],"removing":[202],"cases":[204],"reduces":[208],"LLM\u2019s":[210],"4.7%":[214],"8.6%":[216],"on":[217],"average,":[218],"respectively,":[219],"while":[220],"explicit":[221],"instructions":[222,250],"rank":[224,252],"statements":[226,254],"improves":[227],"6.9%.":[230],"These":[231],"observations":[232],"suggest":[233],"offer":[236],"localisation.":[242],"Furthermore,":[243],"utilising":[244],"providing":[249],"can":[255],"significantly":[256],"enhance":[257],"LLMs.":[263]},"counts_by_year":[],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-31T00:00:00"}