{"id":"https://openalex.org/W7167413474","doi":"https://doi.org/10.1145/3805760.3814892","title":"Using Mutation-Analysis to Examine an LLM's Ability to Summarize Code","display_name":"Using Mutation-Analysis to Examine an LLM's Ability to Summarize Code","publication_year":2026,"publication_date":"2026-07-05","ids":{"openalex":"https://openalex.org/W7167413474","doi":"https://doi.org/10.1145/3805760.3814892"},"language":null,"primary_location":{"id":"doi:10.1145/3805760.3814892","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805760.3814892","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805760.3814892","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119872370","display_name":"Lara Khatib","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Lara Khatib","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0009-0004-4770-8900","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140080204","display_name":"Michael Pu","orcid":"https://orcid.org/0009-0006-3374-8957"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Michael Pu","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0009-0006-3374-8957","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050821883","display_name":"Bogdan Vasilescu","orcid":"https://orcid.org/0000-0003-4418-5783"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bogdan Vasilescu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0000-0003-4418-5783","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029916309","display_name":"Meiyappan Nagappan","orcid":"https://orcid.org/0000-0003-4533-4728"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Meiyappan Nagappan","raw_affiliation_strings":["University of Waterloo, Waterloo, Canada"],"raw_orcid":"https://orcid.org/0000-0003-4533-4728","affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"51","last_page":"60"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.4778999984264374,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.4778999984264374,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.38119998574256897,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.013899999670684338,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.613099992275238},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.49219998717308044},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4537999927997589},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.36970001459121704},{"id":"https://openalex.org/keywords/enhanced-data-rates-for-gsm-evolution","display_name":"Enhanced Data Rates for GSM Evolution","score":0.3149999976158142}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7148000001907349},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.613099992275238},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.49219998717308044},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4537999927997589},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.38690000772476196},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.36970001459121704},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36239999532699585},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.3149999976158142},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3147999942302704},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.31060001254081726},{"id":"https://openalex.org/C2988963302","wikidata":"https://www.wikidata.org/wiki/Q629206","display_name":"Program code","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C163565370","wikidata":"https://www.wikidata.org/wiki/Q4308623","display_name":"Mutation testing","level":4,"score":0.30090001225471497},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29820001125335693},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2903999984264374},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2689000070095062},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26409998536109924},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2513999938964844}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805760.3814892","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805760.3814892","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805760.3814892","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805760.3814892","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM International Conference on AI-Powered Software","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.5100382566452026,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W2102742419","https://openalex.org/W2741643261","https://openalex.org/W2806532810","https://openalex.org/W3195727321","https://openalex.org/W4247854836","https://openalex.org/W4283705032","https://openalex.org/W4377028986","https://openalex.org/W4382654294","https://openalex.org/W4391876619","https://openalex.org/W4394745253","https://openalex.org/W4394745967"],"related_works":[],"abstract_inverted_index":{"As":[0],"developers":[1],"increasingly":[2],"rely":[3],"on":[4,114,167],"LLM-generated":[5,254],"code":[6,36],"summaries":[7,20,255],"for":[8,136,146,151,251],"documentation,":[9],"testing,":[10],"and":[11,87,128,157,209,220,270],"review,":[12],"it":[13,39,54,102],"is":[14],"important":[15],"to":[16,95,149,218,224,232],"study":[17],"whether":[18,67,253],"these":[19],"accurately":[21],"reflect":[22,96,256],"what":[23,34,53],"the":[24,35,72,85,90,97,172,180],"program":[25,257],"actually":[26,55],"does.":[27],"LLMs":[28],"often":[29,187],"produce":[30],"confident":[31],"descriptions":[32],"of":[33,201],"looks":[37],"like":[38],"should":[40],"do":[41],"(intent),":[42],"while":[43,154,204],"missing":[44],"subtle":[45],"edge":[46],"cases":[47],"or":[48],"logic":[49],"changes":[50],"that":[51,64],"define":[52],"does":[56],"(behavior).":[57],"We":[58,100,133],"present":[59],"a":[60,68,78,81,197,205,212,248],"mutation-based":[61],"evaluation":[62],"methodology":[63],"directly":[65],"tests":[66],"summary":[69,94,138,198],"truly":[70],"matches":[71],"code\u2019s":[73],"logic.":[74],"Our":[75],"approach":[76,250],"generates":[77],"summary,":[79],"injects":[80],"targeted":[82],"mutation":[83,155,245],"into":[84],"code,":[86],"checks":[88],"if":[89],"LLM":[91],"updates":[92],"its":[93],"new":[98],"behavior.":[99],"validate":[101],"through":[103],"three":[104],"experiments":[105],"totalling":[106],"624":[107],"mutation\u2013summary":[108],"evaluations":[109],"across":[110,267],"62":[111],"programs.":[112],"First,":[113],"12":[115],"controlled":[116],"synthetic":[117],"programs":[118,170],"with":[119,142,196,234],"324":[120],"mutations":[121,226],"varying":[122],"in":[123],"type":[124,156],"(statement,":[125],"value,":[126],"decision)":[127],"location":[129,158],"(beginning,":[130],"middle,":[131],"end).":[132],"find":[134],"that,":[135],"GPT-4,":[137],"accuracy":[139,199],"decreases":[140],"sharply":[141],"complexity":[143],"from":[144,171,238],"76.5%":[145],"single":[147],"functions":[148],"17.3%":[150],"multi-threaded":[152],"systems,":[153],"exhibit":[159],"weaker":[160],"effects.":[161],"Second,":[162],"testing":[163],"150":[164],"mutated":[165,194],"samples":[166],"50":[168],"human-written":[169],"Less":[173],"Basic":[174],"Python":[175],"Problems":[176],"(LBPP)":[177],"dataset":[178],"confirms":[179],"same":[181],"failure":[182],"patterns":[183],"persist":[184],"as":[185,227,247],"models":[186,230],"describe":[188],"algorithmic":[189,240],"intent":[190],"rather":[191,259],"than":[192,260],"actual":[193],"behavior":[195,258],"rate":[200],"49.3%.":[202],"Furthermore,":[203],"comparison":[206,266],"between":[207],"GPT-4":[208],"GPT-5.2":[210],"shows":[211],"substantial":[213],"performance":[214],"leap":[215],"(from":[216],"49.3%":[217],"85.3%)":[219],"an":[221],"improved":[222],"ability":[223],"identify":[225],"\u201dbugs\u201d,":[228],"both":[229],"continue":[231],"struggle":[233],"distinguishing":[235],"implementation":[236],"details":[237],"standard":[239],"patterns.":[241],"This":[242],"work":[243],"establishes":[244],"analysis":[246],"systematic":[249],"assessing":[252],"superficial":[261],"textual":[262],"patterns,":[263],"enabling":[264],"direct":[265],"models,":[268],"prompts,":[269],"datasets.":[271]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-07-06T00:00:00"}