{"id":"https://openalex.org/W7131067248","doi":"https://doi.org/10.48550/arxiv.2602.17838","title":"Using Mutation-Analysis to Examine an LLM's Ability to Summarize Code","display_name":"Using Mutation-Analysis to Examine an LLM's Ability to Summarize Code","publication_year":2026,"publication_date":"2026-02-19","ids":{"openalex":"https://openalex.org/W7131067248","doi":"https://doi.org/10.48550/arxiv.2602.17838"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.17838","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119872370","display_name":"Lara Khatib","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khatib, Lara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126610692","display_name":"Micheal Pu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pu, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102428026","display_name":"Bogdan Vasilescu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vasilescu, Bogdan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5029916309","display_name":"Meiyappan Nagappan","orcid":"https://orcid.org/0000-0003-4533-4728"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nagappan, Meiyappan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18205945,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.4528999924659729,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.4528999924659729,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.43459999561309814,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.010200000368058681,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.555400013923645},{"id":"https://openalex.org/keywords/mutation","display_name":"Mutation","score":0.48080000281333923},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.44200000166893005},{"id":"https://openalex.org/keywords/enhanced-data-rates-for-gsm-evolution","display_name":"Enhanced Data Rates for GSM Evolution","score":0.3100999891757965},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.30799999833106995}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6535000205039978},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.555400013923645},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.48080000281333923},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.44200000166893005},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36579999327659607},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3199000060558319},{"id":"https://openalex.org/C162307627","wikidata":"https://www.wikidata.org/wiki/Q204833","display_name":"Enhanced Data Rates for GSM Evolution","level":2,"score":0.3100999891757965},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.30799999833106995},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30799999833106995},{"id":"https://openalex.org/C2988963302","wikidata":"https://www.wikidata.org/wiki/Q629206","display_name":"Program code","level":2,"score":0.303600013256073},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.29649999737739563},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27889999747276306},{"id":"https://openalex.org/C110332635","wikidata":"https://www.wikidata.org/wiki/Q629498","display_name":"Genetic programming","level":2,"score":0.26159998774528503},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26080000400543213}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.17838","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.17838","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.17838","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.17838","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.4826650321483612,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"developers":[1],"increasingly":[2],"rely":[3],"on":[4,114,165],"LLM-generated":[5,252],"code":[6,36],"summaries":[7,20,253],"for":[8,144,149,249],"documentation,":[9],"testing,":[10],"and":[11,87,128,155,207,218],"review,":[12],"it":[13,39,54,102],"is":[14],"important":[15],"to":[16,95,147,216,222,230],"study":[17],"whether":[18,67,251],"these":[19],"accurately":[21],"reflect":[22,96,254],"what":[23,34,53],"the":[24,35,72,85,90,97,170,178],"program":[25,255],"actually":[26,55],"does.":[27],"LLMs":[28],"often":[29,185],"produce":[30],"confident":[31],"descriptions":[32],"of":[33,199],"looks":[37],"like":[38],"should":[40],"do":[41],"(intent),":[42],"while":[43,152,202],"missing":[44],"subtle":[45],"edge":[46],"cases":[47],"or":[48],"logic":[49],"changes":[50],"that":[51,64,135],"define":[52],"does":[56],"(behavior).":[57],"We":[58,100,133],"present":[59],"a":[60,68,78,81,195,203,210,246],"mutation-based":[61],"evaluation":[62],"methodology":[63],"directly":[65],"tests":[66],"summary":[69,94,136,196],"truly":[70],"matches":[71],"code's":[73],"logic.":[74],"Our":[75],"approach":[76,248],"generates":[77],"summary,":[79],"injects":[80],"targeted":[82],"mutation":[83,153,243],"into":[84],"code,":[86],"checks":[88],"if":[89],"LLM":[91],"updates":[92],"its":[93],"new":[98],"behavior.":[99],"validate":[101],"through":[103],"three":[104],"experiments":[105],"totalling":[106],"624":[107],"mutation-summary":[108],"evaluations":[109],"across":[110],"62":[111],"programs.":[112],"First,":[113],"12":[115],"controlled":[116],"synthetic":[117],"programs":[118,168],"with":[119,140,194,232],"324":[120],"mutations":[121,224],"varying":[122],"in":[123],"type":[124,154],"(statement,":[125],"value,":[126],"decision)":[127],"location":[129,156],"(beginning,":[130],"middle,":[131],"end).":[132],"find":[134],"accuracy":[137,197],"decreases":[138],"sharply":[139],"complexity":[141],"from":[142,169,236],"76.5%":[143],"single":[145],"functions":[146],"17.3%":[148],"multi-threaded":[150],"systems,":[151],"exhibit":[157],"weaker":[158],"effects.":[159],"Second,":[160],"testing":[161],"150":[162],"mutated":[163,192],"samples":[164],"50":[166],"human-written":[167],"Less":[171],"Basic":[172],"Python":[173],"Problems":[174],"(LBPP)":[175],"dataset":[176],"confirms":[177],"same":[179],"failure":[180],"patterns":[181],"persist":[182],"as":[183,225,245],"models":[184,228],"describe":[186],"algorithmic":[187,238],"intent":[188],"rather":[189,257],"than":[190,258],"actual":[191],"behavior":[193,256],"rate":[198],"49.3%.":[200],"Furthermore,":[201],"comparison":[204],"between":[205],"GPT-4":[206],"GPT-5.2":[208],"shows":[209],"substantial":[211],"performance":[212],"leap":[213],"(from":[214],"49.3%":[215],"85.3%)":[217],"an":[219],"improved":[220],"ability":[221],"identify":[223],"\"bugs\",":[226],"both":[227],"continue":[229],"struggle":[231],"distinguishing":[233],"implementation":[234],"details":[235],"standard":[237],"patterns.":[239,261],"This":[240],"work":[241],"establishes":[242],"analysis":[244],"systematic":[247],"assessing":[250],"superficial":[259],"textual":[260]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-24T00:00:00"}