{"id":"https://openalex.org/W7155571223","doi":"https://doi.org/10.48550/arxiv.2604.21579","title":"A Metamorphic Testing Approach to Diagnosing Memorization in LLM-Based Program Repair","display_name":"A Metamorphic Testing Approach to Diagnosing Memorization in LLM-Based Program Repair","publication_year":2026,"publication_date":"2026-04-23","ids":{"openalex":"https://openalex.org/W7155571223","doi":"https://doi.org/10.48550/arxiv.2604.21579"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.21579","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.21579","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.21579","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119604212","display_name":"Milan de Koning","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"De Koning, Milan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115630858","display_name":"Ali Asgari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Asgari, Ali","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034023299","display_name":"Pouria Derakhshanfar","orcid":"https://orcid.org/0000-0003-3549-9019"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Derakhshanfar, Pouria","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5067127346","display_name":"Annibale Panichella","orcid":"https://orcid.org/0000-0002-7395-3588"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Panichella, Annibale","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7387999892234802,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.7387999892234802,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.07850000262260437,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.05620000138878822,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memorization","display_name":"Memorization","score":0.8260999917984009},{"id":"https://openalex.org/keywords/debugging","display_name":"Debugging","score":0.7681000232696533},{"id":"https://openalex.org/keywords/discriminative-model","display_name":"Discriminative model","score":0.4650000035762787},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.3682999908924103},{"id":"https://openalex.org/keywords/random-testing","display_name":"Random testing","score":0.35760000348091125},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.2775999903678894}],"concepts":[{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.8260999917984009},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.7681000232696533},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6256999969482422},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.4650000035762787},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41670000553131104},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38839998841285706},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.3682999908924103},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.35760000348091125},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3474000096321106},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3206999897956848},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.30970001220703125},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2930000126361847},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.2754000127315521},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2621999979019165},{"id":"https://openalex.org/C2991870787","wikidata":"https://www.wikidata.org/wiki/Q181982","display_name":"Western hemisphere","level":2,"score":0.2567000091075897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.21579","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.21579","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.21579","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.21579","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM-based":[0,203],"automated":[1],"program":[2],"repair":[3,99],"(APR)":[4],"techniques":[5],"have":[6,174],"shown":[7],"promising":[8],"results":[9,16,121],"in":[10,69,131,202],"reducing":[11],"debugging":[12],"costs.":[13],"However,":[14],"prior":[15,70],"can":[17,51,197],"be":[18],"affected":[19],"by":[20,56,81],"data":[21,54,191],"leakage:":[22],"large":[23],"language":[24],"models":[25,164],"(LLMs)":[26],"may":[27],"memorize":[28],"bug":[29],"fixes":[30],"when":[31],"evaluation":[32],"benchmarks":[33,80],"overlap":[34],"with":[35,61,156,182],"their":[36],"pretraining":[37],"data,":[38],"leading":[39],"to":[40,85,144,173],"inflated":[41],"performance":[42,116],"estimates.":[43],"In":[44],"this":[45,152],"paper,":[46],"we":[47,50,96,149],"investigate":[48],"whether":[49],"better":[52,166],"reveal":[53],"leakage":[55],"combining":[57,180],"metamorphic":[58,194],"testing":[59,195],"(MT)":[60],"negative":[62],"log-likelihood":[63],"(NLL),":[64],"which":[65],"has":[66],"been":[67],"used":[68,88],"work":[71],"as":[72],"a":[73],"proxy":[74],"for":[75,142,146],"memorization.":[76],"We":[77],"construct":[78],"variant":[79],"applying":[82],"semantics-preserving":[83],"transformations":[84],"two":[86],"widely":[87],"datasets,":[89],"Defects4J":[90],"and":[91,108,111,118,186],"GitBug-Java.":[92],"Using":[93],"these":[94],"benchmarks,":[95,138,161],"evaluate":[97],"the":[98,113,159],"success":[100,134],"rates":[101,135],"of":[102,190],"seven":[103],"LLMs":[104,127],"on":[105,136,158,167],"both":[106],"original":[107,160],"transformed":[109,137],"versions,":[110],"analyze":[112],"relationship":[114],"between":[115],"degradation":[117,153],"NLL.":[119],"Our":[120],"show":[122,178],"that":[123,151,163,179],"all":[124],"evaluated":[125],"state-of-the-art":[126],"exhibit":[128],"substantial":[129],"drops":[130],"patch":[132],"generation":[133],"ranging":[139],"from":[140],"-4.1%":[141],"GPT-4o":[143],"-15.98%":[145],"Llama-3.1.":[147],"Furthermore,":[148],"find":[150],"strongly":[154],"correlates":[155],"NLL":[157,183],"suggesting":[162],"perform":[165],"instances":[168],"they":[169],"are":[170],"more":[171,187],"likely":[172],"memorized.":[175],"These":[176],"findings":[177],"MT":[181],"provides":[184],"stronger":[185],"reliable":[188],"evidence":[189],"leakage,":[192],"while":[193],"alone":[196],"help":[198],"mitigate":[199],"its":[200],"effects":[201],"APR":[204],"evaluations.":[205]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-25T00:00:00"}