{"id":"https://openalex.org/W7164046446","doi":"https://doi.org/10.48550/arxiv.2606.08588","title":"LLM vs. Human Unit Tests: Fault Detection on Real Python Bugs","display_name":"LLM vs. Human Unit Tests: Fault Detection on Real Python Bugs","publication_year":2026,"publication_date":"2026-06-07","ids":{"openalex":"https://openalex.org/W7164046446","doi":"https://doi.org/10.48550/arxiv.2606.08588"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.08588","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08588","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.08588","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138216725","display_name":"Phouvadeth Vathana","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vathana, Phouvadeth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019378384","display_name":"Prapti Bhatt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bhatt, Prapti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138246731","display_name":"Rishi Patel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Patel, Rishi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138205908","display_name":"Nasir U. Eisty","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eisty, Nasir U.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8223000168800354,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8223000168800354,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03420000150799751,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.026599999517202377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.7477999925613403},{"id":"https://openalex.org/keywords/unit-testing","display_name":"Unit testing","score":0.7080000042915344},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5306000113487244},{"id":"https://openalex.org/keywords/fault-detection-and-isolation","display_name":"Fault detection and isolation","score":0.5144000053405762},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4318000078201294},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.3522999882698059},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.302700012922287}],"concepts":[{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.7477999925613403},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7430999875068665},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.7080000042915344},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5306000113487244},{"id":"https://openalex.org/C152745839","wikidata":"https://www.wikidata.org/wiki/Q5438153","display_name":"Fault detection and isolation","level":3,"score":0.5144000053405762},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4410000145435333},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4318000078201294},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.3522999882698059},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3334999978542328},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3278000056743622},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3037000000476837},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.302700012922287},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.29580000042915344},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.2840000092983246},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.27630001306533813},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.2752000093460083},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.27059999108314514},{"id":"https://openalex.org/C126953365","wikidata":"https://www.wikidata.org/wiki/Q5438152","display_name":"Fault coverage","level":3,"score":0.26010000705718994},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C106159264","wikidata":"https://www.wikidata.org/wiki/Q17146789","display_name":"Random testing","level":4,"score":0.25540000200271606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.08588","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08588","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.08588","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.08588","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"have":[4],"shown":[5],"considerable":[6],"promise":[7],"for":[8,111,150],"automated":[9],"unit":[10,47],"test":[11],"generation,":[12],"yet":[13],"their":[14,163],"practical":[15],"effectiveness":[16],"relative":[17],"to":[18,109],"human-written":[19,46,113],"tests":[20,48,98,114],"remains":[21],"poorly":[22],"understood.":[23],"Existing":[24],"evaluations":[25],"commonly":[26],"rely":[27],"on":[28],"coverage-oriented":[29],"benchmarks":[30],"that":[31,86,144],"do":[32],"not":[33],"assess":[34],"fault-detection":[35,151],"capability":[36],"directly.":[37],"We":[38,153],"present":[39],"an":[40,147],"empirical":[41],"comparison":[42],"of":[43,106,171],"LLM-generated":[44,97],"and":[45,66,68,126,139,166,174],"across":[49],"three":[50],"complementary":[51,164],"Python":[52],"benchmarks:":[53],"29":[54],"real":[55],"historical":[56],"bugs":[57],"from":[58,64],"BugsInPy,":[59],"a":[60,69,81],"function-level":[61],"benchmark":[62,176],"drawn":[63],"python-slugify":[65],"packaging,":[67],"controlled":[70],"paired":[71],"benchmark.":[72],"Our":[73],"generation":[74,91],"pipeline":[75],"couples":[76],"Gemini":[77],"2.5":[78],"Flash":[79],"with":[80,99],"lightweight":[82],"lexical":[83],"retrieval":[84,172],"mechanism":[85],"supplies":[87],"bug-relevant":[88],"context":[89,101,173],"at":[90],"time.":[92],"Across":[93],"eight":[94],"quality":[95],"dimensions,":[96],"retrieval-augmented":[100],"detect":[102],"faults":[103],"in":[104,178],"69%":[105],"cases":[107],"compared":[108],"17.2%":[110],"general-purpose":[112],"(Fisher's":[115],"exact,":[116],"$p":[117],"<":[118],"0.001$,":[119],"Cohen's":[120],"$h":[121],"=":[122],"1.10$).":[123],"Critically,":[124],"line":[125],"branch":[127],"coverage":[128,145],"are":[129],"nearly":[130],"identical":[131],"between":[132],"the":[133,155,168],"two":[134],"approaches":[135],"(84.8%":[136],"vs.":[137,141],"88.5%":[138],"75.2%":[140],"82.1%),":[142],"confirming":[143],"is":[146],"insufficient":[148],"proxy":[149],"capability.":[152],"discuss":[154],"conditions":[156],"under":[157],"which":[158],"each":[159],"approach":[160],"excels,":[161],"characterize":[162],"strengths,":[165],"identify":[167],"critical":[169],"role":[170],"reproducible":[175],"construction":[177],"meaningful":[179],"test-quality":[180],"evaluation.":[181]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}