{"id":"https://openalex.org/W7223661145","doi":"https://doi.org/10.48550/arxiv.2610.32548","title":"Beyond Coverage and Mutation: A Blind-Spot Analysis of LLM-Generated Unit Tests","display_name":"Beyond Coverage and Mutation: A Blind-Spot Analysis of LLM-Generated Unit Tests","publication_year":2026,"publication_date":"2026-10-23","ids":{"openalex":"https://openalex.org/W7223661145","doi":"https://doi.org/10.48550/arxiv.2610.32548"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.32548","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.32548","pdf_url":"https://arxiv.org/pdf/2610.32548","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.32548","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5159669365","display_name":"Xiaoyu Shi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Shi, Xiaoyu","raw_affiliation_strings":["Aalto University , Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalto University , Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159689740","display_name":"Wenqiang Luo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]},{"id":"https://openalex.org/I39333907","display_name":"Yanshan University","ror":"https://ror.org/02txfnf15","country_code":"CN","type":"education","lineage":["https://openalex.org/I39333907"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Luo, Wenqiang","raw_affiliation_strings":["City University of Hong Kong , China BOYANG YANG ,","Yanshan University , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"City University of Hong Kong , China BOYANG YANG ,","institution_ids":["https://openalex.org/I168719708"]},{"raw_affiliation_string":"Yanshan University , China","institution_ids":["https://openalex.org/I39333907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159732178","display_name":"Boyang Yang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Boyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5159730753","display_name":"Haoye Tian","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Tian, Haoye","raw_affiliation_strings":["Aalto University , Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Aalto University , Finland","institution_ids":["https://openalex.org/I9927081"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/java-benchmarks","display_name":"Java benchmarks","score":0.628000020980835},{"id":"https://openalex.org/keywords/code-complexity","display_name":"code complexity","score":0.6299999952316284},{"id":"https://openalex.org/keywords/execution-reliability","display_name":"execution reliability","score":0.5360000133514404}],"concepts":[{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.8302000164985657},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.6805999875068665},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.598800003528595},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.5918999910354614},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5767999887466431},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.574999988079071},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.4754999876022339},{"id":"https://openalex.org/C148027188","wikidata":"https://www.wikidata.org/wiki/Q907375","display_name":"Unit testing","level":3,"score":0.40459999442100525},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.388700008392334},{"id":"https://openalex.org/C150921843","wikidata":"https://www.wikidata.org/wiki/Q1170431","display_name":"Resampling","level":2,"score":0.3598000109195709},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.35109999775886536},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.34540000557899475},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.34200000762939453},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.3418999910354614},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3353999853134155},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.3149999976158142},{"id":"https://openalex.org/C122637931","wikidata":"https://www.wikidata.org/wiki/Q118084","display_name":"Unit (ring theory)","level":2,"score":0.31369999051094055},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3019999861717224},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.29249998927116394},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.2919999957084656},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.2827000021934509},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2565000057220459},{"id":"https://openalex.org/C80478641","wikidata":"https://www.wikidata.org/wiki/Q195771","display_name":"Sequential analysis","level":2,"score":0.2542000114917755}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.32548","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.32548","pdf_url":"https://arxiv.org/pdf/2610.32548","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.32548","doi":"https://doi.org/10.48550/arxiv.2610.32548","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.32548","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.32548","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.32548","pdf_url":"https://arxiv.org/pdf/2610.32548","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7223661145.pdf","grobid_xml":"https://content.openalex.org/works/W7223661145.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1964962870","https://openalex.org/W1971650562","https://openalex.org/W1992639336","https://openalex.org/W1996290308","https://openalex.org/W2003385991","https://openalex.org/W2013711971","https://openalex.org/W2041713059","https://openalex.org/W2106072155","https://openalex.org/W2114208144","https://openalex.org/W2135841285","https://openalex.org/W2135937266","https://openalex.org/W2147002252","https://openalex.org/W2164777277","https://openalex.org/W2174442795","https://openalex.org/W2466388701","https://openalex.org/W2490092854","https://openalex.org/W2586946380","https://openalex.org/W2725449579","https://openalex.org/W2788962378","https://openalex.org/W2884302104","https://openalex.org/W3164133168","https://openalex.org/W3177813494","https://openalex.org/W4384304747","https://openalex.org/W4393305539","https://openalex.org/W4394015350","https://openalex.org/W4399668074","https://openalex.org/W4400484796","https://openalex.org/W4400582690","https://openalex.org/W4400582872","https://openalex.org/W4403536268","https://openalex.org/W4409796230","https://openalex.org/W6967084484","https://openalex.org/W7079557847","https://openalex.org/W7141844560","https://openalex.org/W7171561727"],"related_works":["https://openalex.org/W7212041177","https://openalex.org/W4404318737","https://openalex.org/W7142196394","https://openalex.org/W7171561727","https://openalex.org/W7162811052","https://openalex.org/W4403536388","https://openalex.org/W4416874128","https://openalex.org/W7125903811","https://openalex.org/W4400267179","https://openalex.org/W4413212461"],"abstract_inverted_index":{"Existing":[0],"evaluation":[1,46],"protocols":[2],"for":[3,108,144,199],"unit":[4],"tests":[5,93,164],"rely":[6],"on":[7,174,192],"aggregate":[8,242],"metrics":[9,18],"such":[10],"as":[11],"branch":[12],"coverage":[13,115,125,141],"and":[14,62,83,129,136,148,158,177,184,204,233],"mutation":[15],"score.":[16],"These":[17,206],"quantify":[19],"the":[20,34,50,166],"extent":[21],"to":[22],"which":[23],"a":[24,76],"test":[25,111],"suite":[26],"exercises":[27],"program":[28],"behavior,":[29],"but":[30,194],"do":[31],"not":[32,118,220],"identify":[33,229],"behavioral":[35],"obligations":[36],"it":[37],"systematically":[38],"leaves":[39],"unchecked.":[40,187],"We":[41,72],"propose":[42],"BlindSpotEval,":[43],"an":[44,67],"obligation-based":[45],"framework":[47,54],"aligned":[48],"with":[49,87,139,196],"Arrange-Act-Assert":[51],"pattern.":[52],"The":[53],"comprises":[55],"three":[56],"metrics,":[57],"Boundary-Value":[58],"Adequacy,":[59,61],"Control-Flow":[60,152],"Oracle":[63,188],"Strength,":[64],"complemented":[65],"by":[66,241],"assessment":[68],"of":[69,110,126,155],"execution":[70],"reliability.":[71],"also":[73,209],"introduce":[74],"ObligBench,":[75],"complexity-stratified":[77],"Java":[78],"benchmark":[79],"comprising":[80],"895":[81],"complex":[82],"700":[84],"low-complexity":[85,212],"functions":[86],"source-side":[88],"obligation":[89,231],"annotations.":[90],"Evaluating":[91],"14,320":[92],"from":[94],"four":[95,98],"LLMs":[96],"under":[97],"prompting":[99],"strategies":[100],"reveals":[101],"systematic":[102],"blind":[103,207],"spots.":[104],"Incorrect":[105],"assertions":[106,198],"account":[107],"15.6%":[109],"failures,":[112],"illustrating":[113],"that":[114,215,228],"alone":[116,218],"does":[117,219],"establish":[119],"oracle":[120],"correctness.":[121],"Among":[122],"executable":[123],"tests,":[124,172],"null,":[127],"negative,":[128],"empty":[130],"arguments":[131],"reaches":[132],"only":[133,156],"34.8%,":[134],"23.8%,":[135],"38.8%,":[137],"compared":[138],"average":[140,193],"above":[142],"93%":[143],"booleans.":[145],"Catch":[146],"blocks":[147],"default":[149],"groups":[150],"have":[151],"Adequacy":[153],"scores":[154,243],"0.100":[157],"0.312.":[159],"Across":[160],"both":[161],"dimensions,":[162],"LLM-generated":[163],"exhibit":[165],"happy-path":[167],"bias":[168],"observed":[169],"in":[170,211],"human-written":[171],"concentrating":[173],"normal":[175],"inputs":[176],"expected":[178],"flows":[179],"while":[180],"leaving":[181],"unusual":[182],"values":[183,203],"exceptional":[185],"paths":[186],"strength":[189],"is":[190],"high":[191],"uneven,":[195],"weaker":[197],"object":[200],"return":[201],"property":[202],"exceptions.":[205],"spots":[208],"occur":[210],"functions,":[213],"suggesting":[214],"code":[216],"complexity":[217],"explain":[221],"them.":[222],"BlindSpotEval":[223],"provides":[224],"construct-level":[225],"adequacy":[226],"profiles":[227],"under-tested":[230],"types":[232],"can":[234],"guide":[235],"targeted":[236],"improvements":[237],"beyond":[238],"those":[239],"supported":[240],"alone.":[244]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-27T00:00:00"}