{"id":"https://openalex.org/W7204112943","doi":"https://doi.org/10.1109/tr.2026.3716597","title":"LLMKernelBench: Benchmarking Large Language Models on Software Vulnerability Detection in Linux Kernel","display_name":"LLMKernelBench: Benchmarking Large Language Models on Software Vulnerability Detection in Linux Kernel","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7204112943","doi":"https://doi.org/10.1109/tr.2026.3716597"},"language":"en","primary_location":{"id":"doi:10.1109/tr.2026.3716597","doi":"https://doi.org/10.1109/tr.2026.3716597","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tr.2026.3716597","pdf_url":null,"source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-c","cwts-core","dongbi-b","jufo-2","norway-1","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/tr.2026.3716597","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114387275","display_name":"Arastoo Zibaeirad","orcid":"https://orcid.org/0009-0003-4213-9940","observed_orcids":["https://orcid.org/0009-0003-4213-9940"]},"institutions":[{"id":"https://openalex.org/I102149020","display_name":"University of North Carolina at Charlotte","ror":"https://ror.org/04dawnj30","country_code":"US","type":"education","lineage":["https://openalex.org/I102149020"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arastoo Zibaeirad","raw_affiliation_strings":["University of North Carolina at Charlotte"],"raw_orcid":"https://orcid.org/0009-0003-4213-9940","affiliations":[{"raw_affiliation_string":"University of North Carolina at Charlotte","institution_ids":["https://openalex.org/I102149020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108152227","display_name":"Rodrigo Pato Nogueira","orcid":"https://orcid.org/0009-0007-8486-3805","observed_orcids":["https://orcid.org/0009-0007-8486-3805"]},"institutions":[{"id":"https://openalex.org/I102149020","display_name":"University of North Carolina at Charlotte","ror":"https://ror.org/04dawnj30","country_code":"US","type":"education","lineage":["https://openalex.org/I102149020"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rodrigo Pato Nogueira","raw_affiliation_strings":["University of North Carolina at Charlotte"],"raw_orcid":"https://orcid.org/0009-0007-8486-3805","affiliations":[{"raw_affiliation_string":"University of North Carolina at Charlotte","institution_ids":["https://openalex.org/I102149020"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034213030","display_name":"Marco Vieira","orcid":"https://orcid.org/0000-0001-5103-8541","observed_orcids":["https://orcid.org/0000-0001-5103-8541"]},"institutions":[{"id":"https://openalex.org/I102149020","display_name":"University of North Carolina at Charlotte","ror":"https://ror.org/04dawnj30","country_code":"US","type":"education","lineage":["https://openalex.org/I102149020"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marco Vieira","raw_affiliation_strings":["University of North Carolina at Charlotte"],"raw_orcid":"https://orcid.org/0000-0001-5103-8541","affiliations":[{"raw_affiliation_string":"University of North Carolina at Charlotte","institution_ids":["https://openalex.org/I102149020"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I102149020"],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":{"value":2645,"currency":"USD","value_usd":2645},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"75","issue":null,"first_page":"3178","last_page":"3192"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.7792999744415283,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.7792999744415283,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.09690000116825104,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.0494999997317791,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9509999752044678},{"id":"https://openalex.org/keywords/software-vulnerability-detection","display_name":"software vulnerability detection","score":0.984000027179718},{"id":"https://openalex.org/keywords/linux-kernel","display_name":"Linux kernel","score":0.9369999766349792},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.7129999995231628},{"id":"https://openalex.org/keywords/common-weakness-enumeration","display_name":"Common Weakness Enumeration","score":0.8149999976158142},{"id":"https://openalex.org/keywords/binary-classification","display_name":"binary classification","score":0.38199999928474426},{"id":"https://openalex.org/keywords/training-data-leakage","display_name":"training data leakage","score":0.5529999732971191}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7922999858856201},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5960999727249146},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.5271999835968018},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5149999856948853},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.484499990940094},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.44130000472068787},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3589000105857849},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.33559998869895935},{"id":"https://openalex.org/C553261973","wikidata":"https://www.wikidata.org/wiki/Q14579","display_name":"Linux kernel","level":2,"score":0.3255999982357025},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3221000134944916},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.31619998812675476},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.31189998984336853},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2944999933242798},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28540000319480896},{"id":"https://openalex.org/C71134354","wikidata":"https://www.wikidata.org/wiki/Q458825","display_name":"Kernel density estimation","level":3,"score":0.28060001134872437},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.27549999952316284},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.25099998712539673}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tr.2026.3716597","doi":"https://doi.org/10.1109/tr.2026.3716597","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tr.2026.3716597","pdf_url":null,"source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-c","cwts-core","dongbi-b","jufo-2","norway-1","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1109/tr.2026.3716597","doi":"https://doi.org/10.1109/tr.2026.3716597","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tr.2026.3716597","pdf_url":null,"source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","ccf-c","cwts-core","dongbi-b","jufo-2","norway-1","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1520352740","https://openalex.org/W1825242443","https://openalex.org/W2117539524","https://openalex.org/W2168745915","https://openalex.org/W2928702109","https://openalex.org/W2964241064","https://openalex.org/W3086100325","https://openalex.org/W3098605233","https://openalex.org/W3163206498","https://openalex.org/W3183469243","https://openalex.org/W3191226037","https://openalex.org/W3215250117","https://openalex.org/W4206767299","https://openalex.org/W4226287673","https://openalex.org/W4308627645","https://openalex.org/W4318969196","https://openalex.org/W4384345694","https://openalex.org/W4388422146","https://openalex.org/W4391974543","https://openalex.org/W4392414327","https://openalex.org/W4399837262","https://openalex.org/W4400484796","https://openalex.org/W4402264467","https://openalex.org/W4410537167","https://openalex.org/W6959659194"],"related_works":["https://openalex.org/W4403969644","https://openalex.org/W7133399245","https://openalex.org/W7221017868","https://openalex.org/W7087433862","https://openalex.org/W7167261681","https://openalex.org/W7167910292","https://openalex.org/W7134170360","https://openalex.org/W7212289778","https://openalex.org/W7155641943","https://openalex.org/W7124557342"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,138,202],"(LLMs)":[3],"demonstrate":[4],"strong":[5],"capabilities":[6],"in":[7,13,116],"code-related":[8],"tasks,":[9],"however":[10],"their":[11],"effectiveness":[12],"software":[14],"vulnerability":[15,128],"detection":[16,129],"(SVD)":[17],"remains":[18,184],"poorly":[19],"understood":[20],"due":[21],"to":[22,188,196,231],"inadequate":[23],"evaluation":[24,35,105],"frameworks.":[25],"Existing":[26],"benchmarks":[27],"suffer":[28],"from":[29],"training":[30],"data":[31],"contamination,":[32],"isolated":[33],"function":[34],"without":[36,47],"cross-component":[37],"context,":[38],"lack":[39],"of":[40,59,160],"strict":[41],"vulnerable-patched":[42],"pairing,":[43],"and":[44,93,110,124,134,164,186,200,206],"binary":[45,182],"classification":[46],"hierarchy-aware":[48,111],"common":[49],"weakness":[50],"enumeration":[51],"(CWE)":[52],"assessment,":[53],"which":[54,226],"prevents":[55],"a":[56,71,86,94,165],"reliable":[57,172],"measurement":[58],"security":[60],"reasoning":[61],"versus":[62],"pattern":[63],"matching":[64],"on":[65,208,217,222],"leaked":[66],"data.":[67],"We":[68,118],"introduce":[69],"LLMKernelBench,":[70],"rigorous":[72],"benchmark":[73,88],"comprising":[74],"417":[75],"real-world":[76],"Linux":[77],"kernel":[78],"vulnerabilities":[79],"across":[80],"74":[81],"CWE":[82,150],"types,":[83],"split":[84],"into":[85],"primary":[87],"dataset":[89,97],"(PBD;":[90],"314":[91],"samples,$\\leq$2024)":[92],"leakage":[95],"free":[96],"(LFD;":[98],"103":[99],"samples,":[100],"2025":[101,180],"post-cutoff),":[102],"with":[103,155,198,229],"context-aware":[104],"at":[106],"three":[107],"granularity":[108],"levels":[109],"metrics":[112],"quantifying":[113],"semantic":[114],"proximity":[115,168],"misclassifications.":[117],"evaluate":[119],"seven":[120],"LLM":[121],"spanning":[122],"code-specialized":[123,199],"general-purpose":[125,201],"architectures.":[126],"Binary":[127],"is":[130,152,191,215,227],"near-random":[131,185],"($\\sim\\!\\!":[132],"50\\%$accuracy)":[133],"strongly":[135],"biased:":[136],"some":[137],"label$>75\\%$of":[139],"samples":[140],"as":[141,148],"vulnerable,":[142],"while":[143],"others":[144],"mostly":[145],"label":[146],"them":[147],"nonvulnerable.":[149],"prediction":[151],"effectively":[153],"unusable,":[154],"an":[156,238],"average":[157],"Top-1":[158],"accuracy":[159,183],"1.4%":[161],"(best:":[162],"3.3%)":[163],"12.3%":[166],"hierarchy":[167],"score,":[169],"providing":[170],"little":[171],"exact":[173],"or":[174],"taxonomy-level":[175],"signal.":[176],"On":[177],"the":[178],"leakage-free":[179],"split,":[181],"robustness":[187],"multifile":[189],"abstraction":[190],"model-specific":[192],"rather":[193],"than":[194,221],"tied":[195],"specialization,":[197],"degrading":[203],"by":[204],"2.6%":[205],"4.5%":[207],"average,":[209],"respectively.":[210],"The":[211],"micro-to-macro":[212],"CWE-accuracy":[213],"gap":[214],"larger":[216],"LFD":[218],"(6.6":[219],"points)":[220],"PBD":[223],"(0.9":[224],"points),":[225],"consistent":[228],"sensitivity":[230],"class":[232],"frequency":[233],"but":[234],"does":[235],"not":[236],"identify":[237],"internal":[239],"model":[240],"mechanism.":[241]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-25T00:00:00"}