{"id":"https://openalex.org/W4415746025","doi":"https://doi.org/10.1109/icsme64153.2025.00023","title":"Toward Realistic Evaluations of Just-In-Time Vulnerability Prediction","display_name":"Toward Realistic Evaluations of Just-In-Time Vulnerability Prediction","publication_year":2025,"publication_date":"2025-09-07","ids":{"openalex":"https://openalex.org/W4415746025","doi":"https://doi.org/10.1109/icsme64153.2025.00023"},"language":null,"primary_location":{"id":"doi:10.1109/icsme64153.2025.00023","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icsme64153.2025.00023","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Software Maintenance and Evolution (ICSME)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107179038","display_name":"Duong Nguyen","orcid":null},"institutions":[{"id":"https://openalex.org/I94518387","display_name":"Hanoi University of Science and Technology","ror":"https://ror.org/04nyv3z04","country_code":"VN","type":"education","lineage":["https://openalex.org/I94518387"]}],"countries":["VN"],"is_corresponding":true,"raw_author_name":"Duong Nguyen","raw_affiliation_strings":["Hanoi University of Science and Technology,School of Communication and Information Technology,Hanoi,Vietnam"],"affiliations":[{"raw_affiliation_string":"Hanoi University of Science and Technology,School of Communication and Information Technology,Hanoi,Vietnam","institution_ids":["https://openalex.org/I94518387"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013306670","display_name":"Thanh Le-Cong","orcid":"https://orcid.org/0000-0002-9566-324X"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Thanh Le-Cong","raw_affiliation_strings":["The University of Melbourne,School of Computing and Information Systems,Melbourne,Australia"],"affiliations":[{"raw_affiliation_string":"The University of Melbourne,School of Computing and Information Systems,Melbourne,Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017801401","display_name":"Triet Huynh Minh Le","orcid":"https://orcid.org/0000-0003-1935-037X"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Triet Huynh Minh Le","raw_affiliation_strings":["The University of Adelaide,School of Computer and Mathematical Sciences,Adelaide,Australia"],"affiliations":[{"raw_affiliation_string":"The University of Adelaide,School of Computer and Mathematical Sciences,Adelaide,Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088992365","display_name":"M. Ali Babar","orcid":null},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"M. Ali Babar","raw_affiliation_strings":["The University of Adelaide,School of Computer and Mathematical Sciences,Adelaide,Australia"],"affiliations":[{"raw_affiliation_string":"The University of Adelaide,School of Computer and Mathematical Sciences,Adelaide,Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057073070","display_name":"Quyet\u2010Thang Huynh","orcid":null},"institutions":[{"id":"https://openalex.org/I94518387","display_name":"Hanoi University of Science and Technology","ror":"https://ror.org/04nyv3z04","country_code":"VN","type":"education","lineage":["https://openalex.org/I94518387"]}],"countries":["VN"],"is_corresponding":false,"raw_author_name":"Quyet-Thang Huynh","raw_affiliation_strings":["Hanoi University of Science and Technology,School of Communication and Information Technology,Hanoi,Vietnam"],"affiliations":[{"raw_affiliation_string":"Hanoi University of Science and Technology,School of Communication and Information Technology,Hanoi,Vietnam","institution_ids":["https://openalex.org/I94518387"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5107179038"],"corresponding_institution_ids":["https://openalex.org/I94518387"],"apc_list":null,"apc_paid":null,"fwci":3.6264,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.94851691,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.41260001063346863,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.41260001063346863,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.335999995470047,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.05249999836087227,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.6349999904632568},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4828000068664551},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4526999890804291},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.38920000195503235},{"id":"https://openalex.org/keywords/predictive-modelling","display_name":"Predictive modelling","score":0.37119999527931213},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.3587999939918518}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6881999969482422},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.6349999904632568},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4828000068664551},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.47540000081062317},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4526999890804291},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.38920000195503235},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.37119999527931213},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.3587999939918518},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34049999713897705},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.329800009727478},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3255999982357025},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.31360000371932983},{"id":"https://openalex.org/C167063184","wikidata":"https://www.wikidata.org/wiki/Q1400839","display_name":"Vulnerability assessment","level":3,"score":0.313400000333786},{"id":"https://openalex.org/C149629883","wikidata":"https://www.wikidata.org/wiki/Q660926","display_name":"Fraction (chemistry)","level":2,"score":0.30820000171661377},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.30230000615119934},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2879999876022339},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.26350000500679016},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.257099986076355}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icsme64153.2025.00023","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icsme64153.2025.00023","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE International Conference on Software Maintenance and Evolution (ICSME)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W1985871103","https://openalex.org/W1992114977","https://openalex.org/W1997236144","https://openalex.org/W2069268700","https://openalex.org/W2126166995","https://openalex.org/W2147386665","https://openalex.org/W2147751596","https://openalex.org/W2157353183","https://openalex.org/W2166336492","https://openalex.org/W2599212561","https://openalex.org/W2781491433","https://openalex.org/W2783078595","https://openalex.org/W2792987835","https://openalex.org/W2800788706","https://openalex.org/W2885030880","https://openalex.org/W2885541650","https://openalex.org/W2953482317","https://openalex.org/W2955991060","https://openalex.org/W2963351448","https://openalex.org/W2967556797","https://openalex.org/W2999309192","https://openalex.org/W3030391949","https://openalex.org/W3098605233","https://openalex.org/W3126095862","https://openalex.org/W3147362533","https://openalex.org/W3162494388","https://openalex.org/W3166095789","https://openalex.org/W3178061567","https://openalex.org/W3184339106","https://openalex.org/W3195612455","https://openalex.org/W4214525227","https://openalex.org/W4284709537","https://openalex.org/W4308643023","https://openalex.org/W4308732630","https://openalex.org/W4312492777","https://openalex.org/W4312640079","https://openalex.org/W4312793626","https://openalex.org/W4381304075","https://openalex.org/W4384345689","https://openalex.org/W4384345694","https://openalex.org/W4384345705","https://openalex.org/W4388422146","https://openalex.org/W4388483502","https://openalex.org/W4389165160","https://openalex.org/W4390604801","https://openalex.org/W4392301058","https://openalex.org/W4394746043","https://openalex.org/W4399125977","https://openalex.org/W4399667873","https://openalex.org/W4400350633","https://openalex.org/W4402443177","https://openalex.org/W4403413408","https://openalex.org/W4407137448"],"related_works":[],"abstract_inverted_index":{"Modern":[0],"software":[1],"systems":[2],"are":[3,48,193],"increasingly":[4],"complex,":[5],"presenting":[6],"significant":[7,114],"challenges":[8],"in":[9,116,148,195,200,222],"quality":[10],"assurance.":[11],"Just-intime":[12],"vulnerability":[13],"prediction":[14],"(JIT-VP)":[15],"is":[16,140],"a":[17,71,84,89,113,156],"proactive":[18],"approach":[19],"to":[20,121,136,143,218],"identifying":[21],"vulnerable":[22],"commits":[23,97,153],"and":[24,55,79,100,183,212],"providing":[25],"early":[26],"warnings":[27],"about":[28],"potential":[29],"security":[30],"risks.":[31],"However,":[32],"we":[33,87,166],"observe":[34],"that":[35,75,190],"current":[36],"JIT-VP":[37,68,110,211],"evaluations":[38,209],"rely":[39],"on":[40,129],"an":[41],"idealized":[42],"setting,":[43],"where":[44,151],"the":[45,65,101,126,144,168,197,205,213],"evaluation":[46],"datasets":[47],"artificially":[49],"balanced,":[50],"consisting":[51],"exclusively":[52],"of":[53,67,107,159,170,207,210],"vulnerability-introducing":[54,152],"vulnerability-fixing":[56],"commits.":[57,81,161],"To":[58,82,162],"address":[59,219],"this":[60,62,164],"limitation,":[61],"study":[63],"assesses":[64],"effectiveness":[66,169],"techniques":[69,111,173,192,217],"under":[70],"more":[72],"realistic":[73,208],"setting":[74],"includes":[76],"both":[77],"vulnerability-related":[78],"vulnerability-neutral":[80],"enable":[83],"reliable":[85],"evaluation,":[86],"introduce":[88],"large-scale":[90],"public":[91],"dataset":[92,176],"comprising":[93],"over":[94],"one":[95],"million":[96],"from":[98,134],"FFmpeg":[99],"Linux":[102,130],"kernel.":[103],"Our":[104],"empirical":[105],"analysis":[106],"eight":[108],"state-of-theart":[109],"reveals":[112],"decline":[115],"predictive":[117],"performance":[118],"when":[119],"applied":[120],"real-world":[122,149],"conditions;":[123],"for":[124,174,215],"example,":[125],"average":[127],"PR-AUC":[128],"drops":[131],"98":[132],"%":[133],"0.805":[135],"0.016.":[137],"This":[138],"discrepancy":[139],"mainly":[141],"attributed":[142],"severe":[145],"class":[146],"imbalance":[147,198,221],"datasets,":[150],"constitute":[154],"only":[155],"small":[157],"fraction":[158],"all":[160],"mitigate":[163],"issue,":[165],"explore":[167],"widely":[171],"adopted":[172],"handling":[175],"imbalance,":[177],"including":[178],"customized":[179],"loss":[180],"functions,":[181],"oversampling,":[182],"undersampling.":[184],"Surprisingly,":[185],"our":[186],"experimental":[187],"results":[188],"indicate":[189],"these":[191],"ineffective":[194],"addressing":[196],"problem":[199],"JIT-VP.":[201],"These":[202],"findings":[203],"underscore":[204],"importance":[206],"need":[214],"domain-specific":[216],"data":[220],"such":[223],"scenarios.":[224]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-10-31T00:00:00"}