{"id":"https://openalex.org/W2969223458","doi":"https://doi.org/10.1109/tr.2019.2924932","title":"Large-Scale Empirical Studies on Effort-Aware Security Vulnerability Prediction Methods","display_name":"Large-Scale Empirical Studies on Effort-Aware Security Vulnerability Prediction Methods","publication_year":2019,"publication_date":"2019-08-22","ids":{"openalex":"https://openalex.org/W2969223458","doi":"https://doi.org/10.1109/tr.2019.2924932","mag":"2969223458"},"language":"en","primary_location":{"id":"doi:10.1109/tr.2019.2924932","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tr.2019.2924932","pdf_url":null,"source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100441911","display_name":"Xiang Chen","orcid":"https://orcid.org/0000-0002-1180-3891"},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiang Chen","raw_affiliation_strings":["School of Information Science and Technology, Nantong University, Nantong, China"],"affiliations":[{"raw_affiliation_string":"School of Information Science and Technology, Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089420241","display_name":"Yingquan Zhao","orcid":"https://orcid.org/0000-0003-2998-1052"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yingquan Zhao","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071655887","display_name":"Zhanqi Cui","orcid":"https://orcid.org/0000-0002-5537-9236"},"institutions":[{"id":"https://openalex.org/I78675632","display_name":"Beijing Information Science & Technology University","ror":"https://ror.org/04xnqep60","country_code":"CN","type":"education","lineage":["https://openalex.org/I78675632"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhanqi Cui","raw_affiliation_strings":["Computer School, Beijing Information Science and Technology University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Computer School, Beijing Information Science and Technology University, Beijing, China","institution_ids":["https://openalex.org/I78675632"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017417068","display_name":"Guozhu Meng","orcid":"https://orcid.org/0000-0001-6388-2571"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]},{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210156404","display_name":"Institute of Information Engineering","ror":"https://ror.org/04r53se39","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210156404"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Guozhu Meng","raw_affiliation_strings":["Institute of Information Engineering, Chinese Academy of Sciences, Beijing, China","School of Computer Science and Engineering, Nanyang Technological University, Singapore"],"affiliations":[{"raw_affiliation_string":"Institute of Information Engineering, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210156404","https://openalex.org/I19820366"]},{"raw_affiliation_string":"School of Computer Science and Engineering, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100355692","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0001-7300-9215"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["School of Computer Science and Engineering, Nanyang Technological University, Singapore"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100707447","display_name":"Zan Wang","orcid":"https://orcid.org/0000-0001-6173-8170"},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zan Wang","raw_affiliation_strings":["College of Intelligence and Computing, Tianjin University, Tianjin, China"],"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing, Tianjin University, Tianjin, China","institution_ids":["https://openalex.org/I162868743"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100441911"],"corresponding_institution_ids":["https://openalex.org/I199305430"],"apc_list":null,"apc_paid":null,"fwci":7.751,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":{"value":0.97365722,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"69","issue":"1","first_page":"70","last_page":"87"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7762717008590698},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7598901987075806},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5197931528091431},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5114485025405884},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.497165709733963},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.49126070737838745},{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.4651903212070465},{"id":"https://openalex.org/keywords/software-metric","display_name":"Software metric","score":0.46104854345321655},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4603199362754822},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.29929035902023315},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.1997530460357666}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7762717008590698},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7598901987075806},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5197931528091431},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5114485025405884},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.497165709733963},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.49126070737838745},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.4651903212070465},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.46104854345321655},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4603199362754822},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.29929035902023315},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.1997530460357666},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tr.2019.2924932","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tr.2019.2924932","pdf_url":null,"source":{"id":"https://openalex.org/S87725633","display_name":"IEEE Transactions on Reliability","issn_l":"0018-9529","issn":["0018-9529","1558-1721"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Reliability","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4833460672","display_name":null,"funder_award_id":"61202006","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7385005796","display_name":null,"funder_award_id":"61702041","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7844191210","display_name":null,"funder_award_id":"61602267","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8867248768","display_name":null,"funder_award_id":"QXTCP C201906","funder_id":"https://openalex.org/F4320326666","funder_display_name":"Beijing Information Science and Technology University"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320326666","display_name":"Beijing Information Science and Technology University","ror":"https://ror.org/04xnqep60"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W1615337185","https://openalex.org/W1641872551","https://openalex.org/W1961761736","https://openalex.org/W1969483024","https://openalex.org/W1975040830","https://openalex.org/W1997236144","https://openalex.org/W1997646511","https://openalex.org/W2003529494","https://openalex.org/W2010985317","https://openalex.org/W2043837581","https://openalex.org/W2048899404","https://openalex.org/W2050496630","https://openalex.org/W2051978688","https://openalex.org/W2061686014","https://openalex.org/W2062020999","https://openalex.org/W2067148378","https://openalex.org/W2067436653","https://openalex.org/W2079753286","https://openalex.org/W2090044595","https://openalex.org/W2098162425","https://openalex.org/W2099183630","https://openalex.org/W2104329051","https://openalex.org/W2105672266","https://openalex.org/W2110065044","https://openalex.org/W2118978333","https://openalex.org/W2123704085","https://openalex.org/W2126105956","https://openalex.org/W2133114940","https://openalex.org/W2143637886","https://openalex.org/W2147386665","https://openalex.org/W2149014999","https://openalex.org/W2151666086","https://openalex.org/W2154398797","https://openalex.org/W2163837601","https://openalex.org/W2172232422","https://openalex.org/W2242800359","https://openalex.org/W2244669237","https://openalex.org/W2276400542","https://openalex.org/W2295673959","https://openalex.org/W2343875716","https://openalex.org/W2385483600","https://openalex.org/W2394841101","https://openalex.org/W2408200640","https://openalex.org/W2462192250","https://openalex.org/W2467903332","https://openalex.org/W2474835145","https://openalex.org/W2548915941","https://openalex.org/W2559874352","https://openalex.org/W2560646185","https://openalex.org/W2594132308","https://openalex.org/W2606150376","https://openalex.org/W2617307387","https://openalex.org/W2731743750","https://openalex.org/W2735995639","https://openalex.org/W2740565296","https://openalex.org/W2744611928","https://openalex.org/W2748690817","https://openalex.org/W2766521509","https://openalex.org/W2767894374","https://openalex.org/W2772246504","https://openalex.org/W2774137886","https://openalex.org/W2774919641","https://openalex.org/W2774988559","https://openalex.org/W2795921267","https://openalex.org/W2802138742","https://openalex.org/W2805001156","https://openalex.org/W2895067255","https://openalex.org/W2922408684","https://openalex.org/W2962960733","https://openalex.org/W2963548617","https://openalex.org/W3105203384","https://openalex.org/W4234346598","https://openalex.org/W4250023757"],"related_works":["https://openalex.org/W2095999892","https://openalex.org/W2018764758","https://openalex.org/W2383689843","https://openalex.org/W1550668881","https://openalex.org/W2494189957","https://openalex.org/W1965637956","https://openalex.org/W2116876706","https://openalex.org/W2350290471","https://openalex.org/W2166119067","https://openalex.org/W1792767331"],"abstract_inverted_index":{"Security":[0],"vulnerability":[1,349],"prediction":[2,147],"(SVP)":[3],"can":[4,287,310],"identify":[5],"potential":[6],"vulnerable":[7,370],"modules":[8,167,236,332],"in":[9,77,120,142,170,209,226,240,248,295],"advance":[10],"and":[11,39,44,53,98,112,139,157,237,253,270,274,285,341,358,380,382,387],"then":[12,45],"help":[13],"developers":[14],"to":[15,22,60,122,333],"allocate":[16],"most":[17],"of":[18,29,63,68,79,92,126,198,267,297,319,347],"the":[19,27,36,61,66,104,117,154,158,162,166,171,196,249,254,292,304,320,345,355,359,368],"test":[20],"resources":[21],"these":[23,244,307,326],"modules.":[24],"To":[25,195],"evaluate":[26,243],"performance":[28,48,81,290,299],"different":[30,69,89,192],"SVP":[31,70,90,203,211,245,251,256],"methods,":[32,91,106,119,128,151,164],"we":[33,86,107,129,152,218,302,323,351],"should":[34],"take":[35],"security":[37],"audit":[38],"code":[40,268],"inspection":[41],"into":[42],"account":[43],"consider":[46,87,108,131,153],"effort-aware":[47,80,144,298],"measures":[49],"(such":[50],"as":[51,228],"ACC":[52],"P":[54],"<sub":[55],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[56],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">opt</sub>":[57],").":[58],"However,":[59],"best":[62,197],"our":[64,199,214],"knowledge,":[65,200],"effectiveness":[67],"methods":[71,97,111,134,204,246,264,280,294,309,328,339,357,384],"has":[72],"not":[73,206],"been":[74,207],"thoroughly":[75],"investigated":[76],"terms":[78,296],"measures.":[82,300],"In":[83,213],"this":[84],"article,":[85],"48":[88],"which":[93],"36":[94],"are":[95,100,168,183,190,390],"supervised":[96,105,279,383],"12":[99,180,191],"unsupervised":[101,163,193,263,338,356],"methods.":[102,115,194],"For":[103,116,149,161,314],"34":[109],"software-metric-based":[110,118],"two":[113,262],"text-mining-based":[114,150,388],"addition":[121],"a":[123,176],"large":[124],"number":[125],"classification":[127],"also":[130],"four":[132,275],"state-of-the-art":[133,278],"(i.e.,":[135,281],"EALR,":[136],"OneWay,":[137,283],"CBS,":[138,284],"MULTI)":[140],"proposed":[141,277],"recent":[143],"just-in-time":[145],"defect":[146],"studies.":[148,212],"Bag-of-Word":[155],"model":[156],"term-frequency-inverse-document-frequency":[159],"model.":[160],"all":[165,354],"ranked":[169],"ascendent":[172],"order":[173],"based":[174],"on":[175,377],"specific":[177],"metric.":[178],"Since":[179],"software":[181],"metrics":[182],"considered":[184,208],"when":[185,316],"measuring":[186],"extracted":[187],"modules,":[188],"there":[189],"over":[201],"40":[202],"have":[205,362,374],"previous":[210],"large-scale":[215],"empirical":[216],"studies,":[217],"use":[219],"three":[220,231],"real":[221],"open-source":[222],"web":[223,232],"applications":[224,233],"written":[225],"PHP":[227],"benchmark.":[229],"These":[230],"include":[234],"3466":[235],"223":[238],"vulnerabilities":[239],"total.":[241],"We":[242],"both":[247],"within-project":[250],"scenario":[252],"cross-project":[255],"scenario.":[257],"Empirical":[258],"results":[259],"show":[260],"that":[261,325,353],"[i.e.,":[265],"lines":[266],"(LOC)":[269],"Halstead's":[271],"volume":[272],"(HV)]":[273],"recently":[276],"MULTI,":[282],"EALR)":[286],"achieve":[288,311],"better":[289,312],"than":[291],"other":[293],"Then,":[301],"analyze":[303],"reasons":[305],"why":[306],"six":[308,327],"performance.":[313],"example,":[315],"using":[317],"20%":[318],"entire":[321],"efforts,":[322],"find":[324,352],"always":[329],"require":[330],"more":[331],"be":[334],"inspected,":[335],"especially":[336],"for":[337],"LOC":[340],"HV.":[342],"Finally,":[343],"from":[344],"view":[346],"practical":[348],"localization,":[350],"OneWay":[360],"method":[361],"high":[363],"false":[364],"alarms":[365],"before":[366],"finding":[367],"first":[369],"module.":[371],"This":[372],"may":[373],"an":[375],"impact":[376],"developers'":[378],"confidence":[379],"tolerance,":[381],"(especially":[385],"MULTI":[386],"methods)":[389],"preferred.":[391]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":1}],"updated_date":"2026-04-15T08:11:43.952461","created_date":"2025-10-10T00:00:00"}