{"id":"https://openalex.org/W4403182498","doi":"https://doi.org/10.1109/tse.2024.3470333","title":"Multitask-Based Evaluation of Open-Source LLM on Software Vulnerability","display_name":"Multitask-Based Evaluation of Open-Source LLM on Software Vulnerability","publication_year":2024,"publication_date":"2024-10-07","ids":{"openalex":"https://openalex.org/W4403182498","doi":"https://doi.org/10.1109/tse.2024.3470333"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3470333","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3470333","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101626014","display_name":"Xin Yin","orcid":"https://orcid.org/0009-0005-3396-0571"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Yin","raw_affiliation_strings":["State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0005-3396-0571","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102737361","display_name":"Chao Ni","orcid":"https://orcid.org/0000-0002-2906-0598"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Ni","raw_affiliation_strings":["State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-2906-0598","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Blockchain and Data Security, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070990953","display_name":"Shaohua Wang","orcid":"https://orcid.org/0000-0001-5777-7759"},"institutions":[{"id":"https://openalex.org/I137867983","display_name":"Central University of Finance and Economics","ror":"https://ror.org/008e3hf02","country_code":"CN","type":"education","lineage":["https://openalex.org/I137867983"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaohua Wang","raw_affiliation_strings":["Central University of Finance and Economics, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5777-7759","affiliations":[{"raw_affiliation_string":"Central University of Finance and Economics, Beijing, China","institution_ids":["https://openalex.org/I137867983"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":29.7211,"has_fulltext":false,"cited_by_count":65,"citation_normalized_percentile":{"value":0.99669239,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"50","issue":"11","first_page":"3071","last_page":"3087"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.8784000277519226,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.8784000277519226,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11614","display_name":"Cloud Data Security Solutions","score":0.8073999881744385,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12479","display_name":"Web Application Security Vulnerabilities","score":0.7949000000953674,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8591312170028687},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.5474683046340942},{"id":"https://openalex.org/keywords/open-source-software","display_name":"Open source software","score":0.5406385064125061},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5036606192588806},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.48719948530197144},{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.45845431089401245},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3820343017578125},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.24460220336914062}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8591312170028687},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.5474683046340942},{"id":"https://openalex.org/C2988343187","wikidata":"https://www.wikidata.org/wiki/Q1130645","display_name":"Open source software","level":3,"score":0.5406385064125061},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5036606192588806},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.48719948530197144},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.45845431089401245},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3820343017578125},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.24460220336914062}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2024.3470333","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3470333","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3041480715","display_name":null,"funder_award_id":"2022J184","funder_id":"https://openalex.org/F4320332587","funder_display_name":"Natural Science Foundation of Ningbo"},{"id":"https://openalex.org/G5088726832","display_name":null,"funder_award_id":"62202419","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5286106294","display_name":null,"funder_award_id":"226-202200064","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320332587","display_name":"Natural Science Foundation of Ningbo","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":82,"referenced_works":["https://openalex.org/W2736601468","https://openalex.org/W2781491433","https://openalex.org/W2885030880","https://openalex.org/W2890264517","https://openalex.org/W2892195839","https://openalex.org/W2906845177","https://openalex.org/W2907705732","https://openalex.org/W2954121408","https://openalex.org/W2973379954","https://openalex.org/W2998879504","https://openalex.org/W3023647278","https://openalex.org/W3091588759","https://openalex.org/W3098605233","https://openalex.org/W3101228802","https://openalex.org/W3104656873","https://openalex.org/W3119507053","https://openalex.org/W3161799213","https://openalex.org/W3166095789","https://openalex.org/W3170092793","https://openalex.org/W3177116043","https://openalex.org/W3183469243","https://openalex.org/W3184339106","https://openalex.org/W3185341429","https://openalex.org/W3193682477","https://openalex.org/W3194346579","https://openalex.org/W3195612455","https://openalex.org/W3198685994","https://openalex.org/W3200192855","https://openalex.org/W4210556785","https://openalex.org/W4221033043","https://openalex.org/W4221166942","https://openalex.org/W4223464372","https://openalex.org/W4225592887","https://openalex.org/W4226279206","https://openalex.org/W4226485558","https://openalex.org/W4254635861","https://openalex.org/W4284667406","https://openalex.org/W4285490489","https://openalex.org/W4308643319","https://openalex.org/W4308732630","https://openalex.org/W4312436517","https://openalex.org/W4312969325","https://openalex.org/W4362598291","https://openalex.org/W4376167329","https://openalex.org/W4380993527","https://openalex.org/W4384155466","https://openalex.org/W4384159062","https://openalex.org/W4384304865","https://openalex.org/W4384345698","https://openalex.org/W4384345708","https://openalex.org/W4384345748","https://openalex.org/W4385245566","https://openalex.org/W4385562549","https://openalex.org/W4385572142","https://openalex.org/W4386185625","https://openalex.org/W4389161980","https://openalex.org/W4389519352","https://openalex.org/W4391558363","https://openalex.org/W4391579639","https://openalex.org/W4392563948","https://openalex.org/W4392669753","https://openalex.org/W4393406994","https://openalex.org/W4393763814","https://openalex.org/W4398785927","https://openalex.org/W6682631176","https://openalex.org/W6739585900","https://openalex.org/W6741002519","https://openalex.org/W6767260250","https://openalex.org/W6767288045","https://openalex.org/W6767858076","https://openalex.org/W6778883912","https://openalex.org/W6783227185","https://openalex.org/W6788241328","https://openalex.org/W6809524888","https://openalex.org/W6810604016","https://openalex.org/W6810681707","https://openalex.org/W6810738896","https://openalex.org/W6810874553","https://openalex.org/W6851228291","https://openalex.org/W6852746770","https://openalex.org/W6854013397","https://openalex.org/W6863467715"],"related_works":["https://openalex.org/W4376877853","https://openalex.org/W1493891899","https://openalex.org/W4250928611","https://openalex.org/W166480398","https://openalex.org/W1612808768","https://openalex.org/W167327709","https://openalex.org/W1977393088","https://openalex.org/W4387839566","https://openalex.org/W4210922265","https://openalex.org/W2288962794"],"abstract_inverted_index":{"This":[0,35],"paper":[1],"proposes":[2],"a":[3],"pipeline":[4,157],"for":[5],"quantitatively":[6],"evaluating":[7],"interactive":[8],"Large":[9],"Language":[10,56],"Models":[11,57],"(LLMs)":[12],"using":[13,26],"publicly":[14],"available":[15],"datasets.":[16],"We":[17,47],"carry":[18],"out":[19],"an":[20],"extensive":[21],"technical":[22],"evaluation":[23,36,156],"of":[24,41,100,164],"LLMs":[25,42,63,76,103,126,165],"Big-Vul":[27],"covering":[28],"four":[29],"different":[30],"common":[31],"software":[32,65,70,168],"vulnerability":[33,66,71,97,106],"tasks.":[34],"assesses":[37],"the":[38,50,96,138,145,162],"multi-tasking":[39],"capabilities":[40,99,163],"based":[43],"on":[44],"this":[45],"dataset.":[46],"find":[48],"that":[49],"existing":[51],"state-of-the-art":[52],"approaches":[53],"and":[54,73,79,89,144],"pre-trained":[55,87,122],"(LMs)":[58],"are":[59],"generally":[60],"superior":[61,83],"to":[62,86,112,121,147,150],"in":[64,69,129,136,141,166],"detection.":[67],"However,":[68],"assessment":[72,98],"location,":[74],"certain":[75],"(e.g.,":[77],"CodeLlama":[78],"WizardCoder)":[80],"have":[81],"demonstrated":[82],"performance":[84,119],"compared":[85,120],"LMs,":[88],"providing":[90],"more":[91],"contextual":[92],"information":[93],"can":[94],"enhance":[95],"LLMs.":[101],"Moreover,":[102],"exhibit":[104],"strong":[105],"description":[107],"capabilities,":[108],"but":[109],"their":[110,118,153],"tendency":[111],"produce":[113],"excessive":[114],"output":[115],"significantly":[116],"weakens":[117],"LMs.":[123],"Overall,":[124],"though":[125],"perform":[127],"well":[128],"some":[130],"aspects,":[131],"they":[132],"still":[133],"need":[134],"improvement":[135],"understanding":[137],"subtle":[139],"differences":[140],"code":[142],"vulnerabilities":[143,149],"ability":[146],"describe":[148],"fully":[151],"realize":[152],"potential.":[154],"Our":[155],"provides":[158],"valuable":[159],"insights":[160],"into":[161],"handling":[167],"vulnerabilities.":[169]},"counts_by_year":[{"year":2026,"cited_by_count":17},{"year":2025,"cited_by_count":44},{"year":2024,"cited_by_count":4}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}