{"id":"https://openalex.org/W4407806421","doi":"https://doi.org/10.48550/arxiv.2502.14202","title":"Do LLMs Consider Security? An Empirical Study on Responses to Programming Questions","display_name":"Do LLMs Consider Security? An Empirical Study on Responses to Programming Questions","publication_year":2025,"publication_date":"2025-02-20","ids":{"openalex":"https://openalex.org/W4407806421","doi":"https://doi.org/10.48550/arxiv.2502.14202"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.14202","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.14202","pdf_url":"https://arxiv.org/pdf/2502.14202","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.14202","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102534201","display_name":"Amirali Sajadi","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Sajadi, Amirali","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113782778","display_name":"Binh Le","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Le, Binh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053404847","display_name":"Thu Anh Nguyen","orcid":"https://orcid.org/0000-0002-2089-2902"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Anh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073349866","display_name":"Kostadin Damevski","orcid":"https://orcid.org/0000-0001-7799-2026"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Damevski, Kostadin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5049106181","display_name":"Preetha Chatterjee","orcid":"https://orcid.org/0000-0003-3057-7807"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chatterjee, Preetha","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5102534201"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.9679999947547913,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.9679999947547913,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.46376726031303406},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3738032877445221},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.11118993163108826},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09203976392745972}],"concepts":[{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.46376726031303406},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3738032877445221},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11118993163108826},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09203976392745972}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.14202","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.14202","pdf_url":"https://arxiv.org/pdf/2502.14202","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.14202","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.14202","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.14202","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.14202","pdf_url":"https://arxiv.org/pdf/2502.14202","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"The":[0],"widespread":[1],"adoption":[2],"of":[3,17,49,99,118,146,165,188,211,227],"conversational":[4],"LLMs":[5,66,159,195],"for":[6],"software":[7],"development":[8],"has":[9],"raised":[10],"new":[11],"security":[12,50,100,198],"concerns":[13],"regarding":[14],"the":[15,31,47,83,92,111,115,119,158,206,225],"safety":[16],"LLM-generated":[18],"content.":[19],"Our":[20,126],"motivational":[21],"study":[22,44],"outlines":[23],"ChatGPT's":[24],"potential":[25,116],"in":[26],"volunteering":[27],"context-specific":[28],"information":[29,109,170,204],"to":[30,45,75,82,121,134,149,161,168,214,240],"developers,":[32],"promoting":[33],"safe":[34],"coding":[35],"practices.":[36],"Motivated":[37],"by":[38,53],"this":[39],"finding,":[40],"we":[41,103,219],"conduct":[42],"a":[43,97,143,232],"evaluate":[46,76],"degree":[48,98],"awareness":[51],"exhibited":[52],"three":[54,131],"prominent":[55],"LLMs:":[56],"Claude":[57],"3,":[58],"GPT-4,":[59],"and":[60,114,137,172,209,230],"Llama":[61],"3.":[62],"We":[63,154],"prompt":[64],"these":[65],"with":[67],"Stack":[68,215],"Overflow":[69,216],"questions":[70,84],"that":[71,129,157,236],"contain":[72],"vulnerable":[73],"code":[74],"whether":[77,105],"they":[78,87,200],"merely":[79],"provide":[80,108,202,220],"answers":[81],"or":[85,191],"if":[86],"also":[88,155],"warn":[89,138],"users":[90,139],"about":[91,110,140],"insecure":[93],"code,":[94],"thereby":[95],"demonstrating":[96],"awareness.":[101,125],"Further,":[102],"assess":[104],"LLM":[106,245],"responses":[107],"causes,":[112,207],"exploits,":[113,208],"fixes":[117,210],"vulnerability,":[120],"help":[122],"raise":[123],"users'":[124],"findings":[127,229],"show":[128],"all":[130],"models":[132],"struggle":[133],"accurately":[135],"detect":[136],"vulnerabilities,":[141],"achieving":[142],"detection":[144],"rate":[145],"only":[147],"12.6%":[148],"40%":[150],"across":[151],"our":[152,228],"datasets.":[153],"observe":[156],"tend":[160],"identify":[162],"certain":[163],"types":[164],"vulnerabilities":[166,212],"related":[167],"sensitive":[169],"exposure":[171],"improper":[173],"input":[174],"neutralization":[175],"much":[176],"more":[177,203,243],"frequently":[178],"than":[179],"other":[180],"types,":[181],"such":[182],"as":[183],"those":[184],"involving":[185],"external":[186],"control":[187],"file":[189],"names":[190],"paths.":[192],"Furthermore,":[193],"when":[194],"do":[196],"issue":[197],"warnings,":[199],"often":[201],"on":[205,224],"compared":[213],"responses.":[217,246],"Finally,":[218],"an":[221],"in-depth":[222],"discussion":[223],"implications":[226],"present":[231],"CLI-based":[233],"prompting":[234],"tool":[235],"can":[237],"be":[238],"used":[239],"generate":[241],"significantly":[242],"secure":[244]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}