{"id":"https://openalex.org/W4398766423","doi":"https://doi.org/10.1145/3639476.3639764","title":"Breaking the Silence: the Threats of Using LLMs in Software Engineering","display_name":"Breaking the Silence: the Threats of Using LLMs in Software Engineering","publication_year":2024,"publication_date":"2024-04-14","ids":{"openalex":"https://openalex.org/W4398766423","doi":"https://doi.org/10.1145/3639476.3639764"},"language":"en","primary_location":{"id":"doi:10.1145/3639476.3639764","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639476.3639764","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639764","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639764","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040439036","display_name":"June Sallou","orcid":"https://orcid.org/0000-0003-2230-9351"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"June Sallou","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043941055","display_name":"Thomas Durieux","orcid":"https://orcid.org/0000-0002-1996-6134"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Thomas Durieux","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067127346","display_name":"Annibale Panichella","orcid":"https://orcid.org/0000-0002-7395-3588"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Annibale Panichella","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5040439036"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":32.4726,"has_fulltext":false,"cited_by_count":69,"citation_normalized_percentile":{"value":0.9975145,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"102","last_page":"106"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9736999869346619,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/silence","display_name":"Silence","score":0.6417042016983032},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5103678107261658},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4344305992126465},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.4061678349971771},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.33101946115493774},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.12081381678581238},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.11302396655082703}],"concepts":[{"id":"https://openalex.org/C2781115785","wikidata":"https://www.wikidata.org/wiki/Q502261","display_name":"Silence","level":2,"score":0.6417042016983032},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5103678107261658},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4344305992126465},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.4061678349971771},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.33101946115493774},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.12081381678581238},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11302396655082703},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3639476.3639764","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639476.3639764","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639764","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3639476.3639764","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3639476.3639764","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3639476.3639764","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 ACM/IEEE 44th International Conference on Software Engineering: New Ideas and Emerging Results","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4398766423.pdf"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W1526710119","https://openalex.org/W1599808047","https://openalex.org/W1971650562","https://openalex.org/W2156723666","https://openalex.org/W3176423632","https://openalex.org/W3183304837","https://openalex.org/W3199249334","https://openalex.org/W4220722393","https://openalex.org/W4288080275","https://openalex.org/W4310881984","https://openalex.org/W4312632714","https://openalex.org/W4367059011","https://openalex.org/W4379743664","https://openalex.org/W4381687070","https://openalex.org/W4384304865","https://openalex.org/W4384345708","https://openalex.org/W4384345745","https://openalex.org/W4385187421","https://openalex.org/W4385269554","https://openalex.org/W4389520710"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2140535326","https://openalex.org/W1607271848","https://openalex.org/W4251121070","https://openalex.org/W2734809835","https://openalex.org/W2184961913","https://openalex.org/W1965764303","https://openalex.org/W2750153830","https://openalex.org/W2214987084"],"abstract_inverted_index":{"Large":[0],"Language":[1,101],"Models":[2],"(LLMs)":[3],"have":[4],"gained":[5],"considerable":[6],"traction":[7],"within":[8],"the":[9,44,60,82,112,132],"Software":[10],"Engineering":[11],"(SE)":[12],"community,":[13],"impacting":[14],"various":[15],"SE":[16,98,129],"tasks":[17],"from":[18,24],"code":[19,28],"completion":[20],"to":[21,27,59,105],"test":[22,135],"generation,":[23],"program":[25],"repair":[26],"summarization.":[29],"Despite":[30],"their":[31],"promise,":[32],"researchers":[33,99,130],"must":[34],"still":[35],"be":[36],"careful":[37],"as":[38,68],"numerous":[39],"intricate":[40],"factors":[41],"can":[42],"influence":[43],"outcomes":[45],"of":[46,62,84,94,111,134],"experiments":[47],"involving":[48],"LLMs.":[49],"This":[50],"paper":[51,90],"initiates":[52],"an":[53],"open":[54],"discussion":[55],"on":[56],"potential":[57],"threats":[58],"validity":[61],"LLM-based":[63,85],"research":[64,79],"including":[65],"issues":[66],"such":[67],"closed-source":[69],"models,":[70],"possible":[71],"data":[72,77],"leakage":[73],"between":[74],"LLM":[75,122],"training":[76],"and":[78,81,100,124],"evaluation,":[80],"reproducibility":[83],"findings.":[86],"In":[87],"response,":[88],"this":[89],"proposes":[91],"a":[92,125],"set":[93],"guidelines":[95,113],"tailored":[96],"for":[97,128],"Model":[102],"(LM)":[103],"providers":[104,123],"mitigate":[106],"these":[107],"concerns.":[108],"The":[109],"implications":[110],"are":[114],"illustrated":[115],"using":[116],"existing":[117],"good":[118],"practices":[119],"followed":[120],"by":[121],"practical":[126],"example":[127],"in":[131],"context":[133],"case":[136],"generation.":[137]},"counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":45},{"year":2024,"cited_by_count":15}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}