{"id":"https://openalex.org/W4404609282","doi":"https://doi.org/10.1109/tse.2024.3504286","title":"On Inter-Dataset Code Duplication and Data Leakage in Large Language Models","display_name":"On Inter-Dataset Code Duplication and Data Leakage in Large Language Models","alternate_titles":[],"publication_year":2024,"publication_date":"2024-11-21","ids":{"openalex":"https://openalex.org/W4404609282","doi":"https://doi.org/10.1109/tse.2024.3504286"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3504286","doi":"https://doi.org/10.1109/tse.2024.3504286","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3504286","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","fnege-3","jufo-3","norway-2","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W4404609282"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2401.07930","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008895111","display_name":"Jos\u00e9 Antonio Hern\u00e1ndez L\u00f3pez","orcid":"https://orcid.org/0000-0003-2439-2136","observed_orcids":["https://orcid.org/0000-0003-2439-2136"]},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Jos\u00e9 Antonio Hern\u00e1ndez L\u00f3pez","raw_affiliation_strings":["Link\u00f6ping University, Link\u00f6ping, Sweden","Link\u00f6ping University, Sweden"],"raw_orcid":"https://orcid.org/0000-0003-2439-2136","affiliations":[{"raw_affiliation_string":"Link\u00f6ping University, Link\u00f6ping, Sweden","institution_ids":["https://openalex.org/I102134673"]},{"raw_affiliation_string":"Link\u00f6ping University, Sweden","institution_ids":["https://openalex.org/I102134673"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014096938","display_name":"Boqi Chen","orcid":"https://orcid.org/0000-0002-1451-3603","observed_orcids":["https://orcid.org/0000-0002-1451-3603"]},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Boqi Chen","raw_affiliation_strings":["McGill University, Montreal, Canada","McGill University, Canada"],"raw_orcid":"https://orcid.org/0000-0002-1451-3603","affiliations":[{"raw_affiliation_string":"McGill University, Montreal, Canada","institution_ids":["https://openalex.org/I5023651"]},{"raw_affiliation_string":"McGill University, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004400004","display_name":"Mootez Saad","orcid":"https://orcid.org/0009-0008-8159-3632","observed_orcids":["https://orcid.org/0009-0008-8159-3632"]},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mootez Saad","raw_affiliation_strings":["Dalhousie University, Halifax, Canada","Dalhousie University, Canada"],"raw_orcid":"https://orcid.org/0009-0008-8159-3632","affiliations":[{"raw_affiliation_string":"Dalhousie University, Halifax, Canada","institution_ids":["https://openalex.org/I129902397"]},{"raw_affiliation_string":"Dalhousie University, Canada","institution_ids":["https://openalex.org/I129902397"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023044082","display_name":"Tushar Sharma","orcid":"https://orcid.org/0000-0002-0538-052X","observed_orcids":["https://orcid.org/0000-0002-0538-052X"]},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Tushar Sharma","raw_affiliation_strings":["Dalhousie University, Halifax, Canada","Dalhousie University, Canada"],"raw_orcid":"https://orcid.org/0000-0002-0538-052X","affiliations":[{"raw_affiliation_string":"Dalhousie University, Halifax, Canada","institution_ids":["https://openalex.org/I129902397"]},{"raw_affiliation_string":"Dalhousie University, Canada","institution_ids":["https://openalex.org/I129902397"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064890236","display_name":"D\u00e1niel Varr\u00f3","orcid":"https://orcid.org/0000-0002-8790-252X","observed_orcids":["https://orcid.org/0000-0002-8790-252X"]},"institutions":[{"id":"https://openalex.org/I102134673","display_name":"Link\u00f6ping University","ror":"https://ror.org/05ynxx418","country_code":"SE","type":"education","lineage":["https://openalex.org/I102134673"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"D\u00e1niel Varr\u00f3","raw_affiliation_strings":["Link\u00f6ping University, Link\u00f6ping, Sweden","Link\u00f6ping University, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-8790-252X","affiliations":[{"raw_affiliation_string":"Link\u00f6ping University, Link\u00f6ping, Sweden","institution_ids":["https://openalex.org/I102134673"]},{"raw_affiliation_string":"Link\u00f6ping University, Sweden","institution_ids":["https://openalex.org/I102134673"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2495,"currency":"USD","value_usd":2495},"apc_paid":null,"fwci":8.0503,"has_fulltext":true,"cited_by_count":42,"citation_normalized_percentile":{"value":0.97886276,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":100},"biblio":{"volume":"51","issue":"1","first_page":"192","last_page":"205"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-leakage","display_name":"data leakage","score":0.9900000095367432},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9810000061988831},{"id":"https://openalex.org/keywords/codesearchnet","display_name":"CodeSearchNet","score":0.8790000081062317},{"id":"https://openalex.org/keywords/codebert","display_name":"CodeBERT","score":0.6899999976158142},{"id":"https://openalex.org/keywords/graphcodebert","display_name":"GraphCodeBERT","score":0.9819999933242798},{"id":"https://openalex.org/keywords/unixcoder","display_name":"UniXcoder","score":0.6700000166893005},{"id":"https://openalex.org/keywords/software-engineering-tasks","display_name":"software engineering tasks","score":0.5789999961853027},{"id":"https://openalex.org/keywords/code-search","display_name":"code search","score":0.41200000047683716}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8354595899581909},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.49991583824157715},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4603528678417206},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.42298102378845215},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.337974488735199},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.23357230424880981},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tse.2024.3504286","doi":"https://doi.org/10.1109/tse.2024.3504286","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3504286","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","fnege-3","jufo-3","norway-2","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2401.07930","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.07930","pdf_url":"https://arxiv.org/pdf/2401.07930","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2401.07930","doi":"https://doi.org/10.48550/arxiv.2401.07930","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.07930","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2401.07930","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.07930","pdf_url":"https://arxiv.org/pdf/2401.07930","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1175115578","display_name":null,"funder_award_id":"AEI//10.13039/501100011033/","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G2262748287","display_name":null,"funder_award_id":"501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G300979063","display_name":null,"funder_award_id":"10.13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G34346518","display_name":null,"funder_award_id":"MCIN/AEI/10.13039/501 100 011 033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G3480869486","display_name":null,"funder_award_id":"13039","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G425668148","display_name":null,"funder_award_id":".13039/501100011033","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G5080475149","display_name":null,"funder_award_id":"10.13039","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G7084143925","display_name":null,"funder_award_id":"AEI/10","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G8489140898","display_name":null,"funder_award_id":"TED2021-129381B-C22","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"}],"funders":[{"id":"https://openalex.org/F4320322327","display_name":"Knut och Alice Wallenbergs Stiftelse","ror":"https://ror.org/004hzzk67"},{"id":"https://openalex.org/F4320335598","display_name":"Agencia Estatal de Investigaci\u00f3n","ror":"https://ror.org/003x0zc53"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404609282.pdf","grobid_xml":"https://content.openalex.org/works/W4404609282.grobid-xml"},"referenced_works_count":70,"referenced_works":["https://openalex.org/W1682403713","https://openalex.org/W2065053490","https://openalex.org/W2101105183","https://openalex.org/W2128782367","https://openalex.org/W2156723666","https://openalex.org/W2257123346","https://openalex.org/W2298313545","https://openalex.org/W2516621648","https://openalex.org/W2610548325","https://openalex.org/W2626778328","https://openalex.org/W2728773317","https://openalex.org/W2743316948","https://openalex.org/W2762844179","https://openalex.org/W2794601162","https://openalex.org/W2807964941","https://openalex.org/W2891332953","https://openalex.org/W2896457183","https://openalex.org/W2902875180","https://openalex.org/W2903709763","https://openalex.org/W2914501196","https://openalex.org/W2963341956","https://openalex.org/W2971008324","https://openalex.org/W2989195139","https://openalex.org/W3008088841","https://openalex.org/W3009153459","https://openalex.org/W3086007799","https://openalex.org/W3098605233","https://openalex.org/W3126675481","https://openalex.org/W3168867926","https://openalex.org/W3170092793","https://openalex.org/W3174770825","https://openalex.org/W3177813494","https://openalex.org/W3181341910","https://openalex.org/W3198685994","https://openalex.org/W3217001695","https://openalex.org/W4220672926","https://openalex.org/W4221166942","https://openalex.org/W4224051134","https://openalex.org/W4225716708","https://openalex.org/W4226485558","https://openalex.org/W4284667247","https://openalex.org/W4285225959","https://openalex.org/W4288055447","https://openalex.org/W4299358159","https://openalex.org/W4301136205","https://openalex.org/W4301168982","https://openalex.org/W4303450757","https://openalex.org/W4310428868","https://openalex.org/W4311431146","https://openalex.org/W4312999884","https://openalex.org/W4315706637","https://openalex.org/W4360981376","https://openalex.org/W4376167329","https://openalex.org/W4382246105","https://openalex.org/W4383215759","https://openalex.org/W4384154386","https://openalex.org/W4384304951","https://openalex.org/W4389104713","https://openalex.org/W4389519352","https://openalex.org/W4389980560","https://openalex.org/W4394638297","https://openalex.org/W4398766423","https://openalex.org/W4401906817","https://openalex.org/W4406688098","https://openalex.org/W6774629125","https://openalex.org/W6796581206","https://openalex.org/W6845109718","https://openalex.org/W6846665566","https://openalex.org/W7119001154","https://openalex.org/W7133237827"],"related_works":["https://openalex.org/W7155658653","https://openalex.org/W7127292246","https://openalex.org/W4407386460","https://openalex.org/W4411272397","https://openalex.org/W4400611194","https://openalex.org/W7154744223","https://openalex.org/W4402671023","https://openalex.org/W4411950710","https://openalex.org/W4401202026"],"abstract_inverted_index":{"Motivation.Large":[0],"language":[1],"models":[2,227,279],"(LLMs)":[3],"have":[4],"exhibited":[5],"remarkable":[6],"proficiency":[7],"in":[8,69,102,142,213],"diverse":[9],"software":[10],"engineering":[11],"(SE)":[12],"tasks,":[13],"such":[14,24,280],"as":[15,47],"code":[16,18,21,105,152,257,314],"summarization,":[17],"translation,":[19],"and":[20,40,86,98,154,174,189,217,228],"search.":[22],"Handling":[23],"tasks":[25],"typically":[26],"involves":[27],"acquiring":[28],"foundational":[29],"coding":[30],"knowledge":[31],"on":[32,43,157,289,309],"large,":[33],"general-purpose":[34],"datasets":[35,46,177,191],"during":[36,139],"a":[37,50,66,95,169,193,203,244],"pre-training":[38,172,188,215],"phase,":[39],"subsequently":[41],"refining":[42],"smaller,":[44],"task-specific":[45],"part":[48,238],"of":[49,56,71,77,83,128,131,150,239],"fine-tuning":[51,132,176,190,233,271],"phase.Problem":[52],"statement.Data":[53],"leakagei.e.,using":[54],"information":[55],"the":[57,62,81,84,109,124,129,148,184,187,210,240,248,255,269],"test":[58,133,234],"set":[59],"to":[60,247,301,311],"perform":[61],"model":[63],"training,":[64],"is":[65,80,266],"well-known":[67],"issue":[68,79],"training":[70,85],"machine":[72],"learning":[73],"models.":[74],"A":[75],"manifestation":[76],"this":[78,92,118,264,302],"intersection":[82,93,185,212],"testing":[87],"splits.":[88],"Whileintra-datasetcode":[89],"duplication":[90,153,258],"examines":[91],"within":[94],"given":[96],"dataset":[97],"has":[99],"been":[100],"addressed":[101],"prior":[103,295],"research,inter-dataset":[104],"duplication,":[106],"which":[107,208],"gauges":[108],"overlap":[110],"between":[111,186],"different":[112],"datasets,":[113],"remains":[114],"largely":[115],"unexplored.":[116],"If":[117],"phenomenon":[119,149],"exists,":[120],"it":[121],"could":[122],"compromise":[123],"integrity":[125],"ofLLMevaluations":[126],"because":[127],"inclusion":[130],"samples":[134,235],"that":[135,236,263,277,297],"were":[136],"already":[137],"encountered":[138],"pre-training,":[140],"resulting":[141],"inflated":[143],"performance":[144],"metrics.Contribution.This":[145],"paper":[146],"explores":[147],"inter-dataset":[151,256,286,313],"its":[155,214],"impact":[156],"evaluatingLLMs":[158],"across":[159,251],"diverseSEtasks.Study":[160],"design.We":[161],"conduct":[162],"an":[163],"empirical":[164],"study":[165],"using":[166,192,202,232],"theCodeSearchNetdataset":[167],"(csn),":[168],"widely":[170],"adopted":[171],"dataset,":[173],"five":[175],"used":[178],"for":[179],"variousSEtasks.":[180],"We":[181,260],"first":[182],"identify":[183],"deduplication":[194],"process.":[195],"Next,":[196],"we":[197,224,274,292,305],"pre-train":[198],"two":[199],"versions":[200],"ofLLMs":[201,250],"subset":[204],"ofcsn:":[205],"one":[206,218],"leakyLLM,":[207],"includes":[209],"identified":[211],"set,":[216],"non-leakyLLMthat":[219],"excludes":[220],"these":[221],"samples.":[222],"Finally,":[223],"fine-tune":[225],"both":[226],"compare":[229],"their":[230],"performances":[231],"are":[237],"intersection.Results.Our":[241],"findings":[242],"reveal":[243],"potential":[245],"threat":[246,265],"evaluation":[249],"multipleSEtasks,":[252],"stemming":[253],"from":[254],"phenomenon.":[259],"also":[261],"demonstrate":[262],"accentuated":[267],"by":[268,285],"chosen":[270],"technique.":[272],"Furthermore,":[273],"provide":[275],"evidence":[276],"open-source":[278],"asCodeBERT,GraphCodeBERT,":[281],"andUnixCodercould":[282],"be":[283,299],"affected":[284],"duplication.":[287,315],"Based":[288],"our":[290],"findings,":[291],"delve":[293],"into":[294],"research":[296],"may":[298],"susceptible":[300],"threat.":[303],"Additionally,":[304],"offer":[306],"guidance":[307],"toSEresearchers":[308],"strategies":[310],"prevent":[312]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4404609282","counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":2}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2024-11-22T00:00:00"}
